init

bot/services/chunker.py

@@ -0,0 +1,45 @@
+from bot.models import ChannelMessage
+
+MAX_TOKENS_PER_CHUNK = 25_000
+CHARS_PER_TOKEN = 3
+
+
+def _format_message(msg: ChannelMessage) -> str:
+    parts = [f"[{msg.date:%Y-%m-%d %H:%M}]"]
+    stats = []
+    if msg.views is not None:
+        stats.append(f"{msg.views} views")
+    if msg.forwards is not None:
+        stats.append(f"{msg.forwards} fwd")
+    if msg.replies is not None:
+        stats.append(f"{msg.replies} replies")
+    if stats:
+        parts.append(f"({', '.join(stats)})")
+    parts.append(f"\n{msg.text}")
+    return " ".join(parts)
+
+
+def chunk_messages(messages: list[ChannelMessage]) -> list[str]:
+    """Split messages into token-bounded chunks."""
+    max_chars = MAX_TOKENS_PER_CHUNK * CHARS_PER_TOKEN
+    chunks: list[str] = []
+    current_lines: list[str] = []
+    current_len = 0
+
+    for msg in messages:
+        formatted = _format_message(msg)
+        entry = formatted + "\n---\n"
+        entry_len = len(entry)
+
+        if current_len + entry_len > max_chars and current_lines:
+            chunks.append("".join(current_lines))
+            current_lines = []
+            current_len = 0
+
+        current_lines.append(entry)
+        current_len += entry_len
+
+    if current_lines:
+        chunks.append("".join(current_lines))
+
+    return chunks