init

llm/corpus_logs.py

@@ -0,0 +1,63 @@
+"""Phase 2.5 — mine Maven's real dialogue history → data/cpt_raw/logs/
+
+Highest domain-value bucket but smallest (~2%, ~6M tok). Source format is unknown
+here, so point MAVEN_LOGS at your dialogue store and adjust `extract()` to your
+schema. Emits raw RU utterances as {"text": ...}; corpus_clean.py handles the rest
+(langID drops EN turns, dedup, PII scrub).
+
+Usage:
+    MAVEN_LOGS=/path/to/dialogue.jsonl python corpus_logs.py
+"""
+import os
+import json
+
+from corpus_common import RAW, cyrillic_ratio
+
+
+def extract(obj):
+    """Return utterance text from one log record. ADJUST to your schema.
+
+    Handles common shapes: {"text":...}, {"content":...},
+    {"messages":[{"role","content"}...]}, or a bare string.
+    """
+    if isinstance(obj, str):
+        return [obj]
+    if not isinstance(obj, dict):
+        return []
+    if "messages" in obj:
+        return [m.get("content", "") for m in obj["messages"] if m.get("content")]
+    for k in ("text", "content", "utterance", "body"):
+        if obj.get(k):
+            return [obj[k]]
+    return []
+
+
+def main():
+    src = os.environ.get("MAVEN_LOGS")
+    if not src or not os.path.exists(src):
+        raise SystemExit("set MAVEN_LOGS to your dialogue history (jsonl); "
+                         "this bucket is optional — skip if unavailable")
+
+    out = RAW / "logs" / "shard-000.jsonl"
+    out.parent.mkdir(parents=True, exist_ok=True)
+    kept = 0
+    with open(src, encoding="utf-8") as fin, open(out, "w", encoding="utf-8") as fout:
+        for line in fin:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                obj = json.loads(line)
+            except json.JSONDecodeError:
+                obj = line
+            for utt in extract(obj):
+                utt = (utt or "").strip()
+                # keep only substantive Russian turns; clean.py enforces the rest
+                if len(utt) >= 40 and cyrillic_ratio(utt) >= 0.5:
+                    fout.write(json.dumps({"text": utt}, ensure_ascii=False) + "\n")
+                    kept += 1
+    print(f"[logs] kept {kept} utterances -> {out}")
+
+
+if __name__ == "__main__":
+    main()