64 lines
2.1 KiB
Python
64 lines
2.1 KiB
Python
"""Phase 2.5 — mine Maven's real dialogue history → data/cpt_raw/logs/
|
|
|
|
Highest domain-value bucket but smallest (~2%, ~6M tok). Source format is unknown
|
|
here, so point MAVEN_LOGS at your dialogue store and adjust `extract()` to your
|
|
schema. Emits raw RU utterances as {"text": ...}; corpus_clean.py handles the rest
|
|
(langID drops EN turns, dedup, PII scrub).
|
|
|
|
Usage:
|
|
MAVEN_LOGS=/path/to/dialogue.jsonl python corpus_logs.py
|
|
"""
|
|
import os
|
|
import json
|
|
|
|
from corpus_common import RAW, cyrillic_ratio
|
|
|
|
|
|
def extract(obj):
|
|
"""Return utterance text from one log record. ADJUST to your schema.
|
|
|
|
Handles common shapes: {"text":...}, {"content":...},
|
|
{"messages":[{"role","content"}...]}, or a bare string.
|
|
"""
|
|
if isinstance(obj, str):
|
|
return [obj]
|
|
if not isinstance(obj, dict):
|
|
return []
|
|
if "messages" in obj:
|
|
return [m.get("content", "") for m in obj["messages"] if m.get("content")]
|
|
for k in ("text", "content", "utterance", "body"):
|
|
if obj.get(k):
|
|
return [obj[k]]
|
|
return []
|
|
|
|
|
|
def main():
|
|
src = os.environ.get("MAVEN_LOGS")
|
|
if not src or not os.path.exists(src):
|
|
raise SystemExit("set MAVEN_LOGS to your dialogue history (jsonl); "
|
|
"this bucket is optional — skip if unavailable")
|
|
|
|
out = RAW / "logs" / "shard-000.jsonl"
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
kept = 0
|
|
with open(src, encoding="utf-8") as fin, open(out, "w", encoding="utf-8") as fout:
|
|
for line in fin:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
obj = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
obj = line
|
|
for utt in extract(obj):
|
|
utt = (utt or "").strip()
|
|
# keep only substantive Russian turns; clean.py enforces the rest
|
|
if len(utt) >= 40 and cyrillic_ratio(utt) >= 0.5:
|
|
fout.write(json.dumps({"text": utt}, ensure_ascii=False) + "\n")
|
|
kept += 1
|
|
print(f"[logs] kept {kept} utterances -> {out}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|