129 lines
4.9 KiB
Python
129 lines
4.9 KiB
Python
"""Phase 2.4 — router-generated synthetic RU corpus → data/cpt_raw/synth/
|
|
|
|
Generates raw natural-Russian prose/dialogue (NOT JSON, NOT {response,mood}
|
|
persona data — that is Phase 6, a different dataset) to cover registers the open
|
|
corpora lack: smart-home / assistant / conversational RU.
|
|
|
|
Uses an OpenAI-compatible endpoint = the user's free-provider router. Configure:
|
|
export ROUTER_BASE_URL="https://your-router/v1"
|
|
export ROUTER_API_KEY="..."
|
|
export ROUTER_MODELS="model-a,model-b,model-c" # rotated for diversity
|
|
|
|
HARD CAP: stops at 10% of TARGET_TOKENS (plan §2.4 / hard rule 2). Synthetic-heavy
|
|
CPT collapses. Rotate models so it isn't one model's fingerprint.
|
|
|
|
Every generation is filtered here (langID + mixed-script) before write; the full
|
|
cleaning + cross-corpus dedup still happens in corpus_clean.py.
|
|
|
|
Usage: python corpus_synth.py
|
|
"""
|
|
import os
|
|
import json
|
|
import random
|
|
import itertools
|
|
|
|
from openai import OpenAI
|
|
|
|
from corpus_common import (
|
|
RAW, TARGET_TOKENS, SYNTH_CAP,
|
|
has_mixed_script, cyrillic_ratio, lang_prob,
|
|
)
|
|
|
|
CHARS_PER_TOK = 3.5
|
|
TOPICS_FILE = os.path.join(os.path.dirname(__file__), "data", "topics.txt")
|
|
|
|
STYLES = [
|
|
"живой разговорный монолог",
|
|
"диалог двух людей",
|
|
"подробное объяснение",
|
|
"личная история от первого лица",
|
|
"инструкция или совет по бытовой теме",
|
|
]
|
|
# domains open corpora under-represent — the reason synth exists at all.
|
|
EXTRA_TOPICS = [
|
|
"умный дом: свет, розетки, сценарии", "голосовой помощник в квартире",
|
|
"напоминания и списки дел на день", "погода и планы на выходные",
|
|
"готовка ужина и покупки продуктов", "музыка и подкасты дома",
|
|
"будильники, таймеры и распорядок дня", "уборка и бытовые заботы",
|
|
]
|
|
|
|
PROMPT = (
|
|
"Напиши развёрнутый естественный текст на русском языке на тему: «{topic}». "
|
|
"Стиль: {style}. Объём 400–800 слов. Только связный русский текст без разметки, "
|
|
"без английских слов, без списков-маркеров и без JSON."
|
|
)
|
|
|
|
|
|
def load_topics():
|
|
topics = list(EXTRA_TOPICS)
|
|
if os.path.exists(TOPICS_FILE):
|
|
with open(TOPICS_FILE, encoding="utf-8") as f:
|
|
topics += [ln.strip() for ln in f if ln.strip()]
|
|
random.shuffle(topics)
|
|
return topics
|
|
|
|
|
|
def accept(text: str) -> bool:
|
|
"""Local gate before write (cheap). Full clean+dedup is corpus_clean.py."""
|
|
if len(text) < 400:
|
|
return False
|
|
if has_mixed_script(text):
|
|
return False
|
|
if cyrillic_ratio(text) < 0.85:
|
|
return False
|
|
try:
|
|
lang, p = lang_prob(text)
|
|
except FileNotFoundError:
|
|
lang, p = "ru", 1.0 # langID optional here; corpus_clean.py enforces it
|
|
return lang == "ru" and p >= 0.65
|
|
|
|
|
|
def main():
|
|
base_url = os.environ.get("ROUTER_BASE_URL")
|
|
api_key = os.environ.get("ROUTER_API_KEY", "none")
|
|
models = [m for m in os.environ.get("ROUTER_MODELS", "").split(",") if m]
|
|
if not base_url or not models:
|
|
raise SystemExit("set ROUTER_BASE_URL and ROUTER_MODELS (see module docstring)")
|
|
|
|
client = OpenAI(base_url=base_url, api_key=api_key)
|
|
budget_chars = int(TARGET_TOKENS * SYNTH_CAP * CHARS_PER_TOK) # hard cap
|
|
out_path = RAW / "synth" / "shard-000.jsonl"
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
topics = load_topics()
|
|
model_cycle = itertools.cycle(models)
|
|
written = kept = tried = 0
|
|
|
|
with open(out_path, "w", encoding="utf-8") as f:
|
|
for topic in itertools.cycle(topics):
|
|
if written >= budget_chars:
|
|
break
|
|
style = random.choice(STYLES)
|
|
model = next(model_cycle)
|
|
tried += 1
|
|
try:
|
|
resp = client.chat.completions.create(
|
|
model=model,
|
|
messages=[{"role": "user",
|
|
"content": PROMPT.format(topic=topic, style=style)}],
|
|
temperature=0.9, max_tokens=1200,
|
|
)
|
|
text = (resp.choices[0].message.content or "").strip()
|
|
except Exception as e:
|
|
print(f"[synth] gen error on {model}: {e}")
|
|
continue
|
|
if not accept(text):
|
|
continue
|
|
f.write(json.dumps({"text": text, "model": model}, ensure_ascii=False) + "\n")
|
|
written += len(text)
|
|
kept += 1
|
|
if kept % 50 == 0:
|
|
print(f"[synth] kept {kept}/{tried} tried, "
|
|
f"{written/1e6:.1f}M / {budget_chars/1e6:.1f}M chars cap")
|
|
|
|
print(f"[synth] DONE kept {kept}/{tried}, {written/1e6:.1f}M chars -> {out_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|