72 lines
2.6 KiB
Python
72 lines
2.6 KiB
Python
"""Fetch immutable, human-written RU/EN held-out text for CPT evaluation.
|
|
|
|
Universal Dependencies test splits are independent of Maven's CPT source list.
|
|
Only sentence text from CoNLL-U comments is retained. Re-running is deterministic.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
OUT = Path(__file__).resolve().parent / "data" / "eval"
|
|
SOURCES = {
|
|
"ru": {
|
|
"commit": "0f34b7362ac3c3facd1d6ff4b876d241bb15793e",
|
|
"repo": "UD_Russian-GSD",
|
|
"file": "ru_gsd-ud-test.conllu",
|
|
},
|
|
"en": {
|
|
"commit": "4a4d77f599ea53cc405f85d0cec4b2f14f81d42b",
|
|
"repo": "UD_English-EWT",
|
|
"file": "en_ewt-ud-test.conllu",
|
|
},
|
|
}
|
|
|
|
|
|
def fetch(repo: str, commit: str, filename: str) -> bytes:
|
|
"""Fetch one pinned Git object; git transport is more reliable than raw CDN."""
|
|
with tempfile.TemporaryDirectory(prefix="maven-eval-") as tmp:
|
|
target = Path(tmp) / "repo"
|
|
subprocess.run(
|
|
["git", "clone", "--quiet", "--filter=blob:none", "--no-checkout",
|
|
f"https://github.com/UniversalDependencies/{repo}.git", str(target)],
|
|
check=True,
|
|
)
|
|
return subprocess.run(
|
|
["git", "-C", str(target), "show", f"{commit}:{filename}"],
|
|
check=True, capture_output=True,
|
|
).stdout
|
|
|
|
|
|
def sentences(raw: str) -> list[str]:
|
|
return [line[9:].strip() for line in raw.splitlines() if line.startswith("# text = ")]
|
|
|
|
|
|
def main() -> None:
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
manifest = {"schema_version": 1, "license": "UD treebanks; see source repositories"}
|
|
for lang, source in SOURCES.items():
|
|
url = (f"https://raw.githubusercontent.com/UniversalDependencies/"
|
|
f"{source['repo']}/{source['commit']}/{source['file']}")
|
|
raw = fetch(source["repo"], source["commit"], source["file"])
|
|
lines = sentences(raw.decode("utf-8"))
|
|
if len(lines) < 500:
|
|
raise SystemExit(f"unexpectedly small {lang} evaluation set: {len(lines)}")
|
|
output = OUT / f"{lang}_ud_test.txt"
|
|
output.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
manifest[lang] = {
|
|
**source, "url": url, "sentences": len(lines),
|
|
"raw_sha256": hashlib.sha256(raw).hexdigest(),
|
|
"text_sha256": hashlib.sha256(output.read_bytes()).hexdigest(),
|
|
}
|
|
(OUT / "manifest.json").write_text(
|
|
json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
print(json.dumps(manifest, indent=2, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|