114 lines
4.5 KiB
Python
114 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Slice 22 emit: five-way residual non-action semantic router — data files
|
|
========================================================================
|
|
|
|
Slice 21 accepted the deterministic execution-frame guard (docs/evals/
|
|
2026-09-07-execution-frame-guard.md). Slice 22 returns to the coarse non-action
|
|
router that the guard hands to: after TryFastPath misses and the guard passes,
|
|
the remaining utterance is one of five non-action semantics — conversation,
|
|
knowledge, memory_write, system, uncertain. Action rows never reach this
|
|
router; they are usable only as out-of-domain probes, never in primary metrics.
|
|
|
|
This script only repackages the frozen dev pool for the Go legacy baseline and
|
|
the Python experiment. It reuses slice 18's loader/filters and slice 19's
|
|
normalizers verbatim, so the population here is the same one slices 18-21
|
|
measured. It writes:
|
|
|
|
/tmp/mvn-s22/pool.json residual non-action dev rows: idx, text, n_text,
|
|
route, tags, cv_fold, split_group, family_id,
|
|
source_id (1652 rows)
|
|
/tmp/mvn-s22/ood.json residual ACTION dev rows (766): same shape; OOD
|
|
probes only, never primary metrics
|
|
/tmp/mvn-s22/stats.json population summary (routes, families, folds)
|
|
|
|
idx is the row's position among dev_pool rows in dev-pool order, so the Python
|
|
experiment can align the embedding vectors from /tmp/mvn-experiment/embeddings.json
|
|
by index exactly as slice19.load_dev does.
|
|
|
|
No training happens here and no label is changed.
|
|
|
|
Population (verified 2026-09-08 from the frozen file):
|
|
dev_pool 2490
|
|
dev residual 2418 (= dev_pool minus fast_path_resolved)
|
|
residual non-action 1652 knowledge 715 / memory_write 553 / system 184 /
|
|
uncertain 107 / conversation 93
|
|
residual action 766 (OOD probes only)
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.insert(0, HERE)
|
|
|
|
import slice18_sparse # noqa: E402 (normalize_match_text, load_data, filters)
|
|
|
|
OUT_DIR = "/tmp/mvn-s22"
|
|
|
|
ROUTES = ["conversation", "knowledge", "memory_write", "system", "uncertain"]
|
|
|
|
|
|
def main():
|
|
meta, examples = slice18_sparse.load_data()
|
|
dev = slice18_sparse.filter_dev_pool(examples)
|
|
print(f"dev pool: {len(dev)} rows "
|
|
f"(meta declares dev_count={meta.get('dev_count')})")
|
|
|
|
rows = []
|
|
for i, e in enumerate(dev):
|
|
if not e["fast_path_resolved"]:
|
|
rows.append({
|
|
"idx": i,
|
|
"text": e["text"],
|
|
"n_text": slice18_sparse.normalize_match_text(e["text"]),
|
|
"route": e["route"],
|
|
"tags": sorted(set(e.get("tags", []))),
|
|
"cv_fold": e["cv_fold"],
|
|
"split_group": e["split_group"],
|
|
"family_id": e["family_id"],
|
|
"source_id": e["source_id"],
|
|
})
|
|
|
|
na = [r for r in rows if r["route"] != "action"]
|
|
ood = [r for r in rows if r["route"] == "action"]
|
|
print(f"residual rows: {len(rows)} non-action: {len(na)} action(OOD): {len(ood)}")
|
|
|
|
by_route = {}
|
|
for r in na:
|
|
by_route[r["route"]] = by_route.get(r["route"], 0) + 1
|
|
print("routes:", by_route)
|
|
assert sum(by_route.values()) == len(na)
|
|
assert set(ROUTES) == set(by_route), "route set must be the five-way"
|
|
|
|
by_family = {}
|
|
for r in na:
|
|
by_family[r["family_id"]] = by_family.get(r["family_id"], 0) + 1
|
|
by_fold = {}
|
|
for r in na:
|
|
by_fold[r["cv_fold"]] = by_fold.get(r["cv_fold"], 0) + 1
|
|
print(f"family_ids: {len(by_family)} split_groups: {len(set(r['split_group'] for r in na))}")
|
|
print("folds:", by_fold)
|
|
|
|
os.makedirs(OUT_DIR, exist_ok=True)
|
|
with open(os.path.join(OUT_DIR, "pool.json"), "w") as f:
|
|
json.dump(na, f, ensure_ascii=False, indent=1)
|
|
with open(os.path.join(OUT_DIR, "ood.json"), "w") as f:
|
|
json.dump(ood, f, ensure_ascii=False, indent=1)
|
|
with open(os.path.join(OUT_DIR, "stats.json"), "w") as f:
|
|
json.dump({
|
|
"dev_count": len(dev),
|
|
"residual_count": len(rows),
|
|
"non_action_count": len(na),
|
|
"action_ood_count": len(ood),
|
|
"routes": by_route,
|
|
"family_ids": len(by_family),
|
|
"folds": by_fold,
|
|
"top_family": dict(sorted(by_family.items(), key=lambda kv: -kv[1])[:15]),
|
|
}, f, ensure_ascii=False, indent=1)
|
|
print(f"wrote {OUT_DIR}/{{pool,ood,stats}}.json")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |