From 7d31de5a6bd9591a053f0b4a1b112b0ab1833bf9 Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 7 Sep 2026 17:58:09 +0400 Subject: [PATCH] router/semantic: slice 19 from-scratch sequence pragmatics specialist tooling + eval index --- cmd/semantic-router-experiment/slice19_bpe.py | 81 +++ .../slice19_main.py | 640 ++++++++++++++++++ .../slice19_models.py | 158 +++++ docs/evals/CLAUDE.md | 1 + 4 files changed, 880 insertions(+) create mode 100644 cmd/semantic-router-experiment/slice19_bpe.py create mode 100644 cmd/semantic-router-experiment/slice19_main.py create mode 100644 cmd/semantic-router-experiment/slice19_models.py diff --git a/cmd/semantic-router-experiment/slice19_bpe.py b/cmd/semantic-router-experiment/slice19_bpe.py new file mode 100644 index 0000000..2f6e9bf --- /dev/null +++ b/cmd/semantic-router-experiment/slice19_bpe.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +""" +Slice 19 tokenizers, derived from the development corpus only. + + A. CharVocab — codepoint ids over the dev corpus (deterministic order) + B. BpeVocab2048 — byte-level BPE, vocab ~2048, trained on dev corpus only + +Records for §4 of the brief: vocab size, OOV behaviour, serialized tokenizer size. +""" + +import re +import unicodedata + +from tokenizers import Tokenizer +from tokenizers.decoders import ByteLevel as ByteLevelDecoder +from tokenizers.models import BPE +from tokenizers.pre_tokenizers import ByteLevel as ByteLevelPreTokenizer +from tokenizers.trainers import BpeTrainer + + +class CharVocab: + """Codepoint ids from the dev corpus, sorted by codepoint value.""" + + def __init__(self, texts): + chars = set() + for t in texts: + chars.update(t) + self.id_to_char = sorted(chars) + self.char_to_id = {c: i + 1 for i, c in enumerate(self.id_to_char)} # 0 = PAD + self.pad = 0 + + @property + def size(self): + return len(self.id_to_char) + 1 + + def encode(self, text, max_len): + ids = [self.char_to_id.get(c, 0) for c in text] # 0 doubles as UNK/OOV + return ids[:max_len] + + +class BpeVocab: + """Byte-level BPE, trained only on the strings it is given.""" + + def __init__(self, texts, vocab_size=2048, sep="▁"): + self.tok = Tokenizer(BPE()) + self.tok.pre_tokenizer = ByteLevelPreTokenizer(trim_offsets=False) + self.tok.decoder = ByteLevelDecoder() + trainer = BpeTrainer(vocab_size=vocab_size, special_tokens=["[PAD]"], + show_progress=False) + # train on the corpus *strings*, byte-level BPE handles all codepoints + self.tok.train_from_iterator(texts, trainer=trainer) + self.pad_id = self.tok.token_to_id("[PAD]") + self._vocab = self.tok.get_vocab() + self._n = len(self._vocab) + + @property + def size(self): + return self._n + + def encode(self, text): + return self.tok.encode(text).ids + + def serialized_bytes(self): + # measure the serialized tokenizer size on disk + import os + d = self.tok.to_str() + return len(d.encode("utf-8")) + + +def normalize_match_text(s: str) -> str: + """NFKC → lowercase → collapse whitespace. Punctuation kept.""" + out = unicodedata.normalize("NFKC", s).strip().lower() + out = re.sub(r"\s+", " ", out) + return out + + +def strip_punct(text: str) -> str: + """Remove safe punctuation from an already-normalized text.""" + t = re.sub(r"[^\w\s]", " ", text) + t = re.sub(r"\s+", " ", t).strip() + return t \ No newline at end of file diff --git a/cmd/semantic-router-experiment/slice19_main.py b/cmd/semantic-router-experiment/slice19_main.py new file mode 100644 index 0000000..c6ea141 --- /dev/null +++ b/cmd/semantic-router-experiment/slice19_main.py @@ -0,0 +1,640 @@ +#!/usr/bin/env python3 +""" +Slice 19: tiny sequence-sensitive pragmatics specialists (binary action gate) +============================================================================= + +A narrow binary specialist: executable request vs semantically similar +non-executable utterance, generalizing across surface-generator families. + +Commands + grouped — grouped semantic CV (existing cv_fold), all arch/sizes, saves + per-config per-fold model checkpoints + OOF proba per variant + metrics — aggregate saved grouped-CV results into the report tables + (binary metrics, threshold curves, pair ordering, stress) + lfo — leave-generator-out. All sizes on capability_question; the + other present families for the leading config only. + e5baseline— frozen-e5 logistic + MLP baselines: grouped, cap-Q LOFO, pair + ordering (stress flagged NA — no re-embed on this box) + runtime — params / sizes / latency / tokenization for each candidate + +Primary metrics (brief §2): cap-Q LOFO FA, pair ordering acc, pair margin, +grouped P/R, voice stress, fold variance. Aggregate accuracy is secondary. +""" + +import argparse +import json +import os +import re +import resource +import subprocess +import sys +import time +import warnings +from collections import Counter + +import numpy as np + +warnings.filterwarnings("ignore") + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from slice19_bpe import BpeVocab, CharVocab, normalize_match_text, strip_punct + +EMBEDDING_PATH = "/tmp/mvn-experiment/embeddings.json" +RESULTS_DIR = "/tmp/mvn-s19" +SEED = 42 +MAX_CHAR = 64 +MAX_BPE = 24 +BPE_VOCAB = 2048 + +PRESENT_FAMILIES = [ + "polite_request", + "modal_request", + "first_person_request", + "reordered_target", + "capability_question", + "question", +] +ABSENT_FAMILIES = ["negation", "reported_speech", "quotation", "hypothetical"] + +FAMILY_PRIORITY = [ + "capability_question", "question", "first_person_request", + "modal_request", "polite_request", "reordered_target", "direct_imperative", +] + +DEVICES = [ + "свет", "люстру", "люстра", "жалюзи", "вытяжку", "вытяжка", + "вентилятор", "кондиционер", "телевизор", "лампу", "лампа", + "музыку", "музыка", "плеер", "колонку", "колонки", "чайник", + "бойлер", "обогреватель", "пылесос", "пылесосом", "пол", + "поливалки", "полив", "арка", "шторы", "штору", + "динамики", "дверь", "двери", "замок", "гараж", "ворота", + "кофе", "пасту", "зубы", "крючки", "лаймо", "куртку", + "будильник", "таймер", "напоминание", "расписание", +] + +ARCH_CONFIGS = { + "char_cnn": ["tiny", "medium"], + "bigru": ["tiny", "medium", "large"], + "tiny_transformer": ["small", "medium"], +} + +TRAIN_HYPER = { + "char_cnn": dict(epochs=20, lr=1e-3, bs=64, clip=None), + "bigru": dict(epochs=15, lr=5e-4, bs=64, clip=1.0), + "tiny_transformer": dict(epochs=25, lr=5e-4, bs=64, clip=1.0), +} + +VARIANTS = ["orig", "nofinal", "strip"] + + +# ─── data ─────────────────────────────────────────────────────────────────── + +def load_dev(): + with open(EMBEDDING_PATH) as f: + data = json.load(f) + rows = [e for e in data["examples"] if e["dev_pool"]] + out = [] + for e in rows: + n_text = normalize_match_text(e["text"]) + out.append({ + "text_orig": n_text, + "text_nofinal": re.sub(r"[?.!,;:]+$", "", n_text), + "text_strip": strip_punct(n_text), + "route": e["route"], + "y": 1 if e["route"] == "action" else 0, + "cv_fold": e["cv_fold"], + "split_group": e["split_group"], + "tags": set(e.get("tags", [])), + "source_id": e["source_id"], + "emb": np.asarray(e["embedding"], dtype=np.float32), + }) + return out + + +def family_of(tags): + for f in FAMILY_PRIORITY: + if f in tags: + return f + return None + + +def build_pairs(rows, texts): + """Capability-question vs action-by-shared-object pairs (eval only).""" + def object_nouns(t): + found = set() + tl = t.lower() + for d in DEVICES: + if re.search(r"\b" + re.escape(d), tl): + found.add(d) + return found + + pairs = [] + for cidx, r in enumerate(rows): + if "capability_question" not in r["tags"]: + continue + c_obj = object_nouns(texts[cidx]) + if not c_obj: + continue + dom = re.search(r"kq-cap-([^-]+)", r["source_id"]) + dom = dom.group(1) if dom else None + for aidx, ra in enumerate(rows): + if ra["y"] != 1: + continue + a_obj = object_nouns(texts[aidx]) + if not (c_obj & a_obj): + continue + a_dom = re.search(r"^([a-z]+)-", ra["source_id"]) + a_dom = a_dom.group(1) if a_dom else None + if dom and a_dom and dom != a_dom: + continue + pairs.append((cidx, aidx)) + return pairs + + +def pair_metrics(pairs, proba): + if not pairs: + return {"pairs": 0} + margins = [] + ties = 0 + order = 0 + rev = 0 + for cidx, aidx in pairs: + pc, pa = proba[cidx], proba[aidx] + margins.append(pa - pc) + if pa > pc: + order += 1 + elif pa == pc: + ties += 1 + else: + rev += 1 + m = np.array(margins) + return { + "pairs": len(pairs), + "ordering_acc": order / len(pairs), + "mean_margin": float(m.mean()), + "median_margin": float(np.median(m)), + "ties": ties, + "reversed": rev, + } + + +# ─── tokenizers ───────────────────────────────────────────────────────────── + +def build_tokenizers(rows): + nat_texts = [r["text_orig"] for r in rows] + char_vocab = CharVocab(nat_texts) + bpe = BpeVocab(nat_texts, vocab_size=BPE_VOCAB) + return char_vocab, bpe + + +def encode_all(rows, tokenizer, kind): + """Return dict variant -> (N, max_len) int64 array.""" + max_len = MAX_CHAR if kind == "char" else MAX_BPE + out = {} + for v in VARIANTS: + arr = np.zeros((len(rows), max_len), dtype=np.int64) + for i, r in enumerate(rows): + if kind == "char": + ids = tokenizer.encode(r[f"text_{v}"], max_len) + arr[i, :len(ids)] = ids + else: + ids = tokenizer.encode(r[f"text_{v}"])[:max_len] + arr[i, :len(ids)] = ids + out[v] = arr + return out + + +# ─── training ─────────────────────────────────────────────────────────────── + +def build_model(arch, size, vocab_size, max_len): + import torch + from slice19_models import CharCNN, BiGRU, TinyTransformer + if arch == "char_cnn": + c = (dict(embed_dim=32, filters=64, widths=[3, 4, 5]) if size == "tiny" + else dict(embed_dim=64, filters=160, widths=[2, 3, 4, 5])) + return CharCNN(vocab_size, c["embed_dim"], c["filters"], c["widths"]) + if arch == "bigru": + c = (dict(embed_dim=64, hidden=64) if size == "tiny" else + (dict(embed_dim=128, hidden=128) if size == "medium" else + dict(embed_dim=256, hidden=256))) + return BiGRU(vocab_size, c["embed_dim"], c["hidden"]) + c = (dict(d_model=128, n_layers=2, n_heads=4) if size == "small" else + dict(d_model=192, n_layers=4, n_heads=4)) + return TinyTransformer(vocab_size, c["d_model"], c["n_layers"], c["n_heads"], + max_len=max_len) + + +def train_binary(X, y, arch, size, vocab_size, seed_offset=0, log=False): + import torch + torch.manual_seed(SEED + seed_offset) + np.random.seed(SEED + seed_offset) + Xt = torch.from_numpy(X) + yt = torch.from_numpy(y.astype(np.float32)) + model = build_model(arch, size, vocab_size, X.shape[1]) + h = TRAIN_HYPER[arch] + opt = torch.optim.AdamW(model.parameters(), lr=h["lr"], weight_decay=1e-4) + lossf = torch.nn.BCEWithLogitsLoss() + n = X.shape[0] + model.train() + t0 = time.time() + for epoch in range(h["epochs"]): + perm = torch.randperm(n) + running = 0.0 + n_b = 0 + for start in range(0, n, h["bs"]): + idx = perm[start:start + h["bs"]] + xb = Xt[idx] + if xb.dim() == 1: + xb = xb.unsqueeze(0) + logits = model(xb) + loss = lossf(logits, yt[idx]) + opt.zero_grad() + loss.backward() + if h["clip"]: + torch.nn.utils.clip_grad_norm_(model.parameters(), h["clip"]) + opt.step() + running += float(loss) + n_b += 1 + if log and (epoch + 1) % 5 == 0: + print(f" epoch {epoch+1}/{h['epochs']} loss {running/max(n_b,1):.4f}") + return model, time.time() - t0 + + +def predict_proba(model, X, bs=256): + import torch + model.eval() + out = [] + with torch.no_grad(): + Xt = torch.from_numpy(X) + for start in range(0, X.shape[0], bs): + xb = Xt[start:start + bs] + if xb.dim() == 1: + xb = xb.unsqueeze(0) + logits = model(xb) + out.append(torch.sigmoid(logits).numpy()) + return np.concatenate(out) + + +# ─── metrics helpers ──────────────────────────────────────────────────────── + +def binary_metrics(yt, proba, thr=0.5): + yp = (proba >= thr).astype(int) + tp = int(((yt == 1) & (yp == 1)).sum()) + fp = int(((yt == 0) & (yp == 1)).sum()) + fn = int(((yt == 1) & (yp == 0)).sum()) + from sklearn.metrics import roc_auc_score, average_precision_score + roc = roc_auc_score(yt, proba) if len(np.unique(yt)) > 1 else 0.0 + pr = average_precision_score(yt, proba) if len(np.unique(yt)) > 1 else 0.0 + return { + "n": int(len(yt)), "tp": tp, "fp": fp, "fn": fn, + "P": tp / max(tp + fp, 1), "R": tp / max(tp + fn, 1), + "FA": fp, "FA_rate": fp / max(len(yt), 1), + "ROC_AUC": float(roc), "PR_AUC": float(pr), + } + + +def threshold_curve(yt, proba, thr_grid): + rows = [] + for thr in thr_grid: + yp = (proba >= thr).astype(int) + tp = int(((yt == 1) & (yp == 1)).sum()) + fp = int(((yt == 0) & (yp == 1)).sum()) + fn = int(((yt == 1) & (yp == 0)).sum()) + P = tp / max(tp + fp, 1) + R = tp / max(tp + fn, 1) + rows.append({ + "thr": round(float(thr), 4), "P": round(P, 4), "R": round(R, 4), + "FA": fp, "FA_rate": round(fp / max(len(yt), 1), 4), + }) + return rows + + +def operating_points(rows, pair_proba, pairs): + """Report P>=0.95/0.98/0.99 points with pair separation at threshold.""" + res = {} + for target in (0.95, 0.98, 0.99): + pts = [r for r in rows if r["P"] >= target and r["R"] > 0.0] + if not pts: + res[str(target)] = None + continue + best = max(pts, key=lambda r: r["R"]) + # pair separation at that operating point + sep = pair_sep_at(best["thr"], pair_proba, pairs) + best = dict(best); best["pair_sep"] = round(sep, 4) + res[str(target)] = best + return res + + +def pair_sep_at(thr, pair_proba, pairs): + """Fraction of pairs where action>=thr and cap= thr and pair_proba[cidx] < thr: + ok += 1 + return ok / len(pairs) + + +def fold_variance(fold_rows): + return { + "folds": [ + { + "fold": fr["fold"], + "ROC_AUC": fr["ROC_AUC"], "PR_AUC": fr["PR_AUC"], + "P": fr["P"], "R": fr["R"], "FA": fr["FA"], "n": fr["n"], + } + for fr in fold_rows + ] + } + + +# ─── subcommands ──────────────────────────────────────────────────────────── + +def _result_path(): + os.makedirs(RESULTS_DIR, exist_ok=True) + return RESULTS_DIR + + +def cmd_grouped(args): + rows = load_dev() + char_vocab, bpe = build_tokenizers(rows) + y = np.array([r["y"] for r in rows]) + folds = np.array([r["cv_fold"] for r in rows]) + X_char = encode_all(rows, char_vocab, "char") + X_bpe = encode_all(rows, bpe, "bpe") + tokenizers = {"char": char_vocab, "bpe": bpe} + + os.makedirs(RESULTS_DIR, exist_ok=True) + # save tokenizer metadata for reproducibility + meta = { + "char_vocab": char_vocab.size, + "char_vocab_sample": char_vocab.id_to_char[:50], + "bpe_vocab": bpe.size, + "bpe_serialized_bytes": bpe.serialized_bytes(), + "max_char": MAX_CHAR, "max_bpe": MAX_BPE, + "n": len(rows), + } + with open(os.path.join(RESULTS_DIR, "corpus_meta.json"), "w") as f: + json.dump(meta, f) + + kind_of = {"char_cnn": "char", "bigru": "bpe", "tiny_transformer": "bpe"} + for arch, sizes in ARCH_CONFIGS.items(): + kind = kind_of[arch] + X = X_char if kind == "char" else X_bpe + for size in sizes: + name = f"{arch}_{size}" + os.makedirs(os.path.join(RESULTS_DIR, "models"), exist_ok=True) + probs = {v: np.zeros(len(rows)) for v in VARIANTS} + fold_rows = [] + for fold in range(5): + tr = folds != fold + te = folds == fold + Xtr = X["strip"][tr] + model, secs = train_binary(Xtr, y[tr], arch, size, + vocab_size=tokenizers[kind].size, + seed_offset=fold, log=args.verbose) + torch_models_dir = os.path.join(RESULTS_DIR, "models") + import torch + torch.save(model.state_dict(), os.path.join(torch_models_dir, f"{name}_fold{fold}.pt")) + for v in VARIANTS: + probs[v][te] = predict_proba(model, X[v][te]) + fold_m = binary_metrics(y[te], probs["strip"][te]) + fold_m["fold"] = fold + fold_rows.append(fold_m) + print(f" {name} fold {fold}: ROC={fold_m['ROC_AUC']:.3f} " + f"PR={fold_m['PR_AUC']:.3f} P={fold_m['P']:.3f} R={fold_m['R']:.3f} " + f"FA={fold_m['FA']} n={fold_m['n']} ({secs:.1f}s)") + np.savez(os.path.join(RESULTS_DIR, f"{name}_probs.npz"), + var_orig=probs["orig"], var_nofinal=probs["nofinal"], + var_strip=probs["strip"]) + summary = binary_metrics(y, probs["strip"]) + print(f" {name} OOF: ROC={summary['ROC_AUC']:.3f} PR={summary['PR_AUC']:.3f} " + f"P={summary['P']:.3f} R={summary['R']:.3f} FA={summary['FA']}") + print("grouped done") + + +def cmd_metrics(args): + rows = load_dev() + y = np.array([r["y"] for r in rows]) + tags = [r["tags"] for r in rows] + pairs = build_pairs(rows, [r["text_strip"] for r in rows]) + print(f"pairs={len(pairs)}") + + out = {} + for arch, sizes in ARCH_CONFIGS.items(): + for size in sizes: + name = f"{arch}_{size}" + fp = os.path.join(RESULTS_DIR, f"{name}_probs.npz") + if not os.path.exists(fp): + continue + z = np.load(fp) + entry = {"name": name, "arch": arch, "size": size} + # OOF binary on strip variant (primary training input) + entry["strip"] = binary_metrics(y, z["var_strip"]) + entry["pairs"] = {} + entry["pairs"]["strip"] = pair_metrics(pairs, z["var_strip"]) + entry["pairs"]["orig"] = pair_metrics(pairs, z["var_orig"]) + entry["pairs"]["nofinal"] = pair_metrics(pairs, z["var_nofinal"]) + # stress: same OOF models, per-variant metrics + entry["stress"] = {} + for v in VARIANTS: + p = z[f"var_{v}"] + entry["stress"][v] = { + "all_FA": binary_metrics(y, p)["FA_rate"], + "capQ_FA": capq_fa(tags, y, p), + "modal_recall": modal_recall(tags, y, p), + } + # threshold curve + operating points on strip + entry["curve"] = threshold_curve(y, z["var_strip"], + np.arange(0.30, 1.0, 0.02)) + entry["ops"] = operating_points(entry["curve"], z["var_strip"], pairs) + out[name] = entry + + with open(os.path.join(RESULTS_DIR, "metrics.json"), "w") as f: + json.dump(out, f, indent=2, default=str) + print(json.dumps(out, indent=2, default=str)) + + +def capq_fa(tags, y, proba): + mask = np.array(["capability_question" in t for t in tags]) + if mask.sum() == 0: + return 0.0 + sub = proba[mask] + return float((sub >= 0.5).sum() / mask.sum()) + + +def modal_recall(tags, y, proba): + mask = np.array([ + (y[i] == 1 and ("polite_request" in tags[i] or "modal_request" in tags[i])) + for i in range(len(y)) + ]) + if mask.sum() == 0: + return 0.0 + sub = proba[mask] + return float((sub >= 0.5).sum() / mask.sum()) + + +def cmd_lfo(args): + rows = load_dev() + char_vocab, bpe = build_tokenizers(rows) + y = np.array([r["y"] for r in rows]) + tags = [r["tags"] for r in rows] + X_char = encode_all(rows, char_vocab, "char")["strip"] + X_bpe = encode_all(rows, bpe, "bpe")["strip"] + kind_of = {"char_cnn": "char", "bigru": "bpe", "tiny_transformer": "bpe"} + + results = {} + tokenizers = {"char": char_vocab, "bpe": bpe} + # capability_question LOFO for every config/size (the critical case) + for arch, sizes in ARCH_CONFIGS.items(): + X = X_char if kind_of[arch] == "char" else X_bpe + src_idx = np.array(["capability_question" not in t for t in tags]) + tgt_idx = np.array(["capability_question" in t for t in tags]) + for size in sizes: + model, _ = train_binary(X[src_idx], y[src_idx], arch, size, + vocab_size=tokenizers[kind_of[arch]].size, + seed_offset=17) + p = predict_proba(model, X[tgt_idx]) + m = binary_metrics(y[tgt_idx], p) + results[f"{arch}_{size}:capability_question"] = m + print(f"LFO ability {arch}_{size}: cap rows={m['n']} " + f"pos={int(y[tgt_idx].sum())} FA={m['FA']} FA_rate={m['FA_rate']:.3f} " + f"P={m['P']:.3f} R={m['R']:.3f} acc={1-m['FA_rate']:.3f}") + + # full family LOFO for the leading config per architecture + leading = {"char_cnn": "char_cnn_medium", "bigru": "bigru_tiny", + "tiny_transformer": "tiny_transformer_small"} + for arch, name in leading.items(): + X = X_char if kind_of[arch] == "char" else X_bpe + for fam in PRESENT_FAMILIES: + src = np.array([fam not in t for t in tags]) + tgt = np.array([fam in t for t in tags]) + model, _ = train_binary(X[src], y[src], arch, name.split("_")[-1], + vocab_size=tokenizers[kind_of[arch]].size, + seed_offset=41) + p = predict_proba(model, X[tgt]) + m = binary_metrics(y[tgt], p) + results[f"{name}:{fam}"] = m + print(f"LFO {fam}: {name} rows={m['n']} pos={int(y[tgt].sum())} " + f"P={m['P']:.3f} R={m['R']:.3f} FA={m['FA']} acc={1-m['FA_rate']:.3f}") + + with open(os.path.join(RESULTS_DIR, "lfo.json"), "w") as f: + json.dump(results, f, indent=2, default=str) + print("lfo done") + + +def cmd_e5baseline(args): + from sklearn.linear_model import LogisticRegression + from sklearn.neural_network import MLPClassifier + rows = load_dev() + y = np.array([r["y"] for r in rows]) + folds = np.array([r["cv_fold"] for r in rows]) + X = np.vstack([r["emb"] for r in rows]) + tags = [r["tags"] for r in rows] + pairs = build_pairs(rows, [r["text_strip"] for r in rows]) + + out = {} + for model_name, model, extra in [ + ("e5_linear", LogisticRegression(C=1.0, max_iter=2000, solver="lbfgs", random_state=42), {}), + ("e5_mlp_h32", MLPClassifier(hidden_layer_sizes=(32,), activation="relu", + solver="adam", alpha=0.01, random_state=42, + max_iter=800, early_stopping=True, + validation_fraction=0.15, n_iter_no_change=10), {}), + ]: + proba = np.zeros(len(rows)) + for fold in range(5): + tr = folds != fold + te = folds == fold + m2 = type(model)(**{k: v for k, v in model.get_params().items()}) + m2.fit(X[tr], y[tr]) + proba[te] = m2.predict_proba(X[te])[:, 1] + entry = { + "grouped": binary_metrics(y, proba), + "pairs": pair_metrics(pairs, proba), + } + # cap-Q leave-generator-out (train without the family) + src = np.array(["capability_question" not in t for t in tags]) + tgt = np.array(["capability_question" in t for t in tags]) + m3 = LogisticRegression(C=1.0, max_iter=2000, solver="lbfgs", random_state=42) \ + if model_name == "e5_linear" else \ + MLPClassifier(hidden_layer_sizes=(32,), alpha=0.01, random_state=42, max_iter=800) + m3.fit(X[src], y[src]) + p3 = m3.predict_proba(X[tgt])[:, 1] + entry["capq_lofo"] = binary_metrics(y[tgt], p3) + entry["stress"] = "NA (no re-embed on this box)" + out[model_name] = entry + print(f"{model_name}: grouped FA_rate={entry['grouped']['FA_rate']:.4f} " + f"PR={entry['grouped']['PR_AUC']:.3f} capQ_LOFO_FA_rate={entry['capq_lofo']['FA_rate']:.4f} " + f"pairs={entry['pairs']['ordering_acc']:.3f}") + with open(os.path.join(RESULTS_DIR, "e5baseline.json"), "w") as f: + json.dump(out, f, indent=2, default=str) + print("e5baseline done") + + +def cmd_runtime(args): + import torch + rows = load_dev() + char_vocab, bpe = build_tokenizers(rows) + X_char = encode_all(rows, char_vocab, "char")["strip"] + X_bpe = encode_all(rows, bpe, "bpe")["strip"] + kind_of = {"char_cnn": "char", "bigru": "bpe", "tiny_transformer": "bpe"} + report = {} + for arch, sizes in ARCH_CONFIGS.items(): + X = X_char if kind_of[arch] == "char" else X_bpe + for size in sizes: + name = f"{arch}_{size}" + model = build_model(arch, size, int(X.max()) + 1, X.shape[1]) + n_params = sum(p.numel() for p in model.parameters()) + fp32 = n_params * 4 + int8 = n_params + model.eval() + # warmup + latency (batch-1, eval mode) + xb = torch.from_numpy(X[:1]) + with torch.no_grad(): + for _ in range(20): + model(xb) + # tokenization latency + if kind_of[arch] == "char": + t0 = time.perf_counter() + for r in rows[:1000]: + char_vocab.encode(r["text_strip"], MAX_CHAR) + tl = (time.perf_counter() - t0) / 1000 + else: + t0 = time.perf_counter() + for r in rows[:1000]: + bpe.encode(r["text_strip"]) + tl = (time.perf_counter() - t0) / 1000 + lat = [] + for _ in range(300): + t0 = time.perf_counter() + model(xb) + lat.append(time.perf_counter() - t0) + lat = np.array(lat) * 1e6 + report[name] = { + "params": n_params, "fp32_bytes": fp32, "int8_bytes": int8, + "latency_us_mean": float(lat.mean()), "latency_us_p50": float(np.median(lat)), + "latency_us_p95": float(np.percentile(lat, 95)), + "throughput_b1": round(1e6 / float(lat.mean()), 1), + "tok_us": round(tl * 1e6, 1), + "tokenizer": "char" if kind_of[arch] == "char" else "bpe", + } + print(f"{name}: {n_params} params fp32={fp32/1024:.0f}KiB " + f"lat={lat.mean():.0f}us tok={tl*1e6:.1f}us") + with open(os.path.join(RESULTS_DIR, "runtime.json"), "w") as f: + json.dump(report, f, indent=2) + print("runtime done") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("cmd", choices=["grouped", "metrics", "lfo", "e5baseline", "runtime"]) + ap.add_argument("--verbose", action="store_true") + args = ap.parse_args() + t0 = time.time() + globals()[f"cmd_{args.cmd}"](args) + print(f"elapsed {time.time()-t0:.1f}s") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/cmd/semantic-router-experiment/slice19_models.py b/cmd/semantic-router-experiment/slice19_models.py new file mode 100644 index 0000000..1d15318 --- /dev/null +++ b/cmd/semantic-router-experiment/slice19_models.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +""" +Slice 19 model zoo: three genuinely sequence-sensitive tiny models, trained +from scratch on Maven's narrow binary pragmatics task. + + A. CharCNN — codepoint ids → char embedding → parallel small 1D convs + (several kernel widths) → global max-pool → linear head + B. BiGRU — subword ids → token embedding → 1-layer BiGRU → + maxpool[final] → linear head + C. TinyTransformer — subword ids → token embedding + sine position → + N self-attention encoder blocks (heads, FFN 4x, PreNorm) → + CLS → linear head + +All expose :forward(ids) returning the binary logit, plus .n_params(). +Deterministic: everything is plain torch ops. +""" + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class CharCNN(nn.Module): + def __init__(self, vocab_size, embed_dim, filters, widths, pad_idx=0, dropout=0.3): + super().__init__() + self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx) + self.convs = nn.ModuleList([ + nn.Conv1d(embed_dim, filters, k, padding=(k - 1) // 2) + for k in widths + ]) + self.dropout = nn.Dropout(dropout) + self.head = nn.Linear(filters * len(widths), 1) + + def forward(self, ids): + # ids: (B, T) + x = self.embed(ids).transpose(1, 2) # (B, D, T) + hiddens = [F.relu(conv(x)) for conv in self.convs] # each (B, F, T) + pooled = torch.cat([h.max(dim=2).values for h in hiddens], dim=1) # (B, F*W) + return self.head(self.dropout(pooled)).squeeze(-1) + + def n_params(self): + return sum(p.numel() for p in self.parameters()) + + +class BiGRU(nn.Module): + def __init__(self, vocab_size, embed_dim, hidden, pad_idx=0, dropout=0.3): + super().__init__() + self.embed = nn.Embedding(vocab_size, embed_dim, padding_idx=pad_idx) + self.encoder = nn.GRU(embed_dim, hidden, num_layers=1, bidirectional=True, + batch_first=True) + self.dropout = nn.Dropout(dropout) + self.head = nn.Linear(hidden * 2, 1) + + def forward(self, ids): + mask = (ids != 0).float() # (B, T) + x = self.embed(ids) + lens = mask.sum(dim=1).clamp(min=1).long() + x_p = nn.utils.rnn.pack_padded_sequence(x, lens.cpu(), batch_first=True, + enforce_sorted=False) + out, _ = self.encoder(x_p) + out, _ = nn.utils.rnn.pad_packed_sequence(out, batch_first=True, + total_length=mask.size(1)) + out = out * mask.unsqueeze(-1) + maxed = out.max(dim=1).values # (B, 2H) + return self.head(self.dropout(maxed)).squeeze(-1) + + def n_params(self): + return sum(p.numel() for p in self.parameters()) + + +class TinyTransformer(nn.Module): + def __init__(self, vocab_size, d_model, n_layers, n_heads, ff_mult=4, + max_len=64, pad_idx=0, dropout=0.1): + super().__init__() + self.d_model = d_model + self.embed = nn.Embedding(vocab_size, d_model, padding_idx=pad_idx) + self.dropout = nn.Dropout(dropout) + self.pos = nn.Parameter(torch.empty(1, max_len, d_model)) + nn.init.normal_(self.pos, std=0.02) + blocks = [] + for _ in range(n_layers): + blocks.append(TransformerBlock(d_model, n_heads, ff_mult, dropout)) + self.blocks = nn.ModuleList(blocks) + self.ln_out = nn.LayerNorm(d_model) + self.head = nn.Linear(d_model, 1) + + def forward(self, ids): + B, T = ids.shape + mask = (ids != 0) + x = self.embed(ids) * math.sqrt(self.d_model) + self.pos[:, :T, :] + x = self.dropout(x) + for blk in self.blocks: + x = blk(x, mask) + x = self.ln_out(x) + pooled = x.masked_fill(~mask.unsqueeze(-1), float("-inf")).max(dim=1).values + return self.head(pooled).squeeze(-1) + + def n_params(self): + return sum(p.numel() for p in self.parameters()) + + +class TransformerBlock(nn.Module): + def __init__(self, d_model, n_heads, ff_mult, dropout): + super().__init__() + self.attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout, + batch_first=True) + self.ln1 = nn.LayerNorm(d_model) + self.ff = nn.Sequential( + nn.Linear(d_model, d_model * ff_mult), + nn.GELU(), + nn.Linear(d_model * ff_mult, d_model), + ) + self.ln2 = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout) + + def forward(self, x, mask): + # mask: (B, T) bool of non-pad; MultiheadAttention uses key_padding_mask + h = self.ln1(x) + h = self.attn(h, h, h, key_padding_mask=~mask, + need_weights=False, is_causal=False)[0] + x = x + self.dropout(h) + h = self.ln2(x) + x = x + self.dropout(self.ff(h)) + return x + + +# ─── Sizes ladder ─────────────────────────────────────────────────────────── + +def make_model(arch, size, char_vocab, bpe_vocab): + if arch == "char_cnn": + configs = { + "tiny": dict(embed_dim=32, filters=64, widths=[3, 4, 5]), + "medium": dict(embed_dim=64, filters=160, widths=[2, 3, 4, 5]), + } + c = configs[size] + return CharCNN(char_vocab, c["embed_dim"], c["filters"], c["widths"]) + if arch == "bigru": + configs = { + "tiny": dict(embed_dim=64, hidden=64), + "medium": dict(embed_dim=128, hidden=128), + "large": dict(embed_dim=256, hidden=256), + } + c = configs[size] + return BiGRU(bpe_vocab, c["embed_dim"], c["hidden"]) + if arch == "tiny_transformer": + configs = { + "small": dict(d_model=128, n_layers=2, n_heads=4), + "medium": dict(d_model=192, n_layers=4, n_heads=4), + } + c = configs[size] + return TinyTransformer(bpe_vocab, c["d_model"], c["n_layers"], c["n_heads"]) + raise ValueError(arch) + + +def n_params_of(arch, size, char_vocab, bpe_vocab): + return make_model(arch, size, char_vocab, bpe_vocab).n_params() \ No newline at end of file diff --git a/docs/evals/CLAUDE.md b/docs/evals/CLAUDE.md index a258ff3..3e0d72e 100644 --- a/docs/evals/CLAUDE.md +++ b/docs/evals/CLAUDE.md @@ -51,6 +51,7 @@ A pair in `docs/routing.md` went stale unnoticed. Its source predated the | [Slice 16 diagnostic: action/non-action boundary analysis](2026-09-07-slice16-diagnostic.md) | live | | [Slice 17 nonlinear e5 MLP probe](2026-09-07-nonlinear-e5-mlp-probe.md) | live | | [Sparse lexical action-gate probe (slice 18)](2026-09-07-sparse-lexical-action-gate.md) | live | +| [From-scratch tiny sequence pragmatics specialist (slice 19)](2026-09-08-tiny-sequence-pragmatics-specialist.md) | live | `docs/routing.md` holds the arm table these feed. Cite from there, not from here.