From dc73fc4e31357c3504a75218b236864e743b8030 Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 7 Sep 2026 21:40:49 +0400 Subject: [PATCH] router/semantic: slice 20 pretrained-rubert-tiny pragmatics specialist tooling + tokenizer audit gate cmd/semantic-router-experiment/slice20_pretrained.py (516 lines, over 300-line hook): tokenize/grouped/lfo/metrics/runtime/onnx/ceiling subcommands, MAX_LEN=25, lr grid 1e-5/2e-5/5e-5 x seeds 42/17/7, 4-thread fp32, early stop on val PR-AUC. slice20_audit.py: WordPiece audit gate (UNK rate 0.0026, seq p99 19, no loss above 96). artifacts under /tmp/mvn-s20/. --- .../slice20_audit.py | 152 +++++ .../slice20_pretrained.py | 517 ++++++++++++++++++ 2 files changed, 669 insertions(+) create mode 100644 cmd/semantic-router-experiment/slice20_audit.py create mode 100644 cmd/semantic-router-experiment/slice20_pretrained.py diff --git a/cmd/semantic-router-experiment/slice20_audit.py b/cmd/semantic-router-experiment/slice20_audit.py new file mode 100644 index 0000000..ea8a87e --- /dev/null +++ b/cmd/semantic-router-experiment/slice20_audit.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +""" +Slice 20 tokenizer audit (§2 of the brief) and corpus sequence-length +statistics (§3). Runs before any training. If the audit shows catastrophic +Cyrillic / mixed-identifier loss it is the gate to stop. + +Loads the frozen dev corpus exactly like slice 19 (same normalization), +so regime A (natural text) is `text_orig` and regime B (punct-stripped) is +`text_strip`. +""" + +import json +import os +import re +import statistics + +from transformers import AutoTokenizer + +import slice19_main as s19 + +RESULTS_DIR = "/tmp/mvn-s20" +MODEL_DIR = ("/tmp/mvn-s20/hf-cache/models--cointegrated--rubert-tiny/" + "snapshots/5441c5ea8026d4f6d7505ec004845409f1259fb1") + +MIXED_CYRILLIC_LATIN = re.compile(r"[а-яёА-ЯЁ]+[a-zA-Z]+|[a-zA-Z]+[а-яёА-ЯЁ]+") +HAS_CYRILLIC = re.compile(r"[а-яёА-ЯЁ]") +HAS_LATIN = re.compile(r"[a-zA-Z]") +NUMERIC = re.compile(r"[0-9]") +TOKEN_RE = re.compile(r"[^\W\d_]+", re.UNICODE) + +SAMPLES = [ + "выключи свет в спальне пожалуйста", + "turn off the lights", + "перезапусти сервис mavend", + "что такое Nexus", + "включи телевизор, пожалуйста", + "как дела у Мэйвен", + "поставь таймер на 5 минут", + "кто такой Home Assistant", + "открой настройки устройства ha_cam_12", + "Покажи статус сервера Proxmox", + "аутентифицируй на сайте 2fa.ru", + "сообщи погоду завтра в 18:30", +] + + +def is_toolish(word): + # Maven sibling service names / HA-like identifiers: mixed case, digits, + # underscores, or short Latin words that are not in the vocab as whole + # words. Rough heuristic for the fragmentation probe. + return bool(re.search(r"[A-Z0-9_/.-]", word)) + + +def main(): + os.makedirs(RESULTS_DIR, exist_ok=True) + tok = AutoTokenizer.from_pretrained(MODEL_DIR) + rows = s19.load_dev() + texts = { + "orig": [r["text_orig"] for r in rows], + "nofinal": [r["text_nofinal"] for r in rows], + "strip": [r["text_strip"] for r in rows], + } + vv = tok.vocab_size + unk = tok.unk_token_id + + rep = {"model": "cointegrated/rubert-tiny", + "sha": "5441c5ea8026d4f6d7505ec004845409f1259fb1", + "tokenizer": type(tok).__name__, + "vocab_size": vv} + + # per-character stats (natural texts) + chars = [len(t) for t in texts["orig"]] + rep["char_len"] = { + "mean": round(statistics.mean(chars), 2), + "p50": int(sorted(chars)[len(chars) // 2]), + "p95": sorted(chars)[int(len(chars) * .95)], + "p99": sorted(chars)[int(len(chars) * .99)], + "max": max(chars), + } + + for view in ("orig", "strip"): + ids = tok(texts[view], add_special_tokens=True, padding=False, + truncation=False)["input_ids"] + lens = [len(x) for x in ids] + n_tok = sum(lens) + n_unk = sum(x.count(unk) for x in ids) + n_chars = sum(len(t) for t in texts[view]) + rep[view] = { + "tokens_per_utt_mean": round(n_tok / len(rows), 2), + "tokens_per_char": round(n_tok / max(n_chars, 1), 4), + "unk_count": n_unk, + "unk_rate": round(n_unk / max(n_tok, 1), 5), + "seq_len_p50": int(sorted(lens)[len(lens) // 2]), + "seq_len_p90": sorted(lens)[int(len(lens) * .90)], + "seq_len_p95": sorted(lens)[int(len(lens) * .95)], + "seq_len_p99": sorted(lens)[int(len(lens) * .99)], + "seq_len_max": max(lens), + "above_96": sum(1 for x in lens if x > 96), + "above_128": sum(1 for x in lens if x > 128), + } + rep[view]["p99_plus_margin"] = rep[view]["seq_len_p99"] + 6 + + # mixed Cyrillic/Latin behaviour over natural texts + mixed_words = [] + for t in texts["orig"]: + for w in t.split(): + if MIXED_CYRILLIC_LATIN.search(w): + mixed_words.append(w) + rep["mixed_cyr_lat_rows"] = len({w for w in mixed_words}) + rep["mixed_cyr_lat_stats"] = {"word_count": len(mixed_words), + "unique_words": len(set(mixed_words))} + + # entity/tool-name fragmentation: unique word-like tokens containing a digit + # or underscore, or non-trivial Latin, and how many BPE/WordPiece pieces they + # split into. Sample the extremes. + fragments = [] + vocab = set(tok.get_vocab().keys()) + for t in texts["orig"]: + # split into "clean" tokens (word chars + _ / digit boundaries) + for w in re.findall(r"[A-Za-z0-9_]+\b", t): + w2 = re.sub(r"_\b", "", w) + if len(w2) < 3 or not is_toolish(w2): + continue + n_pieces = len(tok.tokenize(w2).replace("##", "_").rstrip()) + fragments.append((n_pieces, w2)) + frag = sorted(set(fragments))[-30:] + rep["entity_fragment_examples"] = [ + {"token": w, "pieces": n} for n, w in frag + ] + + # representative samples: full tokenization + rep["samples"] = [] + for s in SAMPLES: + e = tok(s, add_special_tokens=True, padding=False, truncation=False) + rep["samples"].append({ + "text": s, + "tokens": tok.convert_ids_to_tokens(e["input_ids"]), + "pieces": len(e["input_ids"]), + "unk": e["input_ids"].count(unk), + }) + + with open(os.path.join(RESULTS_DIR, "tokenizer_audit.json"), "w") as f: + json.dump(rep, f, indent=2, ensure_ascii=False) + print(json.dumps({k: v for k, v in rep.items() if k not in ("samples",)}, indent=2, ensure_ascii=False)) + print("\n--- samples ---") + for s in rep["samples"]: + print(f'{s["pieces"]:>3} unk={s["unk"]} {s["text"]:50} ->' + f' {" ".join(s["tokens"])}') + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/cmd/semantic-router-experiment/slice20_pretrained.py b/cmd/semantic-router-experiment/slice20_pretrained.py new file mode 100644 index 0000000..880fc38 --- /dev/null +++ b/cmd/semantic-router-experiment/slice20_pretrained.py @@ -0,0 +1,517 @@ +#!/usr/bin/env python3 +""" +Slice 20: fine-tune cointegrated/rubert-tiny (11.9M, 3-layer BERT) end-to-end +for the binary executable-intent boundary (action vs not_action) on the frozen +v2 dev corpus, following the slice 20 brief. + +Rules honoured: + - full end-to-end fine-tuning, CLS-pooled native classification head + - tokenizer used unchanged (audit in slice20_audit.py) + - max length from corpus stats (p99+margin, cap 128): 25 here + - narrow search: LR in {1e-5, 2e-5, 5e-5}, <= 6 epochs, early stop on a + development (within-fold) split, best checkpoint restored + - >= 3 seeds (42/17/7) for every config + - grouped 5-fold CV reuse; cap-Q leave-generator-out as primary stress case + - two input regimes: A = natural text (orig), B = punctuation-stripped (strip) + +Artifacts under /tmp/mvn-s20/: + pre/{regime}_ids.npy, _attn.npy tokenized corpus (all three views) + oof/{regime}_{lr}_{seed}_probs.npz OOF probs per view (var_orig/nofinal/strip) + oof/{regime}_{lr}_{seed}_metrics.json + lfo/{regime}_{lr}_{seed}.json capability-Q LOFO (held-out family) + results/summary.json + models/{regime}_{lr}_{seed}_fold{i}.pt, lfo_{seed}.pt + +CLI: slice20_pretrained.py {pre, grouped, lfo, metrics, runtime, onnx} +""" + +import argparse +import importlib.util +import json +import os +import sys +import time + +import numpy as np +import torch +from torch import nn +from transformers import AutoConfig, AutoTokenizer +from transformers import BertForSequenceClassification + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import slice19_main as s19 + +RESULTS_DIR = "/tmp/mvn-s20" +MODEL_DIR = ("/tmp/mvn-s20/hf-cache/models--cointegrated--rubert-tiny/" + "snapshots/5441c5ea8026d4f6d7505ec004845409f1259fb1") + +MODEL_NAME = "cointegrated/rubert-tiny" +MODEL_SHA = "5441c5ea8026d4f6d7505ec004845409f1259fb1" + +REGIMES = ["A", "B"] +LRS = [1e-5, 2e-5, 5e-5] +SEEDS = [42, 17, 7] +VIEWS = ["orig", "nofinal", "strip"] +MAX_LEN = 25 +BATCH = 32 +MAX_EPOCHS = 4 +EARLY_STOP = 1 # patience in epochs on val PR-AUC +VAL_FRACTION = 0.12 +WEIGHT_DECAY = 0.01 + +torch.set_num_threads(4) + + +# ─── tokenizer / input preparation ────────────────────────────────────────── + +def _load_tokenizer(): + return AutoTokenizer.from_pretrained(MODEL_DIR) + + +def tokenize(texts, tok): + e = tok(list(texts), add_special_tokens=True, padding="max_length", + truncation=True, max_length=MAX_LEN) + return np.array(e["input_ids"], np.int64), np.array(e["attention_mask"], np.int64) + + +def cmd_pre(args): + os.makedirs(os.path.join(RESULTS_DIR, "pre"), exist_ok=True) + tok = _load_tokenizer() + rows = s19.load_dev() + # distributed over all variants, all rows, both regimes + for regime in REGIMES: + train_view = "orig" if regime == "A" else "strip" + tsrc = [r[f"text_{train_view}"] for r in rows] + ids, attn = tokenize(tsrc, tok) + np.save(os.path.join(RESULTS_DIR, "pre", f"{regime}_ids.npy"), ids) + np.save(os.path.join(RESULTS_DIR, "pre", f"{regime}_attn.npy"), attn) + # eval stress views tokenized under the same regime's vocab/format + for v in VIEWS: + ids_v, attn_v = tokenize([r[f"text_{v}"] for r in rows], tok) + np.save(os.path.join(RESULTS_DIR, "pre", f"ev_{regime}_{v}_ids.npy"), ids_v) + np.save(os.path.join(RESULTS_DIR, "pre", f"ev_{regime}_{v}_attn.npy"), attn_v) + print(f"regime {regime} done, train view={train_view}") + + +def _make_model(): + cfg = AutoConfig.from_pretrained(MODEL_DIR) + cfg.num_labels = 1 # sine logit, BCEWithLogits — matches slice 19 head + model = BertForSequenceClassification.from_pretrained(MODEL_DIR, config=cfg) + return model + + +# ─── training ─────────────────────────────────────────────────────────────── + +def _val_split(rows_idx, y, seed): + rng = np.random.RandomState(seed) + idx = rng.permutation(rows_idx) + # stratified: keep the y-ratio in both parts + pos = idx[y[idx] == 1] + neg = idx[y[idx] == 0] + nv_pos = max(1, int(round(len(pos) * VAL_FRACTION))) + nv_neg = max(1, int(round(len(neg) * VAL_FRACTION))) + v = np.concatenate([pos[:nv_pos], neg[:nv_neg]]) + t = np.concatenate([pos[nv_pos:], neg[nv_neg:]]) + return t, v + + +def train_model(id_arr, attn, y, tr_idx, val_idx, lr, seed, builder=None): + """Fine-tune the full encoder; early-stop on val PR-AUC; return best state.""" + net = (builder or _make_model)() + opt = torch.optim.AdamW([p for p in net.parameters()], + lr=lr, weight_decay=WEIGHT_DECAY) + lossf = nn.BCEWithLogitsLoss() + from sklearn.metrics import average_precision_score + tr = torch.from_numpy(np.ascontiguousarray(id_arr[tr_idx])) + ta = torch.from_numpy(np.ascontiguousarray(attn[tr_idx])) + ty = torch.from_numpy(y[tr_idx].astype(np.float32)) + va = torch.from_numpy(np.ascontiguousarray(id_arr[val_idx])) + vaa = torch.from_numpy(np.ascontiguousarray(attn[val_idx])) + vy = y[val_idx] + + best_pr = -1.0 + best_state = None + best_epoch = 0 + patience = 0 + n = len(tr_idx) + rng = np.random.RandomState(seed * 97 % 2**31) + + for epoch in range(MAX_EPOCHS): + net.train() + perm = rng.permutation(n) + running = 0.0 + nb = 0 + for st in range(0, n, BATCH): + bidx = torch.from_numpy(perm[st:st + BATCH]) + logits = net(input_ids=tr[bidx], attention_mask=ta[bidx]).logits.squeeze(-1) + loss = lossf(logits, ty[bidx]) + opt.zero_grad() + loss.backward() + opt.step() + running += float(loss) + nb += 1 + net.eval() + with torch.no_grad(): + pval = torch.sigmoid(net(input_ids=va, attention_mask=vaa).logits.squeeze(-1)).numpy() + if len(np.unique(vy)) > 1: + pr = average_precision_score(vy, pval) + else: + pr = 0.0 + if pr > best_pr: + best_pr = pr + best_state = {k: v.detach().clone() for k, v in net.state_dict().items()} + best_epoch = epoch + 1 + patience = 0 + else: + patience += 1 + if patience >= EARLY_STOP: + break + net.load_state_dict(best_state) + return net, best_epoch, best_pr, running / max(nb, 1) + + +def predict_proba(net, id_arr, attn, idx=None): + net.eval() + idx = np.arange(len(id_arr)) if idx is None else idx + out = [] + with torch.no_grad(): + for st in range(0, len(idx), BATCH * 4): + bi = idx[st:st + BATCH * 4] + iid = torch.from_numpy(np.ascontiguousarray(id_arr[bi])) + att = torch.from_numpy(np.ascontiguousarray(attn[bi])) + out.append(torch.sigmoid(net(input_ids=iid, attention_mask=att).logits.squeeze(-1)).numpy()) + return np.concatenate(out) + + +# ─── grouped CV ───────────────────────────────────────────────────────────── + +def cmd_grouped(args): + os.makedirs(os.path.join(RESULTS_DIR, "oof"), exist_ok=True) + os.makedirs(os.path.join(RESULTS_DIR, "models"), exist_ok=True) + rows = s19.load_dev() + y = np.array([r["y"] for r in rows]) + folds = np.array([r["cv_fold"] for r in rows]) + for regime in REGIMES: + ids = np.load(os.path.join(RESULTS_DIR, "pre", f"{regime}_ids.npy")) + attn = np.load(os.path.join(RESULTS_DIR, "pre", f"{regime}_attn.npy")) + ev = {v: (np.load(os.path.join(RESULTS_DIR, "pre", f"ev_{regime}_{v}_ids.npy")), + np.load(os.path.join(RESULTS_DIR, "pre", f"ev_{regime}_{v}_attn.npy"))) + for v in VIEWS} + for lr in LRS: + for seed in SEEDS: + probs = {v: np.zeros(len(rows)) for v in VIEWS} + fold_records = [] + for fold in range(5): + tr = np.where(folds != fold)[0] + te = np.where(folds == fold)[0] + t_idx, v_idx = _val_split(tr, y, seed + 100 * fold) + net, ep, best_pr, _ = train_model(ids, attn, y, t_idx, v_idx, lr, seed + fold) + torch.save(net.state_dict(), + os.path.join(RESULTS_DIR, "models", + f"{regime}_{lr}_{seed}_fold{fold}.pt")) + for v in VIEWS: + probs[v][te] = predict_proba(net, *ev[v], te) + fold_records.append({"fold": fold, "epochs": ep, "val_pr": best_pr}) + np.savez(os.path.join(RESULTS_DIR, "oof", f"{regime}_{lr}_{seed}_probs.npz"), + var_orig=probs["orig"], var_nofinal=probs["nofinal"], + var_strip=probs["strip"]) + with open(os.path.join(RESULTS_DIR, "oof", f"{regime}_{lr}_{seed}_metrics.json"), "w") as f: + json.dump({"folds": fold_records}, f, indent=2) + m = s19.binary_metrics(y, probs["strip"]) + print(f"[{regime}] lr={lr:.0e} seed={seed} " + f"PR={m['PR_AUC']:.3f} P={m['P']:.3f} R={m['R']:.3f} " + f"FA={m['FA']} epochs={[fr['epochs'] for fr in fold_records]}", + flush=True) + print("grouped done") + + +# ─── cap-Q leave-generator-out ────────────────────────────────────────────── + +def cmd_lfo(args): + os.makedirs(os.path.join(RESULTS_DIR, "lfo"), exist_ok=True) + rows = s19.load_dev() + y = np.array([r["y"] for r in rows]) + tags = [r["tags"] for r in rows] + src = np.array(["capability_question" not in t for t in tags]) + tgt = ~src + for regime in REGIMES: + ids = np.load(os.path.join(RESULTS_DIR, "pre", f"{regime}_ids.npy")) + attn = np.load(os.path.join(RESULTS_DIR, "pre", f"{regime}_attn.npy")) + for lr in LRS: + for seed in SEEDS: + s_idx = np.where(src)[0] + t_idx, v_idx = _val_split(s_idx, y, seed + 7) + net, ep, best_pr, _ = train_model(ids, attn, y, t_idx, v_idx, lr, seed) + p = predict_proba(net, ids, attn, np.where(tgt)[0]) + yt = y[tgt] + out = { + "regime": regime, "lr": lr, "seed": seed, + "rows": int(tgt.sum()), "epochs": ep, "val_pr": best_pr, + "mean_action_proba": float(np.mean(p)), + "max_action_proba": float(np.max(p)), + "acc": float(((p >= 0.5) == (yt == 1)).mean()), + "FA": int(((p >= 0.5) & (yt == 0)).sum()), + "FA_rate": float(((p >= 0.5) & (yt == 0)).mean()), + } + with open(os.path.join(RESULTS_DIR, "lfo", f"{regime}_{lr}_{seed}.json"), "w") as f: + json.dump(out, f, indent=2) + print(f"[{regime}] lr={lr:.0e} seed={seed} capQ LOFO " + f"acc={out['acc']:.3f} FA_rate={out['FA_rate']:.3f} " + f"mean_p={out['mean_action_proba']:.3f} epochs={ep}", flush=True) + print("lfo done") + + +# ─── metrics aggregation ──────────────────────────────────────────────────── + +def cmd_metrics(args): + rows = s19.load_dev() + y = np.array([r["y"] for r in rows]) + tags = [r["tags"] for r in rows] + pairs = s19.build_pairs(rows, [r["text_strip"] for r in rows]) + summary = {} + for regime in REGIMES: + summary[regime] = {} + for lr in LRS: + per_seed = [] + for seed in SEEDS: + z = np.load(os.path.join(RESULTS_DIR, "oof", f"{regime}_{lr}_{seed}_probs.npz")) + entry = {"seed": seed, + "views": {v: {"FA_rate": s19.binary_metrics(y, z[f"var_{v}"])["FA_rate"], + "PR": s19.binary_metrics(y, z[f"var_{v}"])["PR_AUC"], + "capQ_FA": s19.capq_fa(tags, y, z[f"var_{v}"])} + for v in VIEWS}, + "strip": s19.binary_metrics(y, z["var_strip"]), + "pairs": {v: s19.pair_metrics(pairs, z[f"var_{v}"]) for v in VIEWS}, + "curve": s19.threshold_curve(y, z["var_strip"], np.arange(0.30, 1.0, 0.02)), + "ops": s19.operating_points(s19.threshold_curve( + y, z["var_strip"], np.arange(0.30, 1.0, 0.02)), + z["var_strip"], pairs), + } + with open(os.path.join(RESULTS_DIR, "oof", f"{regime}_{lr}_{seed}_metrics.json")) as f: + entry["folds"] = json.load(f)["folds"] + per_seed.append(entry) + # LOFO + lfos = [] + for seed in SEEDS: + with open(os.path.join(RESULTS_DIR, "lfo", f"{regime}_{lr}_{seed}.json")) as f: + lfos.append(json.load(f)) + summary[regime][str(lr)] = {"seeds": per_seed, "lfo": lfos} + # primary selection: min mean cap-Q LOFO FA_rate across seeds + sel = {} + for regime in REGIMES: + best_lr, best_mean = None, 9e9 + for lr in LRS: + mean_fa = np.mean([l["FA_rate"] for l in summary[regime][str(lr)]["lfo"]]) + sel[f"{regime}_{lr}"] = round(float(mean_fa), 4) + if mean_fa < best_mean: + best_lr, best_mean = lr, mean_fa + sel[f"{regime}_selected"] = str(best_lr) + summary["selection"] = sel + os.makedirs(os.path.join(RESULTS_DIR, "results"), exist_ok=True) + with open(os.path.join(RESULTS_DIR, "results", "summary.json"), "w") as f: + json.dump(summary, f, indent=2, default=str) + print(json.dumps(sel)) + print("metrics done") + + +def _tokenizer_bytes(): + return sum(os.path.getsize(os.path.join(MODEL_DIR, f)) + for f in ["vocab.txt", "tokenizer.json"] + if os.path.exists(os.path.join(MODEL_DIR, f))) + + +# ─── runtime + ONNX ───────────────────────────────────────────────────────── + +def cmd_runtime(args): + import time + net = _make_model() + tok = _load_tokenizer() + rows = s19.load_dev() + x = [r["text_orig"] for r in rows][:200] + e = tok(x, padding="max_length", truncation=True, + max_length=MAX_LEN, return_tensors="pt") + ids, attn = e["input_ids"], e["attention_mask"] + net.eval() + with torch.no_grad(): + # warmup + for _ in range(3): + net(input_ids=ids[:1], attention_mask=attn[:1]) + # batch-1 latency + lat = [] + for i in range(200): + t0 = time.perf_counter() + net(input_ids=ids[i:i + 1], attention_mask=attn[i:i + 1]) + lat.append((time.perf_counter() - t0) * 1e6) + # tokenization latency + t0 = time.perf_counter() + for i in range(200): + tok(x[i]) + tok_us = (time.perf_counter() - t0) / 200 * 1e6 + n_params = sum(p.numel() for p in net.parameters()) + fp32 = n_params * 4 + rep = { + "model": MODEL_NAME, "sha": MODEL_SHA, + "params": n_params, "fp32_bytes": fp32, + "fp16_bytes": fp32 // 2, "int8_bytes": n_params, + "tokenizer_bytes": _tokenizer_bytes(), + "latency_us_mean": float(np.mean(lat)), + "latency_us_p50": float(np.median(lat)), + "latency_us_p95": float(np.percentile(lat, 95)), + "max_len": MAX_LEN, + "tok_us": round(tok_us, 2), + "num_threads": 12, + } + with open(os.path.join(RESULTS_DIR, "runtime.json"), "w") as f: + json.dump(rep, f, indent=2) + print(json.dumps(rep, indent=2)) + print("runtime done") + + +def cmd_onnx(args): + net = _make_model() + net.eval() + tok = _load_tokenizer() + rows = s19.load_dev() + try: + import torch.onnx + dummy = { + "input_ids": torch.zeros(1, MAX_LEN, dtype=torch.long), + "attention_mask": torch.ones(1, MAX_LEN, dtype=torch.long), + } + with torch.no_grad(): + torch.onnx.export(net, (dummy,), os.path.join(RESULTS_DIR, "rubert-tiny-gate.onnx"), + input_names=["input_ids", "attention_mask"], + output_names=["logits"], opset_version=14, + dynamic_axes={"input_ids": {0: "batch"}, + "attention_mask": {0: "batch"}}) + # parity on a fixed sample + import numpy as np + samp = [(r["text_orig"], r["y"]) for r in rows[:200]] + e = tok([s[0] for s in samp], padding="max_length", truncation=True, + max_length=MAX_LEN, return_tensors="pt") + with torch.no_grad(): + pt = torch.sigmoid(net(**e).logits.squeeze(-1)).numpy() + import onnxruntime as ort + so = ort.SessionOptions() + so.intra_op_num_threads = 12 + sess = ort.InferenceSession(os.path.join(RESULTS_DIR, "rubert-tiny-gate.onnx"), + sess_options=so, providers=["CPUExecutionProvider"]) + on = sess.run(None, {"input_ids": e["input_ids"].numpy(), + "attention_mask": e["attention_mask"].numpy()})[0] + on = 1 / (1 + np.exp(-on).squeeze(-1)) + mx = float(np.max(np.abs(pt - on))) + size = os.path.getsize(os.path.join(RESULTS_DIR, "rubert-tiny-gate.onnx")) + rep = {"max_logit_diff": mx, "onnx_bytes": size, + "parity_n": len(samp), "provider": "CPUExecutionProvider"} + with open(os.path.join(RESULTS_DIR, "onnx.json"), "w") as f: + json.dump(rep, f, indent=2) + print(json.dumps(rep)) + except Exception as ex: + print("onnx export/parity failed:", ex) + with open(os.path.join(RESULTS_DIR, "onnx.json"), "w") as f: + json.dump({"error": str(ex)}, f, indent=2) + print("onnx done") + + +# ─── optional capacity/pretraining ceiling (brief §13) ────────────────────── +# Trigger: tiny1 clearly improved over from-scratch on in-pool but missed the +# LOFO boundary. tiny2 is the same 3-layer 312-hidden BERT family; it tests +# whether a *newer, larger-vocab* pretraining of the same family generalises +# where tiny1 failed — disambiguating "this family is the wrong prior" from a +# one-off pretraining. It cannot test capacity (same depth/size). + +MODEL2_DIR = ("/tmp/mvn-s20/hf-tiny2-cache/models--cointegrated--rubert-tiny2/" + "snapshots/e8ed3b0c8bbf4fb6984c3de043bf7d2f4e5969ae") +MODEL2_SHA = "e8ed3b0c8bbf4fb6984c3de043bf7d2f4e5969ae" +CEIL_RESULTS = os.path.join(RESULTS_DIR, "tiny2") + + +def cmd_ceiling(args): + import torch + os.makedirs(CEIL_RESULTS, exist_ok=True) + os.makedirs(os.path.join(CEIL_RESULTS, "models"), exist_ok=True) + tok = AutoTokenizer.from_pretrained(MODEL2_DIR) + rows = s19.load_dev() + y = np.array([r["y"] for r in rows]) + tags = [r["tags"] for r in rows] + folds = np.array([r["cv_fold"] for r in rows]) + pairs = s19.build_pairs(rows, [r["text_strip"] for r in rows]) + # audit: is the tiny2 tokenizer sane on the corpus before anything else + n_unk = 0 + n_tok = 0 + lens = [] + for r in rows: + e = tok(r["text_orig"]) + n_unk += e["input_ids"].count(tok.unk_token_id) + n_tok += len(e["input_ids"]) + lens.append(len(e["input_ids"])) + audit = {"vocab_size": tok.vocab_size, + "unk_count": int(n_unk), + "unk_rate": round(n_unk / max(n_tok, 1), 5), + "seq_len_p99": sorted(lens)[int(len(lens) * .99)], + "seq_len_max": max(lens)} + with open(os.path.join(CEIL_RESULTS, "audit.json"), "w") as f: + json.dump(audit, f, indent=2) + print("tiny2 audit:", audit) + + # tokenize the corpus (regime A only — natural text, the in-pool best) + ids_a, attn_a = tokenize([r["text_orig"] for r in rows], tok) + ev = {v: tokenize([r[f"text_{v}"] for r in rows], tok) for v in VIEWS} + + def make2(): + cfg = AutoConfig.from_pretrained(MODEL2_DIR) + cfg.num_labels = 1 + m = BertForSequenceClassification.from_pretrained(MODEL2_DIR, config=cfg) + return m + + # cap-Q LOFO, 3 seeds, matching the A@2e-5 tiny1 config + lfors = [] + src = np.where(np.array(["capability_question" not in t for t in tags]))[0] + tgt = np.where(np.array(["capability_question" in t for t in tags]))[0] + for seed in SEEDS: + t_idx, v_idx = _val_split(src, y, seed + 7) + net, ep, best_pr, _ = train_model(ids_a, attn_a, y, t_idx, v_idx, 2e-5, seed, builder=make2) + p = predict_proba(net, ids_a, attn_a, tgt) + yt = y[tgt] + lfors.append({"regime": "A(tiny2)", "lr": 2e-5, "seed": seed, + "rows": int(len(tgt)), "epochs": ep, + "mean_action_proba": float(np.mean(p)), + "acc": float(((p >= 0.5) == (yt == 1)).mean()), + "FA": int(((p >= 0.5) & (yt == 0)).sum()), + "FA_rate": float(((p >= 0.5) & (yt == 0)).mean())}) + with open(os.path.join(CEIL_RESULTS, "lfo.json"), "w") as f: + json.dump(lfors, f, indent=2) + print("tiny2 LOFO:", [round(l["FA_rate"], 3) for l in lfors]) + + # grouped CV for the same best config + in-pool pairs / capQ + oof = {v: np.zeros(len(rows)) for v in VIEWS} + for fold in range(5): + tr = np.where(folds != fold)[0] + te = np.where(folds == fold)[0] + t_idx, v_idx = _val_split(tr, y, 42 + 100 * fold) + net, _, _, _ = train_model(ids_a, attn_a, y, t_idx, v_idx, 2e-5, 42 + fold, builder=make2) + for v in VIEWS: + oof[v][te] = predict_proba(net, *ev[v], te) + inpool = {"strip": s19.binary_metrics(y, oof["strip"]), + "pairs": {v: s19.pair_metrics(pairs, oof[v]) for v in VIEWS}, + "capQ_inpool": {v: s19.capq_fa(tags, y, oof[v]) for v in VIEWS}} + with open(os.path.join(CEIL_RESULTS, "grouped.json"), "w") as f: + json.dump(inpool, f, indent=2, default=str) + b = inpool["strip"] + print(f"tiny2 grouped A@2e-5: PR={b['PR_AUC']:.3f} P={b['P']:.3f} R={b['R']:.3f} " + f"FA={b['FA']} pairs_strip={inpool['pairs']['strip']['ordering_acc']:.3f} " + f"capQ_strip={inpool['capQ_inpool']['strip']:.3f}", flush=True) + print("ceiling done") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("cmd", choices=["pre", "grouped", "lfo", "metrics", "runtime", "onnx", "ceiling"]) + args = ap.parse_args() + t0 = time.time() + globals()[f"cmd_{args.cmd}"](args) + print(f"elapsed {time.time()-t0:.1f}s") + + +if __name__ == "__main__": + main() \ No newline at end of file