router/semantic: slice 20 pretrained-rubert-tiny pragmatics specialist tooling + tokenizer audit gate
cmd/semantic-router-experiment/slice20_pretrained.py (516 lines, over 300-line hook): tokenize/grouped/lfo/metrics/runtime/onnx/ceiling subcommands, MAX_LEN=25, lr grid 1e-5/2e-5/5e-5 x seeds 42/17/7, 4-thread fp32, early stop on val PR-AUC. slice20_audit.py: WordPiece audit gate (UNK rate 0.0026, seq p99 19, no loss above 96). artifacts under /tmp/mvn-s20/.
This commit is contained in:
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Slice 20 tokenizer audit (§2 of the brief) and corpus sequence-length
|
||||
statistics (§3). Runs before any training. If the audit shows catastrophic
|
||||
Cyrillic / mixed-identifier loss it is the gate to stop.
|
||||
|
||||
Loads the frozen dev corpus exactly like slice 19 (same normalization),
|
||||
so regime A (natural text) is `text_orig` and regime B (punct-stripped) is
|
||||
`text_strip`.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
import slice19_main as s19
|
||||
|
||||
RESULTS_DIR = "/tmp/mvn-s20"
|
||||
MODEL_DIR = ("/tmp/mvn-s20/hf-cache/models--cointegrated--rubert-tiny/"
|
||||
"snapshots/5441c5ea8026d4f6d7505ec004845409f1259fb1")
|
||||
|
||||
MIXED_CYRILLIC_LATIN = re.compile(r"[а-яёА-ЯЁ]+[a-zA-Z]+|[a-zA-Z]+[а-яёА-ЯЁ]+")
|
||||
HAS_CYRILLIC = re.compile(r"[а-яёА-ЯЁ]")
|
||||
HAS_LATIN = re.compile(r"[a-zA-Z]")
|
||||
NUMERIC = re.compile(r"[0-9]")
|
||||
TOKEN_RE = re.compile(r"[^\W\d_]+", re.UNICODE)
|
||||
|
||||
SAMPLES = [
|
||||
"выключи свет в спальне пожалуйста",
|
||||
"turn off the lights",
|
||||
"перезапусти сервис mavend",
|
||||
"что такое Nexus",
|
||||
"включи телевизор, пожалуйста",
|
||||
"как дела у Мэйвен",
|
||||
"поставь таймер на 5 минут",
|
||||
"кто такой Home Assistant",
|
||||
"открой настройки устройства ha_cam_12",
|
||||
"Покажи статус сервера Proxmox",
|
||||
"аутентифицируй на сайте 2fa.ru",
|
||||
"сообщи погоду завтра в 18:30",
|
||||
]
|
||||
|
||||
|
||||
def is_toolish(word):
|
||||
# Maven sibling service names / HA-like identifiers: mixed case, digits,
|
||||
# underscores, or short Latin words that are not in the vocab as whole
|
||||
# words. Rough heuristic for the fragmentation probe.
|
||||
return bool(re.search(r"[A-Z0-9_/.-]", word))
|
||||
|
||||
|
||||
def main():
|
||||
os.makedirs(RESULTS_DIR, exist_ok=True)
|
||||
tok = AutoTokenizer.from_pretrained(MODEL_DIR)
|
||||
rows = s19.load_dev()
|
||||
texts = {
|
||||
"orig": [r["text_orig"] for r in rows],
|
||||
"nofinal": [r["text_nofinal"] for r in rows],
|
||||
"strip": [r["text_strip"] for r in rows],
|
||||
}
|
||||
vv = tok.vocab_size
|
||||
unk = tok.unk_token_id
|
||||
|
||||
rep = {"model": "cointegrated/rubert-tiny",
|
||||
"sha": "5441c5ea8026d4f6d7505ec004845409f1259fb1",
|
||||
"tokenizer": type(tok).__name__,
|
||||
"vocab_size": vv}
|
||||
|
||||
# per-character stats (natural texts)
|
||||
chars = [len(t) for t in texts["orig"]]
|
||||
rep["char_len"] = {
|
||||
"mean": round(statistics.mean(chars), 2),
|
||||
"p50": int(sorted(chars)[len(chars) // 2]),
|
||||
"p95": sorted(chars)[int(len(chars) * .95)],
|
||||
"p99": sorted(chars)[int(len(chars) * .99)],
|
||||
"max": max(chars),
|
||||
}
|
||||
|
||||
for view in ("orig", "strip"):
|
||||
ids = tok(texts[view], add_special_tokens=True, padding=False,
|
||||
truncation=False)["input_ids"]
|
||||
lens = [len(x) for x in ids]
|
||||
n_tok = sum(lens)
|
||||
n_unk = sum(x.count(unk) for x in ids)
|
||||
n_chars = sum(len(t) for t in texts[view])
|
||||
rep[view] = {
|
||||
"tokens_per_utt_mean": round(n_tok / len(rows), 2),
|
||||
"tokens_per_char": round(n_tok / max(n_chars, 1), 4),
|
||||
"unk_count": n_unk,
|
||||
"unk_rate": round(n_unk / max(n_tok, 1), 5),
|
||||
"seq_len_p50": int(sorted(lens)[len(lens) // 2]),
|
||||
"seq_len_p90": sorted(lens)[int(len(lens) * .90)],
|
||||
"seq_len_p95": sorted(lens)[int(len(lens) * .95)],
|
||||
"seq_len_p99": sorted(lens)[int(len(lens) * .99)],
|
||||
"seq_len_max": max(lens),
|
||||
"above_96": sum(1 for x in lens if x > 96),
|
||||
"above_128": sum(1 for x in lens if x > 128),
|
||||
}
|
||||
rep[view]["p99_plus_margin"] = rep[view]["seq_len_p99"] + 6
|
||||
|
||||
# mixed Cyrillic/Latin behaviour over natural texts
|
||||
mixed_words = []
|
||||
for t in texts["orig"]:
|
||||
for w in t.split():
|
||||
if MIXED_CYRILLIC_LATIN.search(w):
|
||||
mixed_words.append(w)
|
||||
rep["mixed_cyr_lat_rows"] = len({w for w in mixed_words})
|
||||
rep["mixed_cyr_lat_stats"] = {"word_count": len(mixed_words),
|
||||
"unique_words": len(set(mixed_words))}
|
||||
|
||||
# entity/tool-name fragmentation: unique word-like tokens containing a digit
|
||||
# or underscore, or non-trivial Latin, and how many BPE/WordPiece pieces they
|
||||
# split into. Sample the extremes.
|
||||
fragments = []
|
||||
vocab = set(tok.get_vocab().keys())
|
||||
for t in texts["orig"]:
|
||||
# split into "clean" tokens (word chars + _ / digit boundaries)
|
||||
for w in re.findall(r"[A-Za-z0-9_]+\b", t):
|
||||
w2 = re.sub(r"_\b", "", w)
|
||||
if len(w2) < 3 or not is_toolish(w2):
|
||||
continue
|
||||
n_pieces = len(tok.tokenize(w2).replace("##", "_").rstrip())
|
||||
fragments.append((n_pieces, w2))
|
||||
frag = sorted(set(fragments))[-30:]
|
||||
rep["entity_fragment_examples"] = [
|
||||
{"token": w, "pieces": n} for n, w in frag
|
||||
]
|
||||
|
||||
# representative samples: full tokenization
|
||||
rep["samples"] = []
|
||||
for s in SAMPLES:
|
||||
e = tok(s, add_special_tokens=True, padding=False, truncation=False)
|
||||
rep["samples"].append({
|
||||
"text": s,
|
||||
"tokens": tok.convert_ids_to_tokens(e["input_ids"]),
|
||||
"pieces": len(e["input_ids"]),
|
||||
"unk": e["input_ids"].count(unk),
|
||||
})
|
||||
|
||||
with open(os.path.join(RESULTS_DIR, "tokenizer_audit.json"), "w") as f:
|
||||
json.dump(rep, f, indent=2, ensure_ascii=False)
|
||||
print(json.dumps({k: v for k, v in rep.items() if k not in ("samples",)}, indent=2, ensure_ascii=False))
|
||||
print("\n--- samples ---")
|
||||
for s in rep["samples"]:
|
||||
print(f'{s["pieces"]:>3} unk={s["unk"]} {s["text"]:50} ->'
|
||||
f' {" ".join(s["tokens"])}')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,517 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Slice 20: fine-tune cointegrated/rubert-tiny (11.9M, 3-layer BERT) end-to-end
|
||||
for the binary executable-intent boundary (action vs not_action) on the frozen
|
||||
v2 dev corpus, following the slice 20 brief.
|
||||
|
||||
Rules honoured:
|
||||
- full end-to-end fine-tuning, CLS-pooled native classification head
|
||||
- tokenizer used unchanged (audit in slice20_audit.py)
|
||||
- max length from corpus stats (p99+margin, cap 128): 25 here
|
||||
- narrow search: LR in {1e-5, 2e-5, 5e-5}, <= 6 epochs, early stop on a
|
||||
development (within-fold) split, best checkpoint restored
|
||||
- >= 3 seeds (42/17/7) for every config
|
||||
- grouped 5-fold CV reuse; cap-Q leave-generator-out as primary stress case
|
||||
- two input regimes: A = natural text (orig), B = punctuation-stripped (strip)
|
||||
|
||||
Artifacts under /tmp/mvn-s20/:
|
||||
pre/{regime}_ids.npy, _attn.npy tokenized corpus (all three views)
|
||||
oof/{regime}_{lr}_{seed}_probs.npz OOF probs per view (var_orig/nofinal/strip)
|
||||
oof/{regime}_{lr}_{seed}_metrics.json
|
||||
lfo/{regime}_{lr}_{seed}.json capability-Q LOFO (held-out family)
|
||||
results/summary.json
|
||||
models/{regime}_{lr}_{seed}_fold{i}.pt, lfo_{seed}.pt
|
||||
|
||||
CLI: slice20_pretrained.py {pre, grouped, lfo, metrics, runtime, onnx}
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import AutoConfig, AutoTokenizer
|
||||
from transformers import BertForSequenceClassification
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
import slice19_main as s19
|
||||
|
||||
RESULTS_DIR = "/tmp/mvn-s20"
|
||||
MODEL_DIR = ("/tmp/mvn-s20/hf-cache/models--cointegrated--rubert-tiny/"
|
||||
"snapshots/5441c5ea8026d4f6d7505ec004845409f1259fb1")
|
||||
|
||||
MODEL_NAME = "cointegrated/rubert-tiny"
|
||||
MODEL_SHA = "5441c5ea8026d4f6d7505ec004845409f1259fb1"
|
||||
|
||||
REGIMES = ["A", "B"]
|
||||
LRS = [1e-5, 2e-5, 5e-5]
|
||||
SEEDS = [42, 17, 7]
|
||||
VIEWS = ["orig", "nofinal", "strip"]
|
||||
MAX_LEN = 25
|
||||
BATCH = 32
|
||||
MAX_EPOCHS = 4
|
||||
EARLY_STOP = 1 # patience in epochs on val PR-AUC
|
||||
VAL_FRACTION = 0.12
|
||||
WEIGHT_DECAY = 0.01
|
||||
|
||||
torch.set_num_threads(4)
|
||||
|
||||
|
||||
# ─── tokenizer / input preparation ──────────────────────────────────────────
|
||||
|
||||
def _load_tokenizer():
|
||||
return AutoTokenizer.from_pretrained(MODEL_DIR)
|
||||
|
||||
|
||||
def tokenize(texts, tok):
|
||||
e = tok(list(texts), add_special_tokens=True, padding="max_length",
|
||||
truncation=True, max_length=MAX_LEN)
|
||||
return np.array(e["input_ids"], np.int64), np.array(e["attention_mask"], np.int64)
|
||||
|
||||
|
||||
def cmd_pre(args):
|
||||
os.makedirs(os.path.join(RESULTS_DIR, "pre"), exist_ok=True)
|
||||
tok = _load_tokenizer()
|
||||
rows = s19.load_dev()
|
||||
# distributed over all variants, all rows, both regimes
|
||||
for regime in REGIMES:
|
||||
train_view = "orig" if regime == "A" else "strip"
|
||||
tsrc = [r[f"text_{train_view}"] for r in rows]
|
||||
ids, attn = tokenize(tsrc, tok)
|
||||
np.save(os.path.join(RESULTS_DIR, "pre", f"{regime}_ids.npy"), ids)
|
||||
np.save(os.path.join(RESULTS_DIR, "pre", f"{regime}_attn.npy"), attn)
|
||||
# eval stress views tokenized under the same regime's vocab/format
|
||||
for v in VIEWS:
|
||||
ids_v, attn_v = tokenize([r[f"text_{v}"] for r in rows], tok)
|
||||
np.save(os.path.join(RESULTS_DIR, "pre", f"ev_{regime}_{v}_ids.npy"), ids_v)
|
||||
np.save(os.path.join(RESULTS_DIR, "pre", f"ev_{regime}_{v}_attn.npy"), attn_v)
|
||||
print(f"regime {regime} done, train view={train_view}")
|
||||
|
||||
|
||||
def _make_model():
|
||||
cfg = AutoConfig.from_pretrained(MODEL_DIR)
|
||||
cfg.num_labels = 1 # sine logit, BCEWithLogits — matches slice 19 head
|
||||
model = BertForSequenceClassification.from_pretrained(MODEL_DIR, config=cfg)
|
||||
return model
|
||||
|
||||
|
||||
# ─── training ───────────────────────────────────────────────────────────────
|
||||
|
||||
def _val_split(rows_idx, y, seed):
|
||||
rng = np.random.RandomState(seed)
|
||||
idx = rng.permutation(rows_idx)
|
||||
# stratified: keep the y-ratio in both parts
|
||||
pos = idx[y[idx] == 1]
|
||||
neg = idx[y[idx] == 0]
|
||||
nv_pos = max(1, int(round(len(pos) * VAL_FRACTION)))
|
||||
nv_neg = max(1, int(round(len(neg) * VAL_FRACTION)))
|
||||
v = np.concatenate([pos[:nv_pos], neg[:nv_neg]])
|
||||
t = np.concatenate([pos[nv_pos:], neg[nv_neg:]])
|
||||
return t, v
|
||||
|
||||
|
||||
def train_model(id_arr, attn, y, tr_idx, val_idx, lr, seed, builder=None):
|
||||
"""Fine-tune the full encoder; early-stop on val PR-AUC; return best state."""
|
||||
net = (builder or _make_model)()
|
||||
opt = torch.optim.AdamW([p for p in net.parameters()],
|
||||
lr=lr, weight_decay=WEIGHT_DECAY)
|
||||
lossf = nn.BCEWithLogitsLoss()
|
||||
from sklearn.metrics import average_precision_score
|
||||
tr = torch.from_numpy(np.ascontiguousarray(id_arr[tr_idx]))
|
||||
ta = torch.from_numpy(np.ascontiguousarray(attn[tr_idx]))
|
||||
ty = torch.from_numpy(y[tr_idx].astype(np.float32))
|
||||
va = torch.from_numpy(np.ascontiguousarray(id_arr[val_idx]))
|
||||
vaa = torch.from_numpy(np.ascontiguousarray(attn[val_idx]))
|
||||
vy = y[val_idx]
|
||||
|
||||
best_pr = -1.0
|
||||
best_state = None
|
||||
best_epoch = 0
|
||||
patience = 0
|
||||
n = len(tr_idx)
|
||||
rng = np.random.RandomState(seed * 97 % 2**31)
|
||||
|
||||
for epoch in range(MAX_EPOCHS):
|
||||
net.train()
|
||||
perm = rng.permutation(n)
|
||||
running = 0.0
|
||||
nb = 0
|
||||
for st in range(0, n, BATCH):
|
||||
bidx = torch.from_numpy(perm[st:st + BATCH])
|
||||
logits = net(input_ids=tr[bidx], attention_mask=ta[bidx]).logits.squeeze(-1)
|
||||
loss = lossf(logits, ty[bidx])
|
||||
opt.zero_grad()
|
||||
loss.backward()
|
||||
opt.step()
|
||||
running += float(loss)
|
||||
nb += 1
|
||||
net.eval()
|
||||
with torch.no_grad():
|
||||
pval = torch.sigmoid(net(input_ids=va, attention_mask=vaa).logits.squeeze(-1)).numpy()
|
||||
if len(np.unique(vy)) > 1:
|
||||
pr = average_precision_score(vy, pval)
|
||||
else:
|
||||
pr = 0.0
|
||||
if pr > best_pr:
|
||||
best_pr = pr
|
||||
best_state = {k: v.detach().clone() for k, v in net.state_dict().items()}
|
||||
best_epoch = epoch + 1
|
||||
patience = 0
|
||||
else:
|
||||
patience += 1
|
||||
if patience >= EARLY_STOP:
|
||||
break
|
||||
net.load_state_dict(best_state)
|
||||
return net, best_epoch, best_pr, running / max(nb, 1)
|
||||
|
||||
|
||||
def predict_proba(net, id_arr, attn, idx=None):
|
||||
net.eval()
|
||||
idx = np.arange(len(id_arr)) if idx is None else idx
|
||||
out = []
|
||||
with torch.no_grad():
|
||||
for st in range(0, len(idx), BATCH * 4):
|
||||
bi = idx[st:st + BATCH * 4]
|
||||
iid = torch.from_numpy(np.ascontiguousarray(id_arr[bi]))
|
||||
att = torch.from_numpy(np.ascontiguousarray(attn[bi]))
|
||||
out.append(torch.sigmoid(net(input_ids=iid, attention_mask=att).logits.squeeze(-1)).numpy())
|
||||
return np.concatenate(out)
|
||||
|
||||
|
||||
# ─── grouped CV ─────────────────────────────────────────────────────────────
|
||||
|
||||
def cmd_grouped(args):
|
||||
os.makedirs(os.path.join(RESULTS_DIR, "oof"), exist_ok=True)
|
||||
os.makedirs(os.path.join(RESULTS_DIR, "models"), exist_ok=True)
|
||||
rows = s19.load_dev()
|
||||
y = np.array([r["y"] for r in rows])
|
||||
folds = np.array([r["cv_fold"] for r in rows])
|
||||
for regime in REGIMES:
|
||||
ids = np.load(os.path.join(RESULTS_DIR, "pre", f"{regime}_ids.npy"))
|
||||
attn = np.load(os.path.join(RESULTS_DIR, "pre", f"{regime}_attn.npy"))
|
||||
ev = {v: (np.load(os.path.join(RESULTS_DIR, "pre", f"ev_{regime}_{v}_ids.npy")),
|
||||
np.load(os.path.join(RESULTS_DIR, "pre", f"ev_{regime}_{v}_attn.npy")))
|
||||
for v in VIEWS}
|
||||
for lr in LRS:
|
||||
for seed in SEEDS:
|
||||
probs = {v: np.zeros(len(rows)) for v in VIEWS}
|
||||
fold_records = []
|
||||
for fold in range(5):
|
||||
tr = np.where(folds != fold)[0]
|
||||
te = np.where(folds == fold)[0]
|
||||
t_idx, v_idx = _val_split(tr, y, seed + 100 * fold)
|
||||
net, ep, best_pr, _ = train_model(ids, attn, y, t_idx, v_idx, lr, seed + fold)
|
||||
torch.save(net.state_dict(),
|
||||
os.path.join(RESULTS_DIR, "models",
|
||||
f"{regime}_{lr}_{seed}_fold{fold}.pt"))
|
||||
for v in VIEWS:
|
||||
probs[v][te] = predict_proba(net, *ev[v], te)
|
||||
fold_records.append({"fold": fold, "epochs": ep, "val_pr": best_pr})
|
||||
np.savez(os.path.join(RESULTS_DIR, "oof", f"{regime}_{lr}_{seed}_probs.npz"),
|
||||
var_orig=probs["orig"], var_nofinal=probs["nofinal"],
|
||||
var_strip=probs["strip"])
|
||||
with open(os.path.join(RESULTS_DIR, "oof", f"{regime}_{lr}_{seed}_metrics.json"), "w") as f:
|
||||
json.dump({"folds": fold_records}, f, indent=2)
|
||||
m = s19.binary_metrics(y, probs["strip"])
|
||||
print(f"[{regime}] lr={lr:.0e} seed={seed} "
|
||||
f"PR={m['PR_AUC']:.3f} P={m['P']:.3f} R={m['R']:.3f} "
|
||||
f"FA={m['FA']} epochs={[fr['epochs'] for fr in fold_records]}",
|
||||
flush=True)
|
||||
print("grouped done")
|
||||
|
||||
|
||||
# ─── cap-Q leave-generator-out ──────────────────────────────────────────────
|
||||
|
||||
def cmd_lfo(args):
|
||||
os.makedirs(os.path.join(RESULTS_DIR, "lfo"), exist_ok=True)
|
||||
rows = s19.load_dev()
|
||||
y = np.array([r["y"] for r in rows])
|
||||
tags = [r["tags"] for r in rows]
|
||||
src = np.array(["capability_question" not in t for t in tags])
|
||||
tgt = ~src
|
||||
for regime in REGIMES:
|
||||
ids = np.load(os.path.join(RESULTS_DIR, "pre", f"{regime}_ids.npy"))
|
||||
attn = np.load(os.path.join(RESULTS_DIR, "pre", f"{regime}_attn.npy"))
|
||||
for lr in LRS:
|
||||
for seed in SEEDS:
|
||||
s_idx = np.where(src)[0]
|
||||
t_idx, v_idx = _val_split(s_idx, y, seed + 7)
|
||||
net, ep, best_pr, _ = train_model(ids, attn, y, t_idx, v_idx, lr, seed)
|
||||
p = predict_proba(net, ids, attn, np.where(tgt)[0])
|
||||
yt = y[tgt]
|
||||
out = {
|
||||
"regime": regime, "lr": lr, "seed": seed,
|
||||
"rows": int(tgt.sum()), "epochs": ep, "val_pr": best_pr,
|
||||
"mean_action_proba": float(np.mean(p)),
|
||||
"max_action_proba": float(np.max(p)),
|
||||
"acc": float(((p >= 0.5) == (yt == 1)).mean()),
|
||||
"FA": int(((p >= 0.5) & (yt == 0)).sum()),
|
||||
"FA_rate": float(((p >= 0.5) & (yt == 0)).mean()),
|
||||
}
|
||||
with open(os.path.join(RESULTS_DIR, "lfo", f"{regime}_{lr}_{seed}.json"), "w") as f:
|
||||
json.dump(out, f, indent=2)
|
||||
print(f"[{regime}] lr={lr:.0e} seed={seed} capQ LOFO "
|
||||
f"acc={out['acc']:.3f} FA_rate={out['FA_rate']:.3f} "
|
||||
f"mean_p={out['mean_action_proba']:.3f} epochs={ep}", flush=True)
|
||||
print("lfo done")
|
||||
|
||||
|
||||
# ─── metrics aggregation ────────────────────────────────────────────────────
|
||||
|
||||
def cmd_metrics(args):
|
||||
rows = s19.load_dev()
|
||||
y = np.array([r["y"] for r in rows])
|
||||
tags = [r["tags"] for r in rows]
|
||||
pairs = s19.build_pairs(rows, [r["text_strip"] for r in rows])
|
||||
summary = {}
|
||||
for regime in REGIMES:
|
||||
summary[regime] = {}
|
||||
for lr in LRS:
|
||||
per_seed = []
|
||||
for seed in SEEDS:
|
||||
z = np.load(os.path.join(RESULTS_DIR, "oof", f"{regime}_{lr}_{seed}_probs.npz"))
|
||||
entry = {"seed": seed,
|
||||
"views": {v: {"FA_rate": s19.binary_metrics(y, z[f"var_{v}"])["FA_rate"],
|
||||
"PR": s19.binary_metrics(y, z[f"var_{v}"])["PR_AUC"],
|
||||
"capQ_FA": s19.capq_fa(tags, y, z[f"var_{v}"])}
|
||||
for v in VIEWS},
|
||||
"strip": s19.binary_metrics(y, z["var_strip"]),
|
||||
"pairs": {v: s19.pair_metrics(pairs, z[f"var_{v}"]) for v in VIEWS},
|
||||
"curve": s19.threshold_curve(y, z["var_strip"], np.arange(0.30, 1.0, 0.02)),
|
||||
"ops": s19.operating_points(s19.threshold_curve(
|
||||
y, z["var_strip"], np.arange(0.30, 1.0, 0.02)),
|
||||
z["var_strip"], pairs),
|
||||
}
|
||||
with open(os.path.join(RESULTS_DIR, "oof", f"{regime}_{lr}_{seed}_metrics.json")) as f:
|
||||
entry["folds"] = json.load(f)["folds"]
|
||||
per_seed.append(entry)
|
||||
# LOFO
|
||||
lfos = []
|
||||
for seed in SEEDS:
|
||||
with open(os.path.join(RESULTS_DIR, "lfo", f"{regime}_{lr}_{seed}.json")) as f:
|
||||
lfos.append(json.load(f))
|
||||
summary[regime][str(lr)] = {"seeds": per_seed, "lfo": lfos}
|
||||
# primary selection: min mean cap-Q LOFO FA_rate across seeds
|
||||
sel = {}
|
||||
for regime in REGIMES:
|
||||
best_lr, best_mean = None, 9e9
|
||||
for lr in LRS:
|
||||
mean_fa = np.mean([l["FA_rate"] for l in summary[regime][str(lr)]["lfo"]])
|
||||
sel[f"{regime}_{lr}"] = round(float(mean_fa), 4)
|
||||
if mean_fa < best_mean:
|
||||
best_lr, best_mean = lr, mean_fa
|
||||
sel[f"{regime}_selected"] = str(best_lr)
|
||||
summary["selection"] = sel
|
||||
os.makedirs(os.path.join(RESULTS_DIR, "results"), exist_ok=True)
|
||||
with open(os.path.join(RESULTS_DIR, "results", "summary.json"), "w") as f:
|
||||
json.dump(summary, f, indent=2, default=str)
|
||||
print(json.dumps(sel))
|
||||
print("metrics done")
|
||||
|
||||
|
||||
def _tokenizer_bytes():
|
||||
return sum(os.path.getsize(os.path.join(MODEL_DIR, f))
|
||||
for f in ["vocab.txt", "tokenizer.json"]
|
||||
if os.path.exists(os.path.join(MODEL_DIR, f)))
|
||||
|
||||
|
||||
# ─── runtime + ONNX ─────────────────────────────────────────────────────────
|
||||
|
||||
def cmd_runtime(args):
|
||||
import time
|
||||
net = _make_model()
|
||||
tok = _load_tokenizer()
|
||||
rows = s19.load_dev()
|
||||
x = [r["text_orig"] for r in rows][:200]
|
||||
e = tok(x, padding="max_length", truncation=True,
|
||||
max_length=MAX_LEN, return_tensors="pt")
|
||||
ids, attn = e["input_ids"], e["attention_mask"]
|
||||
net.eval()
|
||||
with torch.no_grad():
|
||||
# warmup
|
||||
for _ in range(3):
|
||||
net(input_ids=ids[:1], attention_mask=attn[:1])
|
||||
# batch-1 latency
|
||||
lat = []
|
||||
for i in range(200):
|
||||
t0 = time.perf_counter()
|
||||
net(input_ids=ids[i:i + 1], attention_mask=attn[i:i + 1])
|
||||
lat.append((time.perf_counter() - t0) * 1e6)
|
||||
# tokenization latency
|
||||
t0 = time.perf_counter()
|
||||
for i in range(200):
|
||||
tok(x[i])
|
||||
tok_us = (time.perf_counter() - t0) / 200 * 1e6
|
||||
n_params = sum(p.numel() for p in net.parameters())
|
||||
fp32 = n_params * 4
|
||||
rep = {
|
||||
"model": MODEL_NAME, "sha": MODEL_SHA,
|
||||
"params": n_params, "fp32_bytes": fp32,
|
||||
"fp16_bytes": fp32 // 2, "int8_bytes": n_params,
|
||||
"tokenizer_bytes": _tokenizer_bytes(),
|
||||
"latency_us_mean": float(np.mean(lat)),
|
||||
"latency_us_p50": float(np.median(lat)),
|
||||
"latency_us_p95": float(np.percentile(lat, 95)),
|
||||
"max_len": MAX_LEN,
|
||||
"tok_us": round(tok_us, 2),
|
||||
"num_threads": 12,
|
||||
}
|
||||
with open(os.path.join(RESULTS_DIR, "runtime.json"), "w") as f:
|
||||
json.dump(rep, f, indent=2)
|
||||
print(json.dumps(rep, indent=2))
|
||||
print("runtime done")
|
||||
|
||||
|
||||
def cmd_onnx(args):
|
||||
net = _make_model()
|
||||
net.eval()
|
||||
tok = _load_tokenizer()
|
||||
rows = s19.load_dev()
|
||||
try:
|
||||
import torch.onnx
|
||||
dummy = {
|
||||
"input_ids": torch.zeros(1, MAX_LEN, dtype=torch.long),
|
||||
"attention_mask": torch.ones(1, MAX_LEN, dtype=torch.long),
|
||||
}
|
||||
with torch.no_grad():
|
||||
torch.onnx.export(net, (dummy,), os.path.join(RESULTS_DIR, "rubert-tiny-gate.onnx"),
|
||||
input_names=["input_ids", "attention_mask"],
|
||||
output_names=["logits"], opset_version=14,
|
||||
dynamic_axes={"input_ids": {0: "batch"},
|
||||
"attention_mask": {0: "batch"}})
|
||||
# parity on a fixed sample
|
||||
import numpy as np
|
||||
samp = [(r["text_orig"], r["y"]) for r in rows[:200]]
|
||||
e = tok([s[0] for s in samp], padding="max_length", truncation=True,
|
||||
max_length=MAX_LEN, return_tensors="pt")
|
||||
with torch.no_grad():
|
||||
pt = torch.sigmoid(net(**e).logits.squeeze(-1)).numpy()
|
||||
import onnxruntime as ort
|
||||
so = ort.SessionOptions()
|
||||
so.intra_op_num_threads = 12
|
||||
sess = ort.InferenceSession(os.path.join(RESULTS_DIR, "rubert-tiny-gate.onnx"),
|
||||
sess_options=so, providers=["CPUExecutionProvider"])
|
||||
on = sess.run(None, {"input_ids": e["input_ids"].numpy(),
|
||||
"attention_mask": e["attention_mask"].numpy()})[0]
|
||||
on = 1 / (1 + np.exp(-on).squeeze(-1))
|
||||
mx = float(np.max(np.abs(pt - on)))
|
||||
size = os.path.getsize(os.path.join(RESULTS_DIR, "rubert-tiny-gate.onnx"))
|
||||
rep = {"max_logit_diff": mx, "onnx_bytes": size,
|
||||
"parity_n": len(samp), "provider": "CPUExecutionProvider"}
|
||||
with open(os.path.join(RESULTS_DIR, "onnx.json"), "w") as f:
|
||||
json.dump(rep, f, indent=2)
|
||||
print(json.dumps(rep))
|
||||
except Exception as ex:
|
||||
print("onnx export/parity failed:", ex)
|
||||
with open(os.path.join(RESULTS_DIR, "onnx.json"), "w") as f:
|
||||
json.dump({"error": str(ex)}, f, indent=2)
|
||||
print("onnx done")
|
||||
|
||||
|
||||
# ─── optional capacity/pretraining ceiling (brief §13) ──────────────────────
|
||||
# Trigger: tiny1 clearly improved over from-scratch on in-pool but missed the
|
||||
# LOFO boundary. tiny2 is the same 3-layer 312-hidden BERT family; it tests
|
||||
# whether a *newer, larger-vocab* pretraining of the same family generalises
|
||||
# where tiny1 failed — disambiguating "this family is the wrong prior" from a
|
||||
# one-off pretraining. It cannot test capacity (same depth/size).
|
||||
|
||||
MODEL2_DIR = ("/tmp/mvn-s20/hf-tiny2-cache/models--cointegrated--rubert-tiny2/"
|
||||
"snapshots/e8ed3b0c8bbf4fb6984c3de043bf7d2f4e5969ae")
|
||||
MODEL2_SHA = "e8ed3b0c8bbf4fb6984c3de043bf7d2f4e5969ae"
|
||||
CEIL_RESULTS = os.path.join(RESULTS_DIR, "tiny2")
|
||||
|
||||
|
||||
def cmd_ceiling(args):
|
||||
import torch
|
||||
os.makedirs(CEIL_RESULTS, exist_ok=True)
|
||||
os.makedirs(os.path.join(CEIL_RESULTS, "models"), exist_ok=True)
|
||||
tok = AutoTokenizer.from_pretrained(MODEL2_DIR)
|
||||
rows = s19.load_dev()
|
||||
y = np.array([r["y"] for r in rows])
|
||||
tags = [r["tags"] for r in rows]
|
||||
folds = np.array([r["cv_fold"] for r in rows])
|
||||
pairs = s19.build_pairs(rows, [r["text_strip"] for r in rows])
|
||||
# audit: is the tiny2 tokenizer sane on the corpus before anything else
|
||||
n_unk = 0
|
||||
n_tok = 0
|
||||
lens = []
|
||||
for r in rows:
|
||||
e = tok(r["text_orig"])
|
||||
n_unk += e["input_ids"].count(tok.unk_token_id)
|
||||
n_tok += len(e["input_ids"])
|
||||
lens.append(len(e["input_ids"]))
|
||||
audit = {"vocab_size": tok.vocab_size,
|
||||
"unk_count": int(n_unk),
|
||||
"unk_rate": round(n_unk / max(n_tok, 1), 5),
|
||||
"seq_len_p99": sorted(lens)[int(len(lens) * .99)],
|
||||
"seq_len_max": max(lens)}
|
||||
with open(os.path.join(CEIL_RESULTS, "audit.json"), "w") as f:
|
||||
json.dump(audit, f, indent=2)
|
||||
print("tiny2 audit:", audit)
|
||||
|
||||
# tokenize the corpus (regime A only — natural text, the in-pool best)
|
||||
ids_a, attn_a = tokenize([r["text_orig"] for r in rows], tok)
|
||||
ev = {v: tokenize([r[f"text_{v}"] for r in rows], tok) for v in VIEWS}
|
||||
|
||||
def make2():
|
||||
cfg = AutoConfig.from_pretrained(MODEL2_DIR)
|
||||
cfg.num_labels = 1
|
||||
m = BertForSequenceClassification.from_pretrained(MODEL2_DIR, config=cfg)
|
||||
return m
|
||||
|
||||
# cap-Q LOFO, 3 seeds, matching the A@2e-5 tiny1 config
|
||||
lfors = []
|
||||
src = np.where(np.array(["capability_question" not in t for t in tags]))[0]
|
||||
tgt = np.where(np.array(["capability_question" in t for t in tags]))[0]
|
||||
for seed in SEEDS:
|
||||
t_idx, v_idx = _val_split(src, y, seed + 7)
|
||||
net, ep, best_pr, _ = train_model(ids_a, attn_a, y, t_idx, v_idx, 2e-5, seed, builder=make2)
|
||||
p = predict_proba(net, ids_a, attn_a, tgt)
|
||||
yt = y[tgt]
|
||||
lfors.append({"regime": "A(tiny2)", "lr": 2e-5, "seed": seed,
|
||||
"rows": int(len(tgt)), "epochs": ep,
|
||||
"mean_action_proba": float(np.mean(p)),
|
||||
"acc": float(((p >= 0.5) == (yt == 1)).mean()),
|
||||
"FA": int(((p >= 0.5) & (yt == 0)).sum()),
|
||||
"FA_rate": float(((p >= 0.5) & (yt == 0)).mean())})
|
||||
with open(os.path.join(CEIL_RESULTS, "lfo.json"), "w") as f:
|
||||
json.dump(lfors, f, indent=2)
|
||||
print("tiny2 LOFO:", [round(l["FA_rate"], 3) for l in lfors])
|
||||
|
||||
# grouped CV for the same best config + in-pool pairs / capQ
|
||||
oof = {v: np.zeros(len(rows)) for v in VIEWS}
|
||||
for fold in range(5):
|
||||
tr = np.where(folds != fold)[0]
|
||||
te = np.where(folds == fold)[0]
|
||||
t_idx, v_idx = _val_split(tr, y, 42 + 100 * fold)
|
||||
net, _, _, _ = train_model(ids_a, attn_a, y, t_idx, v_idx, 2e-5, 42 + fold, builder=make2)
|
||||
for v in VIEWS:
|
||||
oof[v][te] = predict_proba(net, *ev[v], te)
|
||||
inpool = {"strip": s19.binary_metrics(y, oof["strip"]),
|
||||
"pairs": {v: s19.pair_metrics(pairs, oof[v]) for v in VIEWS},
|
||||
"capQ_inpool": {v: s19.capq_fa(tags, y, oof[v]) for v in VIEWS}}
|
||||
with open(os.path.join(CEIL_RESULTS, "grouped.json"), "w") as f:
|
||||
json.dump(inpool, f, indent=2, default=str)
|
||||
b = inpool["strip"]
|
||||
print(f"tiny2 grouped A@2e-5: PR={b['PR_AUC']:.3f} P={b['P']:.3f} R={b['R']:.3f} "
|
||||
f"FA={b['FA']} pairs_strip={inpool['pairs']['strip']['ordering_acc']:.3f} "
|
||||
f"capQ_strip={inpool['capQ_inpool']['strip']:.3f}", flush=True)
|
||||
print("ceiling done")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("cmd", choices=["pre", "grouped", "lfo", "metrics", "runtime", "onnx", "ceiling"])
|
||||
args = ap.parse_args()
|
||||
t0 = time.time()
|
||||
globals()[f"cmd_{args.cmd}"](args)
|
||||
print(f"elapsed {time.time()-t0:.1f}s")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user