diff --git a/cmd/semantic-router-experiment/experiment.py b/cmd/semantic-router-experiment/experiment.py new file mode 100644 index 0000000..266315e --- /dev/null +++ b/cmd/semantic-router-experiment/experiment.py @@ -0,0 +1,856 @@ +#!/usr/bin/env python3 +""" +Semantic Router Linear Head Experiment +====================================== + +Evaluates whether the six-way residual routing boundary is linearly learnable +from Maven's existing e5-small representation. + +Architecture under test: + NormalizedInput.MatchText + → existing multilingual-e5-small embedder + → tiny 6-class linear head (softmax logistic regression) + → SemanticRouteDecision + +No new encoder. No fine-tuning. Linear separability floor only. +""" + +import json +import hashlib +import sys +import warnings +from collections import defaultdict +from pathlib import Path + +import numpy as np +from sklearn.exceptions import ConvergenceWarning +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import ( + accuracy_score, + f1_score, + precision_recall_fscore_support, + confusion_matrix, + brier_score_loss, + log_loss, +) +from sklearn.preprocessing import LabelEncoder + +warnings.filterwarnings("ignore", category=ConvergenceWarning) + +# ─── Constants ────────────────────────────────────────────────────────────── + +ROUTES = ["action", "conversation", "knowledge", "memory_write", "system", "uncertain"] +ROUTE_IDX = {r: i for i, r in enumerate(ROUTES)} + +# Regularization grid +C_VALUES = [0.01, 0.1, 1.0, 10.0, 100.0] + +# Abstention thresholds +THRESHOLDS = [0.40, 0.50, 0.60, 0.70, 0.80, 0.90] + +# Contrast families +CONTRAST_FAMILIES = [ + "negation", "question", "reported_speech", "quotation", + "hypothetical", "capability_question", +] + +# ─── Data Loading ─────────────────────────────────────────────────────────── + +def load_embeddings(path): + with open(path) as f: + data = json.load(f) + meta = data["meta"] + examples = data["examples"] + return meta, examples + + +def filter_dev_pool(examples): + """Return only development pool examples (not frozen holdout).""" + return [e for e in examples if e["dev_pool"]] + + +def filter_residual(examples): + """Return only router-residual examples (fast_path_resolved == false).""" + return [e for e in examples if not e["fast_path_resolved"]] + + +def extract_Xy(examples): + """Extract feature matrix and label vector from cached examples.""" + X = np.array([e["embedding"] for e in examples]) + y = np.array([e["route"] for e in examples]) + return X, y + + +def get_fold_groups(examples): + """Return fold assignment array matching the Go-generated CV folds.""" + return np.array([e["cv_fold"] for e in examples]) + + +# ─── Grouped CV ───────────────────────────────────────────────────────────── + +def grouped_cv_experiment(X, y, fold_ids, C_values, examples_meta): + """ + Run grouped cross-validation with logistic regression. + Returns best C, per-fold results, and out-of-fold predictions. + """ + unique_folds = sorted(set(fold_ids)) + n_classes = len(ROUTES) + + results_by_C = {} + for C in C_values: + fold_metrics = [] + oof_rows = [] + + for test_fold in unique_folds: + train_mask = fold_ids != test_fold + test_mask = fold_ids == test_fold + + X_train, y_train = X[train_mask], y[train_mask] + X_test, y_test = X[test_mask], y[test_mask] + + model = LogisticRegression( + C=C, max_iter=2000, solver="lbfgs", + random_state=42, + ) + model.fit(X_train, y_train) + + y_pred = model.predict(X_test) + y_proba = model.predict_proba(X_test) + classes = model.classes_ + + acc = accuracy_score(y_test, y_pred) + macro_f1 = f1_score(y_test, y_pred, average="macro", zero_division=0) + + prec, rec, f1, sup = precision_recall_fscore_support( + y_test, y_pred, labels=ROUTES, zero_division=0 + ) + + false_action = 0 + for true, pred in zip(y_test, y_pred): + if true != "action" and pred == "action": + false_action += 1 + + fold_metrics.append({ + "fold": test_fold, + "train_size": len(X_train), + "test_size": len(X_test), + "accuracy": acc, + "macro_f1": macro_f1, + "per_class": { + route: {"precision": prec[i], "recall": rec[i], "f1": f1[i]} + for i, route in enumerate(ROUTES) + }, + "false_action": false_action, + }) + + for i, (true, pred) in enumerate(zip(y_test, y_pred)): + test_indices = np.where(test_mask)[0] + meta = examples_meta[test_indices[i]] + proba_dict = {cls: float(y_proba[i][j]) for j, cls in enumerate(classes)} + max_proba = max(proba_dict.values()) if proba_dict else 0.0 + oof_rows.append({ + "source_id": meta["source_id"], + "fold": test_fold, + "true": true, + "predicted": pred, + "correct": true == pred, + "max_proba": max_proba, + "proba": proba_dict, + "fast_path_resolved": meta.get("fast_path_resolved", False), + "tags": meta.get("tags", []), + "text": meta["text"], + }) + + # Aggregate across folds + mean_acc = np.mean([m["accuracy"] for m in fold_metrics]) + mean_f1 = np.mean([m["macro_f1"] for m in fold_metrics]) + std_acc = np.std([m["accuracy"] for m in fold_metrics]) + std_f1 = np.std([m["macro_f1"] for m in fold_metrics]) + total_fa = sum(m["false_action"] for m in fold_metrics) + + results_by_C[C] = { + "mean_accuracy": mean_acc, + "std_accuracy": std_acc, + "mean_macro_f1": mean_f1, + "std_macro_f1": std_f1, + "total_false_action": total_fa, + "fold_metrics": fold_metrics, + "oof_predictions": oof_rows, + } + + # Select best C by mean macro F1 + best_C = max(results_by_C, key=lambda c: results_by_C[c]["mean_macro_f1"]) + return best_C, results_by_C + + +# ─── Metrics Computation ──────────────────────────────────────────────────── + +def compute_full_metrics(y_true, y_pred, y_proba=None): + """Compute all required metrics from out-of-fold predictions.""" + acc = accuracy_score(y_true, y_pred) + macro_f1 = f1_score(y_true, y_pred, average="macro", zero_division=0) + + # Per-class P/R/F1 + prec, rec, f1, sup = precision_recall_fscore_support( + y_true, y_pred, labels=ROUTES, zero_division=0 + ) + + # Confusion matrix + cm = confusion_matrix(y_true, y_pred, labels=ROUTES) + + # False action + false_action = 0 + false_action_predicted = [] + for true, pred in zip(y_true, y_pred): + if true != "action" and pred == "action": + false_action += 1 + + # Action precision/recall + action_tp = sum(1 for t, p in zip(y_true, y_pred) if t == "action" and p == "action") + action_fp = sum(1 for t, p in zip(y_true, y_pred) if t != "action" and p == "action") + action_fn = sum(1 for t, p in zip(y_true, y_pred) if t == "action" and p != "action") + action_precision = action_tp / max(action_tp + action_fp, 1) + action_recall = action_tp / max(action_tp + action_fn, 1) + + # Uncertain precision/recall + unc_tp = sum(1 for t, p in zip(y_true, y_pred) if t == "uncertain" and p == "uncertain") + unc_fp = sum(1 for t, p in zip(y_true, y_pred) if t != "uncertain" and p == "uncertain") + unc_fn = sum(1 for t, p in zip(y_true, y_pred) if t == "uncertain" and p != "uncertain") + unc_precision = unc_tp / max(unc_tp + unc_fp, 1) + unc_recall = unc_tp / max(unc_tp + unc_fn, 1) + + metrics = { + "accuracy": acc, + "macro_f1": macro_f1, + "false_action_count": false_action, + "false_action_rate": false_action / max(len(y_true), 1), + "action_precision": action_precision, + "action_recall": action_recall, + "uncertain_precision": unc_precision, + "uncertain_recall": unc_recall, + "per_class": {}, + "confusion_matrix": cm.tolist(), + } + + for i, route in enumerate(ROUTES): + metrics["per_class"][route] = { + "precision": float(prec[i]), + "recall": float(rec[i]), + "f1": float(f1[i]), + "support": int(sup[i]), + } + + return metrics + + +def compute_calibration(y_true, y_pred, oof_rows): + """Compute ECE, Brier score, and per-threshold abstention curves.""" + # ECE (Expected Calibration Error) with 10 bins + n_bins = 10 + bin_boundaries = np.linspace(0, 1, n_bins + 1) + ece = 0.0 + total = len(y_true) + + confidences = np.array([r["max_proba"] for r in oof_rows]) + correct = np.array([r["correct"] for r in oof_rows]) + + for i in range(n_bins): + lo, hi = bin_boundaries[i], bin_boundaries[i + 1] + mask = (confidences > lo) & (confidences <= hi) + if mask.sum() == 0: + continue + bin_acc = correct[mask].mean() + bin_conf = confidences[mask].mean() + ece += mask.sum() / total * abs(bin_acc - bin_conf) + + # Brier score (multiclass one-hot encoding) + y_true_idx = np.array([ROUTES.index(t) for t in y_true]) + n_samples = len(y_true) + n_classes = len(ROUTES) + y_true_oh = np.zeros((n_samples, n_classes)) + y_true_oh[np.arange(n_samples), y_true_idx] = 1.0 + + # Build probability matrix + y_proba_matrix = np.zeros((n_samples, n_classes)) + for i, r in enumerate(oof_rows): + for j, route in enumerate(ROUTES): + y_proba_matrix[i, j] = r["proba"].get(route, 0.0) + + brier = np.mean(np.sum((y_proba_matrix - y_true_oh) ** 2, axis=1)) + ll = log_loss(y_true, y_proba_matrix, labels=ROUTES) + + # Abstention curves + abstention_curves = [] + for thr in THRESHOLDS: + accepted_mask = confidences >= thr + n_accepted = accepted_mask.sum() + coverage = n_accepted / max(total, 1) + + if n_accepted > 0: + acc_accepted = accuracy_score(y_true[accepted_mask], y_pred[accepted_mask]) + f1_accepted = f1_score( + y_true[accepted_mask], y_pred[accepted_mask], + average="macro", zero_division=0, + ) + fa_count = sum( + 1 for t, p in zip(y_true[accepted_mask], y_pred[accepted_mask]) + if t != "action" and p == "action" + ) + else: + acc_accepted = 0.0 + f1_accepted = 0.0 + fa_count = 0 + + abstention_curves.append({ + "threshold": thr, + "n_accepted": int(n_accepted), + "coverage": coverage, + "accuracy": acc_accepted, + "macro_f1": f1_accepted, + "false_action_count": fa_count, + }) + + return { + "ece": float(ece), + "brier": float(brier), + "log_loss": float(ll), + "abstention_curves": abstention_curves, + } + + +def compute_action_threshold(oof_rows): + """Evaluate action-specific safety gate thresholds.""" + action_curves = [] + for thr in THRESHOLDS: + action_pred = [] + for r in oof_rows: + p = r["predicted"] + proba = r["proba"].get("action", 0.0) + if p == "action" and proba < thr: + # Demote action prediction + # Find next best route that isn't action + sorted_routes = sorted(r["proba"].items(), key=lambda x: -x[1]) + for route, _ in sorted_routes: + if route != "action": + p = route + break + action_pred.append(p) + + y_true = np.array([r["true"] for r in oof_rows]) + y_pred = np.array(action_pred) + + action_tp = sum(1 for t, p in zip(y_true, y_pred) if t == "action" and p == "action") + action_fp = sum(1 for t, p in zip(y_true, y_pred) if t != "action" and p == "action") + action_fn = sum(1 for t, p in zip(y_true, y_pred) if t == "action" and p != "action") + false_action = sum(1 for t, p in zip(y_true, y_pred) if t != "action" and p == "action") + + action_curves.append({ + "threshold": thr, + "action_precision": action_tp / max(action_tp + action_fp, 1), + "action_recall": action_tp / max(action_tp + action_fn, 1), + "false_action_count": false_action, + }) + + return action_curves + + +def compute_disagreement(y_true, y_pred, legacy_pred, oof_rows): + """Analyze disagreements between legacy and learned router.""" + results = { + "legacy_wrong_learned_right": [], + "legacy_right_learned_wrong": [], + "both_wrong_differently": [], + } + + legacy_false_actions = [] + learned_false_actions = [] + shared_false_actions = [] + + for i, r in enumerate(oof_rows): + sid = r["source_id"] + text = r["text"] + true = y_true[i] + learned = y_pred[i] + legacy = legacy_pred[i] + + if legacy != true and learned == true: + results["legacy_wrong_learned_right"].append({ + "source_id": sid, "text": text, + "true": true, "legacy": legacy, "learned": learned, + }) + elif legacy == true and learned != true: + results["legacy_right_learned_wrong"].append({ + "source_id": sid, "text": text, + "true": true, "legacy": legacy, "learned": learned, + }) + elif legacy != true and learned != true and legacy != learned: + results["both_wrong_differently"].append({ + "source_id": sid, "text": text, + "true": true, "legacy": legacy, "learned": learned, + }) + + # False action tracking + if true != "action" and legacy == "action": + legacy_false_actions.append(sid) + if true != "action" and learned == "action": + learned_false_actions.append(sid) + if true != "action" and legacy == "action" and learned == "action": + shared_false_actions.append(sid) + + # Repaired false actions + repaired = [sid for sid in legacy_false_actions if sid not in learned_false_actions] + new_errors = [sid for sid in learned_false_actions if sid not in legacy_false_actions] + shared = shared_false_actions + + return { + "details": results, + "legacy_false_actions": legacy_false_actions, + "learned_false_actions": learned_false_actions, + "repaired": repaired, + "new_errors": new_errors, + "shared": shared, + } + + +def compute_contrast_family(oof_rows): + """Analyze performance per contrast family.""" + family_results = {} + for family in CONTRAST_FAMILIES: + members = [r for r in oof_rows if family in r.get("tags", [])] + if not members: + continue + y_true = [r["true"] for r in members] + y_pred = [r["predicted"] for r in members] + correct = sum(1 for t, p in zip(y_true, y_pred) if t == p) + false_act = sum(1 for t, p in zip(y_true, y_pred) if t != "action" and p == "action") + family_results[family] = { + "count": len(members), + "correct": correct, + "accuracy": correct / len(members), + "false_action": false_act, + } + return family_results + + +def compute_legacy_baseline(examples): + """ + Compute legacy baseline by mapping each example through the known + fast-path and classifier behavior. Since we don't have the actual + router running, we use the corpus metadata: + - fast_path_resolved examples are correct (grammar handles them) + - We simulate the legacy baseline from the test output numbers + """ + # The actual legacy baseline was measured in TestLegacyBaseline: + # overall accuracy: 52.2%, residual: 40.8%, false-action rate: 19.9% + # We need per-example predictions. We'll approximate from the corpus structure. + # + # For fast_path_resolved examples, the legacy router is correct (stage-0 grammar). + # For residual examples, we need to simulate the hash-embedder classifier. + # Since we don't have the hash embedder running, we use the known aggregate. + # + # This is a limitation: the legacy baseline numbers come from the Go test, + # and we only have the aggregate. We'll use the aggregate for comparison. + pass + + +# ─── Report Generation ────────────────────────────────────────────────────── + +def fmt_pct(v, decimals=1): + return f"{100 * v:.{decimals}f}%" + + +def fmt_float(v, decimals=3): + return f"{v:.{decimals}f}" + + +def generate_report(meta, all_results, residual_results, oof_all, oof_residual): + """Generate the full experiment report.""" + lines = [] + lines.append("# Semantic Router Linear Head Experiment — Report") + lines.append("") + lines.append("## 1. Exact e5 representation used") + lines.append("") + lines.append(f"- **Model**: {meta['embedder_id']}") + lines.append(f"- **Checkpoint**: {meta['model_path']}") + lines.append(f"- **Tokenizer**: {meta['tokenizer_path']}") + lines.append(f"- **Dimension**: {meta['dimension']}") + lines.append(f"- **Pooling**: {meta['pooling']}") + lines.append(f"- **Normalization**: {meta['normalization']}") + lines.append(f"- **Input template**: {meta['input_template']}") + lines.append("") + + lines.append("## 2. Development/residual row counts") + lines.append("") + lines.append(f"- Total corpus: {meta['total_examples']}") + lines.append(f"- Frozen holdout: {meta['frozen_count']}") + lines.append(f"- Development pool: {meta['dev_count']}") + lines.append(f"- Fast-path resolved: {meta['fast_path_count']}") + lines.append(f"- Router-residual: {meta['residual_count']}") + lines.append("") + lines.append("Route distribution (full corpus):") + for route, count in sorted(meta["route_counts"].items()): + lines.append(f" - {route}: {count}") + lines.append("") + + lines.append("## 3. Grouped fold composition") + lines.append("") + lines.append(f"Folds: {meta['cv_folds']}") + for fold_id, stats in sorted(meta["fold_composition"].items()): + lines.append(f" - Fold {fold_id}: eval={stats['eval_count']} train={stats['train_count']} routes={stats['eval_routes']}") + lines.append("") + + # Regularization selection + lines.append("## 4. Selected regularization") + lines.append("") + + lines.append("### Experiment A: All development examples") + best_C_all = all_results["best_C"] + lines.append(f"- Best C: {best_C_all}") + lines.append(f"- Mean accuracy: {fmt_pct(all_results['results_by_C'][best_C_all]['mean_accuracy'])} ± {fmt_pct(all_results['results_by_C'][best_C_all]['std_accuracy'])}") + lines.append(f"- Mean macro F1: {fmt_float(all_results['results_by_C'][best_C_all]['mean_macro_f1'])} ± {fmt_float(all_results['results_by_C'][best_C_all]['std_macro_f1'])}") + lines.append(f"- Total false actions (CV): {all_results['results_by_C'][best_C_all]['total_false_action']}") + lines.append("") + + lines.append("### Experiment B: Router-residual only") + best_C_res = residual_results["best_C"] + lines.append(f"- Best C: {best_C_res}") + lines.append(f"- Mean accuracy: {fmt_pct(residual_results['results_by_C'][best_C_res]['mean_accuracy'])} ± {fmt_pct(residual_results['results_by_C'][best_C_res]['std_accuracy'])}") + lines.append(f"- Mean macro F1: {fmt_float(residual_results['results_by_C'][best_C_res]['mean_macro_f1'])} ± {fmt_float(residual_results['results_by_C'][best_C_res]['std_macro_f1'])}") + lines.append(f"- Total false actions (CV): {residual_results['results_by_C'][best_C_res]['total_false_action']}") + lines.append("") + + # Stability across folds + lines.append("### Stability across folds") + lines.append("") + for C_val in C_VALUES: + r = all_results["results_by_C"][C_val] + fold_accs = [m["accuracy"] for m in r["fold_metrics"]] + fold_f1s = [m["macro_f1"] for m in r["fold_metrics"]] + lines.append(f" C={C_val:<6} acc={fmt_pct(r['mean_accuracy'])}±{fmt_pct(r['std_accuracy'])} f1={fmt_float(r['mean_macro_f1'])}±{fmt_float(r['std_macro_f1'])} folds_acc={[fmt_pct(a) for a in fold_accs]}") + lines.append("") + + # Experiment A metrics + lines.append("## 5. All-example CV metrics") + lines.append("") + metrics_all = all_results["full_metrics"] + lines.append(f"- Accuracy: {fmt_pct(metrics_all['accuracy'])}") + lines.append(f"- Macro F1: {fmt_float(metrics_all['macro_f1'])}") + lines.append(f"- False-action count: {metrics_all['false_action_count']}") + lines.append(f"- False-action rate: {fmt_pct(metrics_all['false_action_rate'])}") + lines.append(f"- Action precision: {fmt_float(metrics_all['action_precision'])}") + lines.append(f"- Action recall: {fmt_float(metrics_all['action_recall'])}") + lines.append(f"- Uncertain precision: {fmt_float(metrics_all['uncertain_precision'])}") + lines.append(f"- Uncertain recall: {fmt_float(metrics_all['uncertain_recall'])}") + lines.append("") + lines.append("Per-class metrics:") + for route in ROUTES: + pc = metrics_all["per_class"][route] + lines.append(f" {route:<15} P={fmt_float(pc['precision'])} R={fmt_float(pc['recall'])} F1={fmt_float(pc['f1'])} (n={pc['support']})") + lines.append("") + lines.append("Confusion matrix (rows=expected, cols=predicted):") + header = f"{'':>15}" + "".join(f"{r:>15}" for r in ROUTES) + lines.append(header) + for i, route in enumerate(ROUTES): + row = f"{route:>15}" + "".join(f"{metrics_all['confusion_matrix'][i][j]:>15}" for j in range(len(ROUTES))) + lines.append(row) + lines.append("") + + # Experiment B metrics + lines.append("## 6. Residual-only CV metrics") + lines.append("") + metrics_res = residual_results["full_metrics"] + lines.append(f"- Accuracy: {fmt_pct(metrics_res['accuracy'])}") + lines.append(f"- Macro F1: {fmt_float(metrics_res['macro_f1'])}") + lines.append(f"- False-action count: {metrics_res['false_action_count']}") + lines.append(f"- False-action rate: {fmt_pct(metrics_res['false_action_rate'])}") + lines.append(f"- Action precision: {fmt_float(metrics_res['action_precision'])}") + lines.append(f"- Action recall: {fmt_float(metrics_res['action_recall'])}") + lines.append(f"- Uncertain precision: {fmt_float(metrics_res['uncertain_precision'])}") + lines.append(f"- Uncertain recall: {fmt_float(metrics_res['uncertain_recall'])}") + lines.append("") + lines.append("Per-class metrics:") + for route in ROUTES: + pc = metrics_res["per_class"][route] + lines.append(f" {route:<15} P={fmt_float(pc['precision'])} R={fmt_float(pc['recall'])} F1={fmt_float(pc['f1'])} (n={pc['support']})") + lines.append("") + lines.append("Confusion matrix (rows=expected, cols=predicted):") + header = f"{'':>15}" + "".join(f"{r:>15}" for r in ROUTES) + lines.append(header) + for i, route in enumerate(ROUTES): + row = f"{route:>15}" + "".join(f"{metrics_res['confusion_matrix'][i][j]:>15}" for j in range(len(ROUTES))) + lines.append(row) + lines.append("") + + # Legacy comparison + lines.append("## 7. Legacy-vs-linear comparison") + lines.append("") + lines.append("### All examples") + lines.append(f"{'metric':<25} {'legacy':>10} {'linear e5':>10} {'delta':>10}") + lines.append("-" * 55) + # Legacy baseline from test: 52.2% overall, 40.8% residual, 19.9% false-action + # These are approximate since we don't have per-example legacy predictions + legacy_acc = 0.522 + legacy_fa_rate = 0.199 + legacy_macro_f1 = 0.0 # unknown precisely + lines.append(f"{'accuracy':<25} {fmt_pct(legacy_acc):>10} {fmt_pct(metrics_all['accuracy']):>10} {fmt_pct(metrics_all['accuracy'] - legacy_acc):>10}") + lines.append(f"{'macro F1':<25} {'—':>10} {fmt_float(metrics_all['macro_f1']):>10} {'—':>10}") + lines.append(f"{'action precision':<25} {'—':>10} {fmt_float(metrics_all['action_precision']):>10} {'—':>10}") + lines.append(f"{'false-action rate':<25} {fmt_pct(legacy_fa_rate):>10} {fmt_pct(metrics_all['false_action_rate']):>10} {fmt_pct(metrics_all['false_action_rate'] - legacy_fa_rate):>10}") + lines.append(f"{'uncertain F1':<25} {fmt_float(0.0):>10} {fmt_float(metrics_all['per_class']['uncertain']['f1']):>10} {fmt_float(metrics_all['per_class']['uncertain']['f1']):>10}") + lines.append("") + + lines.append("### Router-residual only") + lines.append(f"{'metric':<25} {'legacy':>10} {'linear e5':>10} {'delta':>10}") + lines.append("-" * 55) + legacy_res_acc = 0.408 + lines.append(f"{'accuracy':<25} {fmt_pct(legacy_res_acc):>10} {fmt_pct(metrics_res['accuracy']):>10} {fmt_pct(metrics_res['accuracy'] - legacy_res_acc):>10}") + lines.append(f"{'macro F1':<25} {'—':>10} {fmt_float(metrics_res['macro_f1']):>10} {'—':>10}") + lines.append(f"{'false-action rate':<25} {'—':>10} {fmt_pct(metrics_res['false_action_rate']):>10} {'—':>10}") + lines.append("") + + # Fold variance + lines.append("## 8. Fold variance") + lines.append("") + lines.append("All-example CV:") + for m in all_results["results_by_C"][best_C_all]["fold_metrics"]: + lines.append(f" Fold {m['fold']}: acc={fmt_pct(m['accuracy'])} f1={fmt_float(m['macro_f1'])} false_action={m['false_action']}") + lines.append("") + lines.append("Residual-only CV:") + for m in residual_results["results_by_C"][best_C_res]["fold_metrics"]: + lines.append(f" Fold {m['fold']}: acc={fmt_pct(m['accuracy'])} f1={fmt_float(m['macro_f1'])} false_action={m['false_action']}") + lines.append("") + + # Disagreement analysis + lines.append("## 9. False-action repair/new-error analysis") + lines.append("") + # The disagreement analysis requires legacy per-example predictions. + # Since we don't have those, we report what we can from the out-of-fold data. + lines.append("Note: Legacy per-example predictions were not available for this experiment.") + lines.append("The legacy baseline was measured in aggregate in the Go test suite.") + lines.append("") + lines.append("Learned router false-action cases (out-of-fold):") + for r in oof_all: + if r["true"] != "action" and r["predicted"] == "action": + lines.append(f" {r['source_id']}: '{r['text']}' (true={r['true']}, proba(action)={r['proba'].get('action', 0):.3f})") + lines.append("") + + # Contrast family + lines.append("## 10. Contrast-family results") + lines.append("") + lines.append("### Experiment A (all dev)") + contrast_all = all_results["contrast_family"] + lines.append(f"{'family':<25} {'count':>6} {'correct':>8} {'accuracy':>10} {'false_act':>10}") + lines.append("-" * 60) + for family in CONTRAST_FAMILIES: + if family in contrast_all: + c = contrast_all[family] + lines.append(f"{family:<25} {c['count']:>6} {c['correct']:>8} {fmt_pct(c['accuracy']):>10} {c['false_action']:>10}") + lines.append("") + + lines.append("### Experiment B (residual only)") + contrast_res = residual_results["contrast_family"] + lines.append(f"{'family':<25} {'count':>6} {'correct':>8} {'accuracy':>10} {'false_act':>10}") + lines.append("-" * 60) + for family in CONTRAST_FAMILIES: + if family in contrast_res: + c = contrast_res[family] + lines.append(f"{family:<25} {c['count']:>6} {c['correct']:>8} {fmt_pct(c['accuracy']):>10} {c['false_action']:>10}") + lines.append("") + + # Calibration + lines.append("## 11. Calibration metrics") + lines.append("") + lines.append("### Experiment A") + cal_all = all_results["calibration"] + lines.append(f"- ECE: {fmt_float(cal_all['ece'])}") + lines.append(f"- Brier score: {fmt_float(cal_all['brier'])}") + lines.append(f"- Log loss: {fmt_float(cal_all['log_loss'])}") + lines.append("") + lines.append("### Experiment B") + cal_res = residual_results["calibration"] + lines.append(f"- ECE: {fmt_float(cal_res['ece'])}") + lines.append(f"- Brier score: {fmt_float(cal_res['brier'])}") + lines.append(f"- Log loss: {fmt_float(cal_res['log_loss'])}") + lines.append("") + + # Abstention curves + lines.append("## 12. Abstention curves") + lines.append("") + lines.append("### Experiment A (all dev)") + lines.append(f"{'threshold':>10} {'n_accepted':>11} {'coverage':>10} {'accuracy':>10} {'macro_f1':>10} {'false_act':>10}") + lines.append("-" * 62) + for curve in cal_all["abstention_curves"]: + lines.append(f"{curve['threshold']:>10.2f} {curve['n_accepted']:>11} {fmt_pct(curve['coverage']):>10} {fmt_pct(curve['accuracy']):>10} {fmt_float(curve['macro_f1']):>10} {curve['false_action_count']:>10}") + lines.append("") + + lines.append("### Experiment B (residual only)") + lines.append(f"{'threshold':>10} {'n_accepted':>11} {'coverage':>10} {'accuracy':>10} {'macro_f1':>10} {'false_act':>10}") + lines.append("-" * 62) + for curve in cal_res["abstention_curves"]: + lines.append(f"{curve['threshold']:>10.2f} {curve['n_accepted']:>11} {fmt_pct(curve['coverage']):>10} {fmt_pct(curve['accuracy']):>10} {fmt_float(curve['macro_f1']):>10} {curve['false_action_count']:>10}") + lines.append("") + + # Action threshold + lines.append("## 13. Action-threshold curve") + lines.append("") + lines.append("### Experiment A") + lines.append(f"{'threshold':>10} {'action_P':>10} {'action_R':>10} {'false_act':>10}") + lines.append("-" * 40) + for curve in all_results["action_threshold"]: + lines.append(f"{curve['threshold']:>10.2f} {fmt_float(curve['action_precision']):>10} {fmt_float(curve['action_recall']):>10} {curve['false_action_count']:>10}") + lines.append("") + + lines.append("### Experiment B") + lines.append(f"{'threshold':>10} {'action_P':>10} {'action_R':>10} {'false_act':>10}") + lines.append("-" * 40) + for curve in residual_results["action_threshold"]: + lines.append(f"{curve['threshold']:>10.2f} {fmt_float(curve['action_precision']):>10} {fmt_float(curve['action_recall']):>10} {curve['false_action_count']:>10}") + lines.append("") + + # Model artifact size + lines.append("## 14. Model artifact size and runtime cost") + lines.append("") + # Logistic regression: 6 classes × 384 features + 6 biases = 2310 parameters + n_params = len(ROUTES) * meta["dimension"] + len(ROUTES) + serialized_bytes = n_params * 4 # float32 + lines.append(f"- Trainable parameters: {n_params}") + lines.append(f" - {len(ROUTES)} classes × {meta['dimension']} features = {len(ROUTES) * meta['dimension']} weights") + lines.append(f" - {len(ROUTES)} bias terms") + lines.append(f"- Serialized head size: {serialized_bytes} bytes ({serialized_bytes / 1024:.1f} KB)") + lines.append(f"- Additional inference FLOPs: {len(ROUTES) * meta['dimension']} multiply-accumulates") + lines.append(f"- Incremental cost (e5 already computed): ~{len(ROUTES) * meta['dimension']} FLOPs, <1µs") + lines.append(f"- Cost if semantic router must trigger its own e5: full ONNX inference (~{meta['dimension']} × 128 × 12 = ~590K FLOPs)") + lines.append("") + + # Recommendation + lines.append("## 16. Recommendation") + lines.append("") + # Decision logic + all_f1 = metrics_all["macro_f1"] + res_f1 = metrics_res["macro_f1"] + res_acc = metrics_res["accuracy"] + + if res_f1 > 0.5 and res_acc > 0.55: + verdict = "linear head sufficient" + detail = (f"Residual macro F1 of {fmt_float(res_f1)} and accuracy of {fmt_pct(res_acc)} " + f"exceed the legacy baseline (40.8% residual accuracy) by a meaningful margin. " + f"A linear head over frozen e5-small embeddings is a viable first production candidate.") + elif all_f1 > 0.5 and res_f1 < 0.5: + verdict = "need more data" + detail = (f"All-example F1 ({fmt_float(all_f1)}) is acceptable but residual-only F1 " + f"({fmt_float(res_f1)}) drops, suggesting the contrast-family examples are " + f"hard for a linear classifier. More contrastive training data may help.") + else: + verdict = "representation inadequate" + detail = (f"Neither all-example ({fmt_float(all_f1)}) nor residual-only ({fmt_float(res_f1)}) " + f"F1 reaches the minimum viable threshold. The e5-small linear separability floor " + f"is insufficient for this 6-way task. Consider a non-linear head or a different encoder.") + + lines.append(f"**{verdict}**") + lines.append("") + lines.append(detail) + lines.append("") + + # Commit hash + lines.append("## 17. Commit hash for experiment tooling") + lines.append("") + import subprocess + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, text=True, cwd="/home/kami/apps/Maven" + ) + lines.append(f"`{result.stdout.strip()}`") + except Exception: + lines.append("(unable to determine)") + lines.append("") + + return "\n".join(lines) + + +# ─── Main ─────────────────────────────────────────────────────────────────── + +def run_experiment(X, y, fold_ids, examples_meta, label): + """Run the full experiment pipeline for one population.""" + print(f"\n{'='*60}") + print(f" Experiment: {label}") + print(f" Samples: {len(y)} Features: {X.shape[1]} Folds: {len(set(fold_ids))}") + print(f"{'='*60}") + + # 1. Grouped CV with regularization search + best_C, results_by_C = grouped_cv_experiment(X, y, fold_ids, C_VALUES, examples_meta) + + # 2. Use best C to get full metrics from OOF predictions + oof_rows = results_by_C[best_C]["oof_predictions"] + oof_true = np.array([r["true"] for r in oof_rows]) + oof_pred = np.array([r["predicted"] for r in oof_rows]) + + full_metrics = compute_full_metrics(oof_true, oof_pred) + + # 3. Calibration + calibration = compute_calibration(oof_true, oof_pred, oof_rows) + + # 4. Action threshold + action_threshold = compute_action_threshold(oof_rows) + + # 5. Contrast family + contrast_family = compute_contrast_family(oof_rows) + + return { + "best_C": best_C, + "results_by_C": results_by_C, + "full_metrics": full_metrics, + "calibration": calibration, + "action_threshold": action_threshold, + "contrast_family": contrast_family, + "oof_rows": oof_rows, + } + + +if __name__ == "__main__": + EMBEDDING_PATH = "/tmp/mvn-experiment/embeddings.json" + REPORT_PATH = "/tmp/mvn-experiment/report.md" + + # Load data + meta, examples = load_embeddings(EMBEDDING_PATH) + print(f"Loaded {len(examples)} examples, embedder={meta['embedder_id']}, dim={meta['dimension']}") + + # Development pool only + dev_examples = filter_dev_pool(examples) + print(f"Development pool: {len(dev_examples)} examples") + + # Extract features + X_all, y_all = extract_Xy(dev_examples) + fold_ids_all = get_fold_groups(dev_examples) + + # Experiment A: all dev examples + all_results = run_experiment(X_all, y_all, fold_ids_all, dev_examples, "All development examples") + + # Experiment B: residual only + dev_residual = filter_residual(dev_examples) + X_res, y_res = extract_Xy(dev_residual) + fold_ids_res = get_fold_groups(dev_residual) + residual_results = run_experiment(X_res, y_res, fold_ids_res, dev_residual, "Router-residual only") + + # Generate report + report = generate_report( + meta, all_results, residual_results, + all_results["oof_rows"], residual_results["oof_rows"], + ) + + with open(REPORT_PATH, "w") as f: + f.write(report) + print(f"\nReport written to {REPORT_PATH}") + + # Also print summary + print("\n" + "="*60) + print(" SUMMARY") + print("="*60) + print(f" All-example: acc={fmt_pct(all_results['full_metrics']['accuracy'])} macro_f1={fmt_float(all_results['full_metrics']['macro_f1'])} false_action={all_results['full_metrics']['false_action_count']}") + print(f" Residual: acc={fmt_pct(residual_results['full_metrics']['accuracy'])} macro_f1={fmt_float(residual_results['full_metrics']['macro_f1'])} false_action={residual_results['full_metrics']['false_action_count']}") + print(f" Best C (all): {all_results['best_C']}") + print(f" Best C (res): {residual_results['best_C']}") + print(f" ECE (all): {fmt_float(all_results['calibration']['ece'])}") + print(f" ECE (res): {fmt_float(residual_results['calibration']['ece'])}") + print(f" Brier (all): {fmt_float(all_results['calibration']['brier'])}") + print(f" Brier (res): {fmt_float(residual_results['calibration']['brier'])}")