claude
9df223907b
Python experiment: 6-class logistic regression over frozen e5 embeddings, grouped cross-validation, calibration, abstention curves, contrast-family analysis, action-threshold sweep.
857 lines
36 KiB
Python
857 lines
36 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Semantic Router Linear Head Experiment
|
||
======================================
|
||
|
||
Evaluates whether the six-way residual routing boundary is linearly learnable
|
||
from Maven's existing e5-small representation.
|
||
|
||
Architecture under test:
|
||
NormalizedInput.MatchText
|
||
→ existing multilingual-e5-small embedder
|
||
→ tiny 6-class linear head (softmax logistic regression)
|
||
→ SemanticRouteDecision
|
||
|
||
No new encoder. No fine-tuning. Linear separability floor only.
|
||
"""
|
||
|
||
import json
|
||
import hashlib
|
||
import sys
|
||
import warnings
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
|
||
import numpy as np
|
||
from sklearn.exceptions import ConvergenceWarning
|
||
from sklearn.linear_model import LogisticRegression
|
||
from sklearn.metrics import (
|
||
accuracy_score,
|
||
f1_score,
|
||
precision_recall_fscore_support,
|
||
confusion_matrix,
|
||
brier_score_loss,
|
||
log_loss,
|
||
)
|
||
from sklearn.preprocessing import LabelEncoder
|
||
|
||
warnings.filterwarnings("ignore", category=ConvergenceWarning)
|
||
|
||
# ─── Constants ──────────────────────────────────────────────────────────────
|
||
|
||
ROUTES = ["action", "conversation", "knowledge", "memory_write", "system", "uncertain"]
|
||
ROUTE_IDX = {r: i for i, r in enumerate(ROUTES)}
|
||
|
||
# Regularization grid
|
||
C_VALUES = [0.01, 0.1, 1.0, 10.0, 100.0]
|
||
|
||
# Abstention thresholds
|
||
THRESHOLDS = [0.40, 0.50, 0.60, 0.70, 0.80, 0.90]
|
||
|
||
# Contrast families
|
||
CONTRAST_FAMILIES = [
|
||
"negation", "question", "reported_speech", "quotation",
|
||
"hypothetical", "capability_question",
|
||
]
|
||
|
||
# ─── Data Loading ───────────────────────────────────────────────────────────
|
||
|
||
def load_embeddings(path):
|
||
with open(path) as f:
|
||
data = json.load(f)
|
||
meta = data["meta"]
|
||
examples = data["examples"]
|
||
return meta, examples
|
||
|
||
|
||
def filter_dev_pool(examples):
|
||
"""Return only development pool examples (not frozen holdout)."""
|
||
return [e for e in examples if e["dev_pool"]]
|
||
|
||
|
||
def filter_residual(examples):
|
||
"""Return only router-residual examples (fast_path_resolved == false)."""
|
||
return [e for e in examples if not e["fast_path_resolved"]]
|
||
|
||
|
||
def extract_Xy(examples):
|
||
"""Extract feature matrix and label vector from cached examples."""
|
||
X = np.array([e["embedding"] for e in examples])
|
||
y = np.array([e["route"] for e in examples])
|
||
return X, y
|
||
|
||
|
||
def get_fold_groups(examples):
|
||
"""Return fold assignment array matching the Go-generated CV folds."""
|
||
return np.array([e["cv_fold"] for e in examples])
|
||
|
||
|
||
# ─── Grouped CV ─────────────────────────────────────────────────────────────
|
||
|
||
def grouped_cv_experiment(X, y, fold_ids, C_values, examples_meta):
|
||
"""
|
||
Run grouped cross-validation with logistic regression.
|
||
Returns best C, per-fold results, and out-of-fold predictions.
|
||
"""
|
||
unique_folds = sorted(set(fold_ids))
|
||
n_classes = len(ROUTES)
|
||
|
||
results_by_C = {}
|
||
for C in C_values:
|
||
fold_metrics = []
|
||
oof_rows = []
|
||
|
||
for test_fold in unique_folds:
|
||
train_mask = fold_ids != test_fold
|
||
test_mask = fold_ids == test_fold
|
||
|
||
X_train, y_train = X[train_mask], y[train_mask]
|
||
X_test, y_test = X[test_mask], y[test_mask]
|
||
|
||
model = LogisticRegression(
|
||
C=C, max_iter=2000, solver="lbfgs",
|
||
random_state=42,
|
||
)
|
||
model.fit(X_train, y_train)
|
||
|
||
y_pred = model.predict(X_test)
|
||
y_proba = model.predict_proba(X_test)
|
||
classes = model.classes_
|
||
|
||
acc = accuracy_score(y_test, y_pred)
|
||
macro_f1 = f1_score(y_test, y_pred, average="macro", zero_division=0)
|
||
|
||
prec, rec, f1, sup = precision_recall_fscore_support(
|
||
y_test, y_pred, labels=ROUTES, zero_division=0
|
||
)
|
||
|
||
false_action = 0
|
||
for true, pred in zip(y_test, y_pred):
|
||
if true != "action" and pred == "action":
|
||
false_action += 1
|
||
|
||
fold_metrics.append({
|
||
"fold": test_fold,
|
||
"train_size": len(X_train),
|
||
"test_size": len(X_test),
|
||
"accuracy": acc,
|
||
"macro_f1": macro_f1,
|
||
"per_class": {
|
||
route: {"precision": prec[i], "recall": rec[i], "f1": f1[i]}
|
||
for i, route in enumerate(ROUTES)
|
||
},
|
||
"false_action": false_action,
|
||
})
|
||
|
||
for i, (true, pred) in enumerate(zip(y_test, y_pred)):
|
||
test_indices = np.where(test_mask)[0]
|
||
meta = examples_meta[test_indices[i]]
|
||
proba_dict = {cls: float(y_proba[i][j]) for j, cls in enumerate(classes)}
|
||
max_proba = max(proba_dict.values()) if proba_dict else 0.0
|
||
oof_rows.append({
|
||
"source_id": meta["source_id"],
|
||
"fold": test_fold,
|
||
"true": true,
|
||
"predicted": pred,
|
||
"correct": true == pred,
|
||
"max_proba": max_proba,
|
||
"proba": proba_dict,
|
||
"fast_path_resolved": meta.get("fast_path_resolved", False),
|
||
"tags": meta.get("tags", []),
|
||
"text": meta["text"],
|
||
})
|
||
|
||
# Aggregate across folds
|
||
mean_acc = np.mean([m["accuracy"] for m in fold_metrics])
|
||
mean_f1 = np.mean([m["macro_f1"] for m in fold_metrics])
|
||
std_acc = np.std([m["accuracy"] for m in fold_metrics])
|
||
std_f1 = np.std([m["macro_f1"] for m in fold_metrics])
|
||
total_fa = sum(m["false_action"] for m in fold_metrics)
|
||
|
||
results_by_C[C] = {
|
||
"mean_accuracy": mean_acc,
|
||
"std_accuracy": std_acc,
|
||
"mean_macro_f1": mean_f1,
|
||
"std_macro_f1": std_f1,
|
||
"total_false_action": total_fa,
|
||
"fold_metrics": fold_metrics,
|
||
"oof_predictions": oof_rows,
|
||
}
|
||
|
||
# Select best C by mean macro F1
|
||
best_C = max(results_by_C, key=lambda c: results_by_C[c]["mean_macro_f1"])
|
||
return best_C, results_by_C
|
||
|
||
|
||
# ─── Metrics Computation ────────────────────────────────────────────────────
|
||
|
||
def compute_full_metrics(y_true, y_pred, y_proba=None):
|
||
"""Compute all required metrics from out-of-fold predictions."""
|
||
acc = accuracy_score(y_true, y_pred)
|
||
macro_f1 = f1_score(y_true, y_pred, average="macro", zero_division=0)
|
||
|
||
# Per-class P/R/F1
|
||
prec, rec, f1, sup = precision_recall_fscore_support(
|
||
y_true, y_pred, labels=ROUTES, zero_division=0
|
||
)
|
||
|
||
# Confusion matrix
|
||
cm = confusion_matrix(y_true, y_pred, labels=ROUTES)
|
||
|
||
# False action
|
||
false_action = 0
|
||
false_action_predicted = []
|
||
for true, pred in zip(y_true, y_pred):
|
||
if true != "action" and pred == "action":
|
||
false_action += 1
|
||
|
||
# Action precision/recall
|
||
action_tp = sum(1 for t, p in zip(y_true, y_pred) if t == "action" and p == "action")
|
||
action_fp = sum(1 for t, p in zip(y_true, y_pred) if t != "action" and p == "action")
|
||
action_fn = sum(1 for t, p in zip(y_true, y_pred) if t == "action" and p != "action")
|
||
action_precision = action_tp / max(action_tp + action_fp, 1)
|
||
action_recall = action_tp / max(action_tp + action_fn, 1)
|
||
|
||
# Uncertain precision/recall
|
||
unc_tp = sum(1 for t, p in zip(y_true, y_pred) if t == "uncertain" and p == "uncertain")
|
||
unc_fp = sum(1 for t, p in zip(y_true, y_pred) if t != "uncertain" and p == "uncertain")
|
||
unc_fn = sum(1 for t, p in zip(y_true, y_pred) if t == "uncertain" and p != "uncertain")
|
||
unc_precision = unc_tp / max(unc_tp + unc_fp, 1)
|
||
unc_recall = unc_tp / max(unc_tp + unc_fn, 1)
|
||
|
||
metrics = {
|
||
"accuracy": acc,
|
||
"macro_f1": macro_f1,
|
||
"false_action_count": false_action,
|
||
"false_action_rate": false_action / max(len(y_true), 1),
|
||
"action_precision": action_precision,
|
||
"action_recall": action_recall,
|
||
"uncertain_precision": unc_precision,
|
||
"uncertain_recall": unc_recall,
|
||
"per_class": {},
|
||
"confusion_matrix": cm.tolist(),
|
||
}
|
||
|
||
for i, route in enumerate(ROUTES):
|
||
metrics["per_class"][route] = {
|
||
"precision": float(prec[i]),
|
||
"recall": float(rec[i]),
|
||
"f1": float(f1[i]),
|
||
"support": int(sup[i]),
|
||
}
|
||
|
||
return metrics
|
||
|
||
|
||
def compute_calibration(y_true, y_pred, oof_rows):
|
||
"""Compute ECE, Brier score, and per-threshold abstention curves."""
|
||
# ECE (Expected Calibration Error) with 10 bins
|
||
n_bins = 10
|
||
bin_boundaries = np.linspace(0, 1, n_bins + 1)
|
||
ece = 0.0
|
||
total = len(y_true)
|
||
|
||
confidences = np.array([r["max_proba"] for r in oof_rows])
|
||
correct = np.array([r["correct"] for r in oof_rows])
|
||
|
||
for i in range(n_bins):
|
||
lo, hi = bin_boundaries[i], bin_boundaries[i + 1]
|
||
mask = (confidences > lo) & (confidences <= hi)
|
||
if mask.sum() == 0:
|
||
continue
|
||
bin_acc = correct[mask].mean()
|
||
bin_conf = confidences[mask].mean()
|
||
ece += mask.sum() / total * abs(bin_acc - bin_conf)
|
||
|
||
# Brier score (multiclass one-hot encoding)
|
||
y_true_idx = np.array([ROUTES.index(t) for t in y_true])
|
||
n_samples = len(y_true)
|
||
n_classes = len(ROUTES)
|
||
y_true_oh = np.zeros((n_samples, n_classes))
|
||
y_true_oh[np.arange(n_samples), y_true_idx] = 1.0
|
||
|
||
# Build probability matrix
|
||
y_proba_matrix = np.zeros((n_samples, n_classes))
|
||
for i, r in enumerate(oof_rows):
|
||
for j, route in enumerate(ROUTES):
|
||
y_proba_matrix[i, j] = r["proba"].get(route, 0.0)
|
||
|
||
brier = np.mean(np.sum((y_proba_matrix - y_true_oh) ** 2, axis=1))
|
||
ll = log_loss(y_true, y_proba_matrix, labels=ROUTES)
|
||
|
||
# Abstention curves
|
||
abstention_curves = []
|
||
for thr in THRESHOLDS:
|
||
accepted_mask = confidences >= thr
|
||
n_accepted = accepted_mask.sum()
|
||
coverage = n_accepted / max(total, 1)
|
||
|
||
if n_accepted > 0:
|
||
acc_accepted = accuracy_score(y_true[accepted_mask], y_pred[accepted_mask])
|
||
f1_accepted = f1_score(
|
||
y_true[accepted_mask], y_pred[accepted_mask],
|
||
average="macro", zero_division=0,
|
||
)
|
||
fa_count = sum(
|
||
1 for t, p in zip(y_true[accepted_mask], y_pred[accepted_mask])
|
||
if t != "action" and p == "action"
|
||
)
|
||
else:
|
||
acc_accepted = 0.0
|
||
f1_accepted = 0.0
|
||
fa_count = 0
|
||
|
||
abstention_curves.append({
|
||
"threshold": thr,
|
||
"n_accepted": int(n_accepted),
|
||
"coverage": coverage,
|
||
"accuracy": acc_accepted,
|
||
"macro_f1": f1_accepted,
|
||
"false_action_count": fa_count,
|
||
})
|
||
|
||
return {
|
||
"ece": float(ece),
|
||
"brier": float(brier),
|
||
"log_loss": float(ll),
|
||
"abstention_curves": abstention_curves,
|
||
}
|
||
|
||
|
||
def compute_action_threshold(oof_rows):
|
||
"""Evaluate action-specific safety gate thresholds."""
|
||
action_curves = []
|
||
for thr in THRESHOLDS:
|
||
action_pred = []
|
||
for r in oof_rows:
|
||
p = r["predicted"]
|
||
proba = r["proba"].get("action", 0.0)
|
||
if p == "action" and proba < thr:
|
||
# Demote action prediction
|
||
# Find next best route that isn't action
|
||
sorted_routes = sorted(r["proba"].items(), key=lambda x: -x[1])
|
||
for route, _ in sorted_routes:
|
||
if route != "action":
|
||
p = route
|
||
break
|
||
action_pred.append(p)
|
||
|
||
y_true = np.array([r["true"] for r in oof_rows])
|
||
y_pred = np.array(action_pred)
|
||
|
||
action_tp = sum(1 for t, p in zip(y_true, y_pred) if t == "action" and p == "action")
|
||
action_fp = sum(1 for t, p in zip(y_true, y_pred) if t != "action" and p == "action")
|
||
action_fn = sum(1 for t, p in zip(y_true, y_pred) if t == "action" and p != "action")
|
||
false_action = sum(1 for t, p in zip(y_true, y_pred) if t != "action" and p == "action")
|
||
|
||
action_curves.append({
|
||
"threshold": thr,
|
||
"action_precision": action_tp / max(action_tp + action_fp, 1),
|
||
"action_recall": action_tp / max(action_tp + action_fn, 1),
|
||
"false_action_count": false_action,
|
||
})
|
||
|
||
return action_curves
|
||
|
||
|
||
def compute_disagreement(y_true, y_pred, legacy_pred, oof_rows):
|
||
"""Analyze disagreements between legacy and learned router."""
|
||
results = {
|
||
"legacy_wrong_learned_right": [],
|
||
"legacy_right_learned_wrong": [],
|
||
"both_wrong_differently": [],
|
||
}
|
||
|
||
legacy_false_actions = []
|
||
learned_false_actions = []
|
||
shared_false_actions = []
|
||
|
||
for i, r in enumerate(oof_rows):
|
||
sid = r["source_id"]
|
||
text = r["text"]
|
||
true = y_true[i]
|
||
learned = y_pred[i]
|
||
legacy = legacy_pred[i]
|
||
|
||
if legacy != true and learned == true:
|
||
results["legacy_wrong_learned_right"].append({
|
||
"source_id": sid, "text": text,
|
||
"true": true, "legacy": legacy, "learned": learned,
|
||
})
|
||
elif legacy == true and learned != true:
|
||
results["legacy_right_learned_wrong"].append({
|
||
"source_id": sid, "text": text,
|
||
"true": true, "legacy": legacy, "learned": learned,
|
||
})
|
||
elif legacy != true and learned != true and legacy != learned:
|
||
results["both_wrong_differently"].append({
|
||
"source_id": sid, "text": text,
|
||
"true": true, "legacy": legacy, "learned": learned,
|
||
})
|
||
|
||
# False action tracking
|
||
if true != "action" and legacy == "action":
|
||
legacy_false_actions.append(sid)
|
||
if true != "action" and learned == "action":
|
||
learned_false_actions.append(sid)
|
||
if true != "action" and legacy == "action" and learned == "action":
|
||
shared_false_actions.append(sid)
|
||
|
||
# Repaired false actions
|
||
repaired = [sid for sid in legacy_false_actions if sid not in learned_false_actions]
|
||
new_errors = [sid for sid in learned_false_actions if sid not in legacy_false_actions]
|
||
shared = shared_false_actions
|
||
|
||
return {
|
||
"details": results,
|
||
"legacy_false_actions": legacy_false_actions,
|
||
"learned_false_actions": learned_false_actions,
|
||
"repaired": repaired,
|
||
"new_errors": new_errors,
|
||
"shared": shared,
|
||
}
|
||
|
||
|
||
def compute_contrast_family(oof_rows):
|
||
"""Analyze performance per contrast family."""
|
||
family_results = {}
|
||
for family in CONTRAST_FAMILIES:
|
||
members = [r for r in oof_rows if family in r.get("tags", [])]
|
||
if not members:
|
||
continue
|
||
y_true = [r["true"] for r in members]
|
||
y_pred = [r["predicted"] for r in members]
|
||
correct = sum(1 for t, p in zip(y_true, y_pred) if t == p)
|
||
false_act = sum(1 for t, p in zip(y_true, y_pred) if t != "action" and p == "action")
|
||
family_results[family] = {
|
||
"count": len(members),
|
||
"correct": correct,
|
||
"accuracy": correct / len(members),
|
||
"false_action": false_act,
|
||
}
|
||
return family_results
|
||
|
||
|
||
def compute_legacy_baseline(examples):
|
||
"""
|
||
Compute legacy baseline by mapping each example through the known
|
||
fast-path and classifier behavior. Since we don't have the actual
|
||
router running, we use the corpus metadata:
|
||
- fast_path_resolved examples are correct (grammar handles them)
|
||
- We simulate the legacy baseline from the test output numbers
|
||
"""
|
||
# The actual legacy baseline was measured in TestLegacyBaseline:
|
||
# overall accuracy: 52.2%, residual: 40.8%, false-action rate: 19.9%
|
||
# We need per-example predictions. We'll approximate from the corpus structure.
|
||
#
|
||
# For fast_path_resolved examples, the legacy router is correct (stage-0 grammar).
|
||
# For residual examples, we need to simulate the hash-embedder classifier.
|
||
# Since we don't have the hash embedder running, we use the known aggregate.
|
||
#
|
||
# This is a limitation: the legacy baseline numbers come from the Go test,
|
||
# and we only have the aggregate. We'll use the aggregate for comparison.
|
||
pass
|
||
|
||
|
||
# ─── Report Generation ──────────────────────────────────────────────────────
|
||
|
||
def fmt_pct(v, decimals=1):
|
||
return f"{100 * v:.{decimals}f}%"
|
||
|
||
|
||
def fmt_float(v, decimals=3):
|
||
return f"{v:.{decimals}f}"
|
||
|
||
|
||
def generate_report(meta, all_results, residual_results, oof_all, oof_residual):
|
||
"""Generate the full experiment report."""
|
||
lines = []
|
||
lines.append("# Semantic Router Linear Head Experiment — Report")
|
||
lines.append("")
|
||
lines.append("## 1. Exact e5 representation used")
|
||
lines.append("")
|
||
lines.append(f"- **Model**: {meta['embedder_id']}")
|
||
lines.append(f"- **Checkpoint**: {meta['model_path']}")
|
||
lines.append(f"- **Tokenizer**: {meta['tokenizer_path']}")
|
||
lines.append(f"- **Dimension**: {meta['dimension']}")
|
||
lines.append(f"- **Pooling**: {meta['pooling']}")
|
||
lines.append(f"- **Normalization**: {meta['normalization']}")
|
||
lines.append(f"- **Input template**: {meta['input_template']}")
|
||
lines.append("")
|
||
|
||
lines.append("## 2. Development/residual row counts")
|
||
lines.append("")
|
||
lines.append(f"- Total corpus: {meta['total_examples']}")
|
||
lines.append(f"- Frozen holdout: {meta['frozen_count']}")
|
||
lines.append(f"- Development pool: {meta['dev_count']}")
|
||
lines.append(f"- Fast-path resolved: {meta['fast_path_count']}")
|
||
lines.append(f"- Router-residual: {meta['residual_count']}")
|
||
lines.append("")
|
||
lines.append("Route distribution (full corpus):")
|
||
for route, count in sorted(meta["route_counts"].items()):
|
||
lines.append(f" - {route}: {count}")
|
||
lines.append("")
|
||
|
||
lines.append("## 3. Grouped fold composition")
|
||
lines.append("")
|
||
lines.append(f"Folds: {meta['cv_folds']}")
|
||
for fold_id, stats in sorted(meta["fold_composition"].items()):
|
||
lines.append(f" - Fold {fold_id}: eval={stats['eval_count']} train={stats['train_count']} routes={stats['eval_routes']}")
|
||
lines.append("")
|
||
|
||
# Regularization selection
|
||
lines.append("## 4. Selected regularization")
|
||
lines.append("")
|
||
|
||
lines.append("### Experiment A: All development examples")
|
||
best_C_all = all_results["best_C"]
|
||
lines.append(f"- Best C: {best_C_all}")
|
||
lines.append(f"- Mean accuracy: {fmt_pct(all_results['results_by_C'][best_C_all]['mean_accuracy'])} ± {fmt_pct(all_results['results_by_C'][best_C_all]['std_accuracy'])}")
|
||
lines.append(f"- Mean macro F1: {fmt_float(all_results['results_by_C'][best_C_all]['mean_macro_f1'])} ± {fmt_float(all_results['results_by_C'][best_C_all]['std_macro_f1'])}")
|
||
lines.append(f"- Total false actions (CV): {all_results['results_by_C'][best_C_all]['total_false_action']}")
|
||
lines.append("")
|
||
|
||
lines.append("### Experiment B: Router-residual only")
|
||
best_C_res = residual_results["best_C"]
|
||
lines.append(f"- Best C: {best_C_res}")
|
||
lines.append(f"- Mean accuracy: {fmt_pct(residual_results['results_by_C'][best_C_res]['mean_accuracy'])} ± {fmt_pct(residual_results['results_by_C'][best_C_res]['std_accuracy'])}")
|
||
lines.append(f"- Mean macro F1: {fmt_float(residual_results['results_by_C'][best_C_res]['mean_macro_f1'])} ± {fmt_float(residual_results['results_by_C'][best_C_res]['std_macro_f1'])}")
|
||
lines.append(f"- Total false actions (CV): {residual_results['results_by_C'][best_C_res]['total_false_action']}")
|
||
lines.append("")
|
||
|
||
# Stability across folds
|
||
lines.append("### Stability across folds")
|
||
lines.append("")
|
||
for C_val in C_VALUES:
|
||
r = all_results["results_by_C"][C_val]
|
||
fold_accs = [m["accuracy"] for m in r["fold_metrics"]]
|
||
fold_f1s = [m["macro_f1"] for m in r["fold_metrics"]]
|
||
lines.append(f" C={C_val:<6} acc={fmt_pct(r['mean_accuracy'])}±{fmt_pct(r['std_accuracy'])} f1={fmt_float(r['mean_macro_f1'])}±{fmt_float(r['std_macro_f1'])} folds_acc={[fmt_pct(a) for a in fold_accs]}")
|
||
lines.append("")
|
||
|
||
# Experiment A metrics
|
||
lines.append("## 5. All-example CV metrics")
|
||
lines.append("")
|
||
metrics_all = all_results["full_metrics"]
|
||
lines.append(f"- Accuracy: {fmt_pct(metrics_all['accuracy'])}")
|
||
lines.append(f"- Macro F1: {fmt_float(metrics_all['macro_f1'])}")
|
||
lines.append(f"- False-action count: {metrics_all['false_action_count']}")
|
||
lines.append(f"- False-action rate: {fmt_pct(metrics_all['false_action_rate'])}")
|
||
lines.append(f"- Action precision: {fmt_float(metrics_all['action_precision'])}")
|
||
lines.append(f"- Action recall: {fmt_float(metrics_all['action_recall'])}")
|
||
lines.append(f"- Uncertain precision: {fmt_float(metrics_all['uncertain_precision'])}")
|
||
lines.append(f"- Uncertain recall: {fmt_float(metrics_all['uncertain_recall'])}")
|
||
lines.append("")
|
||
lines.append("Per-class metrics:")
|
||
for route in ROUTES:
|
||
pc = metrics_all["per_class"][route]
|
||
lines.append(f" {route:<15} P={fmt_float(pc['precision'])} R={fmt_float(pc['recall'])} F1={fmt_float(pc['f1'])} (n={pc['support']})")
|
||
lines.append("")
|
||
lines.append("Confusion matrix (rows=expected, cols=predicted):")
|
||
header = f"{'':>15}" + "".join(f"{r:>15}" for r in ROUTES)
|
||
lines.append(header)
|
||
for i, route in enumerate(ROUTES):
|
||
row = f"{route:>15}" + "".join(f"{metrics_all['confusion_matrix'][i][j]:>15}" for j in range(len(ROUTES)))
|
||
lines.append(row)
|
||
lines.append("")
|
||
|
||
# Experiment B metrics
|
||
lines.append("## 6. Residual-only CV metrics")
|
||
lines.append("")
|
||
metrics_res = residual_results["full_metrics"]
|
||
lines.append(f"- Accuracy: {fmt_pct(metrics_res['accuracy'])}")
|
||
lines.append(f"- Macro F1: {fmt_float(metrics_res['macro_f1'])}")
|
||
lines.append(f"- False-action count: {metrics_res['false_action_count']}")
|
||
lines.append(f"- False-action rate: {fmt_pct(metrics_res['false_action_rate'])}")
|
||
lines.append(f"- Action precision: {fmt_float(metrics_res['action_precision'])}")
|
||
lines.append(f"- Action recall: {fmt_float(metrics_res['action_recall'])}")
|
||
lines.append(f"- Uncertain precision: {fmt_float(metrics_res['uncertain_precision'])}")
|
||
lines.append(f"- Uncertain recall: {fmt_float(metrics_res['uncertain_recall'])}")
|
||
lines.append("")
|
||
lines.append("Per-class metrics:")
|
||
for route in ROUTES:
|
||
pc = metrics_res["per_class"][route]
|
||
lines.append(f" {route:<15} P={fmt_float(pc['precision'])} R={fmt_float(pc['recall'])} F1={fmt_float(pc['f1'])} (n={pc['support']})")
|
||
lines.append("")
|
||
lines.append("Confusion matrix (rows=expected, cols=predicted):")
|
||
header = f"{'':>15}" + "".join(f"{r:>15}" for r in ROUTES)
|
||
lines.append(header)
|
||
for i, route in enumerate(ROUTES):
|
||
row = f"{route:>15}" + "".join(f"{metrics_res['confusion_matrix'][i][j]:>15}" for j in range(len(ROUTES)))
|
||
lines.append(row)
|
||
lines.append("")
|
||
|
||
# Legacy comparison
|
||
lines.append("## 7. Legacy-vs-linear comparison")
|
||
lines.append("")
|
||
lines.append("### All examples")
|
||
lines.append(f"{'metric':<25} {'legacy':>10} {'linear e5':>10} {'delta':>10}")
|
||
lines.append("-" * 55)
|
||
# Legacy baseline from test: 52.2% overall, 40.8% residual, 19.9% false-action
|
||
# These are approximate since we don't have per-example legacy predictions
|
||
legacy_acc = 0.522
|
||
legacy_fa_rate = 0.199
|
||
legacy_macro_f1 = 0.0 # unknown precisely
|
||
lines.append(f"{'accuracy':<25} {fmt_pct(legacy_acc):>10} {fmt_pct(metrics_all['accuracy']):>10} {fmt_pct(metrics_all['accuracy'] - legacy_acc):>10}")
|
||
lines.append(f"{'macro F1':<25} {'—':>10} {fmt_float(metrics_all['macro_f1']):>10} {'—':>10}")
|
||
lines.append(f"{'action precision':<25} {'—':>10} {fmt_float(metrics_all['action_precision']):>10} {'—':>10}")
|
||
lines.append(f"{'false-action rate':<25} {fmt_pct(legacy_fa_rate):>10} {fmt_pct(metrics_all['false_action_rate']):>10} {fmt_pct(metrics_all['false_action_rate'] - legacy_fa_rate):>10}")
|
||
lines.append(f"{'uncertain F1':<25} {fmt_float(0.0):>10} {fmt_float(metrics_all['per_class']['uncertain']['f1']):>10} {fmt_float(metrics_all['per_class']['uncertain']['f1']):>10}")
|
||
lines.append("")
|
||
|
||
lines.append("### Router-residual only")
|
||
lines.append(f"{'metric':<25} {'legacy':>10} {'linear e5':>10} {'delta':>10}")
|
||
lines.append("-" * 55)
|
||
legacy_res_acc = 0.408
|
||
lines.append(f"{'accuracy':<25} {fmt_pct(legacy_res_acc):>10} {fmt_pct(metrics_res['accuracy']):>10} {fmt_pct(metrics_res['accuracy'] - legacy_res_acc):>10}")
|
||
lines.append(f"{'macro F1':<25} {'—':>10} {fmt_float(metrics_res['macro_f1']):>10} {'—':>10}")
|
||
lines.append(f"{'false-action rate':<25} {'—':>10} {fmt_pct(metrics_res['false_action_rate']):>10} {'—':>10}")
|
||
lines.append("")
|
||
|
||
# Fold variance
|
||
lines.append("## 8. Fold variance")
|
||
lines.append("")
|
||
lines.append("All-example CV:")
|
||
for m in all_results["results_by_C"][best_C_all]["fold_metrics"]:
|
||
lines.append(f" Fold {m['fold']}: acc={fmt_pct(m['accuracy'])} f1={fmt_float(m['macro_f1'])} false_action={m['false_action']}")
|
||
lines.append("")
|
||
lines.append("Residual-only CV:")
|
||
for m in residual_results["results_by_C"][best_C_res]["fold_metrics"]:
|
||
lines.append(f" Fold {m['fold']}: acc={fmt_pct(m['accuracy'])} f1={fmt_float(m['macro_f1'])} false_action={m['false_action']}")
|
||
lines.append("")
|
||
|
||
# Disagreement analysis
|
||
lines.append("## 9. False-action repair/new-error analysis")
|
||
lines.append("")
|
||
# The disagreement analysis requires legacy per-example predictions.
|
||
# Since we don't have those, we report what we can from the out-of-fold data.
|
||
lines.append("Note: Legacy per-example predictions were not available for this experiment.")
|
||
lines.append("The legacy baseline was measured in aggregate in the Go test suite.")
|
||
lines.append("")
|
||
lines.append("Learned router false-action cases (out-of-fold):")
|
||
for r in oof_all:
|
||
if r["true"] != "action" and r["predicted"] == "action":
|
||
lines.append(f" {r['source_id']}: '{r['text']}' (true={r['true']}, proba(action)={r['proba'].get('action', 0):.3f})")
|
||
lines.append("")
|
||
|
||
# Contrast family
|
||
lines.append("## 10. Contrast-family results")
|
||
lines.append("")
|
||
lines.append("### Experiment A (all dev)")
|
||
contrast_all = all_results["contrast_family"]
|
||
lines.append(f"{'family':<25} {'count':>6} {'correct':>8} {'accuracy':>10} {'false_act':>10}")
|
||
lines.append("-" * 60)
|
||
for family in CONTRAST_FAMILIES:
|
||
if family in contrast_all:
|
||
c = contrast_all[family]
|
||
lines.append(f"{family:<25} {c['count']:>6} {c['correct']:>8} {fmt_pct(c['accuracy']):>10} {c['false_action']:>10}")
|
||
lines.append("")
|
||
|
||
lines.append("### Experiment B (residual only)")
|
||
contrast_res = residual_results["contrast_family"]
|
||
lines.append(f"{'family':<25} {'count':>6} {'correct':>8} {'accuracy':>10} {'false_act':>10}")
|
||
lines.append("-" * 60)
|
||
for family in CONTRAST_FAMILIES:
|
||
if family in contrast_res:
|
||
c = contrast_res[family]
|
||
lines.append(f"{family:<25} {c['count']:>6} {c['correct']:>8} {fmt_pct(c['accuracy']):>10} {c['false_action']:>10}")
|
||
lines.append("")
|
||
|
||
# Calibration
|
||
lines.append("## 11. Calibration metrics")
|
||
lines.append("")
|
||
lines.append("### Experiment A")
|
||
cal_all = all_results["calibration"]
|
||
lines.append(f"- ECE: {fmt_float(cal_all['ece'])}")
|
||
lines.append(f"- Brier score: {fmt_float(cal_all['brier'])}")
|
||
lines.append(f"- Log loss: {fmt_float(cal_all['log_loss'])}")
|
||
lines.append("")
|
||
lines.append("### Experiment B")
|
||
cal_res = residual_results["calibration"]
|
||
lines.append(f"- ECE: {fmt_float(cal_res['ece'])}")
|
||
lines.append(f"- Brier score: {fmt_float(cal_res['brier'])}")
|
||
lines.append(f"- Log loss: {fmt_float(cal_res['log_loss'])}")
|
||
lines.append("")
|
||
|
||
# Abstention curves
|
||
lines.append("## 12. Abstention curves")
|
||
lines.append("")
|
||
lines.append("### Experiment A (all dev)")
|
||
lines.append(f"{'threshold':>10} {'n_accepted':>11} {'coverage':>10} {'accuracy':>10} {'macro_f1':>10} {'false_act':>10}")
|
||
lines.append("-" * 62)
|
||
for curve in cal_all["abstention_curves"]:
|
||
lines.append(f"{curve['threshold']:>10.2f} {curve['n_accepted']:>11} {fmt_pct(curve['coverage']):>10} {fmt_pct(curve['accuracy']):>10} {fmt_float(curve['macro_f1']):>10} {curve['false_action_count']:>10}")
|
||
lines.append("")
|
||
|
||
lines.append("### Experiment B (residual only)")
|
||
lines.append(f"{'threshold':>10} {'n_accepted':>11} {'coverage':>10} {'accuracy':>10} {'macro_f1':>10} {'false_act':>10}")
|
||
lines.append("-" * 62)
|
||
for curve in cal_res["abstention_curves"]:
|
||
lines.append(f"{curve['threshold']:>10.2f} {curve['n_accepted']:>11} {fmt_pct(curve['coverage']):>10} {fmt_pct(curve['accuracy']):>10} {fmt_float(curve['macro_f1']):>10} {curve['false_action_count']:>10}")
|
||
lines.append("")
|
||
|
||
# Action threshold
|
||
lines.append("## 13. Action-threshold curve")
|
||
lines.append("")
|
||
lines.append("### Experiment A")
|
||
lines.append(f"{'threshold':>10} {'action_P':>10} {'action_R':>10} {'false_act':>10}")
|
||
lines.append("-" * 40)
|
||
for curve in all_results["action_threshold"]:
|
||
lines.append(f"{curve['threshold']:>10.2f} {fmt_float(curve['action_precision']):>10} {fmt_float(curve['action_recall']):>10} {curve['false_action_count']:>10}")
|
||
lines.append("")
|
||
|
||
lines.append("### Experiment B")
|
||
lines.append(f"{'threshold':>10} {'action_P':>10} {'action_R':>10} {'false_act':>10}")
|
||
lines.append("-" * 40)
|
||
for curve in residual_results["action_threshold"]:
|
||
lines.append(f"{curve['threshold']:>10.2f} {fmt_float(curve['action_precision']):>10} {fmt_float(curve['action_recall']):>10} {curve['false_action_count']:>10}")
|
||
lines.append("")
|
||
|
||
# Model artifact size
|
||
lines.append("## 14. Model artifact size and runtime cost")
|
||
lines.append("")
|
||
# Logistic regression: 6 classes × 384 features + 6 biases = 2310 parameters
|
||
n_params = len(ROUTES) * meta["dimension"] + len(ROUTES)
|
||
serialized_bytes = n_params * 4 # float32
|
||
lines.append(f"- Trainable parameters: {n_params}")
|
||
lines.append(f" - {len(ROUTES)} classes × {meta['dimension']} features = {len(ROUTES) * meta['dimension']} weights")
|
||
lines.append(f" - {len(ROUTES)} bias terms")
|
||
lines.append(f"- Serialized head size: {serialized_bytes} bytes ({serialized_bytes / 1024:.1f} KB)")
|
||
lines.append(f"- Additional inference FLOPs: {len(ROUTES) * meta['dimension']} multiply-accumulates")
|
||
lines.append(f"- Incremental cost (e5 already computed): ~{len(ROUTES) * meta['dimension']} FLOPs, <1µs")
|
||
lines.append(f"- Cost if semantic router must trigger its own e5: full ONNX inference (~{meta['dimension']} × 128 × 12 = ~590K FLOPs)")
|
||
lines.append("")
|
||
|
||
# Recommendation
|
||
lines.append("## 16. Recommendation")
|
||
lines.append("")
|
||
# Decision logic
|
||
all_f1 = metrics_all["macro_f1"]
|
||
res_f1 = metrics_res["macro_f1"]
|
||
res_acc = metrics_res["accuracy"]
|
||
|
||
if res_f1 > 0.5 and res_acc > 0.55:
|
||
verdict = "linear head sufficient"
|
||
detail = (f"Residual macro F1 of {fmt_float(res_f1)} and accuracy of {fmt_pct(res_acc)} "
|
||
f"exceed the legacy baseline (40.8% residual accuracy) by a meaningful margin. "
|
||
f"A linear head over frozen e5-small embeddings is a viable first production candidate.")
|
||
elif all_f1 > 0.5 and res_f1 < 0.5:
|
||
verdict = "need more data"
|
||
detail = (f"All-example F1 ({fmt_float(all_f1)}) is acceptable but residual-only F1 "
|
||
f"({fmt_float(res_f1)}) drops, suggesting the contrast-family examples are "
|
||
f"hard for a linear classifier. More contrastive training data may help.")
|
||
else:
|
||
verdict = "representation inadequate"
|
||
detail = (f"Neither all-example ({fmt_float(all_f1)}) nor residual-only ({fmt_float(res_f1)}) "
|
||
f"F1 reaches the minimum viable threshold. The e5-small linear separability floor "
|
||
f"is insufficient for this 6-way task. Consider a non-linear head or a different encoder.")
|
||
|
||
lines.append(f"**{verdict}**")
|
||
lines.append("")
|
||
lines.append(detail)
|
||
lines.append("")
|
||
|
||
# Commit hash
|
||
lines.append("## 17. Commit hash for experiment tooling")
|
||
lines.append("")
|
||
import subprocess
|
||
try:
|
||
result = subprocess.run(
|
||
["git", "rev-parse", "HEAD"],
|
||
capture_output=True, text=True, cwd="/home/kami/apps/Maven"
|
||
)
|
||
lines.append(f"`{result.stdout.strip()}`")
|
||
except Exception:
|
||
lines.append("(unable to determine)")
|
||
lines.append("")
|
||
|
||
return "\n".join(lines)
|
||
|
||
|
||
# ─── Main ───────────────────────────────────────────────────────────────────
|
||
|
||
def run_experiment(X, y, fold_ids, examples_meta, label):
|
||
"""Run the full experiment pipeline for one population."""
|
||
print(f"\n{'='*60}")
|
||
print(f" Experiment: {label}")
|
||
print(f" Samples: {len(y)} Features: {X.shape[1]} Folds: {len(set(fold_ids))}")
|
||
print(f"{'='*60}")
|
||
|
||
# 1. Grouped CV with regularization search
|
||
best_C, results_by_C = grouped_cv_experiment(X, y, fold_ids, C_VALUES, examples_meta)
|
||
|
||
# 2. Use best C to get full metrics from OOF predictions
|
||
oof_rows = results_by_C[best_C]["oof_predictions"]
|
||
oof_true = np.array([r["true"] for r in oof_rows])
|
||
oof_pred = np.array([r["predicted"] for r in oof_rows])
|
||
|
||
full_metrics = compute_full_metrics(oof_true, oof_pred)
|
||
|
||
# 3. Calibration
|
||
calibration = compute_calibration(oof_true, oof_pred, oof_rows)
|
||
|
||
# 4. Action threshold
|
||
action_threshold = compute_action_threshold(oof_rows)
|
||
|
||
# 5. Contrast family
|
||
contrast_family = compute_contrast_family(oof_rows)
|
||
|
||
return {
|
||
"best_C": best_C,
|
||
"results_by_C": results_by_C,
|
||
"full_metrics": full_metrics,
|
||
"calibration": calibration,
|
||
"action_threshold": action_threshold,
|
||
"contrast_family": contrast_family,
|
||
"oof_rows": oof_rows,
|
||
}
|
||
|
||
|
||
if __name__ == "__main__":
|
||
EMBEDDING_PATH = "/tmp/mvn-experiment/embeddings.json"
|
||
REPORT_PATH = "/tmp/mvn-experiment/report.md"
|
||
|
||
# Load data
|
||
meta, examples = load_embeddings(EMBEDDING_PATH)
|
||
print(f"Loaded {len(examples)} examples, embedder={meta['embedder_id']}, dim={meta['dimension']}")
|
||
|
||
# Development pool only
|
||
dev_examples = filter_dev_pool(examples)
|
||
print(f"Development pool: {len(dev_examples)} examples")
|
||
|
||
# Extract features
|
||
X_all, y_all = extract_Xy(dev_examples)
|
||
fold_ids_all = get_fold_groups(dev_examples)
|
||
|
||
# Experiment A: all dev examples
|
||
all_results = run_experiment(X_all, y_all, fold_ids_all, dev_examples, "All development examples")
|
||
|
||
# Experiment B: residual only
|
||
dev_residual = filter_residual(dev_examples)
|
||
X_res, y_res = extract_Xy(dev_residual)
|
||
fold_ids_res = get_fold_groups(dev_residual)
|
||
residual_results = run_experiment(X_res, y_res, fold_ids_res, dev_residual, "Router-residual only")
|
||
|
||
# Generate report
|
||
report = generate_report(
|
||
meta, all_results, residual_results,
|
||
all_results["oof_rows"], residual_results["oof_rows"],
|
||
)
|
||
|
||
with open(REPORT_PATH, "w") as f:
|
||
f.write(report)
|
||
print(f"\nReport written to {REPORT_PATH}")
|
||
|
||
# Also print summary
|
||
print("\n" + "="*60)
|
||
print(" SUMMARY")
|
||
print("="*60)
|
||
print(f" All-example: acc={fmt_pct(all_results['full_metrics']['accuracy'])} macro_f1={fmt_float(all_results['full_metrics']['macro_f1'])} false_action={all_results['full_metrics']['false_action_count']}")
|
||
print(f" Residual: acc={fmt_pct(residual_results['full_metrics']['accuracy'])} macro_f1={fmt_float(residual_results['full_metrics']['macro_f1'])} false_action={residual_results['full_metrics']['false_action_count']}")
|
||
print(f" Best C (all): {all_results['best_C']}")
|
||
print(f" Best C (res): {residual_results['best_C']}")
|
||
print(f" ECE (all): {fmt_float(all_results['calibration']['ece'])}")
|
||
print(f" ECE (res): {fmt_float(residual_results['calibration']['ece'])}")
|
||
print(f" Brier (all): {fmt_float(all_results['calibration']['brier'])}")
|
||
print(f" Brier (res): {fmt_float(residual_results['calibration']['brier'])}")
|