From dd98da5c429a24e94d5b6f9c0793087cd727d888 Mon Sep 17 00:00:00 2001
From: claude <no-reply@agents.claude.kvmx.ru>
Date: Mon, 7 Sep 2026 01:42:39 +0400
Subject: [PATCH] router/semantic: shadow harness for experiment observation
 (slice 12)

ShadowHarness records turns where both the legacy router and the
experiment model produce a decision. Reports agreement/disagreement
split by fast-path vs residual. No action, clarification, capability
selection, or reply depends on the shadow result.
---
 internal/router/semantic/shadow.go | 159 +++++++++++++++++++++++++++++
 1 file changed, 159 insertions(+)
 create mode 100644 internal/router/semantic/shadow.go

diff --git a/internal/router/semantic/shadow.go b/internal/router/semantic/shadow.go
new file mode 100644
index 0000000..24cadce
--- /dev/null
+++ b/internal/router/semantic/shadow.go
@@ -0,0 +1,159 @@
+package semantic
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/kami/maven/internal/router"
+)
+
+// ShadowOutcome — one turn observed in shadow mode. Carries both the
+// authoritative legacy decision and the coarse-route experiment decision
+// for later analysis.
+type ShadowOutcome struct {
+	Text          string
+	LegacyIntent  router.Intent
+	LegacyRoute   SemanticRoute
+	Experiment    SemanticRouteDecision
+	FastPathMatch bool
+	Agree         bool
+	Timestamp     time.Time
+}
+
+// ShadowHarness records turns where both the legacy router and the experiment
+// model produce a decision. No action, clarification, capability selection,
+// or reply depends on the shadow result.
+type ShadowHarness struct {
+	model   SemanticRouter
+	outcome []ShadowOutcome
+}
+
+// NewShadowHarness creates a harness that observes both paths without
+// affecting the authoritative route.
+func NewShadowHarness(model SemanticRouter) *ShadowHarness {
+	return &ShadowHarness{model: model}
+}
+
+// Observe records one turn. The authoritative decision comes from the
+// existing router; the experiment decision comes from the shadow model.
+// fastPathMatch is true when TryFastPath already resolved this turn.
+//
+// Observe never fails the turn — if the shadow model errors, the outcome
+// records RouteUncertain with the error in Experiment.
+func (h *ShadowHarness) Observe(
+	ctx context.Context,
+	text string,
+	legacy router.Decision,
+	fastPathMatch bool,
+) ShadowOutcome {
+	legacyRoute := IntentToRoute(legacy.Intent)
+
+	var exp SemanticRouteDecision
+	if h.model != nil {
+		d, err := h.model.Route(ctx, text)
+		if err != nil {
+			exp = SemanticRouteDecision{
+				Route:   RouteUncertain,
+				Abstain: true,
+			}
+		} else {
+			exp = d
+		}
+	}
+
+	o := ShadowOutcome{
+		Text:          text,
+		LegacyIntent:  legacy.Intent,
+		LegacyRoute:   legacyRoute,
+		Experiment:    exp,
+		FastPathMatch: fastPathMatch,
+		Agree:         legacyRoute == exp.Route,
+		Timestamp:     time.Now(),
+	}
+	h.outcome = append(h.outcome, o)
+	return o
+}
+
+// Outcomes returns a copy of all recorded shadow outcomes.
+func (h *ShadowHarness) Outcomes() []ShadowOutcome {
+	out := make([]ShadowOutcome, len(h.outcome))
+	copy(out, h.outcome)
+	return out
+}
+
+// ShadowReport — aggregate agreement/disagreement statistics.
+type ShadowReport struct {
+	Total         int
+	Agree         int
+	Disagree      int
+	FastPathTotal int
+	FastPathAgree int
+	ResidualTotal int
+	ResidualAgree int
+	// DisagreeByRoute[want][got] counts disagreements by legacy route
+	DisagreeByRoute map[SemanticRoute]map[SemanticRoute]int
+}
+
+// Summarize produces aggregate stats from collected outcomes.
+func (h *ShadowHarness) Summarize() ShadowReport {
+	rep := ShadowReport{
+		DisagreeByRoute: make(map[SemanticRoute]map[SemanticRoute]int),
+	}
+	for _, r := range AllRoutes {
+		rep.DisagreeByRoute[r] = make(map[SemanticRoute]int)
+	}
+
+	for _, o := range h.outcome {
+		rep.Total++
+		if o.Agree {
+			rep.Agree++
+		} else {
+			rep.Disagree++
+			rep.DisagreeByRoute[o.LegacyRoute][o.Experiment.Route]++
+		}
+		if o.FastPathMatch {
+			rep.FastPathTotal++
+			if o.Agree {
+				rep.FastPathAgree++
+			}
+		} else {
+			rep.ResidualTotal++
+			if o.Agree {
+				rep.ResidualAgree++
+			}
+		}
+	}
+	return rep
+}
+
+// String renders the shadow report.
+func (r ShadowReport) String() string {
+	agreeRate := 0.0
+	if r.Total > 0 {
+		agreeRate = float64(r.Agree) / float64(r.Total)
+	}
+	resAgreeRate := 0.0
+	if r.ResidualTotal > 0 {
+		resAgreeRate = float64(r.ResidualAgree) / float64(r.ResidualTotal)
+	}
+	s := fmt.Sprintf("shadow: %d turns, %d agree (%.1f%%), %d disagree\n",
+		r.Total, r.Agree, 100*agreeRate, r.Disagree)
+	s += fmt.Sprintf("  fast-path: %d turns  residual: %d turns (%.1f%% agree)\n",
+		r.FastPathTotal, r.ResidualTotal, 100*resAgreeRate)
+	if r.Disagree > 0 {
+		s += "  disagreements (legacy→experiment):\n"
+		routes := make([]SemanticRoute, 0, len(r.DisagreeByRoute))
+		for route := range r.DisagreeByRoute {
+			routes = append(routes, route)
+		}
+		for _, w := range routes {
+			for _, g := range routes {
+				if c := r.DisagreeByRoute[w][g]; c > 0 {
+					s += fmt.Sprintf("    %s → %s ×%d\n", w, g, c)
+				}
+			}
+		}
+	}
+	return s
+}