package semantic import ( "context" "fmt" "time" "github.com/kami/maven/internal/router" ) // ShadowOutcome — one turn observed in shadow mode. Carries both the // authoritative legacy decision and the coarse-route experiment decision // for later analysis. type ShadowOutcome struct { Text string LegacyIntent router.Intent LegacyRoute SemanticRoute Experiment SemanticRouteDecision FastPathMatch bool Agree bool Timestamp time.Time } // ShadowHarness records turns where both the legacy router and the experiment // model produce a decision. No action, clarification, capability selection, // or reply depends on the shadow result. type ShadowHarness struct { model SemanticRouter outcome []ShadowOutcome } // NewShadowHarness creates a harness that observes both paths without // affecting the authoritative route. func NewShadowHarness(model SemanticRouter) *ShadowHarness { return &ShadowHarness{model: model} } // Observe records one turn. The authoritative decision comes from the // existing router; the experiment decision comes from the shadow model. // fastPathMatch is true when TryFastPath already resolved this turn. // // Observe never fails the turn — if the shadow model errors, the outcome // records RouteUncertain with the error in Experiment. func (h *ShadowHarness) Observe( ctx context.Context, text string, legacy router.Decision, fastPathMatch bool, ) ShadowOutcome { legacyRoute := IntentToRoute(legacy.Intent) var exp SemanticRouteDecision if h.model != nil { d, err := h.model.Route(ctx, text) if err != nil { exp = SemanticRouteDecision{ Route: RouteUncertain, Abstain: true, } } else { exp = d } } o := ShadowOutcome{ Text: text, LegacyIntent: legacy.Intent, LegacyRoute: legacyRoute, Experiment: exp, FastPathMatch: fastPathMatch, Agree: legacyRoute == exp.Route, Timestamp: time.Now(), } h.outcome = append(h.outcome, o) return o } // Outcomes returns a copy of all recorded shadow outcomes. func (h *ShadowHarness) Outcomes() []ShadowOutcome { out := make([]ShadowOutcome, len(h.outcome)) copy(out, h.outcome) return out } // ShadowReport — aggregate agreement/disagreement statistics. type ShadowReport struct { Total int Agree int Disagree int FastPathTotal int FastPathAgree int ResidualTotal int ResidualAgree int // DisagreeByRoute[want][got] counts disagreements by legacy route DisagreeByRoute map[SemanticRoute]map[SemanticRoute]int } // Summarize produces aggregate stats from collected outcomes. func (h *ShadowHarness) Summarize() ShadowReport { rep := ShadowReport{ DisagreeByRoute: make(map[SemanticRoute]map[SemanticRoute]int), } for _, r := range AllRoutes { rep.DisagreeByRoute[r] = make(map[SemanticRoute]int) } for _, o := range h.outcome { rep.Total++ if o.Agree { rep.Agree++ } else { rep.Disagree++ rep.DisagreeByRoute[o.LegacyRoute][o.Experiment.Route]++ } if o.FastPathMatch { rep.FastPathTotal++ if o.Agree { rep.FastPathAgree++ } } else { rep.ResidualTotal++ if o.Agree { rep.ResidualAgree++ } } } return rep } // String renders the shadow report. func (r ShadowReport) String() string { agreeRate := 0.0 if r.Total > 0 { agreeRate = float64(r.Agree) / float64(r.Total) } resAgreeRate := 0.0 if r.ResidualTotal > 0 { resAgreeRate = float64(r.ResidualAgree) / float64(r.ResidualTotal) } s := fmt.Sprintf("shadow: %d turns, %d agree (%.1f%%), %d disagree\n", r.Total, r.Agree, 100*agreeRate, r.Disagree) s += fmt.Sprintf(" fast-path: %d turns residual: %d turns (%.1f%% agree)\n", r.FastPathTotal, r.ResidualTotal, 100*resAgreeRate) if r.Disagree > 0 { s += " disagreements (legacy→experiment):\n" routes := make([]SemanticRoute, 0, len(r.DisagreeByRoute)) for route := range r.DisagreeByRoute { routes = append(routes, route) } for _, w := range routes { for _, g := range routes { if c := r.DisagreeByRoute[w][g]; c > 0 { s += fmt.Sprintf(" %s → %s ×%d\n", w, g, c) } } } } return s }