Maven/internal/router/semantic/shadow.go

package semantic

import (
	"context"
	"fmt"
	"time"

	"github.com/kami/maven/internal/router"
)

// ShadowOutcome — one turn observed in shadow mode. Carries both the
// authoritative legacy decision and the coarse-route experiment decision
// for later analysis.
type ShadowOutcome struct {
	Text          string
	LegacyIntent  router.Intent
	LegacyRoute   SemanticRoute
	Experiment    SemanticRouteDecision
	FastPathMatch bool
	Agree         bool
	Timestamp     time.Time
}

// ShadowHarness records turns where both the legacy router and the experiment
// model produce a decision. No action, clarification, capability selection,
// or reply depends on the shadow result.
type ShadowHarness struct {
	model   SemanticRouter
	outcome []ShadowOutcome
}

// NewShadowHarness creates a harness that observes both paths without
// affecting the authoritative route.
func NewShadowHarness(model SemanticRouter) *ShadowHarness {
	return &ShadowHarness{model: model}
}

// Observe records one turn. The authoritative decision comes from the
// existing router; the experiment decision comes from the shadow model.
// fastPathMatch is true when TryFastPath already resolved this turn.
//
// Observe never fails the turn — if the shadow model errors, the outcome
// records RouteUncertain with the error in Experiment.
func (h *ShadowHarness) Observe(
	ctx context.Context,
	text string,
	legacy router.Decision,
	fastPathMatch bool,
) ShadowOutcome {
	legacyRoute := IntentToRoute(legacy.Intent)

	var exp SemanticRouteDecision
	if h.model != nil {
		d, err := h.model.Route(ctx, text)
		if err != nil {
			exp = SemanticRouteDecision{
				Route:   RouteUncertain,
				Abstain: true,
			}
		} else {
			exp = d
		}
	}

	o := ShadowOutcome{
		Text:          text,
		LegacyIntent:  legacy.Intent,
		LegacyRoute:   legacyRoute,
		Experiment:    exp,
		FastPathMatch: fastPathMatch,
		Agree:         legacyRoute == exp.Route,
		Timestamp:     time.Now(),
	}
	h.outcome = append(h.outcome, o)
	return o
}

// Outcomes returns a copy of all recorded shadow outcomes.
func (h *ShadowHarness) Outcomes() []ShadowOutcome {
	out := make([]ShadowOutcome, len(h.outcome))
	copy(out, h.outcome)
	return out
}

// ShadowReport — aggregate agreement/disagreement statistics.
type ShadowReport struct {
	Total         int
	Agree         int
	Disagree      int
	FastPathTotal int
	FastPathAgree int
	ResidualTotal int
	ResidualAgree int
	// DisagreeByRoute[want][got] counts disagreements by legacy route
	DisagreeByRoute map[SemanticRoute]map[SemanticRoute]int
}

// Summarize produces aggregate stats from collected outcomes.
func (h *ShadowHarness) Summarize() ShadowReport {
	rep := ShadowReport{
		DisagreeByRoute: make(map[SemanticRoute]map[SemanticRoute]int),
	}
	for _, r := range AllRoutes {
		rep.DisagreeByRoute[r] = make(map[SemanticRoute]int)
	}

	for _, o := range h.outcome {
		rep.Total++
		if o.Agree {
			rep.Agree++
		} else {
			rep.Disagree++
			rep.DisagreeByRoute[o.LegacyRoute][o.Experiment.Route]++
		}
		if o.FastPathMatch {
			rep.FastPathTotal++
			if o.Agree {
				rep.FastPathAgree++
			}
		} else {
			rep.ResidualTotal++
			if o.Agree {
				rep.ResidualAgree++
			}
		}
	}
	return rep
}

// String renders the shadow report.
func (r ShadowReport) String() string {
	agreeRate := 0.0
	if r.Total > 0 {
		agreeRate = float64(r.Agree) / float64(r.Total)
	}
	resAgreeRate := 0.0
	if r.ResidualTotal > 0 {
		resAgreeRate = float64(r.ResidualAgree) / float64(r.ResidualTotal)
	}
	s := fmt.Sprintf("shadow: %d turns, %d agree (%.1f%%), %d disagree\n",
		r.Total, r.Agree, 100*agreeRate, r.Disagree)
	s += fmt.Sprintf("  fast-path: %d turns  residual: %d turns (%.1f%% agree)\n",
		r.FastPathTotal, r.ResidualTotal, 100*resAgreeRate)
	if r.Disagree > 0 {
		s += "  disagreements (legacy→experiment):\n"
		routes := make([]SemanticRoute, 0, len(r.DisagreeByRoute))
		for route := range r.DisagreeByRoute {
			routes = append(routes, route)
		}
		for _, w := range routes {
			for _, g := range routes {
				if c := r.DisagreeByRoute[w][g]; c > 0 {
					s += fmt.Sprintf("    %s → %s ×%d\n", w, g, c)
				}
			}
		}
	}
	return s
}