diff --git a/internal/router/semantic/baseline_types.go b/internal/router/semantic/baseline_types.go new file mode 100644 index 0000000..25f7af5 --- /dev/null +++ b/internal/router/semantic/baseline_types.go @@ -0,0 +1,67 @@ +package semantic + +import ( + "context" + "time" + + "github.com/kami/maven/internal/router" +) + +// LegacyRouter — the interface for scoring the current routing machinery. +// *router.Router satisfies this directly. Tests can substitute a stub. +type LegacyRouter interface { + Route(ctx context.Context, input router.NormalizedInput, now time.Time) (router.Decision, error) +} + +// LegacyCase — one scored row from the legacy baseline. +type LegacyCase struct { + Example RouteExample + Decision router.Decision + Predicted SemanticRoute + Agree bool + FastPathHit bool + // PrerouteConsumed is true when a pre-route resolver (e.g. + // command-prohibition grammar) consumed this turn before the general + // cascade. These cases never reach the classifier or the LLM. + PrerouteConsumed bool + Error error +} + +// LegacyReport — aggregate metrics from running the real router against the +// coarse-route corpus. +type LegacyReport struct { + Stats CorpusStats + Total int + Passed int + ByRoute map[SemanticRoute]RouteMetrics + // FalseAction — cases expected non-action but the router predicted action. + FalseAction int + FalseActionRate float64 + // FalseActionCases — the individual cases, for inspection. + FalseActionCases []LegacyCase + // Confusion[want][got] + Confusion map[SemanticRoute]map[SemanticRoute]int + // Fast-path breakdown + FastPathTotal int + FastPathPassed int + ResidualTotal int + ResidualPassed int + // Router-residual breakdown: cases that reach the general cascade + // (not consumed by pre-route resolvers like command-prohibition). + RouterResidualTotal int + RouterResidualPassed int + PreRouteTotal int // cases consumed by command-prohibition grammar + PreRoutePassed int + // Cases by individual outcome + Cases []LegacyCase +} + +// ContrastFamilyReport — per-transform-family breakdown. +type ContrastFamilyReport struct { + Transform string + Examples int + Correct int + FalseAction int // predicted action when expected non-action + OtherErrors int + Cases []LegacyCase +} diff --git a/internal/router/semantic/contract_test.go b/internal/router/semantic/contract_test.go index 2754fc4..d08fe51 100644 --- a/internal/router/semantic/contract_test.go +++ b/internal/router/semantic/contract_test.go @@ -287,3 +287,53 @@ func TestFamilyID(t *testing.T) { t.Error("different base IDs produced same FamilyID") } } + +func TestContrastFamiliesShareSplitGroup(t *testing.T) { + exs, err := LoadCorpus() + if err != nil { + t.Fatal(err) + } + + // Group contrastive examples by their base source_id. + baseGroups := map[string]map[string]bool{} // source_id → {split_group: true} + for _, e := range exs { + if e.Source != "contrastive" { + continue + } + if baseGroups[e.SourceID] == nil { + baseGroups[e.SourceID] = map[string]bool{} + } + baseGroups[e.SourceID][e.SplitGroup] = true + } + + // Every base seed's contrastive variants must share exactly one + // split_group (the base seed's own group). + for sourceID, groups := range baseGroups { + if len(groups) != 1 { + t.Errorf("base %q has contrastive variants in %d split groups: %v", + sourceID, len(groups), groups) + } + } + + // Also verify that contrastive variants share the split_group with + // their base seed from ru_routing_v1. + ruGroups := map[string]string{} // source_id → split_group + for _, e := range exs { + if e.Source == "ru_routing_v1" { + ruGroups[e.SourceID] = e.SplitGroup + } + } + for sourceID, groups := range baseGroups { + var contrastGroup string + for g := range groups { + contrastGroup = g + } + if ruGroup, ok := ruGroups[sourceID]; ok && contrastGroup != ruGroup { + t.Errorf("base %q: ru_routing_v1 split_group=%q, contrastive split_group=%q", + sourceID, ruGroup, contrastGroup) + } + } + + t.Logf("contrast family leakage check passed: %d base seeds, all groups consistent", + len(baseGroups)) +} diff --git a/internal/router/semantic/helpers_test.go b/internal/router/semantic/helpers_test.go new file mode 100644 index 0000000..266aeda --- /dev/null +++ b/internal/router/semantic/helpers_test.go @@ -0,0 +1,87 @@ +package semantic + +import ( + "bufio" + "context" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/kami/maven/internal/router" +) + +const seedDir = "../../../models/seeds" + +var seedIntents = []router.Intent{ + router.IntentAct, router.IntentReminder, router.IntentFact, + router.IntentNote, router.IntentQuery, router.IntentChat, router.IntentSystem, +} + +func actVerbList() []string { + return []string{ + "перезапусти", "перезагрузи", "выключи", "включи", "останови", "запусти", + "закрой", "открой", "сделай", "поставь", + "restart", "reboot", "stop", "start", "turn off", "turn on", "open", "close", + } +} + +func buildSeededClassifier(t *testing.T, emb router.Embedder) *router.Classifier { + t.Helper() + cls := router.NewClassifier(emb) + ctx := context.Background() + seeds := seedsWithIntent(t) + texts := make([]string, 0, len(seeds)) + for text := range seeds { + texts = append(texts, text) + } + sort.Strings(texts) + for _, text := range texts { + if err := cls.AddExample(ctx, seeds[text], text); err != nil { + t.Fatalf("seed %q: %v", text, err) + } + } + return cls +} + +func seedsWithIntent(t *testing.T) map[string]router.Intent { + t.Helper() + files := readSeedFiles(t) + out := map[string]router.Intent{} + for _, intent := range seedIntents { + for _, l := range files[intent] { + if _, dup := out[l]; dup { + continue + } + out[l] = intent + } + } + return out +} + +func readSeedFiles(t *testing.T) map[router.Intent][]string { + t.Helper() + out := map[router.Intent][]string{} + for _, intent := range seedIntents { + path := filepath.Join(seedDir, string(intent)+".txt") + fh, err := os.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + sc := bufio.NewScanner(fh) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + out[intent] = append(out[intent], line) + } + err = sc.Err() + fh.Close() + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + } + return out +}