Maven/internal/router/eval/reach_test.go

package eval

import (
	"context"
	"os"
	"path/filepath"
	"testing"

	"github.com/kami/maven/internal/router"
)

func TestLoadReachFixture(t *testing.T) {
	f, err := LoadReach()
	if err != nil {
		t.Fatalf("LoadReach: %v", err)
	}
	if _, err := f.Now(); err != nil {
		t.Fatalf("Now: %v", err)
	}
	valid := map[Service]bool{ServiceNone: true, ServicePraxis: true, ServiceHexis: true}
	seen := map[string]bool{}
	byService := map[Service]int{}
	for _, c := range f.Cases {
		if c.ID == "" || seen[c.ID] {
			t.Errorf("case %q: empty or duplicate id", c.ID)
		}
		seen[c.ID] = true
		if !valid[c.WantService] {
			t.Errorf("%s: want_service %q, want none|praxis|hexis", c.ID, c.WantService)
		}
		if c.Lang != "ru" && c.Lang != "en" {
			t.Errorf("%s: lang %q, want ru|en", c.ID, c.Lang)
		}
		// A capability only means something on the Praxis side, and it must
		// name an arm that exists — otherwise the case asserts a target the
		// dispatch can never hit.
		if c.WantCapability != "" {
			if c.WantService != ServicePraxis {
				t.Errorf("%s: want_capability on %q", c.ID, c.WantService)
			}
			if PraxisAliases[c.WantCapability] != c.WantCapability {
				t.Errorf("%s: capability %q is not a praxis arm", c.ID, c.WantCapability)
			}
		}
		byService[c.WantService]++
	}
	// Coverage floor. The negative cases matter most: without them a router
	// that sent everything to Hexis would score perfectly.
	for s := range valid {
		if byService[s] < 5 {
			t.Errorf("want_service %q has %d cases, want >= 5", s, byService[s])
		}
	}
}

// TestReachFixtureIsHeldOut — same rule as TestFixtureIsHeldOut. The classifier
// routes by similarity to frozen seeds, so a case copied from models/seeds
// would measure memorisation rather than reach.
func TestReachFixtureIsHeldOut(t *testing.T) {
	f, err := LoadReach()
	if err != nil {
		t.Fatalf("LoadReach: %v", err)
	}
	seeds := loadSeeds(t)
	for _, c := range f.Cases {
		if src, ok := seeds[normalize(c.Utterance)]; ok {
			t.Errorf("%s: %q is verbatim in %s — not held out", c.ID, c.Utterance, src)
		}
	}
}

// TestReachDerivation pins the gate order that ScoreReach depends on. These are
// the four branches of actionAct plus the clarify pre-empt, asserted directly
// so a change to cmd/mavend/actions_act.go that this package no longer mirrors
// fails here rather than silently moving the score.
func TestReachDerivation(t *testing.T) {
	m := router.DefaultActMatcher{Fns: actFns}
	cases := []struct {
		name       string
		dec        router.Decision
		want       Service
		capability string
	}{
		{
			name: "act with a praxis alias in the fn slot",
			dec:  router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "list_attention", HasFn: true}},
			want: ServicePraxis, capability: "list_attention",
		},
		{
			name: "act with a russian praxis alias",
			dec:  router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "готово", HasFn: true}},
			want: ServicePraxis, capability: "resolve_item",
		},
		{
			name: "act with an entity but no fn earns one from the matcher",
			dec:  router.Decision{Intent: router.IntentAct, Slots: router.Slots{Text: "выключи свет в спальне"}},
			want: ServiceHexis,
		},
		{
			name: "act with a non-praxis fn and text goes to hexis",
			dec:  router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "restart", HasFn: true, Text: "перезапусти гитею"}},
			want: ServiceHexis,
		},
		{
			name: "act with a fn and no text stays local",
			dec:  router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "restart", HasFn: true}},
			want: ServiceNone,
		},
		{
			name: "a clarified act with text still reaches hexis",
			dec:  router.Decision{Intent: router.IntentAct, Clarify: true, Slots: router.Slots{Text: "выключи это"}},
			want: ServiceHexis,
		},
		{
			name: "a clarified act that already has a fn does not",
			dec:  router.Decision{Intent: router.IntentAct, Clarify: true, Slots: router.Slots{Fn: "restart", HasFn: true, Text: "перезапусти"}},
			want: ServiceNone,
		},
		{
			name: "no intent but act reaches nothing",
			dec:  router.Decision{Intent: router.IntentQuery, Slots: router.Slots{Fn: "list_attention", HasFn: true, Text: "что требует внимания"}},
			want: ServiceNone,
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			got, capability := Reach(tc.dec, m)
			if got != tc.want {
				t.Errorf("Reach = %q, want %q", got, tc.want)
			}
			if capability != tc.capability {
				t.Errorf("capability = %q, want %q", capability, tc.capability)
			}
		})
	}
}

// TestPraxisAliasesShape — the copy of praxisCapabilities in reach.go is a
// drift risk (see its comment). This does not close it, but it does catch the
// cheap half: an arm losing all its aliases, or an alias pointing at an arm
// that no longer has a canonical name.
func TestPraxisAliasesShape(t *testing.T) {
	arms := map[string]int{}
	for alias, capability := range PraxisAliases {
		if alias == "" || capability == "" {
			t.Errorf("empty alias or capability: %q → %q", alias, capability)
		}
		arms[capability]++
	}
	for _, want := range []string{
		"list_attention", "acknowledge_item", "resolve_item",
		"ignore_item", "pin_item", "list_changes", "entity_attention",
	} {
		if arms[want] == 0 {
			t.Errorf("praxis arm %q has no aliases", want)
		}
		// Every arm must be reachable by its own name, which is the alias the
		// LLM router's fn slot actually emits.
		if PraxisAliases[want] != want {
			t.Errorf("arm %q does not alias to itself", want)
		}
	}
}

// TestReachBaselineHash — the deterministic ratchet. Same deal as
// TestClassifierBaseline: the hash embedder never clears the confidence gate on
// paraphrases, so almost everything lands on clarify, and the number this
// asserts is the overreach count rather than the accuracy.
//
// Overreach is the direction worth a hard assertion. An utterance that should
// stay inside Maven and instead resolves an entity through Nexus is one Hexis
// capability away from executing something nobody asked for, and unlike a miss
// he never gets asked about it.
func TestReachBaselineHash(t *testing.T) {
	f, err := LoadReach()
	if err != nil {
		t.Fatalf("LoadReach: %v", err)
	}
	m := router.DefaultActMatcher{Fns: actFns}
	rep, err := ScoreReach(context.Background(), "reach: classifier+hash",
		newBaselineRouter(t, router.NewHashEmbedder(1024), nil), m, f)
	if err != nil {
		t.Fatalf("ScoreReach: %v", err)
	}
	t.Log("\n" + rep.String() + rep.Failures())

	if rep.Overreach > 4 {
		t.Errorf("%d utterances reached a service they should not have, want <= 4:\n%s",
			rep.Overreach, rep.Failures())
	}
}

// TestReachBaselineONNX — the deployed configuration: the cascade with the
// multilingual e5 embedder. Opt-in via MAVEN_ONNX_LIB, same as TestONNXBaseline,
// because deps/ is gitignored. `make eval-reach` points it at the vendored copy.
//
// Reports rather than asserts. This is the number Vikunja #405 asked for, and a
// threshold invented alongside the first measurement is not a ratchet, it is a
// guess written down twice.
func TestReachBaselineONNX(t *testing.T) {
	lib := os.Getenv("MAVEN_ONNX_LIB")
	if lib == "" {
		t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
	}
	model := filepath.Join("../../..", "models/embedder/multilingual-e5-small/model_quantized.onnx")
	tok := filepath.Join("../../..", "models/embedder/multilingual-e5-small/tokenizer.json")
	for _, p := range []string{lib, model, tok} {
		if _, err := os.Stat(p); err != nil {
			t.Skipf("missing %s: %v", p, err)
		}
	}
	emb, err := router.NewONNXEmbedder(model, tok, lib)
	if err != nil {
		t.Skipf("onnx embedder unavailable: %v", err)
	}
	defer emb.Close()

	f, err := LoadReach()
	if err != nil {
		t.Fatalf("LoadReach: %v", err)
	}
	rep, err := ScoreReach(context.Background(), "reach: classifier+onnx",
		newBaselineRouter(t, emb, nil), router.DefaultActMatcher{Fns: actFns}, f)
	if err != nil {
		t.Fatalf("ScoreReach: %v", err)
	}
	t.Log("\n" + rep.String() + rep.Failures())
}