diff --git a/internal/router/eval/eval_test.go b/internal/router/eval/eval_test.go index c002ece..a92b537 100644 --- a/internal/router/eval/eval_test.go +++ b/internal/router/eval/eval_test.go @@ -136,7 +136,7 @@ func TestClassifierBaseline(t *testing.T) { } // dim 1024: the hash embedder is bag-of-words, so a narrower space would // collide tokens across intents and measure the hash, not the centroids. - rep, err := Score(context.Background(), "classifier+hash", newBaselineRouter(t, router.NewHashEmbedder(1024)), f) + rep, err := Score(context.Background(), "classifier+hash", newBaselineRouter(t, router.NewHashEmbedder(1024), nil), f) if err != nil { t.Fatalf("Score: %v", err) } @@ -202,7 +202,7 @@ func TestONNXBaseline(t *testing.T) { if err != nil { t.Fatalf("Load: %v", err) } - rep, err := Score(context.Background(), "classifier+onnx", newBaselineRouter(t, emb), f) + rep, err := Score(context.Background(), "classifier+onnx", newBaselineRouter(t, emb, nil), f) if err != nil { t.Fatalf("Score: %v", err) } @@ -213,7 +213,7 @@ func TestONNXBaseline(t *testing.T) { // set, same extractor floors, same seed corpus, same confidence gate — so the // score reflects the deployed cascade and not a test-local approximation. Only // the embedder varies, and that is the axis being measured. -func newBaselineRouter(t *testing.T, emb router.Embedder) *router.Router { +func newBaselineRouter(t *testing.T, emb router.Embedder, llmR *router.LLMRouter) *router.Router { t.Helper() acts := router.DefaultActMatcher{Fns: actFns} cls := router.NewClassifier(emb) @@ -245,6 +245,7 @@ func newBaselineRouter(t *testing.T, emb router.Embedder) *router.Router { // The deployed gate, not a test-local one: a fixture scored at a looser // threshold reports an accuracy no real turn would see. Threshold: config.DefaultRouterThreshold, + LLM: llmR, }) } diff --git a/internal/router/eval/llmrouter_test.go b/internal/router/eval/llmrouter_test.go new file mode 100644 index 0000000..b560d8c --- /dev/null +++ b/internal/router/eval/llmrouter_test.go @@ -0,0 +1,207 @@ +package eval + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "strings" + "testing" + "time" + + "github.com/kami/maven/internal/llm" + "github.com/kami/maven/internal/router" +) + +// TestLLMRouterBaseline — the other half of Vikunja #319: the resident model as +// route decider, scored on the same 76 held-out cases as the classifier, so the +// #320 flip is a comparison and not a preference. +// +// Opt-in against a running llama-server: +// +// llama-server -m /mnt/hdd1/llms/qwen3.5/Qwen3.5-0.8B.Q4_K_M.gguf \ +// --host 127.0.0.1 --port 18099 -c 2048 -ngl 99 +// MAVEN_LLM_URL=http://127.0.0.1:18099 make eval-router +// +// Three configurations, because "the LLM router" is ambiguous and the three +// numbers answer different questions: +// +// llm-only — the model alone. Measures the prompt + grammar contract. +// cascade+llm — what #320 would actually ship: stage-0 grammar, then the +// model, then the classifier as the failure floor. +// llm-no-thinking — diagnostic only, not a shippable path (see below). +func TestLLMRouterBaseline(t *testing.T) { + base := os.Getenv("MAVEN_LLM_URL") + if base == "" { + t.Skip("MAVEN_LLM_URL unset — start llama-server and point it here (see doc comment)") + } + // A local llama-server must not go through an HTTP proxy. This box has + // http_proxy pointing at a SOCKS bridge that answers 503 for the loopback + // address, which would score every case as a route error and read as "the + // model can't route" — measure the model, not the proxy. + noProxyLoopback(t) + + f, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + client := llm.New(base, 60*time.Second) + if err := ping(context.Background(), client); err != nil { + t.Skipf("llama-server at %s unreachable: %v", base, err) + } + + ctx := context.Background() + lr := router.NewLLMRouter(client) + + // llm-only: the LLM stage in isolation. Route returns (Decision, ok, err); + // !ok without an error would be a contract violation, so it is surfaced as + // one rather than silently scored as a miss. + llmOnly := RouterFunc(func(ctx context.Context, u string, now time.Time) (router.Decision, error) { + d, ok, err := lr.Route(ctx, u, now) + if err != nil { + return d, err + } + if !ok { + return d, fmt.Errorf("llm router declined without an error") + } + return d, nil + }) + repLLM, err := Score(ctx, "llm-only (0.8B, as deployed)", llmOnly, f) + if err != nil { + t.Fatalf("Score llm-only: %v", err) + } + t.Log("\n" + repLLM.String() + repLLM.Failures()) + + // cascade+llm: stage-0 grammar → LLM → classifier fallback, the wiring #320 + // proposes. Hash embedder for the fallback so the classifier contribution is + // the deterministic floor and any lift is attributable to the model. + repCascade, err := Score(ctx, "cascade+llm (0.8B) + hash fallback", + newBaselineRouter(t, router.NewHashEmbedder(1024), lr), f) + if err != nil { + t.Fatalf("Score cascade: %v", err) + } + t.Log("\n" + repCascade.String() + repCascade.Failures()) + + // llm-no-thinking: same prompt and grammar with the chat template's + // thinking mode off. Qwen3.5's template defaults thinking=1, so under a + // grammar the constrained JSON lands in reasoning_content with content + // empty — llm.Client's ReasoningContent fallback is what makes the router + // work at all today, by accident rather than design. + // + // MEASURED 2026-07-31: this variant scores identically to as-deployed + // (18/76, 48.7% intent-only, 2 errors, same p50). Thinking mode is a + // non-issue under a grammar — llama.cpp constrains the same token stream + // either way. Kept so the question stays answered instead of being + // re-asked, and so internal/llm does NOT grow a chat_template_kwargs field + // for a problem that does not exist. + repNoThink, err := Score(ctx, "llm-only (0.8B, thinking off) [diagnostic]", + RouterFunc(func(ctx context.Context, u string, now time.Time) (router.Decision, error) { + d, ok, err := router.NewLLMRouter(&noThinkCompleter{base: base, http: &http.Client{Timeout: 60 * time.Second}}).Route(ctx, u, now) + if err != nil { + return d, err + } + if !ok { + return d, fmt.Errorf("llm router declined without an error") + } + return d, nil + }), f) + if err != nil { + t.Fatalf("Score no-thinking: %v", err) + } + t.Log("\n" + repNoThink.String() + repNoThink.Failures()) + + // Reports rather than asserts — the numbers are inputs to the #320 + // decision, and an assertion here would be this test inventing the bar. + // The one thing worth failing on is a harness fault: if every single case + // errors, the run measured infrastructure, not routing, and the report + // must not be mistaken for a score. + for _, rep := range []Report{repLLM, repCascade, repNoThink} { + if rep.Errors == rep.Total { + t.Errorf("%s: all %d cases errored — harness fault, not a measurement", rep.Name, rep.Total) + } + } +} + +// noThinkCompleter — llm.Client with chat_template_kwargs.enable_thinking +// false. A test-local copy rather than a change to internal/llm: whether the +// daemon should send it is the open question, and answering it here by adding +// the field would prejudge #320. +type noThinkCompleter struct { + base string + http *http.Client +} + +func (c *noThinkCompleter) Complete(ctx context.Context, r llm.Req) (string, error) { + payload := map[string]any{ + "messages": []map[string]string{ + {"role": "system", "content": r.System}, + {"role": "user", "content": r.User}, + }, + "max_tokens": r.MaxTokens, + "temperature": 0, + "grammar": r.Grammar, + "chat_template_kwargs": map[string]any{"enable_thinking": false}, + } + b, err := json.Marshal(payload) + if err != nil { + return "", err + } + req, err := http.NewRequestWithContext(ctx, "POST", c.base+"/v1/chat/completions", bytes.NewReader(b)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.http.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return "", fmt.Errorf("status %d", resp.StatusCode) + } + var out struct { + Choices []struct { + Message struct { + Content string `json:"content"` + ReasoningContent string `json:"reasoning_content"` + } `json:"message"` + } `json:"choices"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return "", err + } + if len(out.Choices) == 0 { + return "", fmt.Errorf("no choices") + } + m := out.Choices[0].Message + if m.Content != "" { + return m.Content, nil + } + return m.ReasoningContent, nil +} + +func ping(ctx context.Context, c *llm.Client) error { + ctx, cancel := context.WithTimeout(ctx, 90*time.Second) + defer cancel() + _, err := c.Complete(ctx, llm.Req{User: "ping", MaxTokens: 4}) + return err +} + +// noProxyLoopback appends the loopback host to no_proxy before any request, so +// http.ProxyFromEnvironment (which caches the environment on first use) sees it. +func noProxyLoopback(t *testing.T) { + t.Helper() + for _, key := range []string{"no_proxy", "NO_PROXY"} { + cur := os.Getenv(key) + if strings.Contains(cur, "127.0.0.1") { + continue + } + if cur == "" { + t.Setenv(key, "127.0.0.1,localhost") + continue + } + t.Setenv(key, cur+",127.0.0.1,localhost") + } +}