From 576dfd8b4c8985cfde73fef465d25fc0d3331f0e Mon Sep 17 00:00:00 2001 From: claude Date: Wed, 5 Aug 2026 21:03:05 +0400 Subject: [PATCH] eval: the resident model never reaches Praxis either (V-517) V-405 measured reach with the classifier only, and the LLM router is the deployed default, so 16/30 was the floor rather than the shipped behaviour. TestReachWithLLMRouter scores the same 30 cases with the model, gated on MAVEN_LLM_URL like TestLLMRouterBaseline. The open question was whether the model writes a literal Praxis capability into the fn slot and reaches a service the classifier structurally cannot. It does not. Praxis is 0/12 with the model alone, exactly what the classifier alone scores, and all twelve fail the same way: local, empty fn. Nothing in the router prompt names a Praxis capability, so there is no string for it to write. So V-516's stage-0 grammars are the only path to Praxis, not a determinism argument. Through the cascade the model scores 28/30 with praxis 11/12, one point above the classifier baseline. Hexis is 10/10 either way. Overreach is 1 in both configurations, under the 4 the harness asserts. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01SoL7EBdYC5Mhz3DJd49GJy --- CLAUDE.md | 8 ++- docs/evals/2026-08-05-reach-llm-router.md | 70 ++++++++++++++++++++ internal/router/eval/llmrouter_test.go | 78 +++++++++++++++++++++++ 3 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 docs/evals/2026-08-05-reach-llm-router.md diff --git a/CLAUDE.md b/CLAUDE.md index 1151c7c..eb43790 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -227,7 +227,13 @@ llama-server in that run), so judge it again before quoting a cascade number. Praxis taken off the model, 05-08-2026 (V-516). `PraxisGrammars()` (`internal/router/praxis.go`, wired in `buildRouter` before the capture marker because -"отметь" is a capture verb) fills `Slots.Fn` with a Praxis capability name. Praxis reach +"отметь" is a capture verb) fills `Slots.Fn` with a Praxis capability name. +**These grammars are the only path to Praxis, not a faster one.** Measured +2026-08-05 with the resident model as router (V-517, +`docs/evals/2026-08-05-reach-llm-router.md`): the model alone reaches Praxis +**0/12**, the same as the classifier alone, because nothing in the router +prompt names a Praxis capability and there is no string for it to write. +Through the cascade it is 11/12. Deleting these rules costs every point. Praxis reach was **0/12 and structurally so**: `handlePraxisAct` compares `Slots.Fn` to a capability alias, and that slot is filled from the deployment's enabled tool names, which no Praxis alias is on. Measured **16/30 → 27/30 overall, praxis 0/12 → 11/12, lifecycle 0/5 → 5/5** diff --git a/docs/evals/2026-08-05-reach-llm-router.md b/docs/evals/2026-08-05-reach-llm-router.md new file mode 100644 index 0000000..71e262b --- /dev/null +++ b/docs/evals/2026-08-05-reach-llm-router.md @@ -0,0 +1,70 @@ +# Ecosystem reach with the resident model as router + +Date: 2026-08-05. Vikunja #517, split out of #405. +Fixture: `internal/router/eval/ru_ecosystem_v1.json`, 30 held-out Russian cases. +Model: Qwen3-1.7B-UD-Q4_K_XL, llama-server on the host at 127.0.0.1:8899. +Harness: `TestReachWithLLMRouter` in `internal/router/eval/llmrouter_test.go`. + +## The numbers + +| configuration | reached the right place | praxis | hexis | none | +|---|---|---|---|---| +| classifier + hash (V-405 floor) | 16/30 | 0/12 | — | — | +| classifier + ONNX, after the V-516 grammars | 27/30 | 11/12 | — | — | +| **llm-only** (resident model alone) | **17/30 (56.7%)** | **0/12** | 10/10 | 7/8 | +| **cascade + llm** + hash fallback | **28/30 (93.3%)** | **11/12** | 10/10 | 7/8 | + +Latency: llm-only p50 1.29s, p95 1.65s. Cascade p50 1.11s, p95 1.64s. +No case errored in either configuration. + +## The open question is answered: the model never reaches Praxis + +The route grammar lets the model write any string into the `fn` slot. So it +could in principle emit a literal Praxis capability name, and reach a service +the classifier structurally cannot. It does not. **Praxis is 0/12 with the +model alone.** That is exactly what the classifier alone scores. Every one of +the twelve fails the same way: the utterance stays local with an empty `fn`. + +So the stage-0 Praxis grammars from V-516 are not a determinism argument. They +are the only path to Praxis that exists. Deleting them takes reach from 11/12 +back to 0/12 whichever engine is answering. + +The failure is not that the model routes these badly in its own terms. It +spreads them across `query`, `fact`, `system` and `chat`. Those are reasonable +readings of "что требует внимания" and "готово, закрывай" for a model that has +never been told Praxis exists. Nothing in the prompt names a Praxis capability, +so there is no string for it to write. + +## What the model does buy + +Hexis is 10/10 with the model alone, and the mutating tag is 10/15 llm-only +against 15/15 through the cascade. The model reaches everything Hexis owns +without help, which is the half the act allowlist already names in the prompt. + +Cascade + llm scores one point above the classifier baseline: 28/30 against +27/30, the difference being one attention case. That is the same shape as the +routing fixture, where the router buys about 4 points rather than a doubling. + +## The two that still miss + +- `eco-ru-021 "что там с нексусом"`. Routes `query`, stays local, wants Praxis. + Asking after a named service reads as a question about a thing, and no + grammar claims a service name. +- `eco-ru-029 "сделай это"`. Routes `act` and reaches Hexis. The fixture wants + nothing reached, because "это" names no target. This is the overreach case + and it is the one direction worth failing on. The confirmation binding + downstream still resolves a canonical entity id before anything executes. + The fixture is right that the turn should have asked. + +Overreach is 1 in both configurations, under the 4 the harness asserts. + +## How to re-run + +```sh +MAVEN_LLM_URL=http://127.0.0.1:8899 \ + deps/go/go/bin/go test -v -count=1 -timeout 40m \ + -run TestReachWithLLMRouter ./internal/router/eval/ +``` + +The host `http_proxy` answers 503 for 127.0.0.1. `noProxyLoopback` in the test +excludes it. A run that scores every case as a route error measured the proxy. diff --git a/internal/router/eval/llmrouter_test.go b/internal/router/eval/llmrouter_test.go index fe5cc43..d279856 100644 --- a/internal/router/eval/llmrouter_test.go +++ b/internal/router/eval/llmrouter_test.go @@ -112,6 +112,84 @@ func TestLLMRouterBaseline(t *testing.T) { } } +// TestReachWithLLMRouter — the reach fixture scored with the resident model as +// router (Vikunja #517). V-405 measured the classifier only, and the LLM router +// is the deployed default, so 16/30 with praxis 0/12 is the floor rather than +// the shipped behaviour. +// +// The question is specific. The route grammar lets the model write any string +// into the fn slot, so it *could* emit a literal Praxis capability name and +// reach a service the classifier structurally cannot. If it does, V-516's +// stage-0 Praxis grammars are a determinism argument. If it does not, they are +// the only path. +// +// Same gate and same two configurations as TestLLMRouterBaseline, for the same +// reason: "the LLM router" alone and the cascade that actually ships answer +// different questions. +func TestReachWithLLMRouter(t *testing.T) { + base := os.Getenv("MAVEN_LLM_URL") + if base == "" { + t.Skip("MAVEN_LLM_URL unset — start llama-server and point it here (see doc comment)") + } + noProxyLoopback(t) + + f, err := LoadReach() + if err != nil { + t.Fatalf("LoadReach: %v", err) + } + client := llm.New(base, 60*time.Second) + if err := ping(context.Background(), client); err != nil { + t.Skipf("llama-server at %s unreachable: %v", base, err) + } + + ctx := context.Background() + model, err := llm.ModelID(ctx, base) + if err != nil { + t.Logf("could not read model id from %s: %v — reports will say %q", base, err, llm.UnknownModel) + model = llm.UnknownModel + } + t.Logf("scoring reach with model %s at %s", model, base) + lr := router.NewLLMRouter(client) + m := router.DefaultActMatcher{Fns: actFns} + + llmOnly := RouterFunc(func(ctx context.Context, u string, now time.Time) (router.Decision, error) { + d, ok, err := lr.Route(ctx, u, now) + if err != nil { + return d, err + } + if !ok { + return d, fmt.Errorf("llm router declined without an error") + } + return d, nil + }) + repLLM, err := ScoreReach(ctx, "reach: llm-only ("+model+")", llmOnly, m, f) + if err != nil { + t.Fatalf("ScoreReach llm-only: %v", err) + } + t.Log("\n" + repLLM.String() + repLLM.Failures()) + + // Hash embedder for the classifier floor, so any lift is the model's and + // not the embedder's — the same control TestLLMRouterBaseline uses. + repCascade, err := ScoreReach(ctx, "reach: cascade+llm ("+model+") + hash fallback", + newBaselineRouter(t, router.NewHashEmbedder(1024), lr), m, f) + if err != nil { + t.Fatalf("ScoreReach cascade: %v", err) + } + t.Log("\n" + repCascade.String() + repCascade.Failures()) + + for _, rep := range []ReachReport{repLLM, repCascade} { + if rep.Errors == rep.Total { + t.Errorf("%s: all %d cases errored — harness fault, not a measurement", rep.Name, rep.Total) + } + } + // Overreach is the one direction worth failing on, for the reason + // TestReachBaselineHash gives: he never gets asked about it. + if repCascade.Overreach > 4 { + t.Errorf("%d utterances reached a service they should not have, want <= 4:\n%s", + repCascade.Overreach, repCascade.Failures()) + } +} + func ping(ctx context.Context, c *llm.Client) error { ctx, cancel := context.WithTimeout(ctx, 90*time.Second) defer cancel()