diff --git a/Makefile b/Makefile index a7ab586..f837391 100644 --- a/Makefile +++ b/Makefile @@ -74,9 +74,12 @@ test: $(GO) test -race -coverprofile=coverage.out ./internal/... ./cmd/... # eval-router — score the held-out RU routing fixture (internal/router/eval). -# Verbose so the report table lands in the terminal; MAVEN_ONNX_LIB additionally -# runs the prod-representative ONNX baseline (skipped without it). This is the -# measurement Vikunja #319 compares before #320 flips the route decider. +# Verbose so the report tables land in the terminal. MAVEN_ONNX_LIB points the +# prod-representative baseline at the vendored runtime; override it or set it +# empty to run only the deterministic hash ratchet. This is the measurement +# Vikunja #319 compares before #320 flips the route decider. +MAVEN_ONNX_LIB ?= $(shell pwd)/deps/onnxruntime-linux-x64-1.26.0/lib/libonnxruntime.so + eval-router: MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/router/eval/ diff --git a/internal/router/eval/eval.go b/internal/router/eval/eval.go index 35fb623..a9ca05c 100644 --- a/internal/router/eval/eval.go +++ b/internal/router/eval/eval.go @@ -135,6 +135,10 @@ type Report struct { // act is a confident destructive guess. MissedClarify int Errors int + // SlotsDeferred — stage-0 hits whose slot the daemon fills downstream + // (reminder grammar → applyAction's time parser). Not a miss, but not a + // full router-level win either; tracked so the two aren't conflated. + SlotsDeferred int Outcomes []Outcome // Confusion counts want→got intent pairs, decided cases only. Confusion map[string]int @@ -218,7 +222,17 @@ func Score(ctx context.Context, name string, r Router, f Fixture) (Report, error o.Reasons = append(o.Reasons, fmt.Sprintf("intent %q, want %q (%.3f)", d.Intent, c.Intent, d.Confidence)) } if c.WantTime && !d.Slots.HasTime { - o.Reasons = append(o.Reasons, "no time slot, want one") + // Stage 0 skips the extractor by design: ReminderGrammar + // captures the text after "напомни"/"remind me" and the + // daemon's applyAction runs the time parser on it (see + // stage0.go). Charging the router for a slot it was never + // asked to fill would make an exact-match win look like a + // miss — so it is counted, not failed. + if d.Stage == 0 { + rep.SlotsDeferred++ + } else { + o.Reasons = append(o.Reasons, "no time slot, want one") + } } if c.WantFn && !d.Slots.HasFn { o.Reasons = append(o.Reasons, "no fn slot, want an allowlist match") @@ -282,8 +296,8 @@ func (r Report) String() string { var b strings.Builder fmt.Fprintf(&b, "%s: %d/%d cases (%.1f%% full, %.1f%% intent-only)\n", r.Name, r.Passed, r.Total, 100*r.Accuracy(), 100*r.IntentAccuracy()) - fmt.Fprintf(&b, " clarify: %d false (asked, shouldn't) / %d missed (guessed, shouldn't) | errors: %d\n", - r.FalseClarify, r.MissedClarify, r.Errors) + fmt.Fprintf(&b, " clarify: %d false (asked, shouldn't) / %d missed (guessed, shouldn't) | errors: %d | slots deferred to daemon: %d\n", + r.FalseClarify, r.MissedClarify, r.Errors, r.SlotsDeferred) fmt.Fprintf(&b, " latency: p50 %s p95 %s max %s\n", r.P50, r.P95, r.Max) fmt.Fprintf(&b, " by lang: %s\n", renderStats(r.ByLang)) fmt.Fprintf(&b, " by tag: %s\n", renderStats(r.ByTag)) diff --git a/internal/router/eval/eval_test.go b/internal/router/eval/eval_test.go index e6290be..c002ece 100644 --- a/internal/router/eval/eval_test.go +++ b/internal/router/eval/eval_test.go @@ -142,13 +142,13 @@ func TestClassifierBaseline(t *testing.T) { } t.Log("\n" + rep.String() + rep.Failures()) - // 0.10 is under the observed 0.118, not a target. Almost every case here + // 0.15 is under the observed 0.171, not a target. Almost every case here // falls to clarify because the hash embedder's cosine never clears the // 0.55 gate on paraphrases — which is the documented floor behaviour // (AGENTS.md: "Russian recall rarely clears the confidence gate"), not a // bug this fixture is asking anyone to fix. The number worth moving is // TestONNXBaseline's. - const floor = 0.10 + const floor = 0.15 if rep.Accuracy() < floor { t.Errorf("accuracy %.3f below ratchet %.2f — routing regressed", rep.Accuracy(), floor) } @@ -162,10 +162,18 @@ func TestClassifierBaseline(t *testing.T) { // TestONNXBaseline — the number that actually belongs in Vikunja #319: the // deployed cascade with the multilingual ONNX embedder, the configuration -// homesrv runs. Skipped unless the runtime is present, because the .so is a -// gitignored ~200MB download and CI has the hash ratchet above instead. +// homesrv runs. Opt-in via MAVEN_ONNX_LIB because deps/ is gitignored, so the +// runtime is not guaranteed to exist on a fresh clone; CI has the hash ratchet +// above instead. `make eval-router` defaults the variable to the vendored +// deps/onnxruntime-linux-x64-1.26.0 copy. // -// MAVEN_ONNX_LIB=/usr/local/lib/libonnxruntime.so go test -run ONNXBaseline ./internal/router/eval/ +// Measured 2026-07-31 on deps/onnxruntime-linux-x64-1.26.0: 28/76 (36.8%), +// 21 false clarifies, 5 MISSED clarifies, p50 ~30-70ms. The missed clarifies +// are the finding — every one of the six ambiguous utterances scores higher +// cosine under ONNX than under the hash floor, so the 0.55 gate that held them +// back stops holding: "сделай это" routes to act at 0.847. A better embedder +// made the refusal lane worse, which is an argument about the gate, not the +// embedder. // // Reports rather than asserts: the score is an input to the #320 flip decision, // and a threshold invented here would just be a second opinion about the same