diff --git a/internal/router/eval/reach.go b/internal/router/eval/reach.go new file mode 100644 index 0000000..eee6828 --- /dev/null +++ b/internal/router/eval/reach.go @@ -0,0 +1,295 @@ +package eval + +import ( + "context" + _ "embed" + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + "github.com/kami/maven/internal/router" +) + +//go:embed ru_ecosystem_v1.json +var reachFixtureJSON []byte + +// Service — where an utterance arrives. Reach stops at the service boundary on +// purpose: whether Nexus knows the entity and whether Hexis holds a capability +// for it are those services' answers, and a fixture that asserted them would be +// measuring three systems and reporting one number. +type Service string + +const ( + // ServiceNone — the turn stays inside Maven. + ServiceNone Service = "none" + // ServicePraxis — handlePraxisAct dispatched to a capability. + ServicePraxis Service = "praxis" + // ServiceHexis — the text reached resolveEntityReference. + ServiceHexis Service = "hexis" +) + +// ReachCase — one utterance and the service it must arrive at. +// +// WantCapability is informational and unscored: it says which Praxis arm the fn +// slot should land on, so a failure reads as "reached Praxis, wrong arm" rather +// than only "reached Praxis". Scoring it would assert an alias table this +// package cannot import. +type ReachCase struct { + ID string `json:"id"` + Utterance string `json:"utterance"` + Lang string `json:"lang"` + WantService Service `json:"want_service"` + WantCapability string `json:"want_capability"` + Tags []string `json:"tags"` + Note string `json:"note"` +} + +// ReachFixture — the versioned envelope, same shape as Fixture. +type ReachFixture struct { + SchemaVersion int `json:"schema_version"` + Name string `json:"name"` + ReferenceNow string `json:"reference_now"` + Notes []string `json:"notes"` + Cases []ReachCase `json:"cases"` +} + +// LoadReach returns the embedded ecosystem fixture. +func LoadReach() (ReachFixture, error) { + var f ReachFixture + if err := json.Unmarshal(reachFixtureJSON, &f); err != nil { + return ReachFixture{}, fmt.Errorf("parse reach fixture: %w", err) + } + if f.SchemaVersion != SchemaVersion { + return ReachFixture{}, fmt.Errorf("reach fixture schema_version %d, want %d", f.SchemaVersion, SchemaVersion) + } + if len(f.Cases) == 0 { + return ReachFixture{}, fmt.Errorf("reach fixture has no cases") + } + return f, nil +} + +// Now — the fixture's reference clock, same contract as Fixture.Now. +func (f ReachFixture) Now() (time.Time, error) { + t, err := time.Parse(time.RFC3339, f.ReferenceNow) + if err != nil { + return time.Time{}, fmt.Errorf("parse reference_now %q: %w", f.ReferenceNow, err) + } + return t, nil +} + +// PraxisAliases — the fn slots handlePraxisAct dispatches on, copied from +// praxisCapabilities in cmd/mavend/ecosystem_acts.go. +// +// It is a copy because the registry lives in package main and cannot be +// imported. That is a drift risk and it is deliberate: the alternative is +// lifting the whole capability registry out of the daemon, which is a +// refactor this measurement should not be carrying. TestPraxisAliasesShape +// asserts the arms are all present; a new capability adds a line here. +var PraxisAliases = map[string]string{ + "list_attention": "list_attention", + "attention": "list_attention", + "внимание": "list_attention", + "что требует внимания": "list_attention", + "что нового": "list_attention", + "acknowledge_item": "acknowledge_item", + "принято": "acknowledge_item", + "понял": "acknowledge_item", + "поняла": "acknowledge_item", + "resolve_item": "resolve_item", + "сделано": "resolve_item", + "готово": "resolve_item", + "решено": "resolve_item", + "ignore_item": "ignore_item", + "игнорировать": "ignore_item", + "неважно": "ignore_item", + "pin_item": "pin_item", + "закрепить": "pin_item", + "list_changes": "list_changes", + "changes": "list_changes", + "изменения": "list_changes", + "что изменилось": "list_changes", + "entity_attention": "entity_attention", + "entity_status": "entity_attention", +} + +// Reach derives which ecosystem service a decision arrives at, assuming all +// three are configured. It mirrors actionAct in cmd/mavend/actions_act.go and +// hexisBeforeClarify in cmd/mavend/ecosystem_acts.go, in their order: +// +// 1. A clarified act with text and no fn reaches Hexis before the clarify +// question is ever asked. That path runs on the raw slots, so the matcher +// does not get to fill fn first. +// 2. Otherwise the act matcher may earn a fn from the text slot. +// 3. A fn that is a Praxis capability alias dispatches to Praxis. +// 4. Non-empty text reaches Hexis. +// 5. Anything else stays inside Maven. +// +// It returns the service and, for Praxis, the capability the fn landed on. +func Reach(d router.Decision, m router.ActMatcher) (Service, string) { + if d.Intent != router.IntentAct { + return ServiceNone, "" + } + if d.Clarify { + if !d.Slots.HasFn && d.Slots.Text != "" { + return ServiceHexis, "" + } + return ServiceNone, "" + } + fn, hasFn := d.Slots.Fn, d.Slots.HasFn + if !hasFn && d.Slots.Text != "" && m != nil { + if matched, _, ok := m.Match(d.Slots.Text); ok { + fn, hasFn = matched, true + } + } + if hasFn { + if capability, ok := PraxisAliases[strings.ToLower(strings.TrimSpace(fn))]; ok { + return ServicePraxis, capability + } + } + if d.Slots.Text != "" { + return ServiceHexis, "" + } + return ServiceNone, "" +} + +// ReachOutcome — one scored case. +type ReachOutcome struct { + Case ReachCase + Decision router.Decision + Got Service + Capability string + Err error + Latency time.Duration + Pass bool + Reason string +} + +// ReachReport — the aggregate. +// +// The two miss directions are kept apart because they cost different things. +// Missed is an utterance that should have reached a service and did not: he +// asks again, or does it himself. Overreach is an utterance that reached a +// service it had no business reaching, and on the Hexis side that is one +// resolution away from executing a capability nobody asked for. +type ReachReport struct { + Name string + Total int + Passed int + Missed int + Overreach int + // WrongService — reached a service, but the other one. + WrongService int + // WrongCapability — reached Praxis on the wrong arm. Reported, not failed. + WrongCapability int + Errors int + Outcomes []ReachOutcome + // ByService is keyed by the fixture's want_service. + ByService map[string]TagStat + ByTag map[string]TagStat + P50, P95 time.Duration + Max time.Duration +} + +// Accuracy — fraction of cases that arrived where the fixture says they must. +func (r ReachReport) Accuracy() float64 { + if r.Total == 0 { + return 0 + } + return float64(r.Passed) / float64(r.Total) +} + +// ScoreReach runs every case through the router and derives where it lands. It +// never fails the run on a route error: an erroring case scores as a miss and +// is counted, because "the model was down" and "the router was wrong" are +// different numbers. +func ScoreReach(ctx context.Context, name string, r Router, m router.ActMatcher, f ReachFixture) (ReachReport, error) { + now, err := f.Now() + if err != nil { + return ReachReport{}, err + } + rep := ReachReport{ + Name: name, + Total: len(f.Cases), + ByService: map[string]TagStat{}, + ByTag: map[string]TagStat{}, + } + lat := make([]time.Duration, 0, len(f.Cases)) + + for _, c := range f.Cases { + start := time.Now() + d, err := r.Route(ctx, c.Utterance, now) + o := ReachOutcome{Case: c, Decision: d, Err: err, Latency: time.Since(start)} + lat = append(lat, o.Latency) + + switch { + case err != nil: + rep.Errors++ + o.Reason = fmt.Sprintf("route error: %v", err) + default: + o.Got, o.Capability = Reach(d, m) + switch { + case o.Got == c.WantService: + o.Pass = true + if c.WantCapability != "" && o.Capability != c.WantCapability { + rep.WrongCapability++ + o.Reason = fmt.Sprintf("reached praxis on %q, want %q", o.Capability, c.WantCapability) + } + case c.WantService == ServiceNone: + rep.Overreach++ + o.Reason = fmt.Sprintf("reached %s, want none (intent %q, clarify %v, text %q)", + o.Got, d.Intent, d.Clarify, d.Slots.Text) + case o.Got == ServiceNone: + rep.Missed++ + o.Reason = fmt.Sprintf("stayed local, want %s (intent %q, clarify %v, fn %q)", + c.WantService, d.Intent, d.Clarify, d.Slots.Fn) + default: + rep.WrongService++ + o.Reason = fmt.Sprintf("reached %s, want %s (fn %q)", o.Got, c.WantService, d.Slots.Fn) + } + } + if o.Pass { + rep.Passed++ + } + bump(rep.ByService, string(c.WantService), o.Pass) + for _, tag := range c.Tags { + bump(rep.ByTag, tag, o.Pass) + } + rep.Outcomes = append(rep.Outcomes, o) + } + + sort.Slice(lat, func(i, j int) bool { return lat[i] < lat[j] }) + rep.P50, rep.P95 = percentile(lat, 0.50), percentile(lat, 0.95) + if len(lat) > 0 { + rep.Max = lat[len(lat)-1] + } + return rep, nil +} + +// String renders the report. +func (r ReachReport) String() string { + var b strings.Builder + fmt.Fprintf(&b, "%s: %d/%d reached the right place (%.1f%%)\n", r.Name, r.Passed, r.Total, 100*r.Accuracy()) + fmt.Fprintf(&b, " missed: %d (should have reached, didn't) | overreach: %d (reached, shouldn't) | wrong service: %d | wrong praxis arm: %d | errors: %d\n", + r.Missed, r.Overreach, r.WrongService, r.WrongCapability, r.Errors) + fmt.Fprintf(&b, " latency: p50 %s p95 %s max %s\n", r.P50, r.P95, r.Max) + fmt.Fprintf(&b, " by want: %s\n", renderStats(r.ByService)) + fmt.Fprintf(&b, " by tag: %s\n", renderStats(r.ByTag)) + return b.String() +} + +// Failures — the per-case detail, sorted by ID so two runs diff cleanly. A +// passing case with a wrong Praxis arm is listed too: it carries a reason. +func (r ReachReport) Failures() string { + var b strings.Builder + out := append([]ReachOutcome(nil), r.Outcomes...) + sort.Slice(out, func(i, j int) bool { return out[i].Case.ID < out[j].Case.ID }) + for _, o := range out { + if o.Reason == "" { + continue + } + fmt.Fprintf(&b, " %s %q: %s\n", o.Case.ID, o.Case.Utterance, o.Reason) + } + return b.String() +}