package eval import ( "context" _ "embed" "encoding/json" "fmt" "sort" "strings" "time" "github.com/kami/maven/internal/router" ) //go:embed ru_ecosystem_v1.json var reachFixtureJSON []byte // Service — where an utterance arrives. Reach stops at the service boundary on // purpose: whether Nexus knows the entity and whether Hexis holds a capability // for it are those services' answers, and a fixture that asserted them would be // measuring three systems and reporting one number. type Service string const ( // ServiceNone — the turn stays inside Maven. ServiceNone Service = "none" // ServicePraxis — handlePraxisAct dispatched to a capability. ServicePraxis Service = "praxis" // ServiceHexis — the text reached resolveEntityReference. ServiceHexis Service = "hexis" ) // ReachCase — one utterance and the service it must arrive at. // // WantCapability is informational and unscored: it says which Praxis arm the fn // slot should land on, so a failure reads as "reached Praxis, wrong arm" rather // than only "reached Praxis". Scoring it would assert an alias table this // package cannot import. type ReachCase struct { ID string `json:"id"` Utterance string `json:"utterance"` Lang string `json:"lang"` WantService Service `json:"want_service"` WantCapability string `json:"want_capability"` Tags []string `json:"tags"` Note string `json:"note"` } // ReachFixture — the versioned envelope, same shape as Fixture. type ReachFixture struct { SchemaVersion int `json:"schema_version"` Name string `json:"name"` ReferenceNow string `json:"reference_now"` Notes []string `json:"notes"` Cases []ReachCase `json:"cases"` } // LoadReach returns the embedded ecosystem fixture. func LoadReach() (ReachFixture, error) { var f ReachFixture if err := json.Unmarshal(reachFixtureJSON, &f); err != nil { return ReachFixture{}, fmt.Errorf("parse reach fixture: %w", err) } if f.SchemaVersion != SchemaVersion { return ReachFixture{}, fmt.Errorf("reach fixture schema_version %d, want %d", f.SchemaVersion, SchemaVersion) } if len(f.Cases) == 0 { return ReachFixture{}, fmt.Errorf("reach fixture has no cases") } return f, nil } // Now — the fixture's reference clock, same contract as Fixture.Now. func (f ReachFixture) Now() (time.Time, error) { t, err := time.Parse(time.RFC3339, f.ReferenceNow) if err != nil { return time.Time{}, fmt.Errorf("parse reference_now %q: %w", f.ReferenceNow, err) } return t, nil } // PraxisAliases — the fn slots handlePraxisAct dispatches on, copied from // praxisCapabilities in cmd/mavend/ecosystem_acts.go. // // It is a copy because the registry lives in package main and cannot be // imported. That is a drift risk and it is deliberate: the alternative is // lifting the whole capability registry out of the daemon, which is a // refactor this measurement should not be carrying. TestPraxisAliasesShape // asserts the arms are all present; a new capability adds a line here. var PraxisAliases = map[string]string{ "list_attention": "list_attention", "attention": "list_attention", "внимание": "list_attention", "что требует внимания": "list_attention", "что нового": "list_attention", "acknowledge_item": "acknowledge_item", "принято": "acknowledge_item", "понял": "acknowledge_item", "поняла": "acknowledge_item", "resolve_item": "resolve_item", "сделано": "resolve_item", "готово": "resolve_item", "решено": "resolve_item", "ignore_item": "ignore_item", "игнорировать": "ignore_item", "неважно": "ignore_item", "pin_item": "pin_item", "закрепить": "pin_item", "list_changes": "list_changes", "changes": "list_changes", "изменения": "list_changes", "что изменилось": "list_changes", "entity_attention": "entity_attention", "entity_status": "entity_attention", } // Reach derives which ecosystem service a decision arrives at, assuming all // three are configured. It mirrors actionAct in cmd/mavend/actions_act.go and // hexisBeforeClarify in cmd/mavend/ecosystem_acts.go, in their order: // // 1. A clarified act with text and no fn reaches Hexis before the clarify // question is ever asked. That path runs on the raw slots, so the matcher // does not get to fill fn first. // 2. Otherwise the act matcher may earn a fn from the text slot. // 3. A fn that is a Praxis capability alias dispatches to Praxis. // 4. Non-empty text reaches Hexis. // 5. Anything else stays inside Maven. // // It returns the service and, for Praxis, the capability the fn landed on. func Reach(d router.Decision, m router.ActMatcher) (Service, string) { if d.Intent != router.IntentAct { return ServiceNone, "" } if d.Clarify { if !d.Slots.HasFn && d.Slots.Text != "" { return ServiceHexis, "" } return ServiceNone, "" } fn, hasFn := d.Slots.Fn, d.Slots.HasFn if !hasFn && d.Slots.Text != "" && m != nil { if matched, _, ok := m.Match(d.Slots.Text); ok { fn, hasFn = matched, true } } if hasFn { if capability, ok := PraxisAliases[strings.ToLower(strings.TrimSpace(fn))]; ok { return ServicePraxis, capability } } if d.Slots.Text != "" { return ServiceHexis, "" } return ServiceNone, "" } // ReachOutcome — one scored case. type ReachOutcome struct { Case ReachCase Decision router.Decision Got Service Capability string Err error Latency time.Duration Pass bool Reason string } // ReachReport — the aggregate. // // The two miss directions are kept apart because they cost different things. // Missed is an utterance that should have reached a service and did not: he // asks again, or does it himself. Overreach is an utterance that reached a // service it had no business reaching, and on the Hexis side that is one // resolution away from executing a capability nobody asked for. type ReachReport struct { Name string Total int Passed int Missed int Overreach int // WrongService — reached a service, but the other one. WrongService int // WrongCapability — reached Praxis on the wrong arm. Reported, not failed. WrongCapability int Errors int Outcomes []ReachOutcome // ByService is keyed by the fixture's want_service. ByService map[string]TagStat ByTag map[string]TagStat P50, P95 time.Duration Max time.Duration } // Accuracy — fraction of cases that arrived where the fixture says they must. func (r ReachReport) Accuracy() float64 { if r.Total == 0 { return 0 } return float64(r.Passed) / float64(r.Total) } // ScoreReach runs every case through the router and derives where it lands. It // never fails the run on a route error: an erroring case scores as a miss and // is counted, because "the model was down" and "the router was wrong" are // different numbers. func ScoreReach(ctx context.Context, name string, r Router, m router.ActMatcher, f ReachFixture) (ReachReport, error) { now, err := f.Now() if err != nil { return ReachReport{}, err } rep := ReachReport{ Name: name, Total: len(f.Cases), ByService: map[string]TagStat{}, ByTag: map[string]TagStat{}, } lat := make([]time.Duration, 0, len(f.Cases)) for _, c := range f.Cases { start := time.Now() d, err := r.Route(ctx, c.Utterance, now) o := ReachOutcome{Case: c, Decision: d, Err: err, Latency: time.Since(start)} lat = append(lat, o.Latency) switch { case err != nil: rep.Errors++ o.Reason = fmt.Sprintf("route error: %v", err) default: o.Got, o.Capability = Reach(d, m) switch { case o.Got == c.WantService: o.Pass = true if c.WantCapability != "" && o.Capability != c.WantCapability { rep.WrongCapability++ o.Reason = fmt.Sprintf("reached praxis on %q, want %q", o.Capability, c.WantCapability) } case c.WantService == ServiceNone: rep.Overreach++ o.Reason = fmt.Sprintf("reached %s, want none (intent %q, clarify %v, text %q)", o.Got, d.Intent, d.Clarify, d.Slots.Text) case o.Got == ServiceNone: rep.Missed++ o.Reason = fmt.Sprintf("stayed local, want %s (intent %q, clarify %v, fn %q)", c.WantService, d.Intent, d.Clarify, d.Slots.Fn) default: rep.WrongService++ o.Reason = fmt.Sprintf("reached %s, want %s (fn %q)", o.Got, c.WantService, d.Slots.Fn) } } if o.Pass { rep.Passed++ } bump(rep.ByService, string(c.WantService), o.Pass) for _, tag := range c.Tags { bump(rep.ByTag, tag, o.Pass) } rep.Outcomes = append(rep.Outcomes, o) } sort.Slice(lat, func(i, j int) bool { return lat[i] < lat[j] }) rep.P50, rep.P95 = percentile(lat, 0.50), percentile(lat, 0.95) if len(lat) > 0 { rep.Max = lat[len(lat)-1] } return rep, nil } // String renders the report. func (r ReachReport) String() string { var b strings.Builder fmt.Fprintf(&b, "%s: %d/%d reached the right place (%.1f%%)\n", r.Name, r.Passed, r.Total, 100*r.Accuracy()) fmt.Fprintf(&b, " missed: %d (should have reached, didn't) | overreach: %d (reached, shouldn't) | wrong service: %d | wrong praxis arm: %d | errors: %d\n", r.Missed, r.Overreach, r.WrongService, r.WrongCapability, r.Errors) fmt.Fprintf(&b, " latency: p50 %s p95 %s max %s\n", r.P50, r.P95, r.Max) fmt.Fprintf(&b, " by want: %s\n", renderStats(r.ByService)) fmt.Fprintf(&b, " by tag: %s\n", renderStats(r.ByTag)) return b.String() } // Failures — the per-case detail, sorted by ID so two runs diff cleanly. A // passing case with a wrong Praxis arm is listed too: it carries a reason. func (r ReachReport) Failures() string { var b strings.Builder out := append([]ReachOutcome(nil), r.Outcomes...) sort.Slice(out, func(i, j int) bool { return out[i].Case.ID < out[j].Case.ID }) for _, o := range out { if o.Reason == "" { continue } fmt.Fprintf(&b, " %s %q: %s\n", o.Case.ID, o.Case.Utterance, o.Reason) } return b.String() }