eval: derive and score which ecosystem service a turn reaches (V-405)
Reach mirrors actionAct and hexisBeforeClarify: praxis needs an act plus a fn slot equal to a capability alias, hexis needs an act plus non-empty text, and a clarified act with text reaches hexis before the question is asked. The two miss directions are counted apart because they cost different things. Missed means he asks again. Overreach means a turn arrived at a mutating path nobody sent it to, and he never gets asked about that one. PraxisAliases is a copy of the registry in cmd/mavend. The registry lives in package main and cannot be imported, and lifting it out is a refactor this measurement should not be carrying.
This commit is contained in:
@@ -0,0 +1,295 @@
|
||||
package eval
|
||||
|
||||
import (
|
||||
"context"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/kami/maven/internal/router"
|
||||
)
|
||||
|
||||
//go:embed ru_ecosystem_v1.json
|
||||
var reachFixtureJSON []byte
|
||||
|
||||
// Service — where an utterance arrives. Reach stops at the service boundary on
|
||||
// purpose: whether Nexus knows the entity and whether Hexis holds a capability
|
||||
// for it are those services' answers, and a fixture that asserted them would be
|
||||
// measuring three systems and reporting one number.
|
||||
type Service string
|
||||
|
||||
const (
|
||||
// ServiceNone — the turn stays inside Maven.
|
||||
ServiceNone Service = "none"
|
||||
// ServicePraxis — handlePraxisAct dispatched to a capability.
|
||||
ServicePraxis Service = "praxis"
|
||||
// ServiceHexis — the text reached resolveEntityReference.
|
||||
ServiceHexis Service = "hexis"
|
||||
)
|
||||
|
||||
// ReachCase — one utterance and the service it must arrive at.
|
||||
//
|
||||
// WantCapability is informational and unscored: it says which Praxis arm the fn
|
||||
// slot should land on, so a failure reads as "reached Praxis, wrong arm" rather
|
||||
// than only "reached Praxis". Scoring it would assert an alias table this
|
||||
// package cannot import.
|
||||
type ReachCase struct {
|
||||
ID string `json:"id"`
|
||||
Utterance string `json:"utterance"`
|
||||
Lang string `json:"lang"`
|
||||
WantService Service `json:"want_service"`
|
||||
WantCapability string `json:"want_capability"`
|
||||
Tags []string `json:"tags"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
// ReachFixture — the versioned envelope, same shape as Fixture.
|
||||
type ReachFixture struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Name string `json:"name"`
|
||||
ReferenceNow string `json:"reference_now"`
|
||||
Notes []string `json:"notes"`
|
||||
Cases []ReachCase `json:"cases"`
|
||||
}
|
||||
|
||||
// LoadReach returns the embedded ecosystem fixture.
|
||||
func LoadReach() (ReachFixture, error) {
|
||||
var f ReachFixture
|
||||
if err := json.Unmarshal(reachFixtureJSON, &f); err != nil {
|
||||
return ReachFixture{}, fmt.Errorf("parse reach fixture: %w", err)
|
||||
}
|
||||
if f.SchemaVersion != SchemaVersion {
|
||||
return ReachFixture{}, fmt.Errorf("reach fixture schema_version %d, want %d", f.SchemaVersion, SchemaVersion)
|
||||
}
|
||||
if len(f.Cases) == 0 {
|
||||
return ReachFixture{}, fmt.Errorf("reach fixture has no cases")
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Now — the fixture's reference clock, same contract as Fixture.Now.
|
||||
func (f ReachFixture) Now() (time.Time, error) {
|
||||
t, err := time.Parse(time.RFC3339, f.ReferenceNow)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("parse reference_now %q: %w", f.ReferenceNow, err)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// PraxisAliases — the fn slots handlePraxisAct dispatches on, copied from
|
||||
// praxisCapabilities in cmd/mavend/ecosystem_acts.go.
|
||||
//
|
||||
// It is a copy because the registry lives in package main and cannot be
|
||||
// imported. That is a drift risk and it is deliberate: the alternative is
|
||||
// lifting the whole capability registry out of the daemon, which is a
|
||||
// refactor this measurement should not be carrying. TestPraxisAliasesShape
|
||||
// asserts the arms are all present; a new capability adds a line here.
|
||||
var PraxisAliases = map[string]string{
|
||||
"list_attention": "list_attention",
|
||||
"attention": "list_attention",
|
||||
"внимание": "list_attention",
|
||||
"что требует внимания": "list_attention",
|
||||
"что нового": "list_attention",
|
||||
"acknowledge_item": "acknowledge_item",
|
||||
"принято": "acknowledge_item",
|
||||
"понял": "acknowledge_item",
|
||||
"поняла": "acknowledge_item",
|
||||
"resolve_item": "resolve_item",
|
||||
"сделано": "resolve_item",
|
||||
"готово": "resolve_item",
|
||||
"решено": "resolve_item",
|
||||
"ignore_item": "ignore_item",
|
||||
"игнорировать": "ignore_item",
|
||||
"неважно": "ignore_item",
|
||||
"pin_item": "pin_item",
|
||||
"закрепить": "pin_item",
|
||||
"list_changes": "list_changes",
|
||||
"changes": "list_changes",
|
||||
"изменения": "list_changes",
|
||||
"что изменилось": "list_changes",
|
||||
"entity_attention": "entity_attention",
|
||||
"entity_status": "entity_attention",
|
||||
}
|
||||
|
||||
// Reach derives which ecosystem service a decision arrives at, assuming all
|
||||
// three are configured. It mirrors actionAct in cmd/mavend/actions_act.go and
|
||||
// hexisBeforeClarify in cmd/mavend/ecosystem_acts.go, in their order:
|
||||
//
|
||||
// 1. A clarified act with text and no fn reaches Hexis before the clarify
|
||||
// question is ever asked. That path runs on the raw slots, so the matcher
|
||||
// does not get to fill fn first.
|
||||
// 2. Otherwise the act matcher may earn a fn from the text slot.
|
||||
// 3. A fn that is a Praxis capability alias dispatches to Praxis.
|
||||
// 4. Non-empty text reaches Hexis.
|
||||
// 5. Anything else stays inside Maven.
|
||||
//
|
||||
// It returns the service and, for Praxis, the capability the fn landed on.
|
||||
func Reach(d router.Decision, m router.ActMatcher) (Service, string) {
|
||||
if d.Intent != router.IntentAct {
|
||||
return ServiceNone, ""
|
||||
}
|
||||
if d.Clarify {
|
||||
if !d.Slots.HasFn && d.Slots.Text != "" {
|
||||
return ServiceHexis, ""
|
||||
}
|
||||
return ServiceNone, ""
|
||||
}
|
||||
fn, hasFn := d.Slots.Fn, d.Slots.HasFn
|
||||
if !hasFn && d.Slots.Text != "" && m != nil {
|
||||
if matched, _, ok := m.Match(d.Slots.Text); ok {
|
||||
fn, hasFn = matched, true
|
||||
}
|
||||
}
|
||||
if hasFn {
|
||||
if capability, ok := PraxisAliases[strings.ToLower(strings.TrimSpace(fn))]; ok {
|
||||
return ServicePraxis, capability
|
||||
}
|
||||
}
|
||||
if d.Slots.Text != "" {
|
||||
return ServiceHexis, ""
|
||||
}
|
||||
return ServiceNone, ""
|
||||
}
|
||||
|
||||
// ReachOutcome — one scored case.
|
||||
type ReachOutcome struct {
|
||||
Case ReachCase
|
||||
Decision router.Decision
|
||||
Got Service
|
||||
Capability string
|
||||
Err error
|
||||
Latency time.Duration
|
||||
Pass bool
|
||||
Reason string
|
||||
}
|
||||
|
||||
// ReachReport — the aggregate.
|
||||
//
|
||||
// The two miss directions are kept apart because they cost different things.
|
||||
// Missed is an utterance that should have reached a service and did not: he
|
||||
// asks again, or does it himself. Overreach is an utterance that reached a
|
||||
// service it had no business reaching, and on the Hexis side that is one
|
||||
// resolution away from executing a capability nobody asked for.
|
||||
type ReachReport struct {
|
||||
Name string
|
||||
Total int
|
||||
Passed int
|
||||
Missed int
|
||||
Overreach int
|
||||
// WrongService — reached a service, but the other one.
|
||||
WrongService int
|
||||
// WrongCapability — reached Praxis on the wrong arm. Reported, not failed.
|
||||
WrongCapability int
|
||||
Errors int
|
||||
Outcomes []ReachOutcome
|
||||
// ByService is keyed by the fixture's want_service.
|
||||
ByService map[string]TagStat
|
||||
ByTag map[string]TagStat
|
||||
P50, P95 time.Duration
|
||||
Max time.Duration
|
||||
}
|
||||
|
||||
// Accuracy — fraction of cases that arrived where the fixture says they must.
|
||||
func (r ReachReport) Accuracy() float64 {
|
||||
if r.Total == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(r.Passed) / float64(r.Total)
|
||||
}
|
||||
|
||||
// ScoreReach runs every case through the router and derives where it lands. It
|
||||
// never fails the run on a route error: an erroring case scores as a miss and
|
||||
// is counted, because "the model was down" and "the router was wrong" are
|
||||
// different numbers.
|
||||
func ScoreReach(ctx context.Context, name string, r Router, m router.ActMatcher, f ReachFixture) (ReachReport, error) {
|
||||
now, err := f.Now()
|
||||
if err != nil {
|
||||
return ReachReport{}, err
|
||||
}
|
||||
rep := ReachReport{
|
||||
Name: name,
|
||||
Total: len(f.Cases),
|
||||
ByService: map[string]TagStat{},
|
||||
ByTag: map[string]TagStat{},
|
||||
}
|
||||
lat := make([]time.Duration, 0, len(f.Cases))
|
||||
|
||||
for _, c := range f.Cases {
|
||||
start := time.Now()
|
||||
d, err := r.Route(ctx, c.Utterance, now)
|
||||
o := ReachOutcome{Case: c, Decision: d, Err: err, Latency: time.Since(start)}
|
||||
lat = append(lat, o.Latency)
|
||||
|
||||
switch {
|
||||
case err != nil:
|
||||
rep.Errors++
|
||||
o.Reason = fmt.Sprintf("route error: %v", err)
|
||||
default:
|
||||
o.Got, o.Capability = Reach(d, m)
|
||||
switch {
|
||||
case o.Got == c.WantService:
|
||||
o.Pass = true
|
||||
if c.WantCapability != "" && o.Capability != c.WantCapability {
|
||||
rep.WrongCapability++
|
||||
o.Reason = fmt.Sprintf("reached praxis on %q, want %q", o.Capability, c.WantCapability)
|
||||
}
|
||||
case c.WantService == ServiceNone:
|
||||
rep.Overreach++
|
||||
o.Reason = fmt.Sprintf("reached %s, want none (intent %q, clarify %v, text %q)",
|
||||
o.Got, d.Intent, d.Clarify, d.Slots.Text)
|
||||
case o.Got == ServiceNone:
|
||||
rep.Missed++
|
||||
o.Reason = fmt.Sprintf("stayed local, want %s (intent %q, clarify %v, fn %q)",
|
||||
c.WantService, d.Intent, d.Clarify, d.Slots.Fn)
|
||||
default:
|
||||
rep.WrongService++
|
||||
o.Reason = fmt.Sprintf("reached %s, want %s (fn %q)", o.Got, c.WantService, d.Slots.Fn)
|
||||
}
|
||||
}
|
||||
if o.Pass {
|
||||
rep.Passed++
|
||||
}
|
||||
bump(rep.ByService, string(c.WantService), o.Pass)
|
||||
for _, tag := range c.Tags {
|
||||
bump(rep.ByTag, tag, o.Pass)
|
||||
}
|
||||
rep.Outcomes = append(rep.Outcomes, o)
|
||||
}
|
||||
|
||||
sort.Slice(lat, func(i, j int) bool { return lat[i] < lat[j] })
|
||||
rep.P50, rep.P95 = percentile(lat, 0.50), percentile(lat, 0.95)
|
||||
if len(lat) > 0 {
|
||||
rep.Max = lat[len(lat)-1]
|
||||
}
|
||||
return rep, nil
|
||||
}
|
||||
|
||||
// String renders the report.
|
||||
func (r ReachReport) String() string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "%s: %d/%d reached the right place (%.1f%%)\n", r.Name, r.Passed, r.Total, 100*r.Accuracy())
|
||||
fmt.Fprintf(&b, " missed: %d (should have reached, didn't) | overreach: %d (reached, shouldn't) | wrong service: %d | wrong praxis arm: %d | errors: %d\n",
|
||||
r.Missed, r.Overreach, r.WrongService, r.WrongCapability, r.Errors)
|
||||
fmt.Fprintf(&b, " latency: p50 %s p95 %s max %s\n", r.P50, r.P95, r.Max)
|
||||
fmt.Fprintf(&b, " by want: %s\n", renderStats(r.ByService))
|
||||
fmt.Fprintf(&b, " by tag: %s\n", renderStats(r.ByTag))
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// Failures — the per-case detail, sorted by ID so two runs diff cleanly. A
|
||||
// passing case with a wrong Praxis arm is listed too: it carries a reason.
|
||||
func (r ReachReport) Failures() string {
|
||||
var b strings.Builder
|
||||
out := append([]ReachOutcome(nil), r.Outcomes...)
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Case.ID < out[j].Case.ID })
|
||||
for _, o := range out {
|
||||
if o.Reason == "" {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, " %s %q: %s\n", o.Case.ID, o.Case.Utterance, o.Reason)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
Reference in New Issue
Block a user