Maven/internal/router/eval/reach.go

package eval

import (
	"context"
	_ "embed"
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/kami/maven/internal/router"
)

//go:embed ru_ecosystem_v1.json
var reachFixtureJSON []byte

// Service — where an utterance arrives. Reach stops at the service boundary on
// purpose: whether Nexus knows the entity and whether Hexis holds a capability
// for it are those services' answers, and a fixture that asserted them would be
// measuring three systems and reporting one number.
type Service string

const (
	// ServiceNone — the turn stays inside Maven.
	ServiceNone Service = "none"
	// ServicePraxis — handlePraxisAct dispatched to a capability.
	ServicePraxis Service = "praxis"
	// ServiceHexis — the text reached resolveEntityReference.
	ServiceHexis Service = "hexis"
)

// ReachCase — one utterance and the service it must arrive at.
//
// WantCapability is informational and unscored: it says which Praxis arm the fn
// slot should land on, so a failure reads as "reached Praxis, wrong arm" rather
// than only "reached Praxis". Scoring it would assert an alias table this
// package cannot import.
type ReachCase struct {
	ID             string   `json:"id"`
	Utterance      string   `json:"utterance"`
	Lang           string   `json:"lang"`
	WantService    Service  `json:"want_service"`
	WantCapability string   `json:"want_capability"`
	Tags           []string `json:"tags"`
	Note           string   `json:"note"`
}

// ReachFixture — the versioned envelope, same shape as Fixture.
type ReachFixture struct {
	SchemaVersion int         `json:"schema_version"`
	Name          string      `json:"name"`
	ReferenceNow  string      `json:"reference_now"`
	Notes         []string    `json:"notes"`
	Cases         []ReachCase `json:"cases"`
}

// LoadReach returns the embedded ecosystem fixture.
func LoadReach() (ReachFixture, error) {
	var f ReachFixture
	if err := json.Unmarshal(reachFixtureJSON, &f); err != nil {
		return ReachFixture{}, fmt.Errorf("parse reach fixture: %w", err)
	}
	if f.SchemaVersion != SchemaVersion {
		return ReachFixture{}, fmt.Errorf("reach fixture schema_version %d, want %d", f.SchemaVersion, SchemaVersion)
	}
	if len(f.Cases) == 0 {
		return ReachFixture{}, fmt.Errorf("reach fixture has no cases")
	}
	return f, nil
}

// Now — the fixture's reference clock, same contract as Fixture.Now.
func (f ReachFixture) Now() (time.Time, error) {
	t, err := time.Parse(time.RFC3339, f.ReferenceNow)
	if err != nil {
		return time.Time{}, fmt.Errorf("parse reference_now %q: %w", f.ReferenceNow, err)
	}
	return t, nil
}

// PraxisAliases — the fn slots handlePraxisAct dispatches on, copied from
// praxisCapabilities in cmd/mavend/ecosystem_acts.go.
//
// It is a copy because the registry lives in package main and cannot be
// imported. That is a drift risk and it is deliberate: the alternative is
// lifting the whole capability registry out of the daemon, which is a
// refactor this measurement should not be carrying. TestPraxisAliasesShape
// asserts the arms are all present; a new capability adds a line here.
var PraxisAliases = map[string]string{
	"list_attention":       "list_attention",
	"attention":            "list_attention",
	"внимание":             "list_attention",
	"что требует внимания": "list_attention",
	"что нового":           "list_attention",
	"acknowledge_item":     "acknowledge_item",
	"принято":              "acknowledge_item",
	"понял":                "acknowledge_item",
	"поняла":               "acknowledge_item",
	"resolve_item":         "resolve_item",
	"сделано":              "resolve_item",
	"готово":               "resolve_item",
	"решено":               "resolve_item",
	"ignore_item":          "ignore_item",
	"игнорировать":         "ignore_item",
	"неважно":              "ignore_item",
	"pin_item":             "pin_item",
	"закрепить":            "pin_item",
	"list_changes":         "list_changes",
	"changes":              "list_changes",
	"изменения":            "list_changes",
	"что изменилось":       "list_changes",
	"entity_attention":     "entity_attention",
	"entity_status":        "entity_attention",
}

// Reach derives which ecosystem service a decision arrives at, assuming all
// three are configured. It mirrors actionAct in cmd/mavend/actions_act.go and
// hexisBeforeClarify in cmd/mavend/ecosystem_acts.go, in their order:
//
//  1. A clarified act with a named entity target and no fn reaches Hexis before the clarify
//     question is ever asked. That path runs on the raw slots, so the matcher
//     does not get to fill fn first.
//  2. Otherwise the act matcher may earn a fn from the text slot.
//  3. A fn that is a Praxis capability alias dispatches to Praxis.
//  4. An act with a named entity target reaches Hexis.
//  5. Anything else stays inside Maven.
//
// It returns the service and, for Praxis, the capability the fn landed on.
func Reach(d router.Decision, m router.ActMatcher) (Service, string) {
	if d.Intent != router.IntentAct {
		return ServiceNone, ""
	}
	if d.Clarify {
		if !d.Slots.HasFn && router.ActHasEntityTarget(d) {
			return ServiceHexis, ""
		}
		return ServiceNone, ""
	}
	fn, hasFn := d.Slots.Fn, d.Slots.HasFn
	if !hasFn && d.Slots.Text != "" && m != nil {
		if matched, _, ok := m.Match(d.Slots.Text); ok {
			fn, hasFn = matched, true
		}
	}
	if hasFn {
		if capability, ok := PraxisAliases[strings.ToLower(strings.TrimSpace(fn))]; ok {
			return ServicePraxis, capability
		}
	}
	if router.ActHasEntityTarget(d) {
		return ServiceHexis, ""
	}
	return ServiceNone, ""
}

// ReachOutcome — one scored case.
type ReachOutcome struct {
	Case       ReachCase
	Decision   router.Decision
	Got        Service
	Capability string
	Err        error
	Latency    time.Duration
	Pass       bool
	Reason     string
}

// ReachReport — the aggregate.
//
// The two miss directions are kept apart because they cost different things.
// Missed is an utterance that should have reached a service and did not: he
// asks again, or does it himself. Overreach is an utterance that reached a
// service it had no business reaching, and on the Hexis side that is one
// resolution away from executing a capability nobody asked for.
type ReachReport struct {
	Name      string
	Total     int
	Passed    int
	Missed    int
	Overreach int
	// WrongService — reached a service, but the other one.
	WrongService int
	// WrongCapability — reached Praxis on the wrong arm. Reported, not failed.
	WrongCapability int
	Errors          int
	Outcomes        []ReachOutcome
	// ByService is keyed by the fixture's want_service.
	ByService map[string]TagStat
	ByTag     map[string]TagStat
	P50, P95  time.Duration
	Max       time.Duration
}

// Accuracy — fraction of cases that arrived where the fixture says they must.
func (r ReachReport) Accuracy() float64 {
	if r.Total == 0 {
		return 0
	}
	return float64(r.Passed) / float64(r.Total)
}

// ScoreReach runs every case through the router and derives where it lands. It
// never fails the run on a route error: an erroring case scores as a miss and
// is counted, because "the model was down" and "the router was wrong" are
// different numbers.
func ScoreReach(ctx context.Context, name string, r Router, m router.ActMatcher, f ReachFixture) (ReachReport, error) {
	now, err := f.Now()
	if err != nil {
		return ReachReport{}, err
	}
	rep := ReachReport{
		Name:      name,
		Total:     len(f.Cases),
		ByService: map[string]TagStat{},
		ByTag:     map[string]TagStat{},
	}
	lat := make([]time.Duration, 0, len(f.Cases))

	for _, c := range f.Cases {
		start := time.Now()
		d, err := r.Route(ctx, c.Utterance, now)
		o := ReachOutcome{Case: c, Decision: d, Err: err, Latency: time.Since(start)}
		lat = append(lat, o.Latency)

		switch {
		case err != nil:
			rep.Errors++
			o.Reason = fmt.Sprintf("route error: %v", err)
		default:
			o.Got, o.Capability = Reach(d, m)
			switch {
			case o.Got == c.WantService:
				o.Pass = true
				if c.WantCapability != "" && o.Capability != c.WantCapability {
					rep.WrongCapability++
					o.Reason = fmt.Sprintf("reached praxis on %q, want %q", o.Capability, c.WantCapability)
				}
			case c.WantService == ServiceNone:
				rep.Overreach++
				o.Reason = fmt.Sprintf("reached %s, want none (intent %q, clarify %v, text %q)",
					o.Got, d.Intent, d.Clarify, d.Slots.Text)
			case o.Got == ServiceNone:
				rep.Missed++
				o.Reason = fmt.Sprintf("stayed local, want %s (intent %q, clarify %v, fn %q)",
					c.WantService, d.Intent, d.Clarify, d.Slots.Fn)
			default:
				rep.WrongService++
				o.Reason = fmt.Sprintf("reached %s, want %s (fn %q)", o.Got, c.WantService, d.Slots.Fn)
			}
		}
		if o.Pass {
			rep.Passed++
		}
		bump(rep.ByService, string(c.WantService), o.Pass)
		for _, tag := range c.Tags {
			bump(rep.ByTag, tag, o.Pass)
		}
		rep.Outcomes = append(rep.Outcomes, o)
	}

	sort.Slice(lat, func(i, j int) bool { return lat[i] < lat[j] })
	rep.P50, rep.P95 = percentile(lat, 0.50), percentile(lat, 0.95)
	if len(lat) > 0 {
		rep.Max = lat[len(lat)-1]
	}
	return rep, nil
}

// String renders the report.
func (r ReachReport) String() string {
	var b strings.Builder
	fmt.Fprintf(&b, "%s: %d/%d reached the right place (%.1f%%)\n", r.Name, r.Passed, r.Total, 100*r.Accuracy())
	fmt.Fprintf(&b, "  missed: %d (should have reached, didn't) | overreach: %d (reached, shouldn't) | wrong service: %d | wrong praxis arm: %d | errors: %d\n",
		r.Missed, r.Overreach, r.WrongService, r.WrongCapability, r.Errors)
	fmt.Fprintf(&b, "  latency: p50 %s  p95 %s  max %s\n", r.P50, r.P95, r.Max)
	fmt.Fprintf(&b, "  by want: %s\n", renderStats(r.ByService))
	fmt.Fprintf(&b, "  by tag:  %s\n", renderStats(r.ByTag))
	return b.String()
}

// Failures — the per-case detail, sorted by ID so two runs diff cleanly. A
// passing case with a wrong Praxis arm is listed too: it carries a reason.
func (r ReachReport) Failures() string {
	var b strings.Builder
	out := append([]ReachOutcome(nil), r.Outcomes...)
	sort.Slice(out, func(i, j int) bool { return out[i].Case.ID < out[j].Case.ID })
	for _, o := range out {
		if o.Reason == "" {
			continue
		}
		fmt.Fprintf(&b, "  %s %q: %s\n", o.Case.ID, o.Case.Utterance, o.Reason)
	}
	return b.String()
}