bb51c28a19
The router names a position ("2", "last") or a demonstrative ("this"),
because only the daemon has the list. surfacedItems records the item ids
she read out, in the order she said them, and only for items she could
actually say: one Praxis returned without a title has no position in what
he heard.
resolveSurfacedPosition maps the reference to an id before dispatch, and
its second return says whether the turn is still Praxis's. A position that
names nothing keeps the turn and clears the slot, so the capability asks
which пункт -- he said "второй пункт" and deserves to hear there is no
second one. A demonstrative that resolves to nothing gives the turn BACK,
because "я это сделал" was probably never about a пункт. "это" also needs
the list to hold exactly one item: pointing at one of five is a guess, and
a wrong guess here transitions the wrong item.
No TTL, unlike the pending confirmation. A stale position resolves to an
item Praxis will report as already acknowledged, which is a harmless
answer, where a stale confirmation would execute something.
Measured, make eval-reach, classifier + ONNX: 16/30 -> 27/30 overall,
praxis 0/12 -> 11/12, lifecycle 0/5 -> 5/5, attention 0/7 -> 6/7, hexis
and none unchanged, p50 20.6ms -> 16.5ms. make eval-router: 60/84, 0 false
clarifies, and no failure in that list comes from a stage-0 decision.
Details and the two judgement calls in docs/evals/2026-08-05-praxis-reach.md.
370 lines
12 KiB
Go
370 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// capturedRequest is a recorded inbound request against a fake ecosystem
|
|
// server, kept minimal (method/path/body) since that's what tests assert on.
|
|
type capturedRequest struct {
|
|
Method string
|
|
Path string
|
|
Query string
|
|
Body []byte
|
|
Header http.Header
|
|
}
|
|
|
|
// fakeServer is the common shell behind fakeNexus/fakePraxis/fakeHexis: an
|
|
// httptest.Server whose responses are driven by a caller-supplied route
|
|
// table, with every inbound request captured for later assertions and an
|
|
// optional fault (status code) that overrides all routing while active —
|
|
// the degraded-mode lever for these tests.
|
|
type fakeServer struct {
|
|
*httptest.Server
|
|
|
|
mu sync.Mutex
|
|
requests []capturedRequest
|
|
fault int // non-zero: every request gets this HTTP status instead of routing
|
|
routeFaults map[string]int // path prefix → status, for one endpoint failing alone
|
|
garbage string // non-empty: returned 200 verbatim instead of routing (malformed-contract lever)
|
|
delay time.Duration
|
|
}
|
|
|
|
// newFakeServer starts a server dispatching to routes keyed by "METHOD
|
|
// PATH_PREFIX" (matched with HasPrefix on path), falling back to notFound
|
|
// then to a 404 if no route matches. Route handlers are plain
|
|
// http.HandlerFuncs so tests can write arbitrary fixture responses.
|
|
func newFakeServer(t *testing.T, routes map[string]http.HandlerFunc) *fakeServer {
|
|
t.Helper()
|
|
fs := &fakeServer{}
|
|
fs.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
body := make([]byte, 0)
|
|
if r.Body != nil {
|
|
buf := make([]byte, r.ContentLength)
|
|
if r.ContentLength > 0 {
|
|
n, _ := r.Body.Read(buf)
|
|
body = buf[:n]
|
|
}
|
|
}
|
|
fs.mu.Lock()
|
|
fs.requests = append(fs.requests, capturedRequest{
|
|
Method: r.Method,
|
|
Path: r.URL.Path,
|
|
Query: r.URL.RawQuery,
|
|
Body: body,
|
|
Header: r.Header.Clone(),
|
|
})
|
|
fault := fs.fault
|
|
if fault == 0 {
|
|
for prefix, status := range fs.routeFaults {
|
|
if hasPrefix(r.URL.Path, prefix) {
|
|
fault = status
|
|
break
|
|
}
|
|
}
|
|
}
|
|
garbage := fs.garbage
|
|
delay := fs.delay
|
|
fs.mu.Unlock()
|
|
|
|
if delay > 0 {
|
|
select {
|
|
case <-time.After(delay):
|
|
case <-r.Context().Done():
|
|
return
|
|
}
|
|
}
|
|
if fault != 0 {
|
|
http.Error(w, "injected fault", fault)
|
|
return
|
|
}
|
|
if garbage != "" {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Write([]byte(garbage))
|
|
return
|
|
}
|
|
|
|
for key, handler := range routes {
|
|
method, prefix := splitRouteKey(key)
|
|
if r.Method == method && hasPrefix(r.URL.Path, prefix) {
|
|
handler(w, r)
|
|
return
|
|
}
|
|
}
|
|
http.NotFound(w, r)
|
|
}))
|
|
t.Cleanup(fs.Server.Close)
|
|
return fs
|
|
}
|
|
|
|
func splitRouteKey(key string) (method, prefix string) {
|
|
for i := 0; i < len(key); i++ {
|
|
if key[i] == ' ' {
|
|
return key[:i], key[i+1:]
|
|
}
|
|
}
|
|
return key, ""
|
|
}
|
|
|
|
func hasPrefix(s, prefix string) bool {
|
|
return len(s) >= len(prefix) && s[:len(prefix)] == prefix
|
|
}
|
|
|
|
// SetFault makes every subsequent request fail closed with the given HTTP
|
|
// status, simulating the dependency being down. Pass 0 to clear it.
|
|
func (fs *fakeServer) SetFault(status int) {
|
|
fs.mu.Lock()
|
|
defer fs.mu.Unlock()
|
|
fs.fault = status
|
|
}
|
|
|
|
// SetRouteFault fails one endpoint while the rest of the server stays healthy,
|
|
// which is the shape most real outages take: attention answers and pin is
|
|
// down. Pass 0 to clear that route. A server-wide SetFault still wins.
|
|
func (fs *fakeServer) SetRouteFault(pathPrefix string, status int) {
|
|
fs.mu.Lock()
|
|
defer fs.mu.Unlock()
|
|
if fs.routeFaults == nil {
|
|
fs.routeFaults = map[string]int{}
|
|
}
|
|
if status == 0 {
|
|
delete(fs.routeFaults, pathPrefix)
|
|
return
|
|
}
|
|
fs.routeFaults[pathPrefix] = status
|
|
}
|
|
|
|
// SetBody makes every subsequent request answer 200 with the given body,
|
|
// bypassing the route table. Used to serve a malformed or contract-violating
|
|
// payload where the transport itself is healthy. Pass "" to clear it.
|
|
func (fs *fakeServer) SetBody(body string) {
|
|
fs.mu.Lock()
|
|
defer fs.mu.Unlock()
|
|
fs.garbage = body
|
|
}
|
|
|
|
// SetDelay stalls every subsequent request for d before answering, so callers
|
|
// can drive client timeouts and context cancellation deterministically. The
|
|
// delay is abandoned as soon as the client hangs up.
|
|
func (fs *fakeServer) SetDelay(d time.Duration) {
|
|
fs.mu.Lock()
|
|
defer fs.mu.Unlock()
|
|
fs.delay = d
|
|
}
|
|
|
|
// Count returns how many captured requests used the given method and path
|
|
// prefix. "" matches any method.
|
|
func (fs *fakeServer) Count(method, prefix string) int {
|
|
n := 0
|
|
for _, r := range fs.Requests() {
|
|
if (method == "" || r.Method == method) && hasPrefix(r.Path, prefix) {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// Requests returns a snapshot of captured requests, in arrival order.
|
|
func (fs *fakeServer) Requests() []capturedRequest {
|
|
fs.mu.Lock()
|
|
defer fs.mu.Unlock()
|
|
out := make([]capturedRequest, len(fs.requests))
|
|
copy(out, fs.requests)
|
|
return out
|
|
}
|
|
|
|
// ResetRequests drops the captured requests, so a test can assert about one
|
|
// turn without subtracting the setup turn's calls.
|
|
func (fs *fakeServer) ResetRequests() {
|
|
fs.mu.Lock()
|
|
defer fs.mu.Unlock()
|
|
fs.requests = nil
|
|
}
|
|
|
|
func jsonHandler(status int, body string) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
w.Write([]byte(body))
|
|
}
|
|
}
|
|
|
|
// --- Protocol fixtures ---
|
|
// Canned response bodies matching each service's documented contract shape,
|
|
// so individual tests don't hand-roll JSON literals for the common cases.
|
|
|
|
func fixtureNexusResolved(entityID, displayName, entityType string) string {
|
|
return mustJSON(map[string]any{
|
|
"status": "resolved",
|
|
"entity": map[string]any{"id": entityID, "display_name": displayName, "type": entityType},
|
|
})
|
|
}
|
|
|
|
// fixtureNexusResolvedFlat is the flat resolve shape documented in
|
|
// ECOSYSTEM-SPEC.md §1.5 (entity_id/entity_type/display_name at the top
|
|
// level) rather than the nested "entity" object — the older of the two
|
|
// wire shapes Maven must keep accepting.
|
|
func fixtureNexusResolvedFlat(entityID, displayName, entityType string) string {
|
|
return mustJSON(map[string]any{
|
|
"status": "resolved",
|
|
"entity_id": entityID,
|
|
"entity_type": entityType,
|
|
"display_name": displayName,
|
|
})
|
|
}
|
|
|
|
// fixtureNexusResolvedFuture is a resolved response from a hypothetical newer
|
|
// Nexus: same required fields plus unknown ones. Decoding must ignore the
|
|
// extras, not fail — forward compatibility is what lets the ecosystem be
|
|
// upgraded one service at a time.
|
|
func fixtureNexusResolvedFuture(entityID, displayName, entityType string) string {
|
|
return mustJSON(map[string]any{
|
|
"status": "resolved",
|
|
"entity": map[string]any{"id": entityID, "display_name": displayName, "type": entityType, "tenant": "home"},
|
|
"provenance": map[string]any{"resolver": "v3", "graph_epoch": 42},
|
|
"score_breakdown": []any{map[string]any{"signal": "alias", "weight": 0.9}},
|
|
})
|
|
}
|
|
|
|
// fixtureNexusResolvedEmpty is the contract violation that decodes cleanly:
|
|
// Nexus claims a resolve and delivers no entity. It must not read as "no such
|
|
// entity", which would let the caller fall through to local execution with the
|
|
// user's verb intact.
|
|
func fixtureNexusResolvedEmpty() string {
|
|
return `{"status":"resolved"}`
|
|
}
|
|
|
|
func fixtureNexusNotFound() string {
|
|
return `{"status":"not_found"}`
|
|
}
|
|
|
|
func fixtureNexusAmbiguous(candidates ...map[string]string) string {
|
|
cs := make([]map[string]any, len(candidates))
|
|
for i, c := range candidates {
|
|
cs[i] = map[string]any{"entity_id": c["entity_id"], "display_name": c["display_name"]}
|
|
}
|
|
return mustJSON(map[string]any{"status": "ambiguous", "candidates": cs})
|
|
}
|
|
|
|
func fixtureHexisCapabilities(caps ...map[string]any) string {
|
|
return mustJSON(caps)
|
|
}
|
|
|
|
func fixtureHexisExecuted(id, status string) string {
|
|
return mustJSON(map[string]any{"id": id, "status": status})
|
|
}
|
|
|
|
// fixtureHexisExecutionFailed is a well-formed Hexis response reporting that
|
|
// the command itself failed: the call succeeded, the execution did not. Maven
|
|
// must distinguish this from a transport failure and from success.
|
|
func fixtureHexisExecutionFailed(id, message string) string {
|
|
return mustJSON(map[string]any{"id": id, "status": "failed", "error": message})
|
|
}
|
|
|
|
// fixturePraxisAttentionScoped tags each item with an entity_id, which is what
|
|
// a Praxis that understands the entity_id query parameter returns. A Praxis
|
|
// that ignores it answers with untagged items from every entity.
|
|
func fixturePraxisAttentionScoped(entityID string, items ...map[string]any) string {
|
|
for _, item := range items {
|
|
item["entity_id"] = entityID
|
|
}
|
|
return mustJSON(items)
|
|
}
|
|
|
|
func fixturePraxisAttentionItems(items ...map[string]any) string {
|
|
return mustJSON(items)
|
|
}
|
|
|
|
func mustJSON(v any) string {
|
|
b, err := json.Marshal(v)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
// --- Fake clock ---
|
|
// A settable time source satisfying the `func() time.Time` shape reactiveHandler
|
|
// takes as `now`, so tests can control "now" independently of wall-clock time
|
|
// (e.g. asserting age-based digest ordering without sleeping).
|
|
|
|
type fakeClock struct {
|
|
mu sync.Mutex
|
|
t time.Time
|
|
step time.Duration // advanced on every read, so elapsed time is measurable
|
|
}
|
|
|
|
func newFakeClock(start time.Time) *fakeClock {
|
|
return &fakeClock{t: start}
|
|
}
|
|
|
|
// newTickingClock advances by step on every read. Durations measured across
|
|
// hops are then non-zero without sleeping, which is what lets a test tell a
|
|
// trace that measured something from one that measured nothing.
|
|
func newTickingClock(start time.Time, step time.Duration) *fakeClock {
|
|
return &fakeClock{t: start, step: step}
|
|
}
|
|
|
|
func (c *fakeClock) Now() time.Time {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
now := c.t
|
|
c.t = c.t.Add(c.step)
|
|
return now
|
|
}
|
|
|
|
func (c *fakeClock) Advance(d time.Duration) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
c.t = c.t.Add(d)
|
|
}
|
|
|
|
// --- Convenience constructors for the three ecosystem services ---
|
|
|
|
// newFakeNexus starts a fake Nexus exposing /api/v1/resolve. resolveBody is
|
|
// returned verbatim (200) for every resolve call unless a fault is injected.
|
|
func newFakeNexus(t *testing.T, resolveBody string) *fakeServer {
|
|
return newFakeServer(t, map[string]http.HandlerFunc{
|
|
"POST /api/v1/resolve": jsonHandler(http.StatusOK, resolveBody),
|
|
})
|
|
}
|
|
|
|
// newFakePraxis starts a fake Praxis exposing the /api/v1/tools/* surface
|
|
// Maven's praxisClient calls. Every route returns its fixed body until a
|
|
// fault is injected via SetFault.
|
|
func newFakePraxis(t *testing.T, attentionBody string) *fakeServer {
|
|
// One healthy source by default: an empty attention list only means
|
|
// all-clear when something is actually polling (Vikunja #540), and the
|
|
// other tests here are about attention rather than about source health.
|
|
return newFakePraxisWithSources(t, attentionBody, `[{"source_id":"src_ntfy","health":"ok"}]`)
|
|
}
|
|
|
|
// newFakePraxisWithSources is newFakePraxis with the /api/v1/sources body
|
|
// under the test's control, for the degraded and no-sources hedges.
|
|
func newFakePraxisWithSources(t *testing.T, attentionBody, sourcesBody string) *fakeServer {
|
|
return newFakeServer(t, map[string]http.HandlerFunc{
|
|
"GET /api/v1/sources": jsonHandler(http.StatusOK, sourcesBody),
|
|
"GET /api/v1/tools/attention": jsonHandler(http.StatusOK, attentionBody),
|
|
"GET /api/v1/tools/changes": jsonHandler(http.StatusOK, `[]`),
|
|
"POST /api/v1/tools/surface": jsonHandler(http.StatusOK, `{}`),
|
|
"POST /api/v1/tools/acknowledge": jsonHandler(http.StatusOK, `{}`),
|
|
"POST /api/v1/tools/resolve": jsonHandler(http.StatusOK, `{}`),
|
|
"POST /api/v1/tools/ignore": jsonHandler(http.StatusOK, `{}`),
|
|
"POST /api/v1/tools/pin": jsonHandler(http.StatusOK, `{}`),
|
|
})
|
|
}
|
|
|
|
// newFakeHexis starts a fake Hexis exposing /api/v1/capabilities and
|
|
// /api/v1/execute.
|
|
func newFakeHexis(t *testing.T, capsBody, executeBody string) *fakeServer {
|
|
return newFakeServer(t, map[string]http.HandlerFunc{
|
|
"GET /api/v1/capabilities": jsonHandler(http.StatusOK, capsBody),
|
|
"POST /api/v1/execute": jsonHandler(http.StatusOK, executeBody),
|
|
})
|
|
}
|