test: make the ecosystem fault suite fail when the feature is deleted

Several assertions passed against code with the behaviour removed. The independent-outage test shared no state to begin with, the capability fixture used to prove read-only filtering was already mutating, and route-level faults were simulated with a separate fake instead of the shared one. The harness now takes per-route faults and a ticking clock, so durations are measurable and one dead endpoint can be shown not to mute a whole service. New cases cover a resolved reference with no entity, a rejected credential, a malformed Praxis body, foreign items in a scoped response, named truncation, traces staying out of facts, and enrichment making progress while its oldest batch is backed off. Found in review of #82.
2026-08-01 14:14:19 +04:00
parent 802d5961ac
commit 617476772e
5 changed files with 409 additions and 45 deletions
@@ -2,7 +2,6 @@ package main

 import (
 	"context"
-	"net/http"
 	"strings"
 	"testing"
 	"time"
@@ -30,7 +29,7 @@ import (
 func ecoHandler(t *testing.T, nexus, praxis, hexis *fakeServer) *reactiveHandler {
 	t.Helper()
 	st := newTestStore(t)
-	clock := newFakeClock(time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC))
+	clock := newTickingClock(time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC), time.Millisecond)
 	w := &ecosystemWiring{}
 	if nexus != nil {
 		w.nexus = newNexusClient(nexus.URL)
@@ -49,31 +48,43 @@ func ecoHandler(t *testing.T, nexus, praxis, hexis *fakeServer) *reactiveHandler
 	}
 }

-func traceFacts(t *testing.T, h *reactiveHandler) []store.Fact {
+// traces reads the ecosystem trace table. Traces live there and not in facts,
+// so a bounded reader of facts never fills up with machine-rate rows.
+func traces(t *testing.T, h *reactiveHandler) []store.EcosystemTrace {
 	t.Helper()
-	facts, err := h.dataStore.RecentFacts(context.Background(), 50)
+	out, err := h.dataStore.RecentEcosystemTraces(context.Background(), 100)
 	if err != nil {
-		t.Fatalf("read facts: %v", err)
+		t.Fatalf("read traces: %v", err)
 	}
-	var out []store.Fact
-	for _, f := range facts {
-		if f.Source == "praxis:trace" {
-			out = append(out, f)
+	return out
+}
+
+// tracesFor returns the traces recorded for one service+operation.
+func tracesFor(t *testing.T, h *reactiveHandler, service, op string) []store.EcosystemTrace {
+	t.Helper()
+	var out []store.EcosystemTrace
+	for _, tr := range traces(t, h) {
+		if tr.Service == service && tr.Operation == op {
+			out = append(out, tr)
 		}
 	}
 	return out
 }

+// restartCaps is a read-only capability. Restarting a service is a mutation,
+// so the read-only one this suite runs through the happy paths is named for
+// what it is; the mutating restart lives in the confirmation tests.
 func restartCaps() string {
 	return fixtureHexisCapabilities(map[string]any{
-		"id": "cap_restart", "name": "restart", "read_only": true,
+		"id": "cap_status", "name": "restart status", "read_only": true,
 	})
 }

-// TestEcosystem_OutagesAreIndependent: Praxis being down must not disable the
-// Nexus+Hexis action path, and vice versa. A shared "ecosystem is broken"
-// mode would take away working capability for no reason.
-func TestEcosystem_OutagesAreIndependent(t *testing.T) {
+// TestEcosystem_OutagesLeaveNoSharedFailureState: the two act paths share a
+// handler, a store and a clock, so what is worth asserting is that a failure
+// on one leaves nothing behind that degrades the other. Faulting one disjoint
+// call graph and exercising the other only tests the call graph.
+func TestEcosystem_OutagesLeaveNoSharedFailureState(t *testing.T) {
 	ctx := context.Background()
 	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
 	praxis := newFakePraxis(t, fixturePraxisAttentionItems(map[string]any{
@@ -82,17 +93,112 @@ func TestEcosystem_OutagesAreIndependent(t *testing.T) {
 	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
 	h := ecoHandler(t, nexus, praxis, hexis)

-	praxis.SetFault(503)
-	if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") {
-		t.Fatalf("praxis outage must not block the hexis path, got %q", reply)
+	// A Nexus outage during a Hexis act writes a failure trace, and a shared
+	// store is the one thing the Praxis path could inherit it through.
+	nexus.SetFault(503)
+	if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); strings.Contains(reply, "выполнена") {
+		t.Fatalf("nexus outage must not report success, got %q", reply)
+	}
+	if len(tracesFor(t, h, "nexus", "resolve")) == 0 {
+		t.Fatal("the failed resolve must be recorded")
 	}

-	praxis.SetFault(0)
-	hexis.SetFault(503)
-	nexus.SetFault(503)
+	nexus.SetFault(0)
 	reply := h.handlePraxisAct(ctx, praxisActDec("list_attention"))
 	if !strings.Contains(reply, "disk almost full") {
-		t.Fatalf("nexus/hexis outage must not block the praxis digest, got %q", reply)
+		t.Fatalf("a recorded nexus failure must not degrade the praxis digest, got %q", reply)
+	}
+	if got := tracesFor(t, h, "praxis", "list_attention"); len(got) != 1 || got[0].Status != traceOK {
+		t.Fatalf("the praxis digest must trace its own success, got %+v", got)
+	}
+
+	// And the reverse: a Praxis outage mid-session leaves the Hexis path whole.
+	praxis.SetFault(503)
+	if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); strings.Contains(reply, "disk") {
+		t.Fatalf("praxis outage must not serve content, got %q", reply)
+	}
+	if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") {
+		t.Fatalf("a praxis outage must not block the hexis path, got %q", reply)
+	}
+}
+
+// TestEcosystem_OneEndpointDownDoesNotMuteTheService: real outages are usually
+// partial. Attention answering while surface is down must still deliver.
+func TestEcosystem_OneEndpointDownDoesNotMuteTheService(t *testing.T) {
+	ctx := context.Background()
+	praxis := newFakePraxis(t, fixturePraxisAttentionItems(map[string]any{
+		"id": "item_1", "title": "disk almost full", "importance": 3.0,
+	}))
+	h := ecoHandler(t, nil, praxis, nil)
+
+	praxis.SetRouteFault("/api/v1/tools/surface", 503)
+	reply := h.handlePraxisAct(ctx, praxisActDec("list_attention"))
+	if !strings.Contains(reply, "disk almost full") {
+		t.Fatalf("a downed surface endpoint must not mute the digest, got %q", reply)
+	}
+	if praxis.Count("POST", "/api/v1/tools/surface") == 0 {
+		t.Fatal("expected the surface attempt")
+	}
+}
+
+// TestEcosystem_ResolvedWithoutEntityFailsClosed: the contract violation that
+// decodes cleanly. Nexus says "resolved" and delivers no entity; treating that
+// as "no such entity" put the user's verb through to the local executor.
+func TestEcosystem_ResolvedWithoutEntityFailsClosed(t *testing.T) {
+	ctx := context.Background()
+	nexus := newFakeNexus(t, fixtureNexusResolvedEmpty())
+	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
+	h := ecoHandler(t, nexus, nil, hexis)
+
+	reply := h.handleHexisAct(ctx, actDec("muzick indexer"))
+	if reply == "" {
+		t.Fatal("a resolve with no entity must degrade, not fall through to local execution")
+	}
+	if strings.Contains(reply, "выполнена") {
+		t.Fatalf("a resolve with no entity must not report success, got %q", reply)
+	}
+	if hexis.Count("", "/api/v1") != 0 {
+		t.Fatal("hexis must not be contacted after a contract-violating resolve")
+	}
+}
+
+// TestEcosystem_RejectedCredentialSaysSo: 401 and 403 must not read as an
+// outage. "Try again" is advice that never works for a misconfigured token.
+func TestEcosystem_RejectedCredentialSaysSo(t *testing.T) {
+	ctx := context.Background()
+	for _, status := range []int{401, 403} {
+		nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
+		hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
+		h := ecoHandler(t, nexus, nil, hexis)
+		nexus.SetFault(status)
+
+		reply := h.handleHexisAct(ctx, actDec("muzick indexer"))
+		if !strings.Contains(reply, "токен") {
+			t.Fatalf("http %d must read as a credential problem, got %q", status, reply)
+		}
+		tr := tracesFor(t, h, "nexus", "resolve")
+		if len(tr) != 1 || tr[0].Status != traceRefused || tr[0].HTTPStatus != status {
+			t.Fatalf("http %d must trace as refused with its status, got %+v", status, tr)
+		}
+	}
+}
+
+// TestEcosystem_MalformedPraxisBodyDegrades: Praxis has the same decode path
+// Nexus does, and a 200 carrying garbage there is a dependency failure too.
+func TestEcosystem_MalformedPraxisBodyDegrades(t *testing.T) {
+	ctx := context.Background()
+	praxis := newFakePraxis(t, fixturePraxisAttentionItems(map[string]any{
+		"id": "item_1", "title": "disk almost full", "importance": 3.0,
+	}))
+	h := ecoHandler(t, nil, praxis, nil)
+
+	praxis.SetBody(`[{"title":`)
+	reply := h.handlePraxisAct(ctx, praxisActDec("list_attention"))
+	if reply == "" {
+		t.Fatal("a malformed praxis body must not answer with silence")
+	}
+	if strings.Contains(reply, "disk almost full") {
+		t.Fatalf("a malformed body must not produce content, got %q", reply)
 	}
 }

@@ -168,13 +274,60 @@ func TestEcosystem_ExecutionFailureIsNotSuccess(t *testing.T) {
 	if reply == "" {
 		t.Fatal("failed execution must say something")
 	}
-	for _, f := range traceFacts(t, h) {
-		if strings.HasPrefix(f.Key, "praxis:hexis:") {
-			t.Fatalf("failed execution must not write a success trace: %+v", f)
+	for _, tr := range tracesFor(t, h, "hexis", "execute") {
+		if tr.Status == traceOK {
+			t.Fatalf("failed execution must not write a success trace: %+v", tr)
 		}
 	}
 }

+// TestEcosystem_SuccessfulActionWritesATrace is the positive half the failure
+// assertions above depend on: without it, "no success trace" passes with the
+// trace writer deleted. It was, for a while — both writers used a fact kind the
+// store's CHECK constraint rejects and the error was discarded.
+func TestEcosystem_SuccessfulActionWritesATrace(t *testing.T) {
+	ctx := context.Background()
+	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
+	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
+	h := ecoHandler(t, nexus, nil, hexis)
+
+	if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") {
+		t.Fatalf("setup: expected success, got %q", reply)
+	}
+	exec := tracesFor(t, h, "hexis", "execute")
+	if len(exec) != 1 || exec[0].Status != traceOK {
+		t.Fatalf("a successful execution must leave exactly one ok trace, got %+v", exec)
+	}
+	if exec[0].CorrelationID == "" {
+		t.Error("a trace with no correlation id cannot be stitched to anything")
+	}
+}
+
+// TestEcosystem_TracesStayOutOfFacts: traces are written at machine rate and
+// facts at human rate. One act turn used to write four fact rows, which pushed
+// his facts out of every bounded reader (the habit profile's window, memeval's
+// prompt, /dash, /history).
+func TestEcosystem_TracesStayOutOfFacts(t *testing.T) {
+	ctx := context.Background()
+	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
+	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
+	h := ecoHandler(t, nexus, nil, hexis)
+
+	if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") {
+		t.Fatalf("setup: expected success, got %q", reply)
+	}
+	if len(traces(t, h)) == 0 {
+		t.Fatal("setup: expected traces")
+	}
+	facts, err := h.dataStore.RecentFacts(ctx, 100)
+	if err != nil {
+		t.Fatalf("read facts: %v", err)
+	}
+	if len(facts) != 0 {
+		t.Fatalf("an ecosystem act must write no facts at all, got %+v", facts)
+	}
+}
+
 // TestEcosystem_AmbiguousTargetBlocksExecution: ambiguity blocks mutation, and
 // the clarification must name the candidates rather than pick one.
 func TestEcosystem_AmbiguousTargetBlocksExecution(t *testing.T) {
@@ -245,12 +398,10 @@ func TestEcosystem_MutatingCapabilityWaitsForConfirmation(t *testing.T) {
 // downgrades bookkeeping, not the answer.
 func TestEcosystem_SurfaceFailureStillDelivers(t *testing.T) {
 	ctx := context.Background()
-	praxis := newFakeServer(t, map[string]http.HandlerFunc{
-		"GET /api/v1/tools/attention": jsonHandler(200, fixturePraxisAttentionItems(
-			map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0},
-		)),
-		"POST /api/v1/tools/surface": jsonHandler(500, `{"error":"boom"}`),
-	})
+	praxis := newFakePraxis(t, fixturePraxisAttentionItems(
+		map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0},
+	))
+	praxis.SetRouteFault("/api/v1/tools/surface", 500)
 	h := ecoHandler(t, nil, praxis, nil)

 	reply := h.handlePraxisAct(ctx, praxisActDec("list_attention"))
@@ -278,7 +429,7 @@ func TestEcosystem_TotalOutageSaysSoForEveryPath(t *testing.T) {
 		"hexis act":   h.handleHexisAct(ctx, actDec("muzick indexer")),
 		"attention":   h.handlePraxisAct(ctx, praxisActDec("list_attention")),
 		"changes":     h.handlePraxisAct(ctx, praxisActDec("list_changes")),
-		"acknowledge": h.handlePraxisAct(ctx, praxisActDec("acknowledge_item")),
+		"acknowledge": h.handlePraxisAct(ctx, praxisItemDec("acknowledge_item", "item_1")),
 	} {
 		if reply == "" {
 			t.Errorf("%s: total outage must not answer with silence", name)
@@ -287,8 +438,13 @@ func TestEcosystem_TotalOutageSaysSoForEveryPath(t *testing.T) {
 			t.Errorf("%s: total outage must not claim success: %q", name, reply)
 		}
 	}
-	if len(traceFacts(t, h)) != 0 {
-		t.Fatal("a total outage must not leave success traces behind")
+	for _, tr := range traces(t, h) {
+		if tr.Status == traceOK {
+			t.Fatalf("a total outage must not leave success traces behind: %+v", tr)
+		}
+	}
+	if len(tracesFor(t, h, "praxis", "acknowledge")) == 0 {
+		t.Fatal("the acknowledge arm must reach praxis and record the refusal")
 	}
 }