Cover ecosystem degraded modes with a shared fault-injection harness (#276)

Extend the fake Nexus/Praxis/Hexis harness with request header and query
capture, a malformed-body lever, a response delay lever, and a request
counter, then add a degraded-mode suite on top of it: independent outages,
malformed and drifted contracts, cancellation, execution failure vs
transport failure, ambiguous targets, no autonomous Praxis to Hexis
chaining, confirmation for mutating capabilities, and recovery without a
restart.

cmd/mavend/ecosystem_degraded_test.go

@@ -0,0 +1,312 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	hexisclient "github.com/kami/hexis/pkg/client"
+
+	"github.com/kami/maven/internal/ipc"
+	"github.com/kami/maven/internal/store"
+)
+
+// Phase-5 hardening suite (Vikunja #276). Everything here drives the shared
+// fake ecosystem (fakeecosystem_test.go) rather than one-off inline handlers,
+// so the same fault levers — SetFault, SetBody, SetDelay — cover every
+// service. What is asserted is the degraded-mode contract:
+//
+//   - services degrade independently: one outage never mutes the others,
+//   - a degraded reply is never silent, never fabricated, never "success",
+//   - contract drift (old shape, unknown fields, garbage) is survivable,
+//   - Maven never acts on an ambiguous target and never chains
+//     Praxis observation into Hexis execution on its own.
+
+// ecoHandler wires a handler against whichever of the three fakes is given
+// (pass nil to leave a service unconfigured, which is a different state from
+// "configured but down").
+func ecoHandler(t *testing.T, nexus, praxis, hexis *fakeServer) *reactiveHandler {
+	t.Helper()
+	st := newTestStore(t)
+	clock := newFakeClock(time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC))
+	w := &ecosystemWiring{}
+	if nexus != nil {
+		w.nexus = newNexusClient(nexus.URL)
+	}
+	if praxis != nil {
+		w.praxis = newPraxisClient(praxis.URL)
+	}
+	if hexis != nil {
+		w.hexis = hexisclient.New(hexis.URL)
+	}
+	return &reactiveHandler{
+		api:       ipc.NewStoreAPI(st),
+		dataStore: st,
+		now:       clock.Now,
+		ecosystem: w,
+	}
+}
+
+func traceFacts(t *testing.T, h *reactiveHandler) []store.Fact {
+	t.Helper()
+	facts, err := h.dataStore.RecentFacts(context.Background(), 50)
+	if err != nil {
+		t.Fatalf("read facts: %v", err)
+	}
+	var out []store.Fact
+	for _, f := range facts {
+		if f.Source == "praxis:trace" {
+			out = append(out, f)
+		}
+	}
+	return out
+}
+
+func restartCaps() string {
+	return fixtureHexisCapabilities(map[string]any{
+		"id": "cap_restart", "name": "restart", "read_only": true,
+	})
+}
+
+// TestEcosystem_OutagesAreIndependent: Praxis being down must not disable the
+// Nexus+Hexis action path, and vice versa. A shared "ecosystem is broken"
+// mode would take away working capability for no reason.
+func TestEcosystem_OutagesAreIndependent(t *testing.T) {
+	ctx := context.Background()
+	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
+	praxis := newFakePraxis(t, fixturePraxisAttentionItems(map[string]any{
+		"id": "item_1", "title": "disk almost full", "importance": 3.0,
+	}))
+	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
+	h := ecoHandler(t, nexus, praxis, hexis)
+
+	praxis.SetFault(503)
+	if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") {
+		t.Fatalf("praxis outage must not block the hexis path, got %q", reply)
+	}
+
+	praxis.SetFault(0)
+	hexis.SetFault(503)
+	nexus.SetFault(503)
+	reply := h.handlePraxisAct(ctx, praxisActDec("list_attention"))
+	if !strings.Contains(reply, "disk almost full") {
+		t.Fatalf("nexus/hexis outage must not block the praxis digest, got %q", reply)
+	}
+}
+
+// TestEcosystem_MalformedNexusResponseFailsClosed: a 200 carrying garbage is a
+// dependency failure, not "no such entity". It must stop before Hexis.
+func TestEcosystem_MalformedNexusResponseFailsClosed(t *testing.T) {
+	ctx := context.Background()
+	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
+	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
+	h := ecoHandler(t, nexus, nil, hexis)
+
+	nexus.SetBody(`{"status":"resolved","entity":`)
+	reply := h.handleHexisAct(ctx, actDec("muzick indexer"))
+	if reply == "" || strings.Contains(reply, "выполнена") {
+		t.Fatalf("malformed nexus body must degrade, got %q", reply)
+	}
+	if hexis.Count("", "/api/v1") != 0 {
+		t.Fatal("hexis must not be contacted after a malformed nexus response")
+	}
+}
+
+// TestEcosystem_UnknownContractFieldsTolerated: a newer Nexus adding fields
+// must not break an older Maven. Same for the older flat resolve shape.
+func TestEcosystem_UnknownContractFieldsTolerated(t *testing.T) {
+	ctx := context.Background()
+	for name, body := range map[string]string{
+		"future": fixtureNexusResolvedFuture("ent_muzick", "Muzick indexer", "service"),
+		"flat":   fixtureNexusResolvedFlat("ent_muzick", "Muzick indexer", "service"),
+	} {
+		t.Run(name, func(t *testing.T) {
+			nexus := newFakeNexus(t, body)
+			hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
+			h := ecoHandler(t, nexus, nil, hexis)
+			if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") {
+				t.Fatalf("%s contract shape must still resolve and execute, got %q", name, reply)
+			}
+		})
+	}
+}
+
+// TestEcosystem_CancelledContextDegrades: a caller hanging up (turn abandoned,
+// deadline hit) must surface as degradation, never as a fabricated result.
+func TestEcosystem_CancelledContextDegrades(t *testing.T) {
+	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
+	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
+	h := ecoHandler(t, nexus, nil, hexis)
+	nexus.SetDelay(2 * time.Second)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
+	defer cancel()
+	reply := h.handleHexisAct(ctx, actDec("muzick indexer"))
+	if reply == "" || strings.Contains(reply, "выполнена") {
+		t.Fatalf("cancelled resolve must degrade, got %q", reply)
+	}
+	if hexis.Count("", "/api/v1") != 0 {
+		t.Fatal("hexis must not be contacted after a cancelled resolve")
+	}
+}
+
+// TestEcosystem_ExecutionFailureIsNotSuccess: Hexis answering 200 with
+// status=failed is a partial failure — the call worked, the command did not.
+// Maven must report it as a failure and must not write a success trace.
+func TestEcosystem_ExecutionFailureIsNotSuccess(t *testing.T) {
+	ctx := context.Background()
+	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
+	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecutionFailed("exec_1", "unit not found"))
+	h := ecoHandler(t, nexus, nil, hexis)
+
+	reply := h.handleHexisAct(ctx, actDec("muzick indexer"))
+	if strings.Contains(reply, "выполнена") {
+		t.Fatalf("failed execution must not read as success, got %q", reply)
+	}
+	if reply == "" {
+		t.Fatal("failed execution must say something")
+	}
+	for _, f := range traceFacts(t, h) {
+		if strings.HasPrefix(f.Key, "praxis:hexis:") {
+			t.Fatalf("failed execution must not write a success trace: %+v", f)
+		}
+	}
+}
+
+// TestEcosystem_AmbiguousTargetBlocksExecution: ambiguity blocks mutation, and
+// the clarification must name the candidates rather than pick one.
+func TestEcosystem_AmbiguousTargetBlocksExecution(t *testing.T) {
+	ctx := context.Background()
+	nexus := newFakeNexus(t, fixtureNexusAmbiguous(
+		map[string]string{"entity_id": "ent_a", "display_name": "Muzick indexer"},
+		map[string]string{"entity_id": "ent_b", "display_name": "Muzick web"},
+	))
+	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
+	h := ecoHandler(t, nexus, nil, hexis)
+
+	reply := h.handleHexisAct(ctx, actDec("muzick"))
+	if !strings.Contains(reply, "Muzick indexer") || !strings.Contains(reply, "Muzick web") {
+		t.Fatalf("ambiguous resolve must list candidates, got %q", reply)
+	}
+	if hexis.Count("POST", "/api/v1/execute") != 0 {
+		t.Fatal("ambiguous target must never execute")
+	}
+}
+
+// TestEcosystem_NoAutonomousPraxisToHexis: reading the attention digest is an
+// observation. Maven must never turn an observed problem into a Hexis command
+// by herself — she is not autonomous.
+func TestEcosystem_NoAutonomousPraxisToHexis(t *testing.T) {
+	ctx := context.Background()
+	praxis := newFakePraxis(t, fixturePraxisAttentionItems(
+		map[string]any{"id": "item_1", "title": "muzick indexer is down", "importance": 4.0, "rule": "service_down"},
+	))
+	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
+	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
+	h := ecoHandler(t, nexus, praxis, hexis)
+
+	_ = h.handlePraxisAct(ctx, praxisActDec("list_attention"))
+	if hexis.Count("", "/api/v1") != 0 {
+		t.Fatal("attention digest must not contact hexis on its own")
+	}
+	if nexus.Count("", "/api/v1/resolve") != 0 {
+		t.Fatal("attention digest must not resolve targets for autonomous action")
+	}
+}
+
+// TestEcosystem_MutatingCapabilityWaitsForConfirmation: a non-read-only
+// capability parks for an explicit spoken confirm bound to capability+target.
+func TestEcosystem_MutatingCapabilityWaitsForConfirmation(t *testing.T) {
+	ctx := context.Background()
+	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
+	caps := fixtureHexisCapabilities(map[string]any{"id": "cap_restart", "name": "restart", "read_only": false})
+	hexis := newFakeHexis(t, caps, fixtureHexisExecuted("exec_1", "succeeded"))
+	h := ecoHandler(t, nexus, nil, hexis)
+
+	reply := h.handleHexisAct(ctx, actDec("restart"))
+	if !strings.Contains(reply, "restart") || !strings.Contains(reply, "да") {
+		t.Fatalf("mutating capability must ask for confirmation, got %q", reply)
+	}
+	if hexis.Count("POST", "/api/v1/execute") != 0 {
+		t.Fatal("mutating capability must not execute before confirmation")
+	}
+	h.mu.Lock()
+	pending := h.pendingHexis
+	h.mu.Unlock()
+	if pending == nil || pending.capabilityID != "cap_restart" || pending.entityID != "ent_muzick" {
+		t.Fatalf("confirmation must be bound to capability+target, got %+v", pending)
+	}
+}
+
+// TestEcosystem_SurfaceFailureStillDelivers: surfacing is bookkeeping. If the
+// surface call fails the digest must still be spoken — a partial failure
+// downgrades bookkeeping, not the answer.
+func TestEcosystem_SurfaceFailureStillDelivers(t *testing.T) {
+	ctx := context.Background()
+	praxis := newFakeServer(t, map[string]http.HandlerFunc{
+		"GET /api/v1/tools/attention": jsonHandler(200, fixturePraxisAttentionItems(
+			map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0},
+		)),
+		"POST /api/v1/tools/surface": jsonHandler(500, `{"error":"boom"}`),
+	})
+	h := ecoHandler(t, nil, praxis, nil)
+
+	reply := h.handlePraxisAct(ctx, praxisActDec("list_attention"))
+	if !strings.Contains(reply, "disk almost full") {
+		t.Fatalf("failed surface must not swallow the digest, got %q", reply)
+	}
+	if praxis.Count("POST", "/api/v1/tools/surface") == 0 {
+		t.Fatal("expected the surface attempt")
+	}
+}
+
+// TestEcosystem_TotalOutageSaysSoForEveryPath: with all three down, every
+// entry point degrades explicitly instead of returning empty or inventing.
+func TestEcosystem_TotalOutageSaysSoForEveryPath(t *testing.T) {
+	ctx := context.Background()
+	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
+	praxis := newFakePraxis(t, fixturePraxisAttentionItems())
+	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
+	for _, fs := range []*fakeServer{nexus, praxis, hexis} {
+		fs.SetFault(503)
+	}
+	h := ecoHandler(t, nexus, praxis, hexis)
+
+	for name, reply := range map[string]string{
+		"hexis act":   h.handleHexisAct(ctx, actDec("muzick indexer")),
+		"attention":   h.handlePraxisAct(ctx, praxisActDec("list_attention")),
+		"changes":     h.handlePraxisAct(ctx, praxisActDec("list_changes")),
+		"acknowledge": h.handlePraxisAct(ctx, praxisActDec("acknowledge_item")),
+	} {
+		if reply == "" {
+			t.Errorf("%s: total outage must not answer with silence", name)
+		}
+		if strings.Contains(reply, "выполнена") {
+			t.Errorf("%s: total outage must not claim success: %q", name, reply)
+		}
+	}
+	if len(traceFacts(t, h)) != 0 {
+		t.Fatal("a total outage must not leave success traces behind")
+	}
+}
+
+// TestEcosystem_RecoveryAfterOutageNeedsNoRestart: once the dependency comes
+// back the very next turn works — no cached failure state, no restart.
+func TestEcosystem_RecoveryAfterOutageNeedsNoRestart(t *testing.T) {
+	ctx := context.Background()
+	praxis := newFakePraxis(t, fixturePraxisAttentionItems(
+		map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0},
+	))
+	h := ecoHandler(t, nil, praxis, nil)
+
+	praxis.SetFault(503)
+	if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); strings.Contains(reply, "disk") {
+		t.Fatalf("outage must not serve content, got %q", reply)
+	}
+	praxis.SetFault(0)
+	if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); !strings.Contains(reply, "disk almost full") {
+		t.Fatalf("recovery must work on the next turn, got %q", reply)
+	}
+}

cmd/mavend/fakeecosystem_test.go

@@ -14,7 +14,9 @@ import (
 type capturedRequest struct {
 	Method string
 	Path   string
+	Query  string
 	Body   []byte
+	Header http.Header
 }
 
 // fakeServer is the common shell behind fakeNexus/fakePraxis/fakeHexis: an
@@ -27,7 +29,9 @@ type fakeServer struct {
 
 	mu       sync.Mutex
 	requests []capturedRequest
-	fault    int // non-zero: every request gets this HTTP status instead of routing
+	fault    int    // non-zero: every request gets this HTTP status instead of routing
+	garbage  string // non-empty: returned 200 verbatim instead of routing (malformed-contract lever)
+	delay    time.Duration
 }
 
 // newFakeServer starts a server dispatching to routes keyed by "METHOD
@@ -47,14 +51,34 @@ func newFakeServer(t *testing.T, routes map[string]http.HandlerFunc) *fakeServer
 			}
 		}
 		fs.mu.Lock()
-		fs.requests = append(fs.requests, capturedRequest{Method: r.Method, Path: r.URL.Path, Body: body})
+		fs.requests = append(fs.requests, capturedRequest{
+			Method: r.Method,
+			Path:   r.URL.Path,
+			Query:  r.URL.RawQuery,
+			Body:   body,
+			Header: r.Header.Clone(),
+		})
 		fault := fs.fault
+		garbage := fs.garbage
+		delay := fs.delay
 		fs.mu.Unlock()
 
+		if delay > 0 {
+			select {
+			case <-time.After(delay):
+			case <-r.Context().Done():
+				return
+			}
+		}
 		if fault != 0 {
 			http.Error(w, "injected fault", fault)
 			return
 		}
+		if garbage != "" {
+			w.Header().Set("Content-Type", "application/json")
+			w.Write([]byte(garbage))
+			return
+		}
 
 		for key, handler := range routes {
 			method, prefix := splitRouteKey(key)
@@ -90,6 +114,36 @@ func (fs *fakeServer) SetFault(status int) {
 	fs.fault = status
 }
 
+// SetBody makes every subsequent request answer 200 with the given body,
+// bypassing the route table. Used to serve a malformed or contract-violating
+// payload where the transport itself is healthy. Pass "" to clear it.
+func (fs *fakeServer) SetBody(body string) {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+	fs.garbage = body
+}
+
+// SetDelay stalls every subsequent request for d before answering, so callers
+// can drive client timeouts and context cancellation deterministically. The
+// delay is abandoned as soon as the client hangs up.
+func (fs *fakeServer) SetDelay(d time.Duration) {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+	fs.delay = d
+}
+
+// Count returns how many captured requests used the given method and path
+// prefix. "" matches any method.
+func (fs *fakeServer) Count(method, prefix string) int {
+	n := 0
+	for _, r := range fs.Requests() {
+		if (method == "" || r.Method == method) && hasPrefix(r.Path, prefix) {
+			n++
+		}
+	}
+	return n
+}
+
 // Requests returns a snapshot of captured requests, in arrival order.
 func (fs *fakeServer) Requests() []capturedRequest {
 	fs.mu.Lock()
@@ -118,6 +172,32 @@ func fixtureNexusResolved(entityID, displayName, entityType string) string {
 	})
 }
 
+// fixtureNexusResolvedFlat is the flat resolve shape documented in
+// ECOSYSTEM-SPEC.md §1.5 (entity_id/entity_type/display_name at the top
+// level) rather than the nested "entity" object — the older of the two
+// wire shapes Maven must keep accepting.
+func fixtureNexusResolvedFlat(entityID, displayName, entityType string) string {
+	return mustJSON(map[string]any{
+		"status":       "resolved",
+		"entity_id":    entityID,
+		"entity_type":  entityType,
+		"display_name": displayName,
+	})
+}
+
+// fixtureNexusResolvedFuture is a resolved response from a hypothetical newer
+// Nexus: same required fields plus unknown ones. Decoding must ignore the
+// extras, not fail — forward compatibility is what lets the ecosystem be
+// upgraded one service at a time.
+func fixtureNexusResolvedFuture(entityID, displayName, entityType string) string {
+	return mustJSON(map[string]any{
+		"status":          "resolved",
+		"entity":          map[string]any{"id": entityID, "display_name": displayName, "type": entityType, "tenant": "home"},
+		"provenance":      map[string]any{"resolver": "v3", "graph_epoch": 42},
+		"score_breakdown": []any{map[string]any{"signal": "alias", "weight": 0.9}},
+	})
+}
+
 func fixtureNexusNotFound() string {
 	return `{"status":"not_found"}`
 }
@@ -138,6 +218,13 @@ func fixtureHexisExecuted(id, status string) string {
 	return mustJSON(map[string]any{"id": id, "status": status})
 }
 
+// fixtureHexisExecutionFailed is a well-formed Hexis response reporting that
+// the command itself failed: the call succeeded, the execution did not. Maven
+// must distinguish this from a transport failure and from success.
+func fixtureHexisExecutionFailed(id, message string) string {
+	return mustJSON(map[string]any{"id": id, "status": "failed", "error": message})
+}
+
 func fixturePraxisAttentionItems(items ...map[string]any) string {
 	return mustJSON(items)
 }
@@ -191,8 +278,13 @@ func newFakeNexus(t *testing.T, resolveBody string) *fakeServer {
 // fault is injected via SetFault.
 func newFakePraxis(t *testing.T, attentionBody string) *fakeServer {
 	return newFakeServer(t, map[string]http.HandlerFunc{
-		"GET /api/v1/tools/attention": jsonHandler(http.StatusOK, attentionBody),
-		"POST /api/v1/tools/surface":  jsonHandler(http.StatusOK, `{}`),
+		"GET /api/v1/tools/attention":    jsonHandler(http.StatusOK, attentionBody),
+		"GET /api/v1/tools/changes":      jsonHandler(http.StatusOK, `[]`),
+		"POST /api/v1/tools/surface":     jsonHandler(http.StatusOK, `{}`),
+		"POST /api/v1/tools/acknowledge": jsonHandler(http.StatusOK, `{}`),
+		"POST /api/v1/tools/resolve":     jsonHandler(http.StatusOK, `{}`),
+		"POST /api/v1/tools/ignore":      jsonHandler(http.StatusOK, `{}`),
+		"POST /api/v1/tools/pin":         jsonHandler(http.StatusOK, `{}`),
 	})
 }