diff --git a/cmd/mavend/ecosystem_degraded_test.go b/cmd/mavend/ecosystem_degraded_test.go new file mode 100644 index 0000000..16211c7 --- /dev/null +++ b/cmd/mavend/ecosystem_degraded_test.go @@ -0,0 +1,312 @@ +package main + +import ( + "context" + "net/http" + "strings" + "testing" + "time" + + hexisclient "github.com/kami/hexis/pkg/client" + + "github.com/kami/maven/internal/ipc" + "github.com/kami/maven/internal/store" +) + +// Phase-5 hardening suite (Vikunja #276). Everything here drives the shared +// fake ecosystem (fakeecosystem_test.go) rather than one-off inline handlers, +// so the same fault levers — SetFault, SetBody, SetDelay — cover every +// service. What is asserted is the degraded-mode contract: +// +// - services degrade independently: one outage never mutes the others, +// - a degraded reply is never silent, never fabricated, never "success", +// - contract drift (old shape, unknown fields, garbage) is survivable, +// - Maven never acts on an ambiguous target and never chains +// Praxis observation into Hexis execution on its own. + +// ecoHandler wires a handler against whichever of the three fakes is given +// (pass nil to leave a service unconfigured, which is a different state from +// "configured but down"). +func ecoHandler(t *testing.T, nexus, praxis, hexis *fakeServer) *reactiveHandler { + t.Helper() + st := newTestStore(t) + clock := newFakeClock(time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC)) + w := &ecosystemWiring{} + if nexus != nil { + w.nexus = newNexusClient(nexus.URL) + } + if praxis != nil { + w.praxis = newPraxisClient(praxis.URL) + } + if hexis != nil { + w.hexis = hexisclient.New(hexis.URL) + } + return &reactiveHandler{ + api: ipc.NewStoreAPI(st), + dataStore: st, + now: clock.Now, + ecosystem: w, + } +} + +func traceFacts(t *testing.T, h *reactiveHandler) []store.Fact { + t.Helper() + facts, err := h.dataStore.RecentFacts(context.Background(), 50) + if err != nil { + t.Fatalf("read facts: %v", err) + } + var out []store.Fact + for _, f := range facts { + if f.Source == "praxis:trace" { + out = append(out, f) + } + } + return out +} + +func restartCaps() string { + return fixtureHexisCapabilities(map[string]any{ + "id": "cap_restart", "name": "restart", "read_only": true, + }) +} + +// TestEcosystem_OutagesAreIndependent: Praxis being down must not disable the +// Nexus+Hexis action path, and vice versa. A shared "ecosystem is broken" +// mode would take away working capability for no reason. +func TestEcosystem_OutagesAreIndependent(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + praxis := newFakePraxis(t, fixturePraxisAttentionItems(map[string]any{ + "id": "item_1", "title": "disk almost full", "importance": 3.0, + })) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, praxis, hexis) + + praxis.SetFault(503) + if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") { + t.Fatalf("praxis outage must not block the hexis path, got %q", reply) + } + + praxis.SetFault(0) + hexis.SetFault(503) + nexus.SetFault(503) + reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")) + if !strings.Contains(reply, "disk almost full") { + t.Fatalf("nexus/hexis outage must not block the praxis digest, got %q", reply) + } +} + +// TestEcosystem_MalformedNexusResponseFailsClosed: a 200 carrying garbage is a +// dependency failure, not "no such entity". It must stop before Hexis. +func TestEcosystem_MalformedNexusResponseFailsClosed(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + + nexus.SetBody(`{"status":"resolved","entity":`) + reply := h.handleHexisAct(ctx, actDec("muzick indexer")) + if reply == "" || strings.Contains(reply, "выполнена") { + t.Fatalf("malformed nexus body must degrade, got %q", reply) + } + if hexis.Count("", "/api/v1") != 0 { + t.Fatal("hexis must not be contacted after a malformed nexus response") + } +} + +// TestEcosystem_UnknownContractFieldsTolerated: a newer Nexus adding fields +// must not break an older Maven. Same for the older flat resolve shape. +func TestEcosystem_UnknownContractFieldsTolerated(t *testing.T) { + ctx := context.Background() + for name, body := range map[string]string{ + "future": fixtureNexusResolvedFuture("ent_muzick", "Muzick indexer", "service"), + "flat": fixtureNexusResolvedFlat("ent_muzick", "Muzick indexer", "service"), + } { + t.Run(name, func(t *testing.T) { + nexus := newFakeNexus(t, body) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") { + t.Fatalf("%s contract shape must still resolve and execute, got %q", name, reply) + } + }) + } +} + +// TestEcosystem_CancelledContextDegrades: a caller hanging up (turn abandoned, +// deadline hit) must surface as degradation, never as a fabricated result. +func TestEcosystem_CancelledContextDegrades(t *testing.T) { + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + nexus.SetDelay(2 * time.Second) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) + defer cancel() + reply := h.handleHexisAct(ctx, actDec("muzick indexer")) + if reply == "" || strings.Contains(reply, "выполнена") { + t.Fatalf("cancelled resolve must degrade, got %q", reply) + } + if hexis.Count("", "/api/v1") != 0 { + t.Fatal("hexis must not be contacted after a cancelled resolve") + } +} + +// TestEcosystem_ExecutionFailureIsNotSuccess: Hexis answering 200 with +// status=failed is a partial failure — the call worked, the command did not. +// Maven must report it as a failure and must not write a success trace. +func TestEcosystem_ExecutionFailureIsNotSuccess(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecutionFailed("exec_1", "unit not found")) + h := ecoHandler(t, nexus, nil, hexis) + + reply := h.handleHexisAct(ctx, actDec("muzick indexer")) + if strings.Contains(reply, "выполнена") { + t.Fatalf("failed execution must not read as success, got %q", reply) + } + if reply == "" { + t.Fatal("failed execution must say something") + } + for _, f := range traceFacts(t, h) { + if strings.HasPrefix(f.Key, "praxis:hexis:") { + t.Fatalf("failed execution must not write a success trace: %+v", f) + } + } +} + +// TestEcosystem_AmbiguousTargetBlocksExecution: ambiguity blocks mutation, and +// the clarification must name the candidates rather than pick one. +func TestEcosystem_AmbiguousTargetBlocksExecution(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusAmbiguous( + map[string]string{"entity_id": "ent_a", "display_name": "Muzick indexer"}, + map[string]string{"entity_id": "ent_b", "display_name": "Muzick web"}, + )) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + + reply := h.handleHexisAct(ctx, actDec("muzick")) + if !strings.Contains(reply, "Muzick indexer") || !strings.Contains(reply, "Muzick web") { + t.Fatalf("ambiguous resolve must list candidates, got %q", reply) + } + if hexis.Count("POST", "/api/v1/execute") != 0 { + t.Fatal("ambiguous target must never execute") + } +} + +// TestEcosystem_NoAutonomousPraxisToHexis: reading the attention digest is an +// observation. Maven must never turn an observed problem into a Hexis command +// by herself — she is not autonomous. +func TestEcosystem_NoAutonomousPraxisToHexis(t *testing.T) { + ctx := context.Background() + praxis := newFakePraxis(t, fixturePraxisAttentionItems( + map[string]any{"id": "item_1", "title": "muzick indexer is down", "importance": 4.0, "rule": "service_down"}, + )) + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, praxis, hexis) + + _ = h.handlePraxisAct(ctx, praxisActDec("list_attention")) + if hexis.Count("", "/api/v1") != 0 { + t.Fatal("attention digest must not contact hexis on its own") + } + if nexus.Count("", "/api/v1/resolve") != 0 { + t.Fatal("attention digest must not resolve targets for autonomous action") + } +} + +// TestEcosystem_MutatingCapabilityWaitsForConfirmation: a non-read-only +// capability parks for an explicit spoken confirm bound to capability+target. +func TestEcosystem_MutatingCapabilityWaitsForConfirmation(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + caps := fixtureHexisCapabilities(map[string]any{"id": "cap_restart", "name": "restart", "read_only": false}) + hexis := newFakeHexis(t, caps, fixtureHexisExecuted("exec_1", "succeeded")) + h := ecoHandler(t, nexus, nil, hexis) + + reply := h.handleHexisAct(ctx, actDec("restart")) + if !strings.Contains(reply, "restart") || !strings.Contains(reply, "да") { + t.Fatalf("mutating capability must ask for confirmation, got %q", reply) + } + if hexis.Count("POST", "/api/v1/execute") != 0 { + t.Fatal("mutating capability must not execute before confirmation") + } + h.mu.Lock() + pending := h.pendingHexis + h.mu.Unlock() + if pending == nil || pending.capabilityID != "cap_restart" || pending.entityID != "ent_muzick" { + t.Fatalf("confirmation must be bound to capability+target, got %+v", pending) + } +} + +// TestEcosystem_SurfaceFailureStillDelivers: surfacing is bookkeeping. If the +// surface call fails the digest must still be spoken — a partial failure +// downgrades bookkeeping, not the answer. +func TestEcosystem_SurfaceFailureStillDelivers(t *testing.T) { + ctx := context.Background() + praxis := newFakeServer(t, map[string]http.HandlerFunc{ + "GET /api/v1/tools/attention": jsonHandler(200, fixturePraxisAttentionItems( + map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0}, + )), + "POST /api/v1/tools/surface": jsonHandler(500, `{"error":"boom"}`), + }) + h := ecoHandler(t, nil, praxis, nil) + + reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")) + if !strings.Contains(reply, "disk almost full") { + t.Fatalf("failed surface must not swallow the digest, got %q", reply) + } + if praxis.Count("POST", "/api/v1/tools/surface") == 0 { + t.Fatal("expected the surface attempt") + } +} + +// TestEcosystem_TotalOutageSaysSoForEveryPath: with all three down, every +// entry point degrades explicitly instead of returning empty or inventing. +func TestEcosystem_TotalOutageSaysSoForEveryPath(t *testing.T) { + ctx := context.Background() + nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) + praxis := newFakePraxis(t, fixturePraxisAttentionItems()) + hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) + for _, fs := range []*fakeServer{nexus, praxis, hexis} { + fs.SetFault(503) + } + h := ecoHandler(t, nexus, praxis, hexis) + + for name, reply := range map[string]string{ + "hexis act": h.handleHexisAct(ctx, actDec("muzick indexer")), + "attention": h.handlePraxisAct(ctx, praxisActDec("list_attention")), + "changes": h.handlePraxisAct(ctx, praxisActDec("list_changes")), + "acknowledge": h.handlePraxisAct(ctx, praxisActDec("acknowledge_item")), + } { + if reply == "" { + t.Errorf("%s: total outage must not answer with silence", name) + } + if strings.Contains(reply, "выполнена") { + t.Errorf("%s: total outage must not claim success: %q", name, reply) + } + } + if len(traceFacts(t, h)) != 0 { + t.Fatal("a total outage must not leave success traces behind") + } +} + +// TestEcosystem_RecoveryAfterOutageNeedsNoRestart: once the dependency comes +// back the very next turn works — no cached failure state, no restart. +func TestEcosystem_RecoveryAfterOutageNeedsNoRestart(t *testing.T) { + ctx := context.Background() + praxis := newFakePraxis(t, fixturePraxisAttentionItems( + map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0}, + )) + h := ecoHandler(t, nil, praxis, nil) + + praxis.SetFault(503) + if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); strings.Contains(reply, "disk") { + t.Fatalf("outage must not serve content, got %q", reply) + } + praxis.SetFault(0) + if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); !strings.Contains(reply, "disk almost full") { + t.Fatalf("recovery must work on the next turn, got %q", reply) + } +} diff --git a/cmd/mavend/fakeecosystem_test.go b/cmd/mavend/fakeecosystem_test.go index ff08eb6..108860f 100644 --- a/cmd/mavend/fakeecosystem_test.go +++ b/cmd/mavend/fakeecosystem_test.go @@ -14,7 +14,9 @@ import ( type capturedRequest struct { Method string Path string + Query string Body []byte + Header http.Header } // fakeServer is the common shell behind fakeNexus/fakePraxis/fakeHexis: an @@ -27,7 +29,9 @@ type fakeServer struct { mu sync.Mutex requests []capturedRequest - fault int // non-zero: every request gets this HTTP status instead of routing + fault int // non-zero: every request gets this HTTP status instead of routing + garbage string // non-empty: returned 200 verbatim instead of routing (malformed-contract lever) + delay time.Duration } // newFakeServer starts a server dispatching to routes keyed by "METHOD @@ -47,14 +51,34 @@ func newFakeServer(t *testing.T, routes map[string]http.HandlerFunc) *fakeServer } } fs.mu.Lock() - fs.requests = append(fs.requests, capturedRequest{Method: r.Method, Path: r.URL.Path, Body: body}) + fs.requests = append(fs.requests, capturedRequest{ + Method: r.Method, + Path: r.URL.Path, + Query: r.URL.RawQuery, + Body: body, + Header: r.Header.Clone(), + }) fault := fs.fault + garbage := fs.garbage + delay := fs.delay fs.mu.Unlock() + if delay > 0 { + select { + case <-time.After(delay): + case <-r.Context().Done(): + return + } + } if fault != 0 { http.Error(w, "injected fault", fault) return } + if garbage != "" { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(garbage)) + return + } for key, handler := range routes { method, prefix := splitRouteKey(key) @@ -90,6 +114,36 @@ func (fs *fakeServer) SetFault(status int) { fs.fault = status } +// SetBody makes every subsequent request answer 200 with the given body, +// bypassing the route table. Used to serve a malformed or contract-violating +// payload where the transport itself is healthy. Pass "" to clear it. +func (fs *fakeServer) SetBody(body string) { + fs.mu.Lock() + defer fs.mu.Unlock() + fs.garbage = body +} + +// SetDelay stalls every subsequent request for d before answering, so callers +// can drive client timeouts and context cancellation deterministically. The +// delay is abandoned as soon as the client hangs up. +func (fs *fakeServer) SetDelay(d time.Duration) { + fs.mu.Lock() + defer fs.mu.Unlock() + fs.delay = d +} + +// Count returns how many captured requests used the given method and path +// prefix. "" matches any method. +func (fs *fakeServer) Count(method, prefix string) int { + n := 0 + for _, r := range fs.Requests() { + if (method == "" || r.Method == method) && hasPrefix(r.Path, prefix) { + n++ + } + } + return n +} + // Requests returns a snapshot of captured requests, in arrival order. func (fs *fakeServer) Requests() []capturedRequest { fs.mu.Lock() @@ -118,6 +172,32 @@ func fixtureNexusResolved(entityID, displayName, entityType string) string { }) } +// fixtureNexusResolvedFlat is the flat resolve shape documented in +// ECOSYSTEM-SPEC.md §1.5 (entity_id/entity_type/display_name at the top +// level) rather than the nested "entity" object — the older of the two +// wire shapes Maven must keep accepting. +func fixtureNexusResolvedFlat(entityID, displayName, entityType string) string { + return mustJSON(map[string]any{ + "status": "resolved", + "entity_id": entityID, + "entity_type": entityType, + "display_name": displayName, + }) +} + +// fixtureNexusResolvedFuture is a resolved response from a hypothetical newer +// Nexus: same required fields plus unknown ones. Decoding must ignore the +// extras, not fail — forward compatibility is what lets the ecosystem be +// upgraded one service at a time. +func fixtureNexusResolvedFuture(entityID, displayName, entityType string) string { + return mustJSON(map[string]any{ + "status": "resolved", + "entity": map[string]any{"id": entityID, "display_name": displayName, "type": entityType, "tenant": "home"}, + "provenance": map[string]any{"resolver": "v3", "graph_epoch": 42}, + "score_breakdown": []any{map[string]any{"signal": "alias", "weight": 0.9}}, + }) +} + func fixtureNexusNotFound() string { return `{"status":"not_found"}` } @@ -138,6 +218,13 @@ func fixtureHexisExecuted(id, status string) string { return mustJSON(map[string]any{"id": id, "status": status}) } +// fixtureHexisExecutionFailed is a well-formed Hexis response reporting that +// the command itself failed: the call succeeded, the execution did not. Maven +// must distinguish this from a transport failure and from success. +func fixtureHexisExecutionFailed(id, message string) string { + return mustJSON(map[string]any{"id": id, "status": "failed", "error": message}) +} + func fixturePraxisAttentionItems(items ...map[string]any) string { return mustJSON(items) } @@ -191,8 +278,13 @@ func newFakeNexus(t *testing.T, resolveBody string) *fakeServer { // fault is injected via SetFault. func newFakePraxis(t *testing.T, attentionBody string) *fakeServer { return newFakeServer(t, map[string]http.HandlerFunc{ - "GET /api/v1/tools/attention": jsonHandler(http.StatusOK, attentionBody), - "POST /api/v1/tools/surface": jsonHandler(http.StatusOK, `{}`), + "GET /api/v1/tools/attention": jsonHandler(http.StatusOK, attentionBody), + "GET /api/v1/tools/changes": jsonHandler(http.StatusOK, `[]`), + "POST /api/v1/tools/surface": jsonHandler(http.StatusOK, `{}`), + "POST /api/v1/tools/acknowledge": jsonHandler(http.StatusOK, `{}`), + "POST /api/v1/tools/resolve": jsonHandler(http.StatusOK, `{}`), + "POST /api/v1/tools/ignore": jsonHandler(http.StatusOK, `{}`), + "POST /api/v1/tools/pin": jsonHandler(http.StatusOK, `{}`), }) }