package main import ( "context" "strings" "testing" "time" hexisclient "github.com/kami/hexis/pkg/client" "github.com/kami/maven/internal/ipc" "github.com/kami/maven/internal/store" ) // Phase-5 hardening suite (Vikunja #276). Everything here drives the shared // fake ecosystem (fakeecosystem_test.go) rather than one-off inline handlers, // so the same fault levers — SetFault, SetBody, SetDelay — cover every // service. What is asserted is the degraded-mode contract: // // - services degrade independently: one outage never mutes the others, // - a degraded reply is never silent, never fabricated, never "success", // - contract drift (old shape, unknown fields, garbage) is survivable, // - Maven never acts on an ambiguous target and never chains // Praxis observation into Hexis execution on its own. // ecoHandler wires a handler against whichever of the three fakes is given // (pass nil to leave a service unconfigured, which is a different state from // "configured but down"). func ecoHandler(t *testing.T, nexus, praxis, hexis *fakeServer) *reactiveHandler { t.Helper() st := newTestStore(t) clock := newTickingClock(time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC), time.Millisecond) w := &ecosystemWiring{} if nexus != nil { w.nexus = newNexusClient(nexus.URL) } if praxis != nil { w.praxis = newPraxisClient(praxis.URL) } if hexis != nil { w.hexis = hexisclient.New(hexis.URL) } return &reactiveHandler{ api: ipc.NewStoreAPI(st), dataStore: st, now: clock.Now, ecosystem: w, } } // traces reads the ecosystem trace table. Traces live there and not in facts, // so a bounded reader of facts never fills up with machine-rate rows. func traces(t *testing.T, h *reactiveHandler) []store.EcosystemTrace { t.Helper() out, err := h.dataStore.RecentEcosystemTraces(context.Background(), 100) if err != nil { t.Fatalf("read traces: %v", err) } return out } // tracesFor returns the traces recorded for one service+operation. func tracesFor(t *testing.T, h *reactiveHandler, service, op string) []store.EcosystemTrace { t.Helper() var out []store.EcosystemTrace for _, tr := range traces(t, h) { if tr.Service == service && tr.Operation == op { out = append(out, tr) } } return out } // restartCaps is a read-only capability. Restarting a service is a mutation, // so the read-only one this suite runs through the happy paths is named for // what it is; the mutating restart lives in the confirmation tests. func restartCaps() string { return fixtureHexisCapabilities(map[string]any{ "id": "cap_status", "name": "restart status", "read_only": true, }) } // TestEcosystem_OutagesLeaveNoSharedFailureState: the two act paths share a // handler, a store and a clock, so what is worth asserting is that a failure // on one leaves nothing behind that degrades the other. Faulting one disjoint // call graph and exercising the other only tests the call graph. func TestEcosystem_OutagesLeaveNoSharedFailureState(t *testing.T) { ctx := context.Background() nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) praxis := newFakePraxis(t, fixturePraxisAttentionItems(map[string]any{ "id": "item_1", "title": "disk almost full", "importance": 3.0, })) hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) h := ecoHandler(t, nexus, praxis, hexis) // A Nexus outage during a Hexis act writes a failure trace, and a shared // store is the one thing the Praxis path could inherit it through. nexus.SetFault(503) if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); strings.Contains(reply, "выполнена") { t.Fatalf("nexus outage must not report success, got %q", reply) } if len(tracesFor(t, h, "nexus", "resolve")) == 0 { t.Fatal("the failed resolve must be recorded") } nexus.SetFault(0) reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")) if !strings.Contains(reply, "disk almost full") { t.Fatalf("a recorded nexus failure must not degrade the praxis digest, got %q", reply) } if got := tracesFor(t, h, "praxis", "list_attention"); len(got) != 1 || got[0].Status != traceOK { t.Fatalf("the praxis digest must trace its own success, got %+v", got) } // And the reverse: a Praxis outage mid-session leaves the Hexis path whole. praxis.SetFault(503) if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); strings.Contains(reply, "disk") { t.Fatalf("praxis outage must not serve content, got %q", reply) } if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") { t.Fatalf("a praxis outage must not block the hexis path, got %q", reply) } } // TestEcosystem_OneEndpointDownDoesNotMuteTheService: real outages are usually // partial. Attention answering while surface is down must still deliver. func TestEcosystem_OneEndpointDownDoesNotMuteTheService(t *testing.T) { ctx := context.Background() praxis := newFakePraxis(t, fixturePraxisAttentionItems(map[string]any{ "id": "item_1", "title": "disk almost full", "importance": 3.0, })) h := ecoHandler(t, nil, praxis, nil) praxis.SetRouteFault("/api/v1/tools/surface", 503) reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")) if !strings.Contains(reply, "disk almost full") { t.Fatalf("a downed surface endpoint must not mute the digest, got %q", reply) } if praxis.Count("POST", "/api/v1/tools/surface") == 0 { t.Fatal("expected the surface attempt") } } // TestEcosystem_ResolvedWithoutEntityFailsClosed: the contract violation that // decodes cleanly. Nexus says "resolved" and delivers no entity; treating that // as "no such entity" put the user's verb through to the local executor. func TestEcosystem_ResolvedWithoutEntityFailsClosed(t *testing.T) { ctx := context.Background() nexus := newFakeNexus(t, fixtureNexusResolvedEmpty()) hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) h := ecoHandler(t, nexus, nil, hexis) reply := h.handleHexisAct(ctx, actDec("muzick indexer")) if reply == "" { t.Fatal("a resolve with no entity must degrade, not fall through to local execution") } if strings.Contains(reply, "выполнена") { t.Fatalf("a resolve with no entity must not report success, got %q", reply) } if hexis.Count("", "/api/v1") != 0 { t.Fatal("hexis must not be contacted after a contract-violating resolve") } } // TestEcosystem_RejectedCredentialSaysSo: 401 and 403 must not read as an // outage. "Try again" is advice that never works for a misconfigured token. func TestEcosystem_RejectedCredentialSaysSo(t *testing.T) { ctx := context.Background() for _, status := range []int{401, 403} { nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) h := ecoHandler(t, nexus, nil, hexis) nexus.SetFault(status) reply := h.handleHexisAct(ctx, actDec("muzick indexer")) if !strings.Contains(reply, "токен") { t.Fatalf("http %d must read as a credential problem, got %q", status, reply) } tr := tracesFor(t, h, "nexus", "resolve") if len(tr) != 1 || tr[0].Status != traceRefused || tr[0].HTTPStatus != status { t.Fatalf("http %d must trace as refused with its status, got %+v", status, tr) } } } // TestEcosystem_MalformedPraxisBodyDegrades: Praxis has the same decode path // Nexus does, and a 200 carrying garbage there is a dependency failure too. func TestEcosystem_MalformedPraxisBodyDegrades(t *testing.T) { ctx := context.Background() praxis := newFakePraxis(t, fixturePraxisAttentionItems(map[string]any{ "id": "item_1", "title": "disk almost full", "importance": 3.0, })) h := ecoHandler(t, nil, praxis, nil) praxis.SetBody(`[{"title":`) reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")) if reply == "" { t.Fatal("a malformed praxis body must not answer with silence") } if strings.Contains(reply, "disk almost full") { t.Fatalf("a malformed body must not produce content, got %q", reply) } } // TestEcosystem_MalformedNexusResponseFailsClosed: a 200 carrying garbage is a // dependency failure, not "no such entity". It must stop before Hexis. func TestEcosystem_MalformedNexusResponseFailsClosed(t *testing.T) { ctx := context.Background() nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) h := ecoHandler(t, nexus, nil, hexis) nexus.SetBody(`{"status":"resolved","entity":`) reply := h.handleHexisAct(ctx, actDec("muzick indexer")) if reply == "" || strings.Contains(reply, "выполнена") { t.Fatalf("malformed nexus body must degrade, got %q", reply) } if hexis.Count("", "/api/v1") != 0 { t.Fatal("hexis must not be contacted after a malformed nexus response") } } // TestEcosystem_UnknownContractFieldsTolerated: a newer Nexus adding fields // must not break an older Maven. Same for the older flat resolve shape. func TestEcosystem_UnknownContractFieldsTolerated(t *testing.T) { ctx := context.Background() for name, body := range map[string]string{ "future": fixtureNexusResolvedFuture("ent_muzick", "Muzick indexer", "service"), "flat": fixtureNexusResolvedFlat("ent_muzick", "Muzick indexer", "service"), } { t.Run(name, func(t *testing.T) { nexus := newFakeNexus(t, body) hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) h := ecoHandler(t, nexus, nil, hexis) if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") { t.Fatalf("%s contract shape must still resolve and execute, got %q", name, reply) } }) } } // TestEcosystem_CancelledContextDegrades: a caller hanging up (turn abandoned, // deadline hit) must surface as degradation, never as a fabricated result. func TestEcosystem_CancelledContextDegrades(t *testing.T) { nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) h := ecoHandler(t, nexus, nil, hexis) nexus.SetDelay(2 * time.Second) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond) defer cancel() reply := h.handleHexisAct(ctx, actDec("muzick indexer")) if reply == "" || strings.Contains(reply, "выполнена") { t.Fatalf("cancelled resolve must degrade, got %q", reply) } if hexis.Count("", "/api/v1") != 0 { t.Fatal("hexis must not be contacted after a cancelled resolve") } } // TestEcosystem_ExecutionFailureIsNotSuccess: Hexis answering 200 with // status=failed is a partial failure — the call worked, the command did not. // Maven must report it as a failure and must not write a success trace. func TestEcosystem_ExecutionFailureIsNotSuccess(t *testing.T) { ctx := context.Background() nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecutionFailed("exec_1", "unit not found")) h := ecoHandler(t, nexus, nil, hexis) reply := h.handleHexisAct(ctx, actDec("muzick indexer")) if strings.Contains(reply, "выполнена") { t.Fatalf("failed execution must not read as success, got %q", reply) } if reply == "" { t.Fatal("failed execution must say something") } for _, tr := range tracesFor(t, h, "hexis", "execute") { if tr.Status == traceOK { t.Fatalf("failed execution must not write a success trace: %+v", tr) } } } // TestEcosystem_SuccessfulActionWritesATrace is the positive half the failure // assertions above depend on: without it, "no success trace" passes with the // trace writer deleted. It was, for a while — both writers used a fact kind the // store's CHECK constraint rejects and the error was discarded. func TestEcosystem_SuccessfulActionWritesATrace(t *testing.T) { ctx := context.Background() nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) h := ecoHandler(t, nexus, nil, hexis) if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") { t.Fatalf("setup: expected success, got %q", reply) } exec := tracesFor(t, h, "hexis", "execute") if len(exec) != 1 || exec[0].Status != traceOK { t.Fatalf("a successful execution must leave exactly one ok trace, got %+v", exec) } if exec[0].CorrelationID == "" { t.Error("a trace with no correlation id cannot be stitched to anything") } } // TestEcosystem_TracesStayOutOfFacts: traces are written at machine rate and // facts at human rate. One act turn used to write four fact rows, which pushed // his facts out of every bounded reader (the habit profile's window, memeval's // prompt, /dash, /history). func TestEcosystem_TracesStayOutOfFacts(t *testing.T) { ctx := context.Background() nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) h := ecoHandler(t, nexus, nil, hexis) if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !strings.Contains(reply, "выполнена") { t.Fatalf("setup: expected success, got %q", reply) } if len(traces(t, h)) == 0 { t.Fatal("setup: expected traces") } facts, err := h.dataStore.RecentFacts(ctx, 100) if err != nil { t.Fatalf("read facts: %v", err) } if len(facts) != 0 { t.Fatalf("an ecosystem act must write no facts at all, got %+v", facts) } } // TestEcosystem_AmbiguousTargetBlocksExecution: ambiguity blocks mutation, and // the clarification must name the candidates rather than pick one. func TestEcosystem_AmbiguousTargetBlocksExecution(t *testing.T) { ctx := context.Background() nexus := newFakeNexus(t, fixtureNexusAmbiguous( map[string]string{"entity_id": "ent_a", "display_name": "Muzick indexer"}, map[string]string{"entity_id": "ent_b", "display_name": "Muzick web"}, )) hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) h := ecoHandler(t, nexus, nil, hexis) reply := h.handleHexisAct(ctx, actDec("muzick")) if !strings.Contains(reply, "Muzick indexer") || !strings.Contains(reply, "Muzick web") { t.Fatalf("ambiguous resolve must list candidates, got %q", reply) } if hexis.Count("POST", "/api/v1/execute") != 0 { t.Fatal("ambiguous target must never execute") } } // TestEcosystem_NoAutonomousPraxisToHexis: reading the attention digest is an // observation. Maven must never turn an observed problem into a Hexis command // by herself — she is not autonomous. func TestEcosystem_NoAutonomousPraxisToHexis(t *testing.T) { ctx := context.Background() praxis := newFakePraxis(t, fixturePraxisAttentionItems( map[string]any{"id": "item_1", "title": "muzick indexer is down", "importance": 4.0, "rule": "service_down"}, )) nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) h := ecoHandler(t, nexus, praxis, hexis) _ = h.handlePraxisAct(ctx, praxisActDec("list_attention")) if hexis.Count("", "/api/v1") != 0 { t.Fatal("attention digest must not contact hexis on its own") } if nexus.Count("", "/api/v1/resolve") != 0 { t.Fatal("attention digest must not resolve targets for autonomous action") } } // TestEcosystem_MutatingCapabilityWaitsForConfirmation: a non-read-only // capability parks for an explicit spoken confirm bound to capability+target. func TestEcosystem_MutatingCapabilityWaitsForConfirmation(t *testing.T) { ctx := context.Background() nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) caps := fixtureHexisCapabilities(map[string]any{"id": "cap_restart", "name": "restart", "read_only": false}) hexis := newFakeHexis(t, caps, fixtureHexisExecuted("exec_1", "succeeded")) h := ecoHandler(t, nexus, nil, hexis) reply := h.handleHexisAct(ctx, actDec("restart")) if !strings.Contains(reply, "restart") || !strings.Contains(reply, "да") { t.Fatalf("mutating capability must ask for confirmation, got %q", reply) } if hexis.Count("POST", "/api/v1/execute") != 0 { t.Fatal("mutating capability must not execute before confirmation") } h.mu.Lock() pending := h.pendingHexis h.mu.Unlock() if pending == nil || pending.capabilityID != "cap_restart" || pending.entityID != "ent_muzick" { t.Fatalf("confirmation must be bound to capability+target, got %+v", pending) } } // TestEcosystem_SurfaceFailureStillDelivers: surfacing is bookkeeping. If the // surface call fails the digest must still be spoken — a partial failure // downgrades bookkeeping, not the answer. func TestEcosystem_SurfaceFailureStillDelivers(t *testing.T) { ctx := context.Background() praxis := newFakePraxis(t, fixturePraxisAttentionItems( map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0}, )) praxis.SetRouteFault("/api/v1/tools/surface", 500) h := ecoHandler(t, nil, praxis, nil) reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")) if !strings.Contains(reply, "disk almost full") { t.Fatalf("failed surface must not swallow the digest, got %q", reply) } if praxis.Count("POST", "/api/v1/tools/surface") == 0 { t.Fatal("expected the surface attempt") } } // TestEcosystem_TotalOutageSaysSoForEveryPath: with all three down, every // entry point degrades explicitly instead of returning empty or inventing. func TestEcosystem_TotalOutageSaysSoForEveryPath(t *testing.T) { ctx := context.Background() nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service")) praxis := newFakePraxis(t, fixturePraxisAttentionItems()) hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded")) for _, fs := range []*fakeServer{nexus, praxis, hexis} { fs.SetFault(503) } h := ecoHandler(t, nexus, praxis, hexis) for name, reply := range map[string]string{ "hexis act": h.handleHexisAct(ctx, actDec("muzick indexer")), "attention": h.handlePraxisAct(ctx, praxisActDec("list_attention")), "changes": h.handlePraxisAct(ctx, praxisActDec("list_changes")), "acknowledge": h.handlePraxisAct(ctx, praxisItemDec("acknowledge_item", "item_1")), } { if reply == "" { t.Errorf("%s: total outage must not answer with silence", name) } if strings.Contains(reply, "выполнена") { t.Errorf("%s: total outage must not claim success: %q", name, reply) } } for _, tr := range traces(t, h) { if tr.Status == traceOK { t.Fatalf("a total outage must not leave success traces behind: %+v", tr) } } if len(tracesFor(t, h, "praxis", "acknowledge")) == 0 { t.Fatal("the acknowledge arm must reach praxis and record the refusal") } } // TestEcosystem_RecoveryAfterOutageNeedsNoRestart: once the dependency comes // back the very next turn works — no cached failure state, no restart. func TestEcosystem_RecoveryAfterOutageNeedsNoRestart(t *testing.T) { ctx := context.Background() praxis := newFakePraxis(t, fixturePraxisAttentionItems( map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0}, )) h := ecoHandler(t, nil, praxis, nil) praxis.SetFault(503) if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); strings.Contains(reply, "disk") { t.Fatalf("outage must not serve content, got %q", reply) } praxis.SetFault(0) if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); !strings.Contains(reply, "disk almost full") { t.Fatalf("recovery must work on the next turn, got %q", reply) } }