Maven/cmd/mavend/ecosystem_degraded_test.go

package main

import (
	"context"
	"strings"
	"testing"
	"time"

	hexisclient "github.com/kami/hexis/pkg/client"

	"github.com/kami/maven/internal/ipc"
	"github.com/kami/maven/internal/store"
)

// Phase-5 hardening suite (Vikunja #276). Everything here drives the shared
// fake ecosystem (fakeecosystem_test.go) rather than one-off inline handlers,
// so the same fault levers — SetFault, SetBody, SetDelay — cover every
// service. What is asserted is the degraded-mode contract:
//
//   - services degrade independently: one outage never mutes the others,
//   - a degraded reply is never silent, never fabricated, never "success",
//   - contract drift (old shape, unknown fields, garbage) is survivable,
//   - Maven never acts on an ambiguous target and never chains
//     Praxis observation into Hexis execution on its own.

// ecoHandler wires a handler against whichever of the three fakes is given
// (pass nil to leave a service unconfigured, which is a different state from
// "configured but down").
func ecoHandler(t *testing.T, nexus, praxis, hexis *fakeServer) *reactiveHandler {
	t.Helper()
	st := newTestStore(t)
	clock := newTickingClock(time.Date(2026, 8, 1, 9, 0, 0, 0, time.UTC), time.Millisecond)
	w := &ecosystemWiring{}
	if nexus != nil {
		w.nexus = newNexusClient(nexus.URL)
	}
	if praxis != nil {
		w.praxis = newPraxisClient(praxis.URL)
	}
	if hexis != nil {
		w.hexis = hexisclient.New(hexis.URL)
	}
	return &reactiveHandler{
		api:       ipc.NewStoreAPI(st),
		dataStore: st,
		now:       clock.Now,
		ecosystem: w,
	}
}

// traces reads the ecosystem trace table. Traces live there and not in facts,
// so a bounded reader of facts never fills up with machine-rate rows.
func traces(t *testing.T, h *reactiveHandler) []store.EcosystemTrace {
	t.Helper()
	out, err := h.dataStore.RecentEcosystemTraces(context.Background(), 100)
	if err != nil {
		t.Fatalf("read traces: %v", err)
	}
	return out
}

// tracesFor returns the traces recorded for one service+operation.
func tracesFor(t *testing.T, h *reactiveHandler, service, op string) []store.EcosystemTrace {
	t.Helper()
	var out []store.EcosystemTrace
	for _, tr := range traces(t, h) {
		if tr.Service == service && tr.Operation == op {
			out = append(out, tr)
		}
	}
	return out
}

// restartCaps is a read-only capability. Restarting a service is a mutation,
// so the read-only one this suite runs through the happy paths is named for
// what it is; the mutating restart lives in the confirmation tests.
func restartCaps() string {
	return fixtureHexisCapabilities(map[string]any{
		"id": "cap_status", "name": "restart status", "read_only": true,
	})
}

// TestEcosystem_OutagesLeaveNoSharedFailureState: the two act paths share a
// handler, a store and a clock, so what is worth asserting is that a failure
// on one leaves nothing behind that degrades the other. Faulting one disjoint
// call graph and exercising the other only tests the call graph.
func TestEcosystem_OutagesLeaveNoSharedFailureState(t *testing.T) {
	ctx := context.Background()
	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
	praxis := newFakePraxis(t, fixturePraxisAttentionItems(map[string]any{
		"id": "item_1", "title": "disk almost full", "importance": 3.0,
	}))
	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
	h := ecoHandler(t, nexus, praxis, hexis)

	// A Nexus outage during a Hexis act writes a failure trace, and a shared
	// store is the one thing the Praxis path could inherit it through.
	nexus.SetFault(503)
	if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); actRan(reply) {
		t.Fatalf("nexus outage must not report success, got %q", reply)
	}
	if len(tracesFor(t, h, "nexus", "resolve")) == 0 {
		t.Fatal("the failed resolve must be recorded")
	}

	nexus.SetFault(0)
	reply := h.handlePraxisAct(ctx, praxisActDec("list_attention"))
	if !strings.Contains(reply, "disk almost full") {
		t.Fatalf("a recorded nexus failure must not degrade the praxis digest, got %q", reply)
	}
	if got := tracesFor(t, h, "praxis", "list_attention"); len(got) != 1 || got[0].Status != traceOK {
		t.Fatalf("the praxis digest must trace its own success, got %+v", got)
	}

	// And the reverse: a Praxis outage mid-session leaves the Hexis path whole.
	praxis.SetFault(503)
	if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); strings.Contains(reply, "disk") {
		t.Fatalf("praxis outage must not serve content, got %q", reply)
	}
	if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !actRan(reply) {
		t.Fatalf("a praxis outage must not block the hexis path, got %q", reply)
	}
}

// TestEcosystem_OneEndpointDownDoesNotMuteTheService: real outages are usually
// partial. Attention answering while surface is down must still deliver.
func TestEcosystem_OneEndpointDownDoesNotMuteTheService(t *testing.T) {
	ctx := context.Background()
	praxis := newFakePraxis(t, fixturePraxisAttentionItems(map[string]any{
		"id": "item_1", "title": "disk almost full", "importance": 3.0,
	}))
	h := ecoHandler(t, nil, praxis, nil)

	praxis.SetRouteFault("/api/v1/tools/surface", 503)
	reply := h.handlePraxisAct(ctx, praxisActDec("list_attention"))
	if !strings.Contains(reply, "disk almost full") {
		t.Fatalf("a downed surface endpoint must not mute the digest, got %q", reply)
	}
	if praxis.Count("POST", "/api/v1/tools/surface") == 0 {
		t.Fatal("expected the surface attempt")
	}
}

// TestEcosystem_ResolvedWithoutEntityFailsClosed: the contract violation that
// decodes cleanly. Nexus says "resolved" and delivers no entity; treating that
// as "no such entity" put the user's verb through to the local executor.
func TestEcosystem_ResolvedWithoutEntityFailsClosed(t *testing.T) {
	ctx := context.Background()
	nexus := newFakeNexus(t, fixtureNexusResolvedEmpty())
	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
	h := ecoHandler(t, nexus, nil, hexis)

	reply := h.handleHexisAct(ctx, actDec("muzick indexer"))
	if reply == "" {
		t.Fatal("a resolve with no entity must degrade, not fall through to local execution")
	}
	if actRan(reply) {
		t.Fatalf("a resolve with no entity must not report success, got %q", reply)
	}
	if hexis.Count("", "/api/v1") != 0 {
		t.Fatal("hexis must not be contacted after a contract-violating resolve")
	}
}

// TestEcosystem_RejectedCredentialSaysSo: 401 and 403 must not read as an
// outage. "Try again" is advice that never works for a misconfigured token.
func TestEcosystem_RejectedCredentialSaysSo(t *testing.T) {
	ctx := context.Background()
	for _, status := range []int{401, 403} {
		nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
		hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
		h := ecoHandler(t, nexus, nil, hexis)
		nexus.SetFault(status)

		reply := h.handleHexisAct(ctx, actDec("muzick indexer"))
		if !strings.Contains(reply, "токен") {
			t.Fatalf("http %d must read as a credential problem, got %q", status, reply)
		}
		tr := tracesFor(t, h, "nexus", "resolve")
		if len(tr) != 1 || tr[0].Status != traceRefused || tr[0].HTTPStatus != status {
			t.Fatalf("http %d must trace as refused with its status, got %+v", status, tr)
		}
	}
}

// TestEcosystem_MalformedPraxisBodyDegrades: Praxis has the same decode path
// Nexus does, and a 200 carrying garbage there is a dependency failure too.
func TestEcosystem_MalformedPraxisBodyDegrades(t *testing.T) {
	ctx := context.Background()
	praxis := newFakePraxis(t, fixturePraxisAttentionItems(map[string]any{
		"id": "item_1", "title": "disk almost full", "importance": 3.0,
	}))
	h := ecoHandler(t, nil, praxis, nil)

	praxis.SetBody(`[{"title":`)
	reply := h.handlePraxisAct(ctx, praxisActDec("list_attention"))
	if reply == "" {
		t.Fatal("a malformed praxis body must not answer with silence")
	}
	if strings.Contains(reply, "disk almost full") {
		t.Fatalf("a malformed body must not produce content, got %q", reply)
	}
}

// TestEcosystem_MalformedNexusResponseFailsClosed: a 200 carrying garbage is a
// dependency failure, not "no such entity". It must stop before Hexis.
func TestEcosystem_MalformedNexusResponseFailsClosed(t *testing.T) {
	ctx := context.Background()
	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
	h := ecoHandler(t, nexus, nil, hexis)

	nexus.SetBody(`{"status":"resolved","entity":`)
	reply := h.handleHexisAct(ctx, actDec("muzick indexer"))
	if reply == "" || actRan(reply) {
		t.Fatalf("malformed nexus body must degrade, got %q", reply)
	}
	if hexis.Count("", "/api/v1") != 0 {
		t.Fatal("hexis must not be contacted after a malformed nexus response")
	}
}

// TestEcosystem_UnknownContractFieldsTolerated: a newer Nexus adding fields
// must not break an older Maven. Same for the older flat resolve shape.
func TestEcosystem_UnknownContractFieldsTolerated(t *testing.T) {
	ctx := context.Background()
	for name, body := range map[string]string{
		"future": fixtureNexusResolvedFuture("ent_muzick", "Muzick indexer", "service"),
		"flat":   fixtureNexusResolvedFlat("ent_muzick", "Muzick indexer", "service"),
	} {
		t.Run(name, func(t *testing.T) {
			nexus := newFakeNexus(t, body)
			hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
			h := ecoHandler(t, nexus, nil, hexis)
			if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !actRan(reply) {
				t.Fatalf("%s contract shape must still resolve and execute, got %q", name, reply)
			}
		})
	}
}

// TestEcosystem_CancelledContextDegrades: a caller hanging up (turn abandoned,
// deadline hit) must surface as degradation, never as a fabricated result.
func TestEcosystem_CancelledContextDegrades(t *testing.T) {
	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
	h := ecoHandler(t, nexus, nil, hexis)
	nexus.SetDelay(2 * time.Second)

	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
	defer cancel()
	reply := h.handleHexisAct(ctx, actDec("muzick indexer"))
	if reply == "" || actRan(reply) {
		t.Fatalf("cancelled resolve must degrade, got %q", reply)
	}
	if hexis.Count("", "/api/v1") != 0 {
		t.Fatal("hexis must not be contacted after a cancelled resolve")
	}
}

// TestEcosystem_ExecutionFailureIsNotSuccess: Hexis answering 200 with
// status=failed is a partial failure — the call worked, the command did not.
// Maven must report it as a failure and must not write a success trace.
func TestEcosystem_ExecutionFailureIsNotSuccess(t *testing.T) {
	ctx := context.Background()
	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecutionFailed("exec_1", "unit not found"))
	h := ecoHandler(t, nexus, nil, hexis)

	reply := h.handleHexisAct(ctx, actDec("muzick indexer"))
	if actRan(reply) {
		t.Fatalf("failed execution must not read as success, got %q", reply)
	}
	if reply == "" {
		t.Fatal("failed execution must say something")
	}
	for _, tr := range tracesFor(t, h, "hexis", "execute") {
		if tr.Status == traceOK {
			t.Fatalf("failed execution must not write a success trace: %+v", tr)
		}
	}
}

// TestEcosystem_SuccessfulActionWritesATrace is the positive half the failure
// assertions above depend on: without it, "no success trace" passes with the
// trace writer deleted. It was, for a while — both writers used a fact kind the
// store's CHECK constraint rejects and the error was discarded.
func TestEcosystem_SuccessfulActionWritesATrace(t *testing.T) {
	ctx := context.Background()
	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
	h := ecoHandler(t, nexus, nil, hexis)

	if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !actRan(reply) {
		t.Fatalf("setup: expected success, got %q", reply)
	}
	exec := tracesFor(t, h, "hexis", "execute")
	if len(exec) != 1 || exec[0].Status != traceOK {
		t.Fatalf("a successful execution must leave exactly one ok trace, got %+v", exec)
	}
	if exec[0].CorrelationID == "" {
		t.Error("a trace with no correlation id cannot be stitched to anything")
	}
}

// TestEcosystem_TracesStayOutOfFacts: traces are written at machine rate and
// facts at human rate. One act turn used to write four fact rows, which pushed
// his facts out of every bounded reader (the habit profile's window, memeval's
// prompt, /dash, /history).
func TestEcosystem_TracesStayOutOfFacts(t *testing.T) {
	ctx := context.Background()
	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
	h := ecoHandler(t, nexus, nil, hexis)

	if reply := h.handleHexisAct(ctx, actDec("muzick indexer")); !actRan(reply) {
		t.Fatalf("setup: expected success, got %q", reply)
	}
	if len(traces(t, h)) == 0 {
		t.Fatal("setup: expected traces")
	}
	facts, err := h.dataStore.RecentFacts(ctx, 100)
	if err != nil {
		t.Fatalf("read facts: %v", err)
	}
	if len(facts) != 0 {
		t.Fatalf("an ecosystem act must write no facts at all, got %+v", facts)
	}
}

// TestEcosystem_AmbiguousTargetBlocksExecution: ambiguity blocks mutation, and
// the clarification must name the candidates rather than pick one.
func TestEcosystem_AmbiguousTargetBlocksExecution(t *testing.T) {
	ctx := context.Background()
	nexus := newFakeNexus(t, fixtureNexusAmbiguous(
		map[string]string{"entity_id": "ent_a", "display_name": "Muzick indexer"},
		map[string]string{"entity_id": "ent_b", "display_name": "Muzick web"},
	))
	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
	h := ecoHandler(t, nexus, nil, hexis)

	reply := h.handleHexisAct(ctx, actDec("muzick"))
	if !strings.Contains(reply, "Muzick indexer") || !strings.Contains(reply, "Muzick web") {
		t.Fatalf("ambiguous resolve must list candidates, got %q", reply)
	}
	if hexis.Count("POST", "/api/v1/execute") != 0 {
		t.Fatal("ambiguous target must never execute")
	}
}

// TestEcosystem_NoAutonomousPraxisToHexis: reading the attention digest is an
// observation. Maven must never turn an observed problem into a Hexis command
// by herself — she is not autonomous.
func TestEcosystem_NoAutonomousPraxisToHexis(t *testing.T) {
	ctx := context.Background()
	praxis := newFakePraxis(t, fixturePraxisAttentionItems(
		map[string]any{"id": "item_1", "title": "muzick indexer is down", "importance": 4.0, "rule": "service_down"},
	))
	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
	h := ecoHandler(t, nexus, praxis, hexis)

	_ = h.handlePraxisAct(ctx, praxisActDec("list_attention"))
	if hexis.Count("", "/api/v1") != 0 {
		t.Fatal("attention digest must not contact hexis on its own")
	}
	if nexus.Count("", "/api/v1/resolve") != 0 {
		t.Fatal("attention digest must not resolve targets for autonomous action")
	}
}

// TestEcosystem_MutatingCapabilityWaitsForConfirmation: a non-read-only
// capability parks for an explicit spoken confirm bound to capability+target.
func TestEcosystem_MutatingCapabilityWaitsForConfirmation(t *testing.T) {
	ctx := context.Background()
	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
	caps := fixtureHexisCapabilities(map[string]any{"id": "cap_restart", "name": "restart", "read_only": false})
	hexis := newFakeHexis(t, caps, fixtureHexisExecuted("exec_1", "succeeded"))
	h := ecoHandler(t, nexus, nil, hexis)

	reply := h.handleHexisAct(ctx, actDec("restart"))
	if !strings.Contains(reply, "restart") || !strings.Contains(reply, "да") {
		t.Fatalf("mutating capability must ask for confirmation, got %q", reply)
	}
	if hexis.Count("POST", "/api/v1/execute") != 0 {
		t.Fatal("mutating capability must not execute before confirmation")
	}
	h.mu.Lock()
	pending := h.pendingHexis
	h.mu.Unlock()
	if pending == nil || pending.capabilityID != "cap_restart" || pending.entityID != "ent_muzick" {
		t.Fatalf("confirmation must be bound to capability+target, got %+v", pending)
	}
}

// TestEcosystem_SurfaceFailureStillDelivers: surfacing is bookkeeping. If the
// surface call fails the digest must still be spoken — a partial failure
// downgrades bookkeeping, not the answer.
func TestEcosystem_SurfaceFailureStillDelivers(t *testing.T) {
	ctx := context.Background()
	praxis := newFakePraxis(t, fixturePraxisAttentionItems(
		map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0},
	))
	praxis.SetRouteFault("/api/v1/tools/surface", 500)
	h := ecoHandler(t, nil, praxis, nil)

	reply := h.handlePraxisAct(ctx, praxisActDec("list_attention"))
	if !strings.Contains(reply, "disk almost full") {
		t.Fatalf("failed surface must not swallow the digest, got %q", reply)
	}
	if praxis.Count("POST", "/api/v1/tools/surface") == 0 {
		t.Fatal("expected the surface attempt")
	}
}

// TestEcosystem_TotalOutageSaysSoForEveryPath: with all three down, every
// entry point degrades explicitly instead of returning empty or inventing.
func TestEcosystem_TotalOutageSaysSoForEveryPath(t *testing.T) {
	ctx := context.Background()
	nexus := newFakeNexus(t, fixtureNexusResolved("ent_muzick", "Muzick indexer", "service"))
	praxis := newFakePraxis(t, fixturePraxisAttentionItems())
	hexis := newFakeHexis(t, restartCaps(), fixtureHexisExecuted("exec_1", "succeeded"))
	for _, fs := range []*fakeServer{nexus, praxis, hexis} {
		fs.SetFault(503)
	}
	h := ecoHandler(t, nexus, praxis, hexis)

	for name, reply := range map[string]string{
		"hexis act":   h.handleHexisAct(ctx, actDec("muzick indexer")),
		"attention":   h.handlePraxisAct(ctx, praxisActDec("list_attention")),
		"changes":     h.handlePraxisAct(ctx, praxisActDec("list_changes")),
		"acknowledge": h.handlePraxisAct(ctx, praxisItemDec("acknowledge_item", "item_1")),
	} {
		if reply == "" {
			t.Errorf("%s: total outage must not answer with silence", name)
		}
		if actRan(reply) {
			t.Errorf("%s: total outage must not claim success: %q", name, reply)
		}
	}
	for _, tr := range traces(t, h) {
		if tr.Status == traceOK {
			t.Fatalf("a total outage must not leave success traces behind: %+v", tr)
		}
	}
	if len(tracesFor(t, h, "praxis", "acknowledge")) == 0 {
		t.Fatal("the acknowledge arm must reach praxis and record the refusal")
	}
}

// TestEcosystem_RecoveryAfterOutageNeedsNoRestart: once the dependency comes
// back the very next turn works — no cached failure state, no restart.
func TestEcosystem_RecoveryAfterOutageNeedsNoRestart(t *testing.T) {
	ctx := context.Background()
	praxis := newFakePraxis(t, fixturePraxisAttentionItems(
		map[string]any{"id": "item_1", "title": "disk almost full", "importance": 3.0},
	))
	h := ecoHandler(t, nil, praxis, nil)

	praxis.SetFault(503)
	if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); strings.Contains(reply, "disk") {
		t.Fatalf("outage must not serve content, got %q", reply)
	}
	praxis.SetFault(0)
	if reply := h.handlePraxisAct(ctx, praxisActDec("list_attention")); !strings.Contains(reply, "disk almost full") {
		t.Fatalf("recovery must work on the next turn, got %q", reply)
	}
}