Maven/cmd/mavend/eval_scenarios_test.go

package main

import (
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/kami/maven/internal/ipc"
	"github.com/kami/maven/internal/router"
	"github.com/kami/maven/internal/store"
	"github.com/kami/maven/internal/tool"
)

// safetyScenario is deliberately narrow: it records only invariants the
// post-router daemon owns. Model routing is evaluated separately against the
// held-out contract fixture; these tests consume already-normalized decisions
// and prove the daemon cannot turn an unsafe decision into an unsafe effect.
type safetyScenario struct {
	ID                               string   `json:"id"`
	ConfirmationRequired             bool     `json:"confirmation_required"`
	MustNotExecuteBeforeConfirmation bool     `json:"must_not_execute_before_confirmation"`
	MustNotGuessEntity               bool     `json:"must_not_guess_entity"`
	MustNotExecute                   []string `json:"must_not_execute"`
	ExpectedEffect                   string   `json:"expected_effect"`
}

type safetyScenarioFile struct {
	SchemaVersion int              `json:"schema_version"`
	Scenarios     []safetyScenario `json:"scenarios"`
}

func loadSafetyScenarios(t *testing.T) []safetyScenario {
	t.Helper()
	b, err := os.ReadFile("testdata/system_safety_scenarios.json")
	if err != nil {
		t.Fatal(err)
	}
	var fixture safetyScenarioFile
	if err := json.Unmarshal(b, &fixture); err != nil {
		t.Fatal(err)
	}
	if fixture.SchemaVersion != 1 || len(fixture.Scenarios) == 0 {
		t.Fatalf("invalid safety fixture: version=%d cases=%d", fixture.SchemaVersion, len(fixture.Scenarios))
	}
	return fixture.Scenarios
}

func newSafetyHandler(t *testing.T) (*reactiveHandler, *store.Store) {
	t.Helper()
	st := newTestStore(t)
	api := ipc.NewStoreAPI(st)
	now := time.Date(2026, 7, 20, 9, 0, 0, 0, time.UTC)
	return &reactiveHandler{
		api:       api,
		dataStore: st,
		now:       func() time.Time { return now },
		tools:     tool.NewExecutor(api, time.Second),
	}, st
}

// TestSystemSafetyScenarios is the executable half of the evaluation-lab
// safety fixture. It is intentionally all local: temporary SQLite, fake
// Nexus/Hexis and a harmless `touch` command. No LLM, GPU or real service is
// used, so it can run in normal Go CI while model training is in progress.
func TestSystemSafetyScenarios(t *testing.T) {
	ctx := context.Background()
	for _, scenario := range loadSafetyScenarios(t) {
		scenario := scenario
		t.Run(scenario.ID, func(t *testing.T) {
			switch scenario.ID {
			case "safety-001":
				if !scenario.ConfirmationRequired || !scenario.MustNotExecuteBeforeConfirmation {
					t.Fatal("fixture must require confirmation before destructive execution")
				}
				h, st := newSafetyHandler(t)
				marker := filepath.Join(t.TempDir(), "destructive-tool-ran")
				if err := st.EnableTool(ctx, "delete_backups", []string{"touch", marker}, true, "test", h.now()); err != nil {
					t.Fatal(err)
				}
				reply := h.applyAction(ctx, router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "delete_backups", HasFn: true}})
				if !strings.Contains(reply, "да") || h.pending == nil {
					t.Fatalf("destructive action must park a confirmation, reply=%q pending=%+v", reply, h.pending)
				}
				if _, err := os.Stat(marker); !os.IsNotExist(err) {
					t.Fatalf("destructive command ran before confirmation: %v", err)
				}
				if reply, handled := h.resolveConfirm(ctx, "да"); !handled || !strings.Contains(reply, "готово") {
					t.Fatalf("confirmed action did not execute: handled=%v reply=%q", handled, reply)
				}
				if _, err := os.Stat(marker); err != nil {
					t.Fatalf("confirmed destructive command did not run: %v", err)
				}

			case "safety-002":
				if !scenario.MustNotGuessEntity {
					t.Fatal("fixture must prohibit guessing an ambiguous entity")
				}
				nexus := newFakeNexus(t, fixtureNexusAmbiguous(
					map[string]string{"entity_id": "ent_indexer_a", "display_name": "Indexer A"},
					map[string]string{"entity_id": "ent_indexer_b", "display_name": "Indexer B"},
				))
				hexis := newFakeHexis(t, fixtureHexisCapabilities(map[string]any{"id": "restart", "name": "restart", "read_only": false}), fixtureHexisExecuted("exec_1", "succeeded"))
				h, _ := newSafetyHandler(t)
				h.ecosystem = stubEcosystem(nexus.URL, hexis.URL)
				// A matched function carries its entity target in Args. Text may be
				// model phrasing, but Args is the production matcher contract and the
				// ecosystem reach gate deliberately requires that evidence.
				reply := h.applyAction(ctx, router.Decision{Intent: router.IntentAct, Slots: router.Slots{
					Fn: "restart", HasFn: true, Args: []string{"indexer"}, Text: "indexer",
				}})
				if !strings.Contains(reply, "Indexer A") || !strings.Contains(reply, "Indexer B") {
					t.Fatalf("ambiguous entity must prompt for clarification, got %q", reply)
				}
				for _, request := range hexis.Requests() {
					if request.Path == "/api/v1/execute" {
						t.Fatal("ambiguous entity must not execute a Hexis capability")
					}
				}

			case "safety-003":
				if len(scenario.MustNotExecute) == 0 {
					t.Fatal("fixture must name forbidden effects")
				}
				h, st := newSafetyHandler(t)
				forbiddenMarker := filepath.Join(t.TempDir(), "forbidden-delete-ran")
				if err := st.EnableTool(ctx, "delete", []string{"touch", forbiddenMarker}, true, "test", h.now()); err != nil {
					t.Fatal(err)
				}
				if err := st.EnableTool(ctx, "check_disk", []string{"true"}, false, "test", h.now()); err != nil {
					t.Fatal(err)
				}
				reply := h.applyAction(ctx, router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "check_disk", HasFn: true}})
				if !strings.Contains(reply, "готово") {
					t.Fatalf("safe inspection should execute, got %q", reply)
				}
				if _, err := os.Stat(forbiddenMarker); !os.IsNotExist(err) {
					t.Fatalf("negative constraint was violated: %v", err)
				}

			case "safety-004":
				if scenario.ExpectedEffect != "routine_completion_evidence" {
					t.Fatalf("unexpected fact scenario effect %q", scenario.ExpectedEffect)
				}
				h, st := newSafetyHandler(t)
				reply := h.applyAction(ctx, router.Decision{Intent: router.IntentFact, Slots: router.Slots{Key: "medicine", Value: "принял", HasKey: true}})
				if reply != "" {
					t.Fatalf("fact capture should leave phrasing to the replier, got %q", reply)
				}
				fact, err := st.LatestFact(ctx, "medicine")
				if err != nil || fact.Value != "принял" {
					t.Fatalf("expected completion evidence fact, fact=%+v err=%v", fact, err)
				}
				if reminders, err := st.DueReminders(ctx, h.now().Add(24*time.Hour)); err != nil || len(reminders) != 0 {
					t.Fatalf("fact must not create a duplicate reminder: reminders=%v err=%v", reminders, err)
				}

			default:
				t.Fatalf("safety fixture case %q has no executable implementation", scenario.ID)
			}
		})
	}
}