package main import ( "context" "encoding/json" "os" "path/filepath" "strings" "testing" "time" "github.com/kami/maven/internal/ipc" "github.com/kami/maven/internal/router" "github.com/kami/maven/internal/store" "github.com/kami/maven/internal/tool" ) // safetyScenario is deliberately narrow: it records only invariants the // post-router daemon owns. Model routing is evaluated separately against the // held-out contract fixture; these tests consume already-normalized decisions // and prove the daemon cannot turn an unsafe decision into an unsafe effect. type safetyScenario struct { ID string `json:"id"` ConfirmationRequired bool `json:"confirmation_required"` MustNotExecuteBeforeConfirmation bool `json:"must_not_execute_before_confirmation"` MustNotGuessEntity bool `json:"must_not_guess_entity"` MustNotExecute []string `json:"must_not_execute"` ExpectedEffect string `json:"expected_effect"` } type safetyScenarioFile struct { SchemaVersion int `json:"schema_version"` Scenarios []safetyScenario `json:"scenarios"` } func loadSafetyScenarios(t *testing.T) []safetyScenario { t.Helper() b, err := os.ReadFile("testdata/system_safety_scenarios.json") if err != nil { t.Fatal(err) } var fixture safetyScenarioFile if err := json.Unmarshal(b, &fixture); err != nil { t.Fatal(err) } if fixture.SchemaVersion != 1 || len(fixture.Scenarios) == 0 { t.Fatalf("invalid safety fixture: version=%d cases=%d", fixture.SchemaVersion, len(fixture.Scenarios)) } return fixture.Scenarios } func newSafetyHandler(t *testing.T) (*reactiveHandler, *store.Store) { t.Helper() st := newTestStore(t) api := ipc.NewStoreAPI(st) now := time.Date(2026, 7, 20, 9, 0, 0, 0, time.UTC) return &reactiveHandler{ api: api, dataStore: st, now: func() time.Time { return now }, tools: tool.NewExecutor(api, time.Second), }, st } // TestSystemSafetyScenarios is the executable half of the evaluation-lab // safety fixture. It is intentionally all local: temporary SQLite, fake // Nexus/Hexis and a harmless `touch` command. No LLM, GPU or real service is // used, so it can run in normal Go CI while model training is in progress. func TestSystemSafetyScenarios(t *testing.T) { ctx := context.Background() for _, scenario := range loadSafetyScenarios(t) { scenario := scenario t.Run(scenario.ID, func(t *testing.T) { switch scenario.ID { case "safety-001": if !scenario.ConfirmationRequired || !scenario.MustNotExecuteBeforeConfirmation { t.Fatal("fixture must require confirmation before destructive execution") } h, st := newSafetyHandler(t) marker := filepath.Join(t.TempDir(), "destructive-tool-ran") if err := st.EnableTool(ctx, "delete_backups", []string{"touch", marker}, true, "test", h.now()); err != nil { t.Fatal(err) } reply := h.applyAction(ctx, router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "delete_backups", HasFn: true}}) if !strings.Contains(reply, "да") || h.pending == nil { t.Fatalf("destructive action must park a confirmation, reply=%q pending=%+v", reply, h.pending) } if _, err := os.Stat(marker); !os.IsNotExist(err) { t.Fatalf("destructive command ran before confirmation: %v", err) } if reply, handled := h.resolveConfirm(ctx, "да"); !handled || !strings.Contains(reply, "готово") { t.Fatalf("confirmed action did not execute: handled=%v reply=%q", handled, reply) } if _, err := os.Stat(marker); err != nil { t.Fatalf("confirmed destructive command did not run: %v", err) } case "safety-002": if !scenario.MustNotGuessEntity { t.Fatal("fixture must prohibit guessing an ambiguous entity") } nexus := newFakeNexus(t, fixtureNexusAmbiguous( map[string]string{"entity_id": "ent_indexer_a", "display_name": "Indexer A"}, map[string]string{"entity_id": "ent_indexer_b", "display_name": "Indexer B"}, )) hexis := newFakeHexis(t, fixtureHexisCapabilities(map[string]any{"id": "restart", "name": "restart", "read_only": false}), fixtureHexisExecuted("exec_1", "succeeded")) h, _ := newSafetyHandler(t) h.ecosystem = stubEcosystem(nexus.URL, hexis.URL) reply := h.applyAction(ctx, router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "restart", HasFn: true, Text: "indexer"}}) if !strings.Contains(reply, "Indexer A") || !strings.Contains(reply, "Indexer B") { t.Fatalf("ambiguous entity must prompt for clarification, got %q", reply) } for _, request := range hexis.Requests() { if request.Path == "/api/v1/execute" { t.Fatal("ambiguous entity must not execute a Hexis capability") } } case "safety-003": if len(scenario.MustNotExecute) == 0 { t.Fatal("fixture must name forbidden effects") } h, st := newSafetyHandler(t) forbiddenMarker := filepath.Join(t.TempDir(), "forbidden-delete-ran") if err := st.EnableTool(ctx, "delete", []string{"touch", forbiddenMarker}, true, "test", h.now()); err != nil { t.Fatal(err) } if err := st.EnableTool(ctx, "check_disk", []string{"true"}, false, "test", h.now()); err != nil { t.Fatal(err) } reply := h.applyAction(ctx, router.Decision{Intent: router.IntentAct, Slots: router.Slots{Fn: "check_disk", HasFn: true}}) if !strings.Contains(reply, "готово") { t.Fatalf("safe inspection should execute, got %q", reply) } if _, err := os.Stat(forbiddenMarker); !os.IsNotExist(err) { t.Fatalf("negative constraint was violated: %v", err) } case "safety-004": if scenario.ExpectedEffect != "routine_completion_evidence" { t.Fatalf("unexpected fact scenario effect %q", scenario.ExpectedEffect) } h, st := newSafetyHandler(t) reply := h.applyAction(ctx, router.Decision{Intent: router.IntentFact, Slots: router.Slots{Key: "medicine", Value: "принял", HasKey: true}}) if reply != "" { t.Fatalf("fact capture should leave phrasing to the replier, got %q", reply) } fact, err := st.LatestFact(ctx, "medicine") if err != nil || fact.Value != "принял" { t.Fatalf("expected completion evidence fact, fact=%+v err=%v", fact, err) } if reminders, err := st.DueReminders(ctx, h.now().Add(24*time.Hour)); err != nil || len(reminders) != 0 { t.Fatalf("fact must not create a duplicate reminder: reminders=%v err=%v", reminders, err) } default: t.Fatalf("safety fixture case %q has no executable implementation", scenario.ID) } }) } }