package memory import ( "context" "encoding/json" "errors" "strings" "testing" "time" "github.com/kami/maven/internal/llm" ) type locativeCompleteFunc func(context.Context, llm.Req) (string, error) func (f locativeCompleteFunc) Complete(ctx context.Context, req llm.Req) (string, error) { return f(ctx, req) } func TestLocativeAnswerVerifierRequestAndVerdict(t *testing.T) { var got llm.Req v := NewLocativeAnswerVerifier(locativeCompleteFunc(func(_ context.Context, req llm.Req) (string, error) { got = req return `{"target":"настройки nginx","memory_subject":"конфиг nginx","answer":"yes"}`, nil })) verdict, err := v.Evaluate(context.Background(), "где настройки nginx?", "конфиг nginx лежит в /etc/nginx") if err != nil { t.Fatalf("Evaluate: %v", err) } if !verdict.Answerable || verdict.Target != "настройки nginx" || verdict.MemorySubject != "конфиг nginx" { t.Fatalf("verdict = %+v", verdict) } if got.Grammar != locativeVerifierGrammar || got.MaxTokens != locativeVerifierMaxTokens || got.RepeatPenalty != 1.1 { t.Fatalf("request bounds drifted: %+v", got) } var input map[string]string if err := json.Unmarshal([]byte(got.User), &input); err != nil { t.Fatalf("user input is not JSON: %v", err) } if len(input) != 2 || input["question"] != "где настройки nginx?" || input["memory"] != "конфиг nginx лежит в /etc/nginx" { t.Fatalf("model saw fields outside question+memory: %#v", input) } if strings.Contains(got.System, "nginx") { t.Fatal("held-out entity leaked into the static verifier prompt") } } func TestLocativeAnswerVerifierNoAndMalformedFailClosed(t *testing.T) { for _, tc := range []struct { name string raw string want bool err bool }{ {"no", `{"target":"паспорт","memory_subject":"ключ","answer":"no"}`, false, false}, {"bare yes", `yes`, false, true}, {"unknown answer", `{"target":"a","memory_subject":"b","answer":"maybe"}`, false, true}, {"empty target", `{"target":"","memory_subject":"b","answer":"yes"}`, false, true}, {"extra field", `{"target":"a","memory_subject":"b","answer":"yes","why":"guess"}`, false, true}, {"trailing object", `{"target":"a","memory_subject":"b","answer":"yes"}{}`, false, true}, } { t.Run(tc.name, func(t *testing.T) { v := NewLocativeAnswerVerifier(locativeCompleteFunc(func(context.Context, llm.Req) (string, error) { return tc.raw, nil })) got, err := v.Answerable(context.Background(), "q", "m") if got != tc.want || (err != nil) != tc.err { t.Fatalf("Answerable = %v, %v; want %v, err=%v", got, err, tc.want, tc.err) } }) } } func TestLocativeAnswerVerifierUnavailableErrorAndTimeout(t *testing.T) { if v := NewLocativeAnswerVerifier(nil); v != nil { t.Fatal("nil resident model produced a verifier") } boom := errors.New("llama down") v := NewLocativeAnswerVerifier(locativeCompleteFunc(func(context.Context, llm.Req) (string, error) { return "", boom })) if ok, err := v.Answerable(context.Background(), "q", "m"); ok || !errors.Is(err, boom) { t.Fatalf("model error = %v, %v; want false wrapping %v", ok, err, boom) } v = NewLocativeAnswerVerifier(locativeCompleteFunc(func(ctx context.Context, _ llm.Req) (string, error) { <-ctx.Done() return "", ctx.Err() })) ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) defer cancel() if ok, err := v.Answerable(ctx, "q", "m"); ok || !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("timeout = %v, %v; want false deadline", ok, err) } } func TestLocativeVerifierGrammarIsFullyBounded(t *testing.T) { if strings.Contains(locativeVerifierGrammar, "*") || strings.Contains(locativeVerifierGrammar, "+") { t.Fatalf("grammar contains an unbounded repetition:\n%s", locativeVerifierGrammar) } for _, bound := range []string{"{1,80}", "{0,2}"} { if !strings.Contains(locativeVerifierGrammar, bound) { t.Errorf("grammar missing bound %s", bound) } } }