Maven/internal/memory/locativeverify_test.go

package memory

import (
	"context"
	"encoding/json"
	"errors"
	"strings"
	"testing"
	"time"

	"github.com/kami/maven/internal/llm"
)

type locativeCompleteFunc func(context.Context, llm.Req) (string, error)

func (f locativeCompleteFunc) Complete(ctx context.Context, req llm.Req) (string, error) {
	return f(ctx, req)
}

func TestLocativeAnswerVerifierRequestAndVerdict(t *testing.T) {
	var got llm.Req
	v := NewLocativeAnswerVerifier(locativeCompleteFunc(func(_ context.Context, req llm.Req) (string, error) {
		got = req
		return `{"target":"настройки nginx","memory_subject":"конфиг nginx","answer":"yes"}`, nil
	}))
	verdict, err := v.Evaluate(context.Background(), "где настройки nginx?", "конфиг nginx лежит в /etc/nginx")
	if err != nil {
		t.Fatalf("Evaluate: %v", err)
	}
	if !verdict.Answerable || verdict.Target != "настройки nginx" || verdict.MemorySubject != "конфиг nginx" {
		t.Fatalf("verdict = %+v", verdict)
	}
	if got.Grammar != locativeVerifierGrammar || got.MaxTokens != locativeVerifierMaxTokens || got.RepeatPenalty != 1.1 {
		t.Fatalf("request bounds drifted: %+v", got)
	}
	var input map[string]string
	if err := json.Unmarshal([]byte(got.User), &input); err != nil {
		t.Fatalf("user input is not JSON: %v", err)
	}
	if len(input) != 2 || input["question"] != "где настройки nginx?" || input["memory"] != "конфиг nginx лежит в /etc/nginx" {
		t.Fatalf("model saw fields outside question+memory: %#v", input)
	}
	if strings.Contains(got.System, "nginx") {
		t.Fatal("held-out entity leaked into the static verifier prompt")
	}
}

func TestLocativeAnswerVerifierNoAndMalformedFailClosed(t *testing.T) {
	for _, tc := range []struct {
		name string
		raw  string
		want bool
		err  bool
	}{
		{"no", `{"target":"паспорт","memory_subject":"ключ","answer":"no"}`, false, false},
		{"bare yes", `yes`, false, true},
		{"unknown answer", `{"target":"a","memory_subject":"b","answer":"maybe"}`, false, true},
		{"empty target", `{"target":"","memory_subject":"b","answer":"yes"}`, false, true},
		{"extra field", `{"target":"a","memory_subject":"b","answer":"yes","why":"guess"}`, false, true},
		{"trailing object", `{"target":"a","memory_subject":"b","answer":"yes"}{}`, false, true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			v := NewLocativeAnswerVerifier(locativeCompleteFunc(func(context.Context, llm.Req) (string, error) {
				return tc.raw, nil
			}))
			got, err := v.Answerable(context.Background(), "q", "m")
			if got != tc.want || (err != nil) != tc.err {
				t.Fatalf("Answerable = %v, %v; want %v, err=%v", got, err, tc.want, tc.err)
			}
		})
	}
}

func TestLocativeAnswerVerifierUnavailableErrorAndTimeout(t *testing.T) {
	if v := NewLocativeAnswerVerifier(nil); v != nil {
		t.Fatal("nil resident model produced a verifier")
	}
	boom := errors.New("llama down")
	v := NewLocativeAnswerVerifier(locativeCompleteFunc(func(context.Context, llm.Req) (string, error) {
		return "", boom
	}))
	if ok, err := v.Answerable(context.Background(), "q", "m"); ok || !errors.Is(err, boom) {
		t.Fatalf("model error = %v, %v; want false wrapping %v", ok, err, boom)
	}

	v = NewLocativeAnswerVerifier(locativeCompleteFunc(func(ctx context.Context, _ llm.Req) (string, error) {
		<-ctx.Done()
		return "", ctx.Err()
	}))
	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond)
	defer cancel()
	if ok, err := v.Answerable(ctx, "q", "m"); ok || !errors.Is(err, context.DeadlineExceeded) {
		t.Fatalf("timeout = %v, %v; want false deadline", ok, err)
	}
}

func TestLocativeVerifierGrammarIsFullyBounded(t *testing.T) {
	if strings.Contains(locativeVerifierGrammar, "*") || strings.Contains(locativeVerifierGrammar, "+") {
		t.Fatalf("grammar contains an unbounded repetition:\n%s", locativeVerifierGrammar)
	}
	for _, bound := range []string{"{1,80}", "{0,2}"} {
		if !strings.Contains(locativeVerifierGrammar, bound) {
			t.Errorf("grammar missing bound %s", bound)
		}
	}
}