Maven/internal/phraser/eval/talk_test.go

package eval

import (
	"context"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/kami/maven/internal/dialogue"
	"github.com/kami/maven/internal/llm"
	"github.com/kami/maven/internal/persona"
	"github.com/kami/maven/internal/phraser"
	"github.com/kami/maven/internal/router"
)

// perPathMinimum — the resolution floor. A per-path score built on a handful of
// cases moves by 12% when a single reply changes, which cannot distinguish a
// prompt regression from noise.
const perPathMinimum = 8

// TestTalkFixture — the fixture itself has to be sound before any score off it
// means anything.
func TestTalkFixture(t *testing.T) {
	f, err := LoadTalk()
	if err != nil {
		t.Fatalf("LoadTalk: %v", err)
	}

	seen := map[string]bool{}
	byPath := map[string]int{}
	for _, c := range f.Cases {
		if seen[c.ID] {
			t.Errorf("duplicate case id %q", c.ID)
		}
		seen[c.ID] = true

		switch c.Path {
		case PathChat, PathQuery, PathKnowledge:
		case PathReply:
			if c.Intent == "" {
				t.Errorf("%s: reply case has no intent — the replier is phrased from the decision", c.ID)
			}
		default:
			t.Errorf("%s: unknown path %q", c.ID, c.Path)
		}
		byPath[c.Path]++

		if strings.TrimSpace(c.Utterance) == "" {
			t.Errorf("%s: empty utterance", c.ID)
		}
		if len(c.WantAny) == 0 {
			t.Errorf("%s: no want_any — the reply cannot be checked for topic", c.ID)
		}
		// A query case with no notes would silently score the knowledge path.
		if c.Path == PathQuery && len(c.Notes) == 0 {
			t.Errorf("%s: query case has no notes", c.ID)
		}
		if c.Path == PathKnowledge && len(c.Notes) > 0 {
			t.Errorf("%s: knowledge case must have no notes", c.ID)
		}
	}

	for _, p := range TalkPaths {
		if byPath[p] < perPathMinimum {
			t.Errorf("path %s has %d cases, want at least %d", p, byPath[p], perPathMinimum)
		}
	}
}

// fakeTalker — a scripted Talker, so the scorer is testable without a model.
type fakeTalker struct{ reply string }

func (f fakeTalker) PhraseChat(context.Context, string, []dialogue.Turn) (string, error) {
	return f.reply, nil
}

func (f fakeTalker) PhraseQuery(context.Context, string, []string) (string, error) {
	return f.reply, nil
}

func (f fakeTalker) PhraseReply(context.Context, router.Decision) (string, error) {
	return f.reply, nil
}

// TestScoreTalkCounts — a reply that fails on purpose must be counted on every
// path, so a real run cannot report a hidden zero.
func TestScoreTalkCounts(t *testing.T) {
	f, err := LoadTalk()
	if err != nil {
		t.Fatalf("LoadTalk: %v", err)
	}
	// Formal address, off-topic, trailing ellipsis: three checks fail at once.
	rep, err := ScoreTalk(context.Background(), "fake", fakeTalker{"Приходите, я вас жду…"}, f)
	if err != nil {
		t.Fatalf("ScoreTalk: %v", err)
	}
	if rep.Total != len(f.Cases) || rep.Passed != 0 {
		t.Errorf("got %d/%d passing, want 0/%d", rep.Passed, rep.Total, len(f.Cases))
	}
	if rep.ByCheck[CheckAddress] != 0 {
		t.Errorf("formal reply passed the address check %d times", rep.ByCheck[CheckAddress])
	}
	if rep.ByCheck[CheckEllipsis] != 0 {
		t.Errorf("truncated reply passed the ellipsis check %d times", rep.ByCheck[CheckEllipsis])
	}
	for _, p := range TalkPaths {
		if rep.ByPath[p].Total == 0 {
			t.Errorf("path %s missing from the report", p)
		}
	}
	if !strings.Contains(rep.String(), "by path") {
		t.Error("report does not break down by path")
	}
}

// TestLLMTalkBaseline — the resident model on all four phrasing paths.
// Opt-in exactly like TestLLMPhrasingBaseline: CI has no model and a run costs
// minutes on the CPU target.
//
//	MAVEN_LLM_URL=http://127.0.0.1:18099 \
//	  go test -run TestLLMTalkBaseline ./internal/phraser/eval/
//
// Reports, does not assert a quality bar — the numbers are the input to tuning
// the persona prompt. The one thing worth failing on is a harness fault.
func TestLLMTalkBaseline(t *testing.T) {
	base := os.Getenv("MAVEN_LLM_URL")
	if base == "" {
		t.Skip("MAVEN_LLM_URL unset — point it at a running llama-server (see doc comment)")
	}
	noProxyLoopback(t)

	ctx := context.Background()
	f, err := LoadTalk()
	if err != nil {
		t.Fatalf("LoadTalk: %v", err)
	}

	cfg := phraser.DefaultConfig("")
	cfg.Timeout = 5 * time.Minute
	cfg.ContextBlock = func() string { return persona.Facts{}.Block(time.Now()) }
	p := phraser.NewLLMPhraserAt(base, cfg)
	defer p.Close()

	// The model id names the run in the report. Since Vikunja #397 every path
	// returns its errors, so a server that dies mid-run shows up in the Errors
	// column instead of scoring as bad phrasing — the before-and-after probe that
	// used to stand in for that is gone.
	model, err := llm.ModelID(ctx, base)
	if err != nil {
		t.Fatalf("no model at %s: %v", base, err)
	}
	t.Logf("scoring model %s at %s", model, base)

	// The reply path is a separate object in the daemon too: the phraser owns its
	// own llama-server, the replier is handed an llm.Client. Pair scores both.
	block := func() string { return persona.Facts{}.Block(time.Now()) }
	target := Pair{Talker: p, Confirmer: phraser.NewReplier(llm.New(base, cfg.Timeout), block)}

	rep, err := ScoreTalk(ctx, "llm ("+model+", built-in persona)", target, f)
	if err != nil {
		t.Fatalf("ScoreTalk: %v", err)
	}
	t.Log("\n" + rep.String() + "\nreplies:\n" + rep.Replies() + "\nfailures:\n" + rep.Failures())

	// A run where nothing was phrased is not a low score, it is no measurement.
	if rep.Errors == rep.Total {
		t.Fatalf("every case errored — nothing was measured, the score above is not a phrasing result")
	}
	if rep.Errors > 0 {
		t.Logf("%d/%d cases errored — those are model failures, not phrasing failures", rep.Errors, rep.Total)
	}
}