Maven/internal/router/eval/llmrouter_test.go

package eval

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/kami/maven/internal/llm"
	"github.com/kami/maven/internal/router"
)

// TestLLMRouterBaseline — the other half of Vikunja #319: the resident model as
// route decider, scored on the same 76 held-out cases as the classifier, so the
// #320 flip is a comparison and not a preference.
//
// Opt-in against a running llama-server:
//
//	llama-server -m /mnt/hdd1/llms/qwen3.5/Qwen3.5-0.8B.Q4_K_M.gguf \
//	  --host 127.0.0.1 --port 18099 -c 2048 -ngl 99
//	MAVEN_LLM_URL=http://127.0.0.1:18099 make eval-models
//
// Every report name carries the model llama-server reports over /v1/models, so
// a bake-off across checkpoints (#278, #250) produces tables you can tell
// apart. Point the variable at one server at a time.
//
// Three configurations, because "the LLM router" is ambiguous and the three
// numbers answer different questions:
//
//	llm-only        — the model alone. Measures the prompt + grammar contract.
//	cascade+llm     — what #320 would actually ship: stage-0 grammar, then the
//	                  model, then the classifier as the failure floor.
//	llm-no-thinking — diagnostic only, not a shippable path (see below).
func TestLLMRouterBaseline(t *testing.T) {
	base := os.Getenv("MAVEN_LLM_URL")
	if base == "" {
		t.Skip("MAVEN_LLM_URL unset — start llama-server and point it here (see doc comment)")
	}
	// A local llama-server must not go through an HTTP proxy. This box has
	// http_proxy pointing at a SOCKS bridge that answers 503 for the loopback
	// address, which would score every case as a route error and read as "the
	// model can't route" — measure the model, not the proxy.
	noProxyLoopback(t)

	f, err := Load()
	if err != nil {
		t.Fatalf("Load: %v", err)
	}
	client := llm.New(base, 60*time.Second)
	if err := ping(context.Background(), client); err != nil {
		t.Skipf("llama-server at %s unreachable: %v", base, err)
	}

	ctx := context.Background()
	model, err := ModelID(ctx, base)
	if err != nil {
		// Not fatal: an unlabelled score is still a score. But say so loudly,
		// because an unlabelled row in a bake-off table is worthless.
		t.Logf("could not read model id from %s: %v — reports will say %q", base, err, "unknown-model")
		model = "unknown-model"
	}
	t.Logf("scoring model %s at %s", model, base)
	lr := router.NewLLMRouter(client)

	// llm-only: the LLM stage in isolation. Route returns (Decision, ok, err);
	// !ok without an error would be a contract violation, so it is surfaced as
	// one rather than silently scored as a miss.
	llmOnly := RouterFunc(func(ctx context.Context, u string, now time.Time) (router.Decision, error) {
		d, ok, err := lr.Route(ctx, u, now)
		if err != nil {
			return d, err
		}
		if !ok {
			return d, fmt.Errorf("llm router declined without an error")
		}
		return d, nil
	})
	repLLM, err := Score(ctx, "llm-only ("+model+", as deployed)", llmOnly, f)
	if err != nil {
		t.Fatalf("Score llm-only: %v", err)
	}
	t.Log("\n" + repLLM.String() + repLLM.Failures())

	// cascade+llm: stage-0 grammar → LLM → classifier fallback, the wiring #320
	// proposes. Hash embedder for the fallback so the classifier contribution is
	// the deterministic floor and any lift is attributable to the model.
	repCascade, err := Score(ctx, "cascade+llm ("+model+") + hash fallback",
		newBaselineRouter(t, router.NewHashEmbedder(1024), lr), f)
	if err != nil {
		t.Fatalf("Score cascade: %v", err)
	}
	t.Log("\n" + repCascade.String() + repCascade.Failures())

	// llm-no-thinking: same prompt and grammar with the chat template's
	// thinking mode off. Qwen3.5's template defaults thinking=1, so under a
	// grammar the constrained JSON lands in reasoning_content with content
	// empty — llm.Client's ReasoningContent fallback is what makes the router
	// work at all today, by accident rather than design.
	//
	// MEASURED 2026-07-31: this variant scores identically to as-deployed
	// (18/76, 48.7% intent-only, 2 errors, same p50). Thinking mode is a
	// non-issue under a grammar — llama.cpp constrains the same token stream
	// either way. Kept so the question stays answered instead of being
	// re-asked, and so internal/llm does NOT grow a chat_template_kwargs field
	// for a problem that does not exist.
	repNoThink, err := Score(ctx, "llm-only ("+model+", thinking off) [diagnostic]",
		RouterFunc(func(ctx context.Context, u string, now time.Time) (router.Decision, error) {
			d, ok, err := router.NewLLMRouter(&noThinkCompleter{base: base, http: &http.Client{Timeout: 60 * time.Second}}).Route(ctx, u, now)
			if err != nil {
				return d, err
			}
			if !ok {
				return d, fmt.Errorf("llm router declined without an error")
			}
			return d, nil
		}), f)
	if err != nil {
		t.Fatalf("Score no-thinking: %v", err)
	}
	t.Log("\n" + repNoThink.String() + repNoThink.Failures())

	// Reports rather than asserts — the numbers are inputs to the #320
	// decision, and an assertion here would be this test inventing the bar.
	// The one thing worth failing on is a harness fault: if every single case
	// errors, the run measured infrastructure, not routing, and the report
	// must not be mistaken for a score.
	for _, rep := range []Report{repLLM, repCascade, repNoThink} {
		if rep.Errors == rep.Total {
			t.Errorf("%s: all %d cases errored — harness fault, not a measurement", rep.Name, rep.Total)
		}
	}
}

// noThinkCompleter — llm.Client with chat_template_kwargs.enable_thinking
// false. A test-local copy rather than a change to internal/llm: whether the
// daemon should send it is the open question, and answering it here by adding
// the field would prejudge #320.
type noThinkCompleter struct {
	base string
	http *http.Client
}

func (c *noThinkCompleter) Complete(ctx context.Context, r llm.Req) (string, error) {
	payload := map[string]any{
		"messages": []map[string]string{
			{"role": "system", "content": r.System},
			{"role": "user", "content": r.User},
		},
		"max_tokens":           r.MaxTokens,
		"temperature":          0,
		"grammar":              r.Grammar,
		"chat_template_kwargs": map[string]any{"enable_thinking": false},
	}
	b, err := json.Marshal(payload)
	if err != nil {
		return "", err
	}
	req, err := http.NewRequestWithContext(ctx, "POST", c.base+"/v1/chat/completions", bytes.NewReader(b))
	if err != nil {
		return "", err
	}
	req.Header.Set("Content-Type", "application/json")
	resp, err := c.http.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", fmt.Errorf("status %d", resp.StatusCode)
	}
	var out struct {
		Choices []struct {
			Message struct {
				Content          string `json:"content"`
				ReasoningContent string `json:"reasoning_content"`
			} `json:"message"`
		} `json:"choices"`
	}
	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
		return "", err
	}
	if len(out.Choices) == 0 {
		return "", fmt.Errorf("no choices")
	}
	m := out.Choices[0].Message
	if m.Content != "" {
		return m.Content, nil
	}
	return m.ReasoningContent, nil
}

func ping(ctx context.Context, c *llm.Client) error {
	ctx, cancel := context.WithTimeout(ctx, 90*time.Second)
	defer cancel()
	_, err := c.Complete(ctx, llm.Req{User: "ping", MaxTokens: 4})
	return err
}

// noProxyLoopback appends the loopback host to no_proxy before any request, so
// http.ProxyFromEnvironment (which caches the environment on first use) sees it.
func noProxyLoopback(t *testing.T) {
	t.Helper()
	for _, key := range []string{"no_proxy", "NO_PROXY"} {
		cur := os.Getenv(key)
		if strings.Contains(cur, "127.0.0.1") {
			continue
		}
		if cur == "" {
			t.Setenv(key, "127.0.0.1,localhost")
			continue
		}
		t.Setenv(key, cur+",127.0.0.1,localhost")
	}
}