claude
98ab646206
Owner explicitly requested direct commits to master. Keep startup cost benchmarked without turning ambient race/coverage load into a correctness failure; record live reminder proof, stale-task reconciliation, and the temporary delegation quota caveat.
749 lines
29 KiB
Go
749 lines
29 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"math"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/kami/maven/internal/router"
|
|
)
|
|
|
|
func TestPersonalBoundaryLinearHeadSeparatesSemanticDirections(t *testing.T) {
|
|
personal := [][]float32{{1, 0}, {0.9, 0.1}, {0.8, -0.1}}
|
|
world := [][]float32{{-1, 0}, {-0.9, 0.1}, {-0.8, -0.1}}
|
|
head, ok := trainPersonalBoundaryLinearHead(personal, world)
|
|
if !ok {
|
|
t.Fatal("valid training vectors were rejected")
|
|
}
|
|
b := personalBoundary{personal: personal, world: world, head: head, loaded: true}
|
|
for _, tc := range []struct {
|
|
vector []float32
|
|
personal bool
|
|
}{
|
|
{vector: []float32{0.75, 0.2}, personal: true},
|
|
{vector: []float32{-0.75, 0.2}, personal: false},
|
|
} {
|
|
personalScore, worldScore, ok := b.score(tc.vector)
|
|
if !ok {
|
|
t.Fatal("loaded boundary did not score")
|
|
}
|
|
if got := personalScore > worldScore; got != tc.personal {
|
|
t.Fatalf("vector %v classified personal=%v (scores %.4f/%.4f), want %v",
|
|
tc.vector, got, personalScore, worldScore, tc.personal)
|
|
}
|
|
if math.Abs(personalScore+worldScore-1) > 1e-12 {
|
|
t.Fatalf("scores %.8f and %.8f are not complementary probabilities", personalScore, worldScore)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPersonalBoundaryTrainingBalancesClasses(t *testing.T) {
|
|
personal := [][]float32{{1, 0}, {0.8, 0.2}}
|
|
world := [][]float32{{-1, 0}}
|
|
oneWorld, ok := trainPersonalBoundaryLinearHead(personal, world)
|
|
if !ok {
|
|
t.Fatal("valid training vectors were rejected")
|
|
}
|
|
repeatedWorld := make([][]float32, 12)
|
|
for i := range repeatedWorld {
|
|
repeatedWorld[i] = world[0]
|
|
}
|
|
twelveWorld, ok := trainPersonalBoundaryLinearHead(personal, repeatedWorld)
|
|
if !ok {
|
|
t.Fatal("valid repeated training vectors were rejected")
|
|
}
|
|
if math.Abs(oneWorld.bias-twelveWorld.bias) > 1e-10 {
|
|
t.Fatalf("duplicating one class moved bias from %.12f to %.12f", oneWorld.bias, twelveWorld.bias)
|
|
}
|
|
for i := range oneWorld.weights {
|
|
if math.Abs(oneWorld.weights[i]-twelveWorld.weights[i]) > 1e-10 {
|
|
t.Fatalf("duplicating one class moved weight %d from %.12f to %.12f",
|
|
i, oneWorld.weights[i], twelveWorld.weights[i])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPersonalBoundaryTrainingRejectsMixedDimensions(t *testing.T) {
|
|
if _, ok := trainPersonalBoundaryLinearHead(
|
|
[][]float32{{1, 0}},
|
|
[][]float32{{-1, 0, 0}},
|
|
); ok {
|
|
t.Fatal("mixed embedding dimensions were accepted")
|
|
}
|
|
}
|
|
|
|
// The corpus is grouped by sentence shape in personalboundary.go. This test
|
|
// leaves one entire shape out of training at a time, then requires the linear
|
|
// head to classify the omitted examples from the semantics learned from the
|
|
// other shapes. It is ordinary deterministic CI: the small axis vectors stand
|
|
// in for frozen embedding directions, so the test proves the training code
|
|
// generalises across groups rather than memorising one row at a time.
|
|
func TestPersonalBoundaryLinearHeadLeaveOneShapeOut(t *testing.T) {
|
|
type example struct {
|
|
vector []float32
|
|
shape int
|
|
want bool
|
|
}
|
|
const shapeCount = 6
|
|
examples := make([]example, 0, shapeCount*4)
|
|
for shape := 0; shape < shapeCount; shape++ {
|
|
for variant := 0; variant < 2; variant++ {
|
|
personal := make([]float32, shapeCount+1)
|
|
world := make([]float32, shapeCount+1)
|
|
personal[0], world[0] = 1, -1
|
|
personal[shape+1] = float32(0.1 * float64(variant+1))
|
|
world[shape+1] = float32(-0.1 * float64(variant+1))
|
|
examples = append(examples,
|
|
example{vector: personal, shape: shape, want: true},
|
|
example{vector: world, shape: shape, want: false},
|
|
)
|
|
}
|
|
}
|
|
|
|
for omitted := 0; omitted < shapeCount; omitted++ {
|
|
var personal, world [][]float32
|
|
for _, example := range examples {
|
|
if example.shape == omitted {
|
|
continue
|
|
}
|
|
if example.want {
|
|
personal = append(personal, example.vector)
|
|
} else {
|
|
world = append(world, example.vector)
|
|
}
|
|
}
|
|
head, ok := trainPersonalBoundaryLinearHead(personal, world)
|
|
if !ok {
|
|
t.Fatalf("fold %d rejected valid vectors", omitted)
|
|
}
|
|
for _, example := range examples {
|
|
if example.shape != omitted {
|
|
continue
|
|
}
|
|
if got := head.logit(example.vector) > 0; got != example.want {
|
|
t.Errorf("fold %d classified %v as personal=%v, want %v", omitted, example.vector, got, example.want)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPersonalBoundaryTrainingCorpusIsIndependent(t *testing.T) {
|
|
// The strict stratified fixture already enforces this for its 72 rows. The
|
|
// historical regression table lives here, so protect it here too: a future
|
|
// seed addition must not copy a regression sentence into training.
|
|
training := make(map[string]bool, len(personalSeeds)+len(worldSeeds))
|
|
for _, seed := range append(append([]string(nil), personalSeeds...), worldSeeds...) {
|
|
training[normalizePersonalBoundaryTraining(seed)] = true
|
|
}
|
|
for _, regression := range []string{
|
|
"что я говорил про бэкапы?",
|
|
"что я сказал вчера про отпуск",
|
|
"я писал что-нибудь про сервер",
|
|
"я упоминал про конференцию?",
|
|
"что я отмечал по поводу переезда",
|
|
"я рассказывал тебе про новую работу?",
|
|
"во сколько у меня встреча",
|
|
"когда мой следующий отпуск",
|
|
"what did i say about backups",
|
|
"did i tell you about the doctor",
|
|
"как я говорил, почему небо синее",
|
|
"как уже я говорил, какая столица франции",
|
|
"почему трава зелёная",
|
|
"столица франции",
|
|
"как мне сварить борщ",
|
|
"что мне посмотреть вечером",
|
|
"я хочу узнать про рим",
|
|
"кто такой гагарин",
|
|
"how do i boil an egg",
|
|
"во сколько закат сегодня",
|
|
"когда сегодня заканчивается концерт",
|
|
"во сколько завтра открывается аптека",
|
|
"какой сегодня праздник",
|
|
"что интересного произошло сегодня в мире",
|
|
"кто выиграл вчера матч",
|
|
"расскажи про эверест",
|
|
"расскажи про войну 1812 года",
|
|
"объясни что такое инфляция",
|
|
"я рассказывал тебе про байкал?",
|
|
} {
|
|
if training[normalizePersonalBoundaryTraining(regression)] {
|
|
t.Errorf("regression utterance leaked into training: %q", regression)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPersonalBoundaryFrozenHeadDecodes(t *testing.T) {
|
|
head, ok := frozenPersonalBoundaryHead()
|
|
if !ok {
|
|
t.Fatal("frozen head did not decode")
|
|
}
|
|
if len(head.weights) != 384 {
|
|
t.Fatalf("frozen head has %d weights, want 384", len(head.weights))
|
|
}
|
|
}
|
|
|
|
func TestPersonalBoundaryHashFloorFitsAndScores(t *testing.T) {
|
|
b := &personalBoundary{}
|
|
embedder := router.NewHashEmbedder(1024)
|
|
query, err := router.EmbedQuery(context.Background(), embedder, "когда моя встреча")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b.load(context.Background(), embedder)
|
|
if _, _, ok := b.score(query); !ok {
|
|
t.Fatal("hash-floor boundary declined to score")
|
|
}
|
|
if len(b.head.weights) != 1024 {
|
|
t.Fatalf("hash-floor boundary has %d weights, want 1024", len(b.head.weights))
|
|
}
|
|
}
|
|
|
|
// BenchmarkPersonalBoundaryHashFloorFitAndScore keeps startup cost measurable
|
|
// without making ambient CI load a correctness condition. In particular,
|
|
// -race and coverage instrumentation both multiply the cost of this numeric
|
|
// training loop; the functional test above is the deterministic gate.
|
|
func BenchmarkPersonalBoundaryHashFloorFitAndScore(b *testing.B) {
|
|
embedder := router.NewHashEmbedder(1024)
|
|
query, err := router.EmbedQuery(context.Background(), embedder, "когда моя встреча")
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
boundary := &personalBoundary{}
|
|
boundary.load(context.Background(), embedder)
|
|
if _, _, ok := boundary.score(query); !ok {
|
|
b.Fatal("hash-floor boundary declined to score")
|
|
}
|
|
}
|
|
}
|
|
|
|
func normalizePersonalBoundaryTraining(value string) string {
|
|
return strings.Join(strings.Fields(strings.ToLower(value)), " ")
|
|
}
|
|
|
|
// A handler with no embedder never loads the seeds, so the boundary falls back
|
|
// to the possession markers. That is the offline floor and it must keep working
|
|
// — an embedder that fails to load must not open the boundary.
|
|
func TestBoundaryFallsBackToMarkersWithNoEmbedder(t *testing.T) {
|
|
h := personalHandler()
|
|
if !h.isPersonalTurn(context.Background(), &queryTurn{
|
|
dec: router.Decision{Utterance: "во сколько у меня встреча"},
|
|
}) {
|
|
t.Error("no embedder: a possession question must still be personal")
|
|
}
|
|
if h.isPersonalTurn(context.Background(), &queryTurn{
|
|
dec: router.Decision{Utterance: "почему небо синее"},
|
|
}) {
|
|
t.Error("no embedder: a world question must still pass")
|
|
}
|
|
}
|
|
|
|
// TestONNXPersonalBoundary — the number that matters, scored against the
|
|
// embedder homesrv actually runs. Opt-in via MAVEN_ONNX_LIB, exactly like
|
|
// TestONNXRecall in internal/memory/recalleval.
|
|
//
|
|
// Every case here is held out: none of these strings is a seed. The #495
|
|
// regression is the first row — "что я говорил про бэкапы?" reached SearXNG and
|
|
// was answered from a Habr article, and no possession word appears in it.
|
|
func TestONNXPersonalBoundary(t *testing.T) {
|
|
lib := os.Getenv("MAVEN_ONNX_LIB")
|
|
if lib == "" {
|
|
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
|
}
|
|
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
|
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
|
if err != nil {
|
|
t.Skipf("onnx embedder unavailable: %v", err)
|
|
}
|
|
defer emb.Close()
|
|
|
|
cases := []struct {
|
|
utterance string
|
|
personal bool
|
|
}{
|
|
{"что я говорил про бэкапы?", true},
|
|
{"что я сказал вчера про отпуск", true},
|
|
{"я писал что-нибудь про сервер", true},
|
|
{"я упоминал про конференцию?", true},
|
|
{"что я отмечал по поводу переезда", true},
|
|
{"я рассказывал тебе про новую работу?", true},
|
|
{"во сколько у меня встреча", true},
|
|
{"когда мой следующий отпуск", true},
|
|
{"what did i say about backups", true},
|
|
{"did i tell you about the doctor", true},
|
|
{"как я говорил, почему небо синее", false},
|
|
{"как уже я говорил, какая столица франции", false},
|
|
{"почему трава зелёная", false},
|
|
{"столица франции", false},
|
|
{"как мне сварить борщ", false},
|
|
{"что мне посмотреть вечером", false},
|
|
{"я хочу узнать про рим", false},
|
|
{"кто такой гагарин", false},
|
|
{"how do i boil an egg", false},
|
|
// Asking when a public thing happens (Vikunja #553). "во сколько закат
|
|
// сегодня" was answered "не знаю — не нашла у тебя такой записи",
|
|
// because the frame lived only on the personal side. The pair above it
|
|
// is the control: "во сколько у меня встреча" is the same frame about
|
|
// something that IS his, and it has to stay personal.
|
|
{"во сколько закат сегодня", false},
|
|
{"когда сегодня заканчивается концерт", false},
|
|
{"во сколько завтра открывается аптека", false},
|
|
// The "какой сегодня X" frame. These clear the weather topic after the
|
|
// V-553 seeds and were then refused here, which is the same defect one
|
|
// source further down the chain.
|
|
{"какой сегодня праздник", false},
|
|
{"что интересного произошло сегодня в мире", false},
|
|
{"кто выиграл вчера матч", false},
|
|
// The narrative shape, held out from the seeds above (Vikunja #554).
|
|
// The control is the row after them: the same verb about his own words
|
|
// is still his.
|
|
{"расскажи про эверест", false},
|
|
{"расскажи про войну 1812 года", false},
|
|
{"объясни что такое инфляция", false},
|
|
{"я рассказывал тебе про байкал?", true},
|
|
}
|
|
|
|
h := &reactiveHandler{recall: recallWiring{embedder: emb}}
|
|
ctx := context.Background()
|
|
wrong := 0
|
|
for _, c := range cases {
|
|
vec, err := router.EmbedQuery(ctx, emb, c.utterance)
|
|
if err != nil {
|
|
t.Fatalf("embed %q: %v", c.utterance, err)
|
|
}
|
|
turn := &queryTurn{dec: router.Decision{Utterance: c.utterance}, vec: vec}
|
|
got := h.isPersonalTurn(ctx, turn)
|
|
p, w, ok := h.recall.boundary.score(vec)
|
|
if !ok {
|
|
t.Fatal("seeds did not load with a working embedder")
|
|
}
|
|
if got != c.personal {
|
|
wrong++
|
|
t.Errorf("%q: personal=%v want %v (personal %.4f world %.4f)", c.utterance, got, c.personal, p, w)
|
|
}
|
|
t.Logf("personal=%-5v personal %.4f world %.4f delta %+.4f %s", got, p, w, p-w, c.utterance)
|
|
}
|
|
t.Logf("personal boundary: %d/%d held-out utterances correct", len(cases)-wrong, len(cases))
|
|
}
|
|
|
|
func TestONNXPersonalBoundaryFourFold(t *testing.T) {
|
|
lib := os.Getenv("MAVEN_ONNX_LIB")
|
|
if lib == "" {
|
|
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
|
}
|
|
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
|
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
|
if err != nil {
|
|
t.Skipf("onnx embedder unavailable: %v", err)
|
|
}
|
|
defer emb.Close()
|
|
ctx := context.Background()
|
|
embedAll := func(values []string) [][]float32 {
|
|
vectors := make([][]float32, len(values))
|
|
for i, value := range values {
|
|
vector, err := router.EmbedQuery(ctx, emb, value)
|
|
if err != nil {
|
|
t.Fatalf("embed %q: %v", value, err)
|
|
}
|
|
vectors[i] = vector
|
|
}
|
|
return vectors
|
|
}
|
|
personalVectors := embedAll(personalSeeds)
|
|
worldVectors := embedAll(worldSeeds)
|
|
|
|
type group struct {
|
|
name string
|
|
personalStart, personalCount int
|
|
worldStart, worldCount int
|
|
}
|
|
groups := []group{
|
|
{name: "remembered_speech", personalStart: 8, personalCount: 8, worldStart: 20, worldCount: 8},
|
|
{name: "possession", personalStart: 16, personalCount: 10, worldStart: 28, worldCount: 12},
|
|
{name: "narrative", personalStart: 26, personalCount: 8, worldStart: 40, worldCount: 8},
|
|
{name: "first_person_preamble", personalStart: 34, personalCount: 8, worldStart: 48, worldCount: 8},
|
|
{name: "advice_current_info", personalStart: 42, personalCount: 8, worldStart: 56, worldCount: 8},
|
|
{name: "public_proper_nouns", personalStart: 50, personalCount: 10, worldStart: 64, worldCount: 8},
|
|
}
|
|
|
|
const foldCount = 4
|
|
aggregateCorrect, aggregateTotal := 0, 0
|
|
for omittedFold := 0; omittedFold < foldCount; omittedFold++ {
|
|
trainingPersonal := append([][]float32(nil), personalVectors[:8]...)
|
|
trainingWorld := append([][]float32(nil), worldVectors[:20]...)
|
|
var heldPersonal, heldWorld [][]float32
|
|
partition := func(vectors [][]float32, start, count int, training, held *[][]float32) {
|
|
for relative, vector := range vectors[start : start+count] {
|
|
if relative%foldCount == omittedFold {
|
|
*held = append(*held, vector)
|
|
} else {
|
|
*training = append(*training, vector)
|
|
}
|
|
}
|
|
}
|
|
for _, group := range groups {
|
|
partition(personalVectors, group.personalStart, group.personalCount, &trainingPersonal, &heldPersonal)
|
|
partition(worldVectors, group.worldStart, group.worldCount, &trainingWorld, &heldWorld)
|
|
}
|
|
head, ok := trainPersonalBoundaryLinearHead(
|
|
trainingPersonal,
|
|
trainingWorld,
|
|
)
|
|
if !ok {
|
|
t.Fatalf("fold %d: valid training fold rejected", omittedFold)
|
|
}
|
|
correct, total := 0, 0
|
|
for _, vector := range heldPersonal {
|
|
total++
|
|
if head.logit(vector) > 0 {
|
|
correct++
|
|
}
|
|
}
|
|
for _, vector := range heldWorld {
|
|
total++
|
|
if head.logit(vector) <= 0 {
|
|
correct++
|
|
}
|
|
}
|
|
t.Logf("fold %d: %d/%d held-out training examples", omittedFold+1, correct, total)
|
|
aggregateCorrect += correct
|
|
aggregateTotal += total
|
|
}
|
|
t.Logf("four-fold aggregate: %d/%d", aggregateCorrect, aggregateTotal)
|
|
if aggregateCorrect < 99 {
|
|
t.Errorf("four-fold aggregate %d/%d, want at least 99/104", aggregateCorrect, aggregateTotal)
|
|
}
|
|
}
|
|
|
|
func TestONNXPersonalBoundarySemanticGroupHoldout(t *testing.T) {
|
|
lib := os.Getenv("MAVEN_ONNX_LIB")
|
|
if lib == "" {
|
|
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
|
}
|
|
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
|
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
|
if err != nil {
|
|
t.Skipf("onnx embedder unavailable: %v", err)
|
|
}
|
|
defer emb.Close()
|
|
ctx := context.Background()
|
|
embedAll := func(values []string) [][]float32 {
|
|
vectors := make([][]float32, len(values))
|
|
for i, value := range values {
|
|
vector, err := router.EmbedQuery(ctx, emb, value)
|
|
if err != nil {
|
|
t.Fatalf("embed %q: %v", value, err)
|
|
}
|
|
vectors[i] = vector
|
|
}
|
|
return vectors
|
|
}
|
|
personalVectors := embedAll(personalSeeds)
|
|
worldVectors := embedAll(worldSeeds)
|
|
|
|
type group struct {
|
|
name string
|
|
personalStart, personalCount int
|
|
worldStart, worldCount int
|
|
}
|
|
groups := []group{
|
|
{name: "remembered_speech", personalStart: 8, personalCount: 8, worldStart: 20, worldCount: 8},
|
|
{name: "possession", personalStart: 16, personalCount: 10, worldStart: 28, worldCount: 12},
|
|
{name: "narrative", personalStart: 26, personalCount: 8, worldStart: 40, worldCount: 8},
|
|
{name: "first_person_preamble", personalStart: 34, personalCount: 8, worldStart: 48, worldCount: 8},
|
|
{name: "advice_current_info", personalStart: 42, personalCount: 8, worldStart: 56, worldCount: 8},
|
|
{name: "public_proper_nouns", personalStart: 50, personalCount: 10, worldStart: 64, worldCount: 8},
|
|
}
|
|
|
|
aggregateCorrect, aggregateTotal := 0, 0
|
|
for _, omitted := range groups {
|
|
excluding := func(vectors [][]float32, start, count int) [][]float32 {
|
|
result := make([][]float32, 0, len(vectors)-count)
|
|
result = append(result, vectors[:start]...)
|
|
return append(result, vectors[start+count:]...)
|
|
}
|
|
head, ok := trainPersonalBoundaryLinearHead(
|
|
excluding(personalVectors, omitted.personalStart, omitted.personalCount),
|
|
excluding(worldVectors, omitted.worldStart, omitted.worldCount),
|
|
)
|
|
if !ok {
|
|
t.Fatalf("%s: valid training fold rejected", omitted.name)
|
|
}
|
|
correct, total := 0, 0
|
|
for _, vector := range personalVectors[omitted.personalStart : omitted.personalStart+omitted.personalCount] {
|
|
total++
|
|
if head.logit(vector) > 0 {
|
|
correct++
|
|
}
|
|
}
|
|
for _, vector := range worldVectors[omitted.worldStart : omitted.worldStart+omitted.worldCount] {
|
|
total++
|
|
if head.logit(vector) <= 0 {
|
|
correct++
|
|
}
|
|
}
|
|
t.Logf("leave %-21s out: %d/%d", omitted.name, correct, total)
|
|
aggregateCorrect += correct
|
|
aggregateTotal += total
|
|
// Whole-shape holdout is an honest diagnostic, not a 100% release gate:
|
|
// some shapes (notably private-vs-general possession) define a distinct
|
|
// semantic ambiguity. The separately authored challenge set remains the
|
|
// strict generalisation gate.
|
|
}
|
|
if aggregateCorrect < 92 {
|
|
t.Errorf("whole-shape aggregate %d/%d, want at least 92/104", aggregateCorrect, aggregateTotal)
|
|
}
|
|
}
|
|
|
|
// This challenge set was originally authored after the six-shape training
|
|
// corpus and the 72-case matrix were frozen. Its sole miss then informed the
|
|
// regularisation comparison, so it is now a strict regression gate rather than
|
|
// independent evidence. It remains outside the production corpus.
|
|
func TestONNXPersonalBoundaryChallenge(t *testing.T) {
|
|
lib := os.Getenv("MAVEN_ONNX_LIB")
|
|
if lib == "" {
|
|
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
|
}
|
|
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
|
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
|
if err != nil {
|
|
t.Skipf("onnx embedder unavailable: %v", err)
|
|
}
|
|
defer emb.Close()
|
|
|
|
cases := []struct {
|
|
utterance string
|
|
personal bool
|
|
}{
|
|
{"какой пароль приложения я задал для почтового клиента?", true},
|
|
{"на каком порту я решил поднять тестовый сервис?", true},
|
|
{"какую причину я указал, когда отменил бронь?", true},
|
|
{"где в гараже я сложил зимние шины?", true},
|
|
{"какой сериал я бросил после второго сезона?", true},
|
|
{"о чём мы договорились с Олегом на прошлой неделе?", true},
|
|
{"почему мой монитор мерцает при частоте 144 герца?", false},
|
|
{"подойдёт ли кабель Thunderbolt 3 к разъёму USB4?", false},
|
|
{"как вывести запах дыма из моей куртки?", false},
|
|
{"что означают кольца на флаге Олимпиады?", false},
|
|
{"почему после дождя на асфальте видна радуга?", false},
|
|
{"какой формат файлов поддерживает Kindle Paperwhite?", false},
|
|
{"which SSH key did I install on the build server?", true},
|
|
{"what spending limit did I set for the travel card?", true},
|
|
{"where did I store the spare apartment fob?", true},
|
|
{"which objection did I raise during the design review?", true},
|
|
{"what route did I plan for the Sunday hike?", true},
|
|
{"when did I promise Maya I would send the draft?", true},
|
|
{"why does my mechanical keyboard sometimes chatter?", false},
|
|
{"can my USB-C charger safely power a Steam Deck?", false},
|
|
{"how do I stop condensation inside my camera lens?", false},
|
|
{"what caused the Tacoma Narrows Bridge to collapse?", false},
|
|
{"why are some auroras red instead of green?", false},
|
|
{"which codecs does the current Firefox release support?", false},
|
|
}
|
|
|
|
b := &personalBoundary{}
|
|
b.load(context.Background(), emb)
|
|
correct := 0
|
|
minimumMargin := math.Inf(1)
|
|
for _, testCase := range cases {
|
|
vector, err := router.EmbedQuery(context.Background(), emb, testCase.utterance)
|
|
if err != nil {
|
|
t.Fatalf("embed %q: %v", testCase.utterance, err)
|
|
}
|
|
personal, world, ok := b.score(vector)
|
|
if !ok {
|
|
t.Fatal("loaded boundary declined to score")
|
|
}
|
|
got := personal > world
|
|
signedMargin := personal - world
|
|
if !testCase.personal {
|
|
signedMargin = -signedMargin
|
|
}
|
|
if signedMargin < minimumMargin {
|
|
minimumMargin = signedMargin
|
|
}
|
|
if got == testCase.personal {
|
|
correct++
|
|
} else {
|
|
t.Logf("miss %q: personal=%v want %v (%.4f/%.4f)", testCase.utterance, got, testCase.personal, personal, world)
|
|
}
|
|
}
|
|
t.Logf("regularisation challenge: %d/%d, minimum signed margin %+.4f", correct, len(cases), minimumMargin)
|
|
if correct != len(cases) {
|
|
t.Errorf("regularisation challenge %d/%d, want every case correct", correct, len(cases))
|
|
}
|
|
}
|
|
|
|
// TestONNXPersonalBoundaryPostRetuneChallenge was authored only after the L2
|
|
// coefficient and frozen head had been selected using corpus cross-validation.
|
|
// It deliberately returns to private configuration, commitments and stored
|
|
// choices with new objects, and contrasts them with public technical facts,
|
|
// compatibility and maintenance. No result from this table may be used to
|
|
// tune the current head; a miss is evidence for the next independently
|
|
// evaluated model revision.
|
|
func TestONNXPersonalBoundaryPostRetuneChallenge(t *testing.T) {
|
|
lib := os.Getenv("MAVEN_ONNX_LIB")
|
|
if lib == "" {
|
|
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
|
}
|
|
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
|
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
|
if err != nil {
|
|
t.Skipf("onnx embedder unavailable: %v", err)
|
|
}
|
|
defer emb.Close()
|
|
|
|
cases := []struct {
|
|
utterance string
|
|
personal bool
|
|
}{
|
|
{"какое имя я выбрал для гостевой сети Wi-Fi?", true},
|
|
{"на какой день я перенёс техосмотр машины?", true},
|
|
{"какую сумму мы с Мариной согласовали за ремонт кухни?", true},
|
|
{"где я сохранил резервные коды от GitHub?", true},
|
|
{"какой из макетов визитки я одобрил?", true},
|
|
{"что я решил делать со страховкой перед поездкой?", true},
|
|
{"какой диапазон частот использует Wi-Fi 6E?", false},
|
|
{"почему OLED-экраны со временем выгорают?", false},
|
|
{"можно ли подключить монитор DisplayPort к Thunderbolt 4?", false},
|
|
{"чем безопасно чистить замшевые ботинки?", false},
|
|
{"когда появился протокол WebSocket?", false},
|
|
{"почему соль ускоряет таяние льда?", false},
|
|
{"which hostname did I assign to the home NAS?", true},
|
|
{"what date did I move the annual checkup to?", true},
|
|
{"where did I save the recovery phrase for the hardware wallet?", true},
|
|
{"which catering quote did we accept for the party?", true},
|
|
{"what did I decide about renewing the domain?", true},
|
|
{"which paint sample did I approve for the hallway?", true},
|
|
{"does Wi-Fi 7 work with older wireless clients?", false},
|
|
{"why can an SSD slow down when it is nearly full?", false},
|
|
{"how should suede shoes be cleaned?", false},
|
|
{"when was the WebSocket protocol standardized?", false},
|
|
{"what does a hardware-wallet recovery phrase do?", false},
|
|
{"why does road salt damage concrete?", false},
|
|
}
|
|
|
|
b := &personalBoundary{}
|
|
b.load(context.Background(), emb)
|
|
correct := 0
|
|
minimumMargin := math.Inf(1)
|
|
for _, testCase := range cases {
|
|
vector, err := router.EmbedQuery(context.Background(), emb, testCase.utterance)
|
|
if err != nil {
|
|
t.Fatalf("embed %q: %v", testCase.utterance, err)
|
|
}
|
|
personal, world, ok := b.score(vector)
|
|
if !ok {
|
|
t.Fatal("loaded boundary declined to score")
|
|
}
|
|
got := personal > world
|
|
signedMargin := personal - world
|
|
if !testCase.personal {
|
|
signedMargin = -signedMargin
|
|
}
|
|
if signedMargin < minimumMargin {
|
|
minimumMargin = signedMargin
|
|
}
|
|
if got == testCase.personal {
|
|
correct++
|
|
} else {
|
|
t.Logf("miss %q: personal=%v want %v (%.4f/%.4f)", testCase.utterance, got, testCase.personal, personal, world)
|
|
}
|
|
}
|
|
t.Logf("post-retune challenge: %d/%d, minimum signed margin %+.4f", correct, len(cases), minimumMargin)
|
|
if correct != len(cases) {
|
|
t.Errorf("post-retune challenge %d/%d, want every case correct", correct, len(cases))
|
|
}
|
|
}
|
|
|
|
func TestONNXPersonalBoundaryLatency(t *testing.T) {
|
|
lib := os.Getenv("MAVEN_ONNX_LIB")
|
|
if lib == "" {
|
|
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
|
}
|
|
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
|
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
|
if err != nil {
|
|
t.Skipf("onnx embedder unavailable: %v", err)
|
|
}
|
|
defer emb.Close()
|
|
ctx := context.Background()
|
|
query, err := router.EmbedQuery(ctx, emb, "что я решил насчёт переезда?")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
b := &personalBoundary{}
|
|
coldStart := time.Now()
|
|
b.load(ctx, emb)
|
|
if _, _, ok := b.score(query); !ok {
|
|
t.Fatal("loaded boundary declined to score")
|
|
}
|
|
cold := time.Since(coldStart)
|
|
|
|
const iterations = 100000
|
|
steadyStart := time.Now()
|
|
for i := 0; i < iterations; i++ {
|
|
if _, _, ok := b.score(query); !ok {
|
|
t.Fatal("loaded boundary declined to score")
|
|
}
|
|
}
|
|
steady := time.Since(steadyStart) / iterations
|
|
t.Logf("boundary cold load+train+score: %s; steady score: %s/op", cold, steady)
|
|
// This is a user-visible first-turn path. Keep a generous ceiling to avoid
|
|
// noisy CI while making an accidental per-turn training/load regression
|
|
// unmistakable.
|
|
if cold > 5*time.Second {
|
|
t.Errorf("cold boundary load %s exceeds 5s local usability ceiling", cold)
|
|
}
|
|
if steady > 100*time.Microsecond {
|
|
t.Errorf("steady boundary score %s exceeds 100µs ceiling", steady)
|
|
}
|
|
}
|
|
|
|
func TestONNXPersonalBoundaryFrozenHeadMatchesCorpusFit(t *testing.T) {
|
|
lib := os.Getenv("MAVEN_ONNX_LIB")
|
|
if lib == "" {
|
|
t.Skip("MAVEN_ONNX_LIB unset — see AGENTS.md § Embedder model for intent routing")
|
|
}
|
|
dir := filepath.Join("../..", "models/embedder/multilingual-e5-small")
|
|
emb, err := router.NewONNXEmbedder(filepath.Join(dir, "model_quantized.onnx"), filepath.Join(dir, "tokenizer.json"), lib)
|
|
if err != nil {
|
|
t.Skipf("onnx embedder unavailable: %v", err)
|
|
}
|
|
defer emb.Close()
|
|
ctx := context.Background()
|
|
embedAll := func(values []string) [][]float32 {
|
|
vectors := make([][]float32, len(values))
|
|
for i, value := range values {
|
|
vector, err := router.EmbedQuery(ctx, emb, value)
|
|
if err != nil {
|
|
t.Fatalf("embed %q: %v", value, err)
|
|
}
|
|
vectors[i] = vector
|
|
}
|
|
return vectors
|
|
}
|
|
fitted, ok := trainPersonalBoundaryLinearHead(embedAll(personalSeeds), embedAll(worldSeeds))
|
|
if !ok {
|
|
t.Fatal("corpus fit failed")
|
|
}
|
|
frozen, ok := frozenPersonalBoundaryHead()
|
|
if !ok {
|
|
t.Fatal("frozen head did not decode")
|
|
}
|
|
if math.Abs(fitted.bias-frozen.bias) > 1e-9 {
|
|
t.Fatalf("frozen bias %.12f != fitted %.12f", frozen.bias, fitted.bias)
|
|
}
|
|
for i := range fitted.weights {
|
|
if math.Abs(fitted.weights[i]-frozen.weights[i]) > 5e-7 {
|
|
t.Fatalf("frozen weight %d %.12f != fitted %.12f", i, frozen.weights[i], fitted.weights[i])
|
|
}
|
|
}
|
|
}
|