Gate recall on the margin over the runner-up, not just the score
The e5 embedder puts every cosine in one narrow band (0.79-0.89), so the absolute query_min_score gate cannot tell a real hit from a made-up question: any value under the band answers everything, any value above it answers nothing. False recall was 5/5. New gate asks whether one note is clearly the best instead: top1 - top2 > delta. New query_min_margin config knob, default 0.008, read off the sweep in the recall harness. The absolute floor stays as a second check. On the recall fixture with e5: answered 72% -> 68%, false recall 5/5 -> 1/5. Vikunja #359 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
package memory
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestConfidentScores(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
scores []float64
|
||||
minScore float64
|
||||
minMargin float64
|
||||
want bool
|
||||
}{
|
||||
{"no hits", nil, 0.55, 0.008, false},
|
||||
{"below the floor", []float64{0.40, 0.10}, 0.55, 0.008, false},
|
||||
{"clear winner", []float64{0.86, 0.70}, 0.55, 0.008, true},
|
||||
{"runner-up too close", []float64{0.860, 0.858}, 0.55, 0.008, false},
|
||||
// The rule is "beats the runner-up by MORE than delta". Not testing an
|
||||
// exactly-equal margin: no pair of these decimals subtracts to exactly
|
||||
// 0.008 in binary float, so such a test would pin rounding, not the rule.
|
||||
{"margin just under delta", []float64{0.8079, 0.8}, 0.55, 0.008, false},
|
||||
{"margin just over delta", []float64{0.8081, 0.8}, 0.55, 0.008, true},
|
||||
// One hit: nothing to compare against, so only the floor applies.
|
||||
{"single hit clears", []float64{0.86}, 0.55, 0.008, true},
|
||||
{"single hit below floor", []float64{0.10}, 0.55, 0.008, false},
|
||||
// Margin off — the old absolute-only behaviour.
|
||||
{"margin off admits a tie", []float64{0.86, 0.86}, 0.55, 0, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := ConfidentScores(c.scores, c.minScore, c.minMargin); got != c.want {
|
||||
t.Errorf("got %v, want %v", got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfidentReadsResultScores(t *testing.T) {
|
||||
res := []Result{{ID: "a", Score: 0.86}, {ID: "b", Score: 0.858}}
|
||||
if Confident(res, 0.55, 0.008) {
|
||||
t.Error("thin margin passed the gate")
|
||||
}
|
||||
if !Confident(res, 0.55, 0) {
|
||||
t.Error("margin off should fall back to the floor alone")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user