Score the recall fixture and write up what it shows
Real recall is 48% after the gate, and one must-be-silent query gets an answer anyway. Review finding 2 (the score distributions overlap, so no gate separates a real recall from a false one) and finding 4 (the memStore branch at voice.go:776 is unreachable for notes). Adds an embedder cache so the gate sweep does not re-embed the fixture nine times. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CGeSZxh1DCtRxmFVSYVGvJ
This commit is contained in:
@@ -104,6 +104,34 @@ func InMemory() (memory.Store, func(), error) {
|
||||
return memory.NewInMemoryStore(), func() {}, nil
|
||||
}
|
||||
|
||||
// Cache wraps an embedder so repeated text is embedded once. The gate sweep
|
||||
// scores the same fixture at nine thresholds, and every case re-inserts the
|
||||
// filler notes — without this the ONNX run spends minutes re-embedding
|
||||
// identical strings. Latency numbers come from the uncached run.
|
||||
func Cache(inner router.Embedder) router.Embedder {
|
||||
return &cachingEmbedder{inner: inner, seen: map[string][]float32{}}
|
||||
}
|
||||
|
||||
type cachingEmbedder struct {
|
||||
inner router.Embedder
|
||||
seen map[string][]float32
|
||||
}
|
||||
|
||||
func (c *cachingEmbedder) Dim() int { return c.inner.Dim() }
|
||||
func (c *cachingEmbedder) Close() error { return nil } // the caller owns inner
|
||||
|
||||
func (c *cachingEmbedder) Embed(ctx context.Context, text string) ([]float32, error) {
|
||||
if v, ok := c.seen[text]; ok {
|
||||
return v, nil
|
||||
}
|
||||
v, err := c.inner.Embed(ctx, text)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.seen[text] = v
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// Outcome — one scored case.
|
||||
type Outcome struct {
|
||||
Case Case
|
||||
|
||||
@@ -268,7 +268,9 @@ func TestONNXRecall(t *testing.T) {
|
||||
t.Fatalf("Score: %v", err)
|
||||
}
|
||||
t.Log("\n" + rep.String() + rep.Failures())
|
||||
t.Log("\ngate sweep:\n" + sweep(t, emb, f))
|
||||
// Cached for the sweep only: the headline run above must pay the real
|
||||
// embedder cost so its latency numbers mean something.
|
||||
t.Log("\ngate sweep:\n" + sweep(t, Cache(emb), f))
|
||||
}
|
||||
|
||||
// sweep scores the fixture at a range of gates and renders one line each. Two
|
||||
|
||||
Reference in New Issue
Block a user