Merge: an infrastructure error does not claim the turn (#211)
V-568. queryEmbed claimed the turn on an EmbedQuery error and answered QueryFailAnswer from position 12 of querySources, above memory, notes, the personal boundary, search, Kiwix and general knowledge. So one failing embed call made every question below it answer "не смогла ответить", including the ones search and Kiwix would have answered without the embedder at all. The notes source had the same bug one position lower, on a QueryNotes error. Both now log once and pass. Both also gained an empty-vector guard, because scores off a nil vector are not a "there is nothing" answer. The distinction the audit used: a source that looked and found nothing may claim, a source that could not look must pass. day-plan, habits, feeds, calendar, weather, home, network and web keep claiming, because each already matched a question about his own data and nothing below can answer it. Answering a personal question with a paragraph about the world is V-474 and V-479.
This commit is contained in:
@@ -503,7 +503,15 @@ func (h *reactiveHandler) queryWeather(ctx context.Context, t *queryTurn) (strin
|
||||
|
||||
// queryEmbed isn't an answer source — it's the shared cost the two recall
|
||||
// sources below both need, run once, in the position it always ran in. It
|
||||
// only claims the turn when the embedder fails.
|
||||
// never claims the turn.
|
||||
//
|
||||
// It used to claim on an embedder error, and that made a RAG hint a hard gate
|
||||
// over everything below it (V-568): one failing EmbedQuery and the memory, the
|
||||
// notes, the boundary, the search, the ZIMs, the named page and the model all
|
||||
// answered "не смогла ответить", including the questions search and Kiwix
|
||||
// would have answered without ever touching the embedder. A failed embed means
|
||||
// this source cannot claim, not that the turn is over — same shape as
|
||||
// turnVector in topics.go, which had it right.
|
||||
func (h *reactiveHandler) queryEmbed(ctx context.Context, t *queryTurn) (string, bool) {
|
||||
// A topic source above already paid for this one; see turnVector.
|
||||
if len(t.vec) > 0 {
|
||||
@@ -511,8 +519,11 @@ func (h *reactiveHandler) queryEmbed(ctx context.Context, t *queryTurn) (string,
|
||||
}
|
||||
vec, err := router.EmbedQuery(ctx, h.recall.embedder, t.dec.Utterance)
|
||||
if err != nil {
|
||||
// Logged once, here, and the chain walks on. The two recall sources
|
||||
// below read the empty vector and pass; the boundary drops to its
|
||||
// offline floor.
|
||||
log.Printf("voice: embed query: %v", err)
|
||||
return phraser.Q(phraser.QueryFailAnswer, nil), true
|
||||
return "", false
|
||||
}
|
||||
t.vec = vec
|
||||
return "", false
|
||||
@@ -532,6 +543,12 @@ func (h *reactiveHandler) queryMemory(ctx context.Context, t *queryTurn) (string
|
||||
if h.recall.memStore == nil {
|
||||
return "", false
|
||||
}
|
||||
if len(t.vec) == 0 {
|
||||
// No query vector: the embed above failed or there is no embedder.
|
||||
// Searching on an empty vector is not a search, and its scores are not
|
||||
// a "there is nothing" answer — pass rather than gate the chain.
|
||||
return "", false
|
||||
}
|
||||
hits, herr := h.recall.memStore.Search(ctx, t.vec, 3)
|
||||
if herr != nil {
|
||||
log.Printf("voice: memory search: %v", herr)
|
||||
@@ -578,10 +595,18 @@ func (h *reactiveHandler) queryMemory(ctx context.Context, t *queryTurn) (string
|
||||
// band. See memory.Confident. Failing the gate passes the turn on to general
|
||||
// knowledge, which is what "don't read back the runner-up" means here.
|
||||
func (h *reactiveHandler) queryNotes(ctx context.Context, t *queryTurn) (string, bool) {
|
||||
if len(t.vec) == 0 {
|
||||
// Same reason as queryMemory above (V-568): with no query vector this
|
||||
// source could not look, and could-not-look passes.
|
||||
return "", false
|
||||
}
|
||||
notes, err := h.api.QueryNotes(ctx, t.vec, 5)
|
||||
if err != nil {
|
||||
// The store failed, so this source could not look either. It used to
|
||||
// claim here, which stopped the search, the ZIMs and the model from
|
||||
// answering a question that never needed a note (V-568).
|
||||
log.Printf("voice: query notes: %v", err)
|
||||
return phraser.Q(phraser.QueryFailAnswer, nil), true
|
||||
return "", false
|
||||
}
|
||||
t.notes = notes
|
||||
noteScores := make([]float64, len(notes))
|
||||
|
||||
@@ -2,8 +2,11 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -31,6 +34,57 @@ func (f *fixedEmbedder) Embed(_ context.Context, text string) ([]float32, error)
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// brokenEmbedder fails every call, which is what an ONNX session error looks
|
||||
// like from the query chain's side.
|
||||
type brokenEmbedder struct{}
|
||||
|
||||
func (brokenEmbedder) Dim() int { return 4 }
|
||||
func (brokenEmbedder) Close() error { return nil }
|
||||
func (brokenEmbedder) Embed(context.Context, string) ([]float32, error) {
|
||||
return nil, errors.New("onnx: session failed")
|
||||
}
|
||||
|
||||
// TestQueryEmbedFailureDoesNotStopTheChain — V-568. The embed source used to
|
||||
// claim the turn on an embedder error, so one failing EmbedQuery answered every
|
||||
// question below it with "не смогла ответить", including the ones the search
|
||||
// answers without an embedder at all. A source that could not look must pass.
|
||||
func TestQueryEmbedFailureDoesNotStopTheChain(t *testing.T) {
|
||||
const q = "почему небо голубое"
|
||||
|
||||
h, _ := searchHandler(t, searchBody, http.StatusOK)
|
||||
h.api = ipc.NewStoreAPI(newTestStore(t))
|
||||
h.recall = recallWiring{embedder: brokenEmbedder{}, minScore: 0.55, minMargin: 0.008}
|
||||
h.now = time.Now
|
||||
|
||||
// The embed source itself passes rather than claiming.
|
||||
turn := &queryTurn{dec: router.Decision{Intent: router.IntentQuery, Utterance: q}}
|
||||
if reply, ok := h.queryEmbed(context.Background(), turn); ok {
|
||||
t.Fatalf("queryEmbed claimed the turn on an embedder error: %q", reply)
|
||||
}
|
||||
|
||||
// And the whole chain still reaches the search below it.
|
||||
reply := askQuery(t, h, q)
|
||||
if !strings.Contains(reply, "рэлеевского рассеяния") {
|
||||
t.Fatalf("reply = %q, want the search answer", reply)
|
||||
}
|
||||
}
|
||||
|
||||
// The recall sources read the empty vector the failed embed left behind, and
|
||||
// neither of them may turn that into an answer: no vector means they could not
|
||||
// look, which is not the same as looking and finding nothing.
|
||||
func TestQueryRecallPassesWithoutAVector(t *testing.T) {
|
||||
h, _ := buildRecallHandler(t, "где молоко", []recallCase{
|
||||
{text: "молоко стоит в холодильнике", score: 0.90, kind: "note"},
|
||||
})
|
||||
turn := &queryTurn{dec: router.Decision{Intent: router.IntentQuery, Utterance: "где молоко"}}
|
||||
if reply, ok := h.queryMemory(context.Background(), turn); ok {
|
||||
t.Errorf("queryMemory claimed with no vector: %q", reply)
|
||||
}
|
||||
if reply, ok := h.queryNotes(context.Background(), turn); ok {
|
||||
t.Errorf("queryNotes claimed with no vector: %q", reply)
|
||||
}
|
||||
}
|
||||
|
||||
// scoreVec builds a unit vector whose cosine against the query vector
|
||||
// (1,0,0,0) is exactly score.
|
||||
func scoreVec(score float64) []float32 {
|
||||
|
||||
Reference in New Issue
Block a user