Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bce5ed210c | |||
| c31f0d1001 |
@@ -175,3 +175,87 @@ func TestLLMRouterLLMError(t *testing.T) {
|
|||||||
t.Fatal("want ok=false, err!=nil on llm error")
|
t.Fatal("want ok=false, err!=nil on llm error")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- slot extraction on top of an LLM decision --------------------------------
|
||||||
|
|
||||||
|
// newLLMTestRouter — a router whose route always comes from the mock model.
|
||||||
|
func newLLMTestRouter(t *testing.T, out string) *Router {
|
||||||
|
t.Helper()
|
||||||
|
c := NewClassifier(NewHashEmbedder(1024))
|
||||||
|
seedClassifier(t, c)
|
||||||
|
acts := DefaultActMatcher{Fns: []string{"restart", "stop", "run", "backup"}}
|
||||||
|
return New(Config{
|
||||||
|
Classifier: c,
|
||||||
|
Extractor: Extractor{Time: StubDateTimeParser{}, Acts: acts, Facts: DefaultFactParser{}},
|
||||||
|
Threshold: 0.4,
|
||||||
|
LLM: NewLLMRouter(mockLLM{out: out}),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// The model cannot produce a fire time, so without extraction every LLM-routed
|
||||||
|
// reminder was dropped as "no time".
|
||||||
|
func TestLLMDecisionGetsReminderTime(t *testing.T) {
|
||||||
|
r := newLLMTestRouter(t, `{"intent":"reminder","text":"позвонить маме"}`)
|
||||||
|
d, err := r.Route(context.Background(), "напомни позвонить маме через 2 часа", refNow())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("route: %v", err)
|
||||||
|
}
|
||||||
|
if d.Intent != IntentReminder {
|
||||||
|
t.Fatalf("want reminder, got %v", d.Intent)
|
||||||
|
}
|
||||||
|
if !d.Slots.HasTime || !d.Slots.Time.Equal(refNow().Add(2*time.Hour)) {
|
||||||
|
t.Fatalf("want time now+2h, got %+v", d.Slots)
|
||||||
|
}
|
||||||
|
if d.Slots.Text != "позвонить маме" {
|
||||||
|
t.Fatalf("extraction overwrote the model's text: %q", d.Slots.Text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// No time in the utterance ⇒ no time in the slots. Do not invent one; the
|
||||||
|
// daemon says it could not read the time.
|
||||||
|
func TestLLMReminderWithoutTimeStaysEmpty(t *testing.T) {
|
||||||
|
r := newLLMTestRouter(t, `{"intent":"reminder","text":"позвонить маме"}`)
|
||||||
|
d, err := r.Route(context.Background(), "напомни позвонить маме", refNow())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("route: %v", err)
|
||||||
|
}
|
||||||
|
if d.Slots.HasTime {
|
||||||
|
t.Fatalf("invented a time: %v", d.Slots.Time)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// An act decision arrived with no Fn, so the tool never ran.
|
||||||
|
func TestLLMDecisionGetsActFn(t *testing.T) {
|
||||||
|
r := newLLMTestRouter(t, `{"intent":"act","verb":"restart nginx"}`)
|
||||||
|
d, err := r.Route(context.Background(), "слушай, restart nginx пожалуйста", refNow())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("route: %v", err)
|
||||||
|
}
|
||||||
|
if !d.Slots.HasFn || d.Slots.Fn != "restart" || len(d.Slots.Args) != 1 || d.Slots.Args[0] != "nginx" {
|
||||||
|
t.Fatalf("want fn=restart args=[nginx], got %+v", d.Slots)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The model's own slots win; extraction only fills gaps.
|
||||||
|
func TestLLMSlotsWinOverExtraction(t *testing.T) {
|
||||||
|
r := newLLMTestRouter(t, `{"intent":"fact","key":"hydration","value":"выпил"}`)
|
||||||
|
d, err := r.Route(context.Background(), "я выпил воду", refNow())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("route: %v", err)
|
||||||
|
}
|
||||||
|
if d.Slots.Key != "hydration" {
|
||||||
|
t.Fatalf("extraction overwrote the model's key: %q", d.Slots.Key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A fact the model left keyless still gets one from the parser.
|
||||||
|
func TestLLMFactGetsKeyFromParser(t *testing.T) {
|
||||||
|
r := newLLMTestRouter(t, `{"intent":"fact","text":"я выпил воду"}`)
|
||||||
|
d, err := r.Route(context.Background(), "я выпил воду", refNow())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("route: %v", err)
|
||||||
|
}
|
||||||
|
if !d.Slots.HasKey || d.Slots.Key != "water" {
|
||||||
|
t.Fatalf("want key=water, got %+v", d.Slots)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -88,6 +88,7 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De
|
|||||||
if r.llm != nil {
|
if r.llm != nil {
|
||||||
if d, ok, err := r.llm.Route(ctx, utterance, now); err == nil && ok {
|
if d, ok, err := r.llm.Route(ctx, utterance, now); err == nil && ok {
|
||||||
d.Utterance = utterance
|
d.Utterance = utterance
|
||||||
|
r.fillSlots(ctx, &d, now)
|
||||||
return d, nil
|
return d, nil
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
log.Printf("router: llm route fell back to classifier: %v", err)
|
log.Printf("router: llm route fell back to classifier: %v", err)
|
||||||
@@ -118,6 +119,39 @@ func (r *Router) Route(ctx context.Context, utterance string, now time.Time) (De
|
|||||||
return d, nil
|
return d, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// fillSlots — run stage-2 extraction on an LLM decision and fill only the slots
|
||||||
|
// the model left empty. The LLM wins where it answered: it saw the sentence, the
|
||||||
|
// parsers are keyword tables. Extraction covers what the model cannot produce at
|
||||||
|
// all — a parsed reminder time and an allowlist fn.
|
||||||
|
//
|
||||||
|
// If a reminder still has no time, leave it missing. The daemon then says it
|
||||||
|
// could not read the time; inventing one would set a wrong alarm.
|
||||||
|
func (r *Router) fillSlots(ctx context.Context, d *Decision, now time.Time) {
|
||||||
|
ex := r.extractor.Extract(ctx, d.Intent, d.Utterance, now)
|
||||||
|
if !d.Slots.HasTime && ex.HasTime {
|
||||||
|
d.Slots.Time, d.Slots.HasTime = ex.Time, ex.HasTime
|
||||||
|
}
|
||||||
|
if !d.Slots.HasKey && ex.HasKey {
|
||||||
|
d.Slots.Key, d.Slots.Value, d.Slots.HasKey = ex.Key, ex.Value, ex.HasKey
|
||||||
|
}
|
||||||
|
if !d.Slots.HasFn && ex.HasFn {
|
||||||
|
d.Slots.Fn, d.Slots.Args, d.Slots.HasFn = ex.Fn, ex.Args, ex.HasFn
|
||||||
|
}
|
||||||
|
// For an act the model returns the verb in Text ("restart nginx"), which is
|
||||||
|
// often cleaner than the raw utterance ("maven, could you restart nginx").
|
||||||
|
// Try it too when the utterance did not match the allowlist.
|
||||||
|
if d.Intent == IntentAct && !d.Slots.HasFn && r.extractor.Acts != nil &&
|
||||||
|
d.Slots.Text != "" && d.Slots.Text != d.Utterance {
|
||||||
|
if fn, args, ok := r.extractor.Acts.Match(d.Slots.Text); ok {
|
||||||
|
d.Slots.Fn, d.Slots.Args, d.Slots.HasFn = fn, args, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if d.Slots.Text == "" {
|
||||||
|
d.Slots.Text = ex.Text
|
||||||
|
}
|
||||||
|
// Stage stays 1: it says who decided the route, and that was the LLM.
|
||||||
|
}
|
||||||
|
|
||||||
// CorrectMisroute — the user corrected a bad classification. Appends a new
|
// CorrectMisroute — the user corrected a bad classification. Appends a new
|
||||||
// example for the corrected intent (append-only — grows the classifier, no
|
// example for the corrected intent (append-only — grows the classifier, no
|
||||||
// retrain). Same shape as nudges.outcome tuning cooldowns: more reliable over
|
// retrain). Same shape as nudges.outcome tuning cooldowns: more reliable over
|
||||||
|
|||||||
Reference in New Issue
Block a user