kami
c0c11cd36d
- Add llmphraser: LFM-based phraser implementing Phraser interface with PhraseChat, PhraseNudge, PhraseReactive, and PhraseReminder methods. - Add shared internal/llm/client: llama-server completion client used by both the phraser (talking back) and router (routing), sharing one model. - Add LLMReplier in mavend: replaces StubReplier for chat/nudge/reactive replies, falls back to stub on model errors. - Update Phraser interface: add PhraseChat method, update stub to match. - Wire LLM phaser into mavend voice init, plumb LLM config from JSON.
86 lines
2.3 KiB
Go
86 lines
2.3 KiB
Go
// Package llm is the shared llama-server completion client — one seam both the
|
|
// phraser (talking back) and the router (routing) call. It does NOT spawn the
|
|
// server; the daemon owns one llama-server (spawned by the phraser) and hands
|
|
// its base URL here, so a single resident model serves both callers.
|
|
package llm
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
type Client struct {
|
|
base string
|
|
http *http.Client
|
|
}
|
|
|
|
func New(baseURL string, timeout time.Duration) *Client {
|
|
return &Client{base: baseURL, http: &http.Client{Timeout: timeout}}
|
|
}
|
|
|
|
type Req struct {
|
|
System string
|
|
User string
|
|
Grammar string // GBNF; empty ⇒ unconstrained
|
|
MaxTokens int
|
|
// RepeatPenalty > 0 ⇒ penalize token repetition (curbs the sub-1B "тоже
|
|
// тоже тоже" loop). 0 ⇒ server default (no extra penalty).
|
|
RepeatPenalty float64
|
|
// Stop — sequences that end generation early (e.g. newline for a one-liner).
|
|
Stop []string
|
|
}
|
|
|
|
type msg struct {
|
|
Role string `json:"role"`
|
|
Content string `json:"content"`
|
|
}
|
|
type body struct {
|
|
Messages []msg `json:"messages"`
|
|
MaxTokens int `json:"max_tokens,omitempty"`
|
|
Grammar string `json:"grammar,omitempty"`
|
|
Temp float64 `json:"temperature"`
|
|
RepeatPenalty float64 `json:"repeat_penalty,omitempty"`
|
|
Stop []string `json:"stop,omitempty"`
|
|
}
|
|
type resp struct {
|
|
Choices []struct {
|
|
Message msg `json:"message"`
|
|
} `json:"choices"`
|
|
}
|
|
|
|
func (c *Client) Complete(ctx context.Context, r Req) (string, error) {
|
|
b, _ := json.Marshal(body{
|
|
Messages: []msg{{"system", r.System}, {"user", r.User}},
|
|
MaxTokens: r.MaxTokens,
|
|
Grammar: r.Grammar,
|
|
Temp: 0,
|
|
RepeatPenalty: r.RepeatPenalty,
|
|
Stop: r.Stop,
|
|
})
|
|
req, err := http.NewRequestWithContext(ctx, "POST", c.base+"/v1/chat/completions", bytes.NewReader(b))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
httpResp, err := c.http.Do(req)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer httpResp.Body.Close()
|
|
if httpResp.StatusCode != 200 {
|
|
return "", fmt.Errorf("llm: status %d", httpResp.StatusCode)
|
|
}
|
|
var out resp
|
|
if err := json.NewDecoder(httpResp.Body).Decode(&out); err != nil {
|
|
return "", err
|
|
}
|
|
if len(out.Choices) == 0 {
|
|
return "", fmt.Errorf("llm: no choices")
|
|
}
|
|
return out.Choices[0].Message.Content, nil
|
|
}
|