// Package llm is the shared llama-server completion client — one seam both the // phraser (talking back) and the router (routing) call. It does NOT spawn the // server; the daemon owns one llama-server (spawned by the phraser) and hands // its base URL here, so a single resident model serves both callers. package llm import ( "bytes" "context" "encoding/json" "fmt" "net/http" "time" ) type Client struct { base string http *http.Client } func New(baseURL string, timeout time.Duration) *Client { return &Client{base: baseURL, http: &http.Client{Timeout: timeout}} } type Req struct { System string User string Grammar string // GBNF; empty ⇒ unconstrained MaxTokens int // RepeatPenalty > 0 ⇒ penalize token repetition (curbs the sub-1B "тоже // тоже тоже" loop). 0 ⇒ server default (no extra penalty). RepeatPenalty float64 // Stop — sequences that end generation early (e.g. newline for a one-liner). Stop []string } type msg struct { Role string `json:"role"` Content string `json:"content"` Reasoning string `json:"reasoning,omitempty"` ReasoningContent string `json:"reasoning_content,omitempty"` } type body struct { Messages []msg `json:"messages"` MaxTokens int `json:"max_tokens,omitempty"` Grammar string `json:"grammar,omitempty"` Temp float64 `json:"temperature"` RepeatPenalty float64 `json:"repeat_penalty,omitempty"` Stop []string `json:"stop,omitempty"` } type resp struct { Choices []struct { Message msg `json:"message"` } `json:"choices"` } func (c *Client) Complete(ctx context.Context, r Req) (string, error) { b, _ := json.Marshal(body{ Messages: []msg{{Role: "system", Content: r.System}, {Role: "user", Content: r.User}}, MaxTokens: r.MaxTokens, Grammar: r.Grammar, Temp: 0, RepeatPenalty: r.RepeatPenalty, Stop: r.Stop, }) req, err := http.NewRequestWithContext(ctx, "POST", c.base+"/v1/chat/completions", bytes.NewReader(b)) if err != nil { return "", err } req.Header.Set("Content-Type", "application/json") httpResp, err := c.http.Do(req) if err != nil { return "", err } defer httpResp.Body.Close() if httpResp.StatusCode != 200 { return "", fmt.Errorf("llm: status %d", httpResp.StatusCode) } var out resp if err := json.NewDecoder(httpResp.Body).Decode(&out); err != nil { return "", err } if len(out.Choices) == 0 { return "", fmt.Errorf("llm: no choices") } content := out.Choices[0].Message.Content if content == "" { content = out.Choices[0].Message.ReasoningContent } return content, nil }