From 6a5121657a4755ef5be4d9a1d675c94284a1817d Mon Sep 17 00:00:00 2001 From: kami Date: Sat, 11 Jul 2026 22:51:50 +0400 Subject: [PATCH] feat: {response,mood} output contract + router removal, TTS piper plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Daemon side of Decision B: parse {"response","mood"} across the 4 consumers (replier, nudges, reminders, chat), fall back to legacy formats. Drop the LLM router — the classifier handles routing; replier/phraser share one llm.Client (timeout 20s->60s). llm.Client reads reasoning_content when content is empty (thinking models). Docs: TTS piper-student plan (OmniVoice teacher -> piper student, from scratch, phoneme-first). CLAUDE.md training guide. Co-Authored-By: Claude Opus 4.8 --- AGENTS.md | 6 +- CLAUDE.md | 121 ++++++ Dockerfile | 1 + cmd/mavend/replier_llm.go | 49 ++- cmd/mavend/replier_llm_test.go | 8 + cmd/mavend/voice.go | 14 +- deploy/mavend.json | 4 +- docs/plans/2026-07-11-ru-cpt-base.md | 413 +++++++++++++++++++++ docs/plans/2026-07-11-tts-piper-student.md | 249 +++++++++++++ internal/delivery/dispatcher.go | 2 + internal/llm/client.go | 14 +- internal/phraser/llmphraser.go | 120 ++++-- internal/router/knowledge.go | 2 +- 13 files changed, 952 insertions(+), 51 deletions(-) create mode 100644 CLAUDE.md create mode 100644 docs/plans/2026-07-11-ru-cpt-base.md create mode 100644 docs/plans/2026-07-11-tts-piper-student.md diff --git a/AGENTS.md b/AGENTS.md index dbea7ec..b65be95 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -74,7 +74,7 @@ sudo cp onnxruntime-linux-x64-1.15.1/lib/libonnxruntime.so* /usr/local/lib/ Without the embedder block, the daemon uses `HashEmbedder` (works, but weak on Russian recall — you may see many "clarify" responses). -## LFM model for router + phraser +## LFM model for router + phraser (Thinking variant) The daemon uses a single resident LFM (sub-1B) for both routing (intent classification + slot extraction) and phrasing (nudges, reminders, reactive @@ -90,8 +90,8 @@ make download-llm Or manually: ```sh -curl -sL "https://huggingface.co/lfm/LFM2.5-1.2B-Instruct-GGUF/resolve/main/LFM2.5-1.2B-Instruct-Q4_K_M.gguf" \ - -o models/llm/LFM2.5-1.2B-Instruct-Q4_K_M.gguf +curl -sL "https://huggingface.co/lfm/LFM2.5-1.2B-Thinking-GGUF/resolve/main/LFM2.5-1.2B-Thinking-Q4_K_M.gguf" \ + -o models/llm/LFM2.5-1.2B-Thinking-Q4_K_M.gguf ``` **Configure in `deploy/mavend.json`** — the `phraser` block points at this diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..26e4838 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,121 @@ +# Maven RU LoRA — training guide + +Fine-tuning a small LLM so Maven speaks correct, in-character Russian (and English) +without breaking Cyrillic. Training scripts live in +`/home/kami/Programs/esp32-whisper-fine-tune/llm`; the model is consumed by this +daemon. + +## Two machines — don't conflate them + +| Role | Box | Specs | Constraint | +|---|---|---|---| +| **Train** | this workstation | RX 7900 GRE, gfx1100 (RDNA3), **16GB VRAM**, ROCm 7.2 / torch 2.10 HIP, `cuda.is_available()==True` | fits 1.5B bf16 LoRA with huge headroom | +| **Deploy** | homesrv `kami@192.168.1.104` | Ryzen 5 5600U (Zen3 6c/12t), Vega iGPU (Cezanne), **13GB RAM, already swapping** (~8.8GB avail), Ubuntu 24.04 | **CPU-only** inference; caps model at ~1.5B for voice latency | + +**Path duality via SSHFS:** `/mnt/server/home/kami/` is an SSHFS mount from homesrv. So when you're in this repo: +- `/home/kami/Programs/` → **local** to workpc +- `/mnt/server/home/kami/` → **remote** homesrv filesystem +- `/mnt/server/home/kami/apps/Maven/` = this repo = remote filesystem accessed locally + +**Inference router** (`localhost:6446`) isn't running on either machine. Check for a compose file (`docker compose up -d`) on whichever box has it before assuming it's available. + +- The Vega iGPU is **not** worth ROCm inference (gfx90c). Inference is CPU llama.cpp; + Vulkan can offload a few layers (`-ngl 4`-ish) for a small speedup. +- homesrv swaps already → every ~200MB of model footprint matters. Keep `--mlock` on. +- A 4B model at any quant is ~5–8 tok/s here = too slow for voice. **Stay ≤1.5B.** + +## Root cause of broken Russian + +Cyrillic breakage is a **base-model + quantization** problem, **not** a missing LoRA: +- Shipped models (`LFM2.5-1.2B-Thinking`, `Nemotron-3-Nano-4B`) are Western, thin-RU, + run at **Q4_K** → mangled endings, Latin/Cyrillic homoglyph swaps. +- A LoRA **cannot** add spelling knowledge the base lacks. It only steers + format / persona / mood / tool-schema / language-mirroring. + +**Levers, in order of impact:** +1. **RU-native base** — `Vikhrmodels/Vikhr-Qwen-2.5-1.5B-Instruct` (continued RU + pretraining on Qwen2.5). Same latency class as the current 1.2B, doesn't shatter + Cyrillic. *(Verify exact HF repo id before download — Vikhr naming shifts.)* +2. **Less aggressive quant** — deploy Q4_K_M first (~1.0GB, matches current + footprint); bump to Q5_K_M only if Cyrillic still breaks. RU-native base tolerates + Q4 far better than the Western models did. +3. **LoRA** — format/persona/mood/tools, on top of a base that already spells. + +## Output contract: `{response,mood}` (Decision B, 2026-07-11) + +All LLM output paths now produce `{"response":"...","mood":"..."}` — the Go side +(`replier_llm.go`, `llmphraser.go`) parses it in all 4 consumers. Fallback logic +preserves backward compat with plain text and the old `{"body","summary"}` format. + +| Consumer (file) | Now parses | +|---|---| +| Replier (`cmd/mavend/replier_llm.go`) | `{"response","mood"}` via `parseResponseMood` | +| Nudges (`internal/phraser/llmphraser.go`) | `{"response","mood"}` via `parseResponseMood` → `Body`, `Mood` on struct | +| Reminders (`internal/phraser/llmphraser.go`) | same, summary derived from response | +| Chat / query (`internal/phraser/llmphraser.go`) | `{"response","mood"}` → returns plain response text | + +Every training sample's assistant turn must produce `{"response":"...","mood":"..."}`. + +## Data defects measured in current corpus + +- **Mood collapse:** `neutral 1051, thinking 666, happy 280, confused 206, tired 7` + (of 2210). Graceful failure (`tired`) is essentially untrained — the most important + small-assistant behavior. Rebalance to ~10–15% combined `tired`+`confused`. +- **System-prompt drift:** 3 variants in `synthetic_dataset.jsonl` + a 4th in + `function_calling.jsonl`. Pick **one canonical system prompt = the exact string the + daemon sends at inference**, normalize all data to it. Train-prompt ≠ deploy-prompt + is a silent accuracy tax. +- **Dolphin trap:** `dolphin_to_messages` wraps ~3000 free-form paragraph outputs + under a strict-format system prompt → teaches the model the format is optional. + Drop it, or reshape via the distiller. Consistency > volume for format LoRAs. +- **`user-*.jsonl`** are bare message fragments (not wrapped in `{"messages":[...]}`); + the `"messages" in s` filter silently drops them. Wrap/normalize before use. + +## Target data recipe (RU-native base — spelling is NOT the job) + +Job = output contract + persona + mood mapping + tool schema + language mirroring. + +| Bucket | ~Share | Source | +|---|---|---| +| Persona chit-chat, on-contract | 45% | distill (Qwen3-4B / API): topic → canonical prompt → generate → validate (parses, mood∈enum, Cyrillic-clean, length) | +| Graceful failure / clarify | 15% | author unanswerable prompts → `tired`/`confused` (current hole) | +| Tool calls | 20% | expand `function_calling.jsonl` (473) — paraphrase 3–5× RU+EN, vary params | +| Real utterances | 10% | mine Maven dialogue history / logs, relabel | +| English → English | 10% | same buckets, EN in/out, so language-mirroring is trained | + +**Hygiene:** dedup near-dup user turns; hold out a *real* (non-distiller) eval set; +route all data through a validator rejecting mixed-script words + invalid JSON — the +same validator is the regression metric (Cyrillic eval). + +## Training (ROCm) — `esp32-whisper-fine-tune/llm/train_rocm.py` + +Adapted from `train_llama.py`; the ROCm fix is dropping the two CUDA-only pieces: +- **No bitsandbytes** — bf16 full weights (1.5B fits 16GB easily), no 4-bit BnB. +- `optim="adamw_torch_fused"` replaces `paged_adamw_8bit`. +- Base `Vikhr-Qwen-2.5-1.5B`; Qwen ChatML masking (`<|im_start|>assistant` / `<|im_end|>`). +- `attn_implementation="eager"` (flash-attn ROCm wheels flaky). +- `HSA_OVERRIDE_GFX_VERSION=11.0.0` belt-and-suspenders (gfx1100 is officially supported). +- batch 2 × grad-accum 4 (eff. 8), LoRA r16/α32, 3 epochs, grad checkpointing. + +## Deploy path (after training) + +```bash +# 1. merge adapter → bf16 weights +python -c "from peft import AutoPeftModelForCausalLM; import torch; \ +m=AutoPeftModelForCausalLM.from_pretrained('./Vikhr-Qwen-1.5b-ru-lora',torch_dtype=torch.bfloat16); \ +m.merge_and_unload().save_pretrained('./Vikhr-merged'); \ +from transformers import AutoTokenizer; AutoTokenizer.from_pretrained('./Vikhr-Qwen-1.5b-ru-lora').save_pretrained('./Vikhr-merged')" +# 2. HF → GGUF (in llama.cpp) +python convert_hf_to_gguf.py ./Vikhr-merged --outfile vikhr-maven-f16.gguf --outtype f16 +# 3. quantize (start Q4_K_M, test Cyrillic, bump to Q5_K_M if needed) +./llama-quantize vikhr-maven-f16.gguf Vikhr-Maven-1.5B-Q4_K_M.gguf Q4_K_M +# 4. scp to homesrv; point deploy/mavend.json phraser.model_path at it; --mlock, small -ngl on Vulkan +``` + +## Open items + +- [x] **Decide output contract** — **B: `{response,mood}`**, Go side patched (2026-07-11). +- [ ] Confirm `Vikhrmodels/Vikhr-Qwen-2.5-1.5B-Instruct` repo id on HF. +- [ ] Write `gen_data.py` distiller (topic → canonical prompt → validate → balanced JSONL w/ mood quotas). +- [ ] Write Cyrillic-validity + JSON eval (extend `llama-eval-test.py`). +- [ ] Normalize `user-*.jsonl` into `{"messages":[...]}`. diff --git a/Dockerfile b/Dockerfile index 14b366a..b8fa7a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,7 @@ COPY deps/whisper.cpp/ggml/include/ /src/deps/whisper.cpp/ggml/include/ COPY go.mod go.sum ./ RUN go mod download COPY cmd/ ./cmd/ +# force rebuild 2026-07-10T16:50 COPY internal/ ./internal/ # CGO wiring mirrors the Makefile; rpath points at the RUNTIME lib location so diff --git a/cmd/mavend/replier_llm.go b/cmd/mavend/replier_llm.go index a29c013..3bd8b20 100644 --- a/cmd/mavend/replier_llm.go +++ b/cmd/mavend/replier_llm.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "strings" "time" @@ -27,30 +28,45 @@ func newLLMReplier(c completer) *llmReplier { return &llmReplier{c: c, stub: voice.NewStubReplier()} } -const replySystem = `Ты — Maven, домашняя ассистентка (о себе — в женском роде). Подтверди действие РОВНО ОДНИМ коротким предложением (≤120 символов), тепло и по-русски. Не задавай вопросов, не повторяй слова, не добавляй ничего после точки. Без кавычек и пояснений.` +const replySystem = `Ты — Maven, домашняя ассистентка (о себе — в женском роде). Подтверди действие РОВНО ОДНИМ коротким предложением (≤120 символов), тепло и по-русски. Не задавай вопросов, не повторяй слова, не добавляй ничего после точки. Respond ONLY with valid JSON: {"response": "...", "mood": "neutral"}.` func (r *llmReplier) Reply(d router.Decision) string { if d.Clarify { return r.stub.Reply(d) } - ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() out, err := r.c.Complete(ctx, llm.Req{ System: replySystem, User: replyContext(d), - MaxTokens: 48, - RepeatPenalty: 1.3, // curb the sub-1B token loop ("тоже тоже тоже") - Stop: []string{"\n"}, + MaxTokens: 512, + RepeatPenalty: 1.3, }) - if out = firstSentence(out); err != nil || out == "" { + if err != nil { return r.stub.Reply(d) } - return out + out = stripThink(out) + if response, _ := parseResponseMood(out); response != "" { + return response + } + // fallback: try plain-text parsing + if out = firstSentence(out); out != "" { + return out + } + return r.stub.Reply(d) } // firstSentence trims the model's output to a single clean confirmation: first // line, first sentence, whitespace-normalized — the last-line defense against a // small model that rambles past the first period despite the prompt + stop. +// stripThink removes the block that Thinking-variant models emit. +func stripThink(s string) string { + if i := strings.LastIndex(s, ""); i >= 0 { + s = strings.TrimSpace(s[i+8:]) + } + return s +} + func firstSentence(s string) string { s = strings.TrimSpace(s) if i := strings.IndexByte(s, '\n'); i >= 0 { @@ -63,6 +79,25 @@ func firstSentence(s string) string { return strings.TrimSpace(s) } +// parseResponseMood extracts {"response","mood"} from LLM output, tolerant +// of thinking tokens and extra text before/after the JSON block. +func parseResponseMood(raw string) (response, mood string) { + cleaned := strings.TrimSpace(raw) + start := strings.Index(cleaned, "{") + end := strings.LastIndex(cleaned, "}") + if start < 0 || end < 0 || end <= start { + return "", "" + } + var parsed struct { + Response string `json:"response"` + Mood string `json:"mood"` + } + if err := json.Unmarshal([]byte(cleaned[start:end+1]), &parsed); err != nil { + return "", "" + } + return parsed.Response, parsed.Mood +} + // replyContext renders the decision into a compact RU description for the model. func replyContext(d router.Decision) string { switch d.Intent { diff --git a/cmd/mavend/replier_llm_test.go b/cmd/mavend/replier_llm_test.go index 11d9121..b16dd0e 100644 --- a/cmd/mavend/replier_llm_test.go +++ b/cmd/mavend/replier_llm_test.go @@ -14,6 +14,14 @@ type mockCompleter struct{ out string; err error } func (m mockCompleter) Complete(_ context.Context, _ llm.Req) (string, error) { return m.out, m.err } func TestLLMReplierReturnsLLMReply(t *testing.T) { + r := newLLMReplier(mockCompleter{out: `{"response":"записала, кофе закончился","mood":"neutral"}`}) + got := r.Reply(router.Decision{Intent: router.IntentNote, Slots: router.Slots{Text: "кофе закончился"}}) + if got != "записала, кофе закончился" { + t.Errorf("got %q, want %q", got, "записала, кофе закончился") + } +} + +func TestLLMReplierFallsBackToPlainText(t *testing.T) { r := newLLMReplier(mockCompleter{out: "записала, кофе закончился"}) got := r.Reply(router.Decision{Intent: router.IntentNote, Slots: router.Slots{Text: "кофе закончился"}}) if got != "записала, кофе закончился" { diff --git a/cmd/mavend/voice.go b/cmd/mavend/voice.go index d4c8a4f..387ccb5 100644 --- a/cmd/mavend/voice.go +++ b/cmd/mavend/voice.go @@ -192,15 +192,12 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem log.Printf("voice: weather provider: stub (not configured)") } - // ----- LLM router (when the phraser is backed by a real model) ----- - // Both the router and the replier share the same *llm.Client; we build - // it here from the phraser's base URL. - var llmRouter *router.LLMRouter + // The replier uses the same llama-server as the phraser. var llmClient *llm.Client if lp, ok := phr.(*phraser.LLMPhraser); ok { - llmClient = llm.New(lp.BaseURL(), 20*time.Second) - llmRouter = router.NewLLMRouter(llmClient) + llmClient = llm.New(lp.BaseURL(), 60*time.Second) } + // LLM router disabled — the classifier handles routing reliably. // ----- router (the cascade; floor examples seed the classifier) ----- // The act matcher's allowlist is exactly the enabled tool names — the @@ -209,7 +206,7 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem if threshold <= 0 { threshold = config.DefaultRouterThreshold } - rtr := buildRouter(emb, matcher, threshold, llmRouter) + rtr := buildRouter(emb, matcher, threshold, nil) // LLM router disabled // ----- sessions registry (shared with voicesink) ----- sessions := voice.NewSessions() @@ -443,6 +440,7 @@ func (h *reactiveHandler) HandlePushToTalk(ctx context.Context, req voice.PushTo // (and eventually by telegram). Splits out the audio bookends from // HandlePushToTalk so text channels share the same routing logic. func (h *reactiveHandler) handleText(ctx context.Context, text string) string { + log.Printf("voice: handleText: %q", text) // 1b. confirm turn — if a destructive act is parked, this utterance is its // y/n answer. Same check as HandlePushToTalk. if reply, handled := h.resolveConfirm(ctx, text); handled { @@ -458,6 +456,7 @@ func (h *reactiveHandler) handleText(ctx context.Context, text string) string { log.Printf("voice: handleText router error: %v", err) return "не получилось разобрать команду." } + log.Printf("voice: route result: intent=%s slots=%+v", dec.Intent, dec.Slots) // 2b. dialogue — same as HandlePushToTalk. if h.dialogueSessions != nil { @@ -494,6 +493,7 @@ func (h *reactiveHandler) handleText(ctx context.Context, text string) string { // 3. action — execute the decision's intent. replyText := h.applyAction(ctx, dec) + log.Printf("voice: applyAction returned: %q", replyText) // 4. replier — phrase the reply when applyAction returned "". if replyText == "" { diff --git a/deploy/mavend.json b/deploy/mavend.json index 9bf2919..29ee63c 100644 --- a/deploy/mavend.json +++ b/deploy/mavend.json @@ -6,11 +6,11 @@ "state_dir": "/var/lib/maven", "phraser": { - "model_path": "/opt/maven/models/llm/LFM2.5/LFM2.5-1.2B-Instruct-Q4_K_M.gguf", + "model_path": "/opt/maven/models/llm/nemotron3-nano/NVIDIA-Nemotron-3-Nano-4B-UD-Q4_K_XL.gguf", "bin_path": "llama-server", "n_gpu_layers": 99, "n_ctx": 2048, - "timeout": "20s" + "timeout": "60s" }, "telegram": { diff --git a/docs/plans/2026-07-11-ru-cpt-base.md b/docs/plans/2026-07-11-ru-cpt-base.md new file mode 100644 index 0000000..51f7324 --- /dev/null +++ b/docs/plans/2026-07-11-ru-cpt-base.md @@ -0,0 +1,413 @@ +# Plan — DIY RU-native base via Continued Pretraining (CPT) + +> **Goal:** Build our *own* RU-native base model instead of using Vikhr. Take a +> clean newer base (Qwen3-1.7B-Base), continue-pretrain it on a curated Russian +> corpus so it spells Cyrillic natively, then run the existing persona-LoRA → +> merge → GGUF → deploy pipeline on top. We own the recipe, the corpus, and the +> checkpoint — reproducible, no dependency on a third-party RU repo. +> +> **This is a big project. The corpus is 80% of the work.** Read the whole file +> before starting. Do phases in order. Each phase has a **DONE-CHECK** — do not +> advance until it passes. + +--- + +## 0. Context an agent picking this up must know + +**Two machines — never conflate (from `CLAUDE.md`):** + +| Role | Box | Specs | Constraint | +|---|---|---|---| +| **Train** | this workstation | RX 7900 GRE, gfx1100 (RDNA3), **16GB VRAM**, ROCm 7.2 / torch 2.10 HIP, `cuda.is_available()==True` | CPT of 1.7B is *tight* — see Phase 3 VRAM math | +| **Deploy** | homesrv `kami@192.168.1.104` | Ryzen 5 5600U CPU, **13GB RAM already swapping** (~8.8GB avail), Ubuntu 24.04 | CPU-only llama.cpp inference; model must stay ≤1.7B and every 200MB matters | + +**Working directories:** +- Training scripts & data: `/home/kami/Programs/esp32-whisper-fine-tune/llm/` + - existing: `train_rocm.py` (persona LoRA, ChatML), `llama-eval-test.py`, `data/` + - **new files from this plan** also go here. +- Daemon (consumes final GGUF): `/mnt/server/home/kami/apps/Maven/` (this repo). + +**Output contract (from `CLAUDE.md`, Decision B, locked 2026-07-11):** the daemon +parses `{"response":"...","mood":"..."}` in all 4 consumers. This only matters for +the **persona LoRA stage (Phase 6)**, NOT for CPT. CPT is raw next-token text, no +JSON, no chat format. + +**Why CPT and not just a LoRA (from `CLAUDE.md`):** a LoRA cannot add spelling +knowledge the base lacks — it only steers format/persona/mood/tools. CPT changes +the base weights themselves, which is the only lever that can add genuine RU +language competence. That is exactly why this is more than a LoRA project. + +**ROCm gotchas (inherit from `train_rocm.py`):** +- `attn_implementation="eager"` (flash-attn ROCm wheels flaky). +- `HSA_OVERRIDE_GFX_VERSION=11.0.0` in env (gfx1100). +- **No bitsandbytes** — ROCm build is flaky. This forbids 8-bit Adam; see Phase 3. + +--- + +## 1. Pipeline overview (the whole journey) + +``` +Phase 2 Corpus build → clean packed RU token blocks (.arrow/.bin, ~300M tok) +Phase 3 train_cpt.py → CPT'd Qwen3-1.7B-Base (bf16 HF checkpoint) +Phase 4 eval → Cyrillic + JSON validator; CPT base vs raw base numbers +Phase 5 DECISION GATE → did CPT beat raw? if not, stop / re-corpus +Phase 6 persona LoRA → existing train_rocm.py on the CPT'd base (ChatML, {response,mood}) +Phase 7 merge → GGUF → Q8_0 → deploy to homesrv +``` + +Build order is **not** the run order: **build the corpus first (Phase 2)** because +it gates everything and takes the longest. + +--- + +## 2. Corpus — the core deliverable + +**Target: ~300M tokens** (Qwen3 tokenizer), curated. Rationale: <100M won't move +the needle vs Qwen3's already-strong multilingual; >1B is weeks of wall-clock on +one 16GB card. 300M is the "nudge, don't re-pretrain" sweet spot for 1 epoch. + +**Two token sources, combined:** +- **A. Open datasets** (the bulk, real human RU text). +- **B. Router-generated synthetic** (fills domain/persona/conversational gaps the + open corpora lack — smart-home commands, assistant dialogue, Maven's domain). + The user's **router for free inference providers** generates this. + +### 2.1 Corpus mix table (sums to ~300M tokens) + +| # | Bucket | Share | Tokens | Source | Why | +|---|---|---|---|---|---| +| B1 | Cleaned web RU | 55% | ~165M | `uonlp/CulturaX` (ru) — **already cleaned + deduped** | backbone, broad RU | +| B2 | Encyclopedic | 15% | ~45M | `wikimedia/wikipedia` config `20231101.ru` | clean factual RU, correct spelling | +| B3 | Books / long-form | 10% | ~30M | `IlyaGusev/librusec` (or subset) | rich morphology, endings | +| B4 | Conversational | 8% | ~24M | `Helsinki-NLP/open_subtitles` (ru) | spoken register — matches voice assistant | +| B5 | **Router-synthetic** | 10% | ~30M | generated (§2.4) | domain/persona/smart-home RU, absent from open sets | +| B6 | Maven real logs | 2% | ~6M | mine daemon dialogue history / logs | exact deploy-domain match | + +**Cap B5 (synthetic) at 10%.** Synthetic-heavy CPT causes model collapse / +degeneration. The bulk MUST be real human text. This cap is a hard rule. + +### 2.2 Exact acquisition — open datasets (Bucket A: B1–B4) + +All via HuggingFace `datasets`, streaming to avoid disk blowup. Some are gated +(CulturaX, OSCAR) — accept the license on the HF model page while logged in +(`huggingface-cli login`) first. + +```python +# corpus_fetch.py — run per bucket, writes raw .txt.gz shards to data/cpt_raw// +from datasets import load_dataset +# B1 CulturaX (gated — accept license first) +ds = load_dataset("uonlp/CulturaX", "ru", split="train", streaming=True) +# B2 Wikipedia +ds = load_dataset("wikimedia/wikipedia", "20231101.ru", split="train", streaming=True) +# B3 Books +ds = load_dataset("IlyaGusev/librusec", split="train", streaming=True) +# B4 Subtitles +ds = load_dataset("Helsinki-NLP/open_subtitles", lang1="en", lang2="ru", split="train", streaming=True) +``` + +Pull enough raw text per bucket to survive filtering (filtering drops ~20–40%); +over-pull ~1.5× the target token count, then trim after cleaning. Text field +differs per dataset (`text`, `content`, `translation['ru']`) — normalize each to a +single `text` string when writing shards. + +**Fallbacks if a source is unavailable:** B1 → `allenai/c4` (ru) or `oscar-corpus/OSCAR-2301` (ru); +B3 → `IlyaGusev/gazeta` / Taiga corpus; B4 → `Helsinki-NLP/tatoeba` (ru pairs). + +### 2.3 Cleaning pipeline (mandatory — dirty corpus = dirty base) + +One script `corpus_clean.py`, applied to every raw shard, in this order: + +1. **Unicode fix** — `ftfy.fix_text` (repairs mojibake, normalizes to NFC). +2. **Language ID** — fastText `lid.176.bin`; drop any doc where top lang != `ru` + or `p(ru) < 0.65`. (Download `lid.176.bin` from fastText releases.) +3. **Mixed-script rejection** — regex: reject any *word* containing both Cyrillic + and Latin letters (`[а-яё].*[a-z]|[a-z].*[а-яё]` case-insensitive per word). + This is the exact homoglyph problem `CLAUDE.md` names. Keep pure-Latin words + (URLs/names) but drop homoglyph-contaminated ones. +4. **Boilerplate/length** — drop docs <200 chars or >50k chars; strip HTML tags, + nav junk, repeated whitespace; drop docs with >30% non-letter chars. +5. **Dedup** — MinHash LSH via `datasketch` (`MinHashLSH`, threshold 0.8, 128 + perms) across ALL buckets combined. Near-dup docs collapse to one. This is the + single biggest quality lever — do not skip. +6. **PII light-scrub** (optional) — regex phone/email → placeholder, since B6 has + real logs. + +Emit cleaned docs as JSONL `{"text": "..."}` in `data/cpt_clean/`. + +**DONE-CHECK 2.3:** run `corpus_stats.py` → per-bucket doc count, char count, and +**a 100-doc random sample printed for eyeball review**. No mixed-script words in +the sample. If sample looks dirty, fix the filter before proceeding. + +### 2.4 Router-synthetic generation (Bucket B5 — the user's router) + +Purpose: generate ~30M tokens of clean RU text in registers open corpora lack — +smart-home/assistant domain, natural conversational RU, Maven-adjacent topics. +**This is CPT text (raw prose/dialogue), NOT the `{response,mood}` persona data** +(that's Phase 6, a different dataset). + +Protocol (`corpus_synth.py`, calls the free-provider router): +1. **Topic seed list** — reuse `data/topics.txt` (already exists, ~46KB of topics) + + add smart-home / daily-assistant / RU-domestic topics. +2. **Prompt template** (per topic), ask the router model to produce **long-form + natural Russian prose or dialogue** on the topic — NOT JSON, NOT persona, NOT + English. e.g. `"Напиши развёрнутый естественный текст на русском языке на тему: {topic}. 400–800 слов, живой разговорный стиль."` +3. **Diversify:** vary style knob (formal/casual/dialogue/monologue), vary length, + rotate across the router's providers/models so it's not one model's fingerprint + (fingerprint uniformity → collapse risk). +4. **Route every generation through the SAME cleaning pipeline (§2.3)** — language + ID, mixed-script rejection, dedup against the real corpus too. Synthetic that + fails Cyrillic cleanliness is exactly what we must not train on. +5. **Stop at ~30M tokens.** Enforce the 10% cap. + +**DONE-CHECK 2.4:** synthetic passes the same stats + eyeball sample as 2.3; token +count ≤ 10% of total corpus; provider diversity logged. + +### 2.5 Maven real logs (Bucket B6) + +Mine daemon dialogue history / logs for real RU user+assistant utterances. Strip +to plain text, run through §2.3. Small (~6M tok) but highest domain value. +Source location: check `mavend` logs / any dialogue history store in the repo. + +### 2.6 Tokenize + pack (final corpus artifact) + +`corpus_pack.py`: +- Load Qwen3-1.7B-Base tokenizer (`transformers.AutoTokenizer`). +- Concatenate all cleaned JSONL, tokenize, append EOS between docs, **pack into + fixed 2048-token blocks** (drop the ragged tail). No padding — packing is + efficient for CPT. +- Save as a HF `datasets` Arrow dir `data/cpt_packed/` (columns: `input_ids`). +- Print final total token count — **must be ~300M ± 50M**. Adjust bucket pulls if off. + +**DONE-CHECK 2.6:** `data/cpt_packed/` loads; total tokens logged; a decoded +random block reads as clean Russian. + +--- + +## 3. `train_cpt.py` — continued pretraining (fits 16GB) + +New file next to `train_rocm.py`. Inherits the ROCm setup from it. + +### 3.1 VRAM reality (why the config is what it is) + +Full-weight CPT of 1.7B with plain Adam ≈ 21–24GB → **does not fit 16GB**. We +can't use 8-bit Adam (no bitsandbytes on ROCm). Fitting strategy, in preference +order — **try A first, fall back to B**: + +- **A. Full-weight + `adafactor`** — Adafactor has no momentum states, cutting the + optimizer-state VRAM roughly in half. Full-weight bf16 + adafactor + gradient + checkpointing + batch 1 × grad-accum should fit ~16GB. This is the real + "change the base weights" path. **Preferred.** +- **B. High-rank DoRA on all linear layers** — if A OOMs or throughput is + unbearable. Fits trivially. Weaker for language injection but still adapts more + than a small LoRA. Use `r=64, α=128, target all linear`, `use_dora=True`. + +### 3.2 Config + +```python +# train_cpt.py essentials +import os; os.environ["HSA_OVERRIDE_GFX_VERSION"] = "11.0.0" +from transformers import (AutoModelForCausalLM, AutoTokenizer, + TrainingArguments, Trainer, + DataCollatorForLanguageModeling) + +MODEL = "Qwen/Qwen3-1.7B-Base" # BASE, not Instruct +model = AutoModelForCausalLM.from_pretrained( + MODEL, torch_dtype="bfloat16", attn_implementation="eager") +model.gradient_checkpointing_enable() +model.config.use_cache = False + +args = TrainingArguments( + output_dir="./Qwen3-1.7B-ru-cpt", + per_device_train_batch_size=1, + gradient_accumulation_steps=16, # eff. batch 16 × 2048 tok + num_train_epochs=1, # ONE pass — more risks forgetting + learning_rate=1e-5, # LOW — nudging pretrained weights + lr_scheduler_type="cosine", + warmup_ratio=0.03, + optim="adafactor", # path A; no bitsandbytes + bf16=True, + gradient_checkpointing=True, + logging_steps=20, + save_steps=500, # checkpoint often — this runs for DAYS + save_total_limit=3, + report_to="none", +) +collator = DataCollatorForLanguageModeling(tokenizer, mlm=False) # causal LM +``` + +- **lr 1e-5 is critical.** CPT lr must be ~10× lower than LoRA lr. High lr = + catastrophic forgetting (model forgets English, reasoning, everything). If the + eval (Phase 4) shows English/tool ability collapsed, lr was too high. +- **1 epoch.** Multiple epochs over 300M tokens overfits and forgets. +- **Resume/checkpoint is mandatory** — days-long job; support `--resume_from_checkpoint`. + +### 3.3 Run + +```bash +cd /home/kami/Programs/esp32-whisper-fine-tune/llm +HSA_OVERRIDE_GFX_VERSION=11.0.0 python train_cpt.py 2>&1 | tee cpt_run.log +``` + +**DONE-CHECK 3:** training completes 1 epoch without OOM; loss decreases and +plateaus (not NaN, not flat-from-step-0); checkpoint saved to +`./Qwen3-1.7B-ru-cpt/`. If OOM → switch to path B (DoRA). + +--- + +## 4. Eval — did CPT actually help? + +Extend the existing `llama-eval-test.py` into `eval_cyrillic.py`. This is also an +open item in `CLAUDE.md` and is the regression metric for the whole project. + +**Metrics (run on a held-out set of real RU prompts — NOT synthetic, NOT training data):** +1. **Cyrillic validity %** — generate on RU prompts; % of outputs with zero + mixed-script words and no homoglyph swaps (reuse §2.3 rule 3 as the checker). +2. **Perplexity** on a held-out clean RU text set (lower = better RU fit). +3. **English-retention check** — a few EN prompts; confirm the model still answers + in coherent English (catches catastrophic forgetting). +4. **JSON-format sanity** (light, for later) — can it produce `{response,mood}` + when asked. (Real test is post-persona-LoRA.) + +**Run it three ways and tabulate:** +| Model | Cyrillic valid % | RU PPL | EN retained? | +|---|---|---|---| +| raw Qwen3-1.7B-Base | (baseline) | | | +| our CPT'd base | (must beat raw) | | | + +**DONE-CHECK 4:** table filled with real numbers. + +--- + +## 5. DECISION GATE + +- **CPT base beats raw on Cyrillic %/PPL AND English retained** → proceed to Phase 6. +- **No improvement** → CPT didn't earn its compute. Options: (a) more/better + corpus (usually the fix — dirty or too-small corpus), (b) tune lr, (c) accept + raw Qwen3-1.7B as the base and skip CPT. Do NOT proceed to persona LoRA on a + CPT base that lost to raw. +- **English collapsed** → lr too high or too many epochs; re-run Phase 3 with lower lr. + +--- + +## 6. Persona LoRA on the CPT'd base (existing pipeline) + +Now the CPT base is just a better base. Run the **existing** `train_rocm.py` +persona/format LoRA on top, with two changes: +- Point base model at `./Qwen3-1.7B-ru-cpt/` (the CPT checkpoint), not Vikhr/Qwen2.5. +- Confirm ChatML template matches Qwen3 (`<|im_start|>assistant … <|im_end|>`). +- The persona **data** is the `{response,mood}` corpus per `CLAUDE.md` Decision B + (separate from the CPT corpus): 45% persona chit-chat, 15% graceful failure + (`tired`/`confused`), 20% tool calls, 10% real utterances, 10% EN→EN. Mood + quotas + validator per `CLAUDE.md`. **This dataset is still an open item** — see + `CLAUDE.md` "Target data recipe" and `gen_data.py` open item. + +**DONE-CHECK 6:** LoRA trains; eval (Phase 4) re-run on merged model shows +`{response,mood}` parses + Cyrillic clean + moods in enum. + +--- + +## 7. Merge → GGUF → quantize → deploy + +Per `CLAUDE.md` deploy path, with **Q8_0 first** (decision this session: start +high-precision to remove the Cyrillic-at-Q4 gamble, drop to Q5/Q4 only if homesrv +memory chokes): + +```bash +# 1. merge LoRA → bf16 +python -c "from peft import AutoPeftModelForCausalLM; import torch; \ +m=AutoPeftModelForCausalLM.from_pretrained('./Qwen3-1.7B-ru-lora',torch_dtype=torch.bfloat16); \ +m.merge_and_unload().save_pretrained('./Qwen3-maven-merged'); \ +from transformers import AutoTokenizer; AutoTokenizer.from_pretrained('./Qwen3-1.7B-ru-lora').save_pretrained('./Qwen3-maven-merged')" +# 2. HF → GGUF (in llama.cpp) +python convert_hf_to_gguf.py ./Qwen3-maven-merged --outfile qwen3-maven-f16.gguf --outtype f16 +# 3. quantize — Q8_0 first (~1.6GB for 1.7B) +./llama-quantize qwen3-maven-f16.gguf Qwen3-Maven-1.7B-Q8_0.gguf Q8_0 +# 4. deploy: scp to homesrv; measure tok/s + swap with --mlock. +# If RAM chokes → re-quant Q5_K_M, then Q4_K_M. Point deploy/mavend.json +# phraser.model_path at the file; small -ngl on Vulkan for a speedup. +``` + +**DONE-CHECK 7:** model runs on homesrv at acceptable voice tok/s (measure!), +Cyrillic clean in live output, `{response,mood}` parsed by the daemon. + +--- + +## 8. File manifest (what this plan creates) + +All in `/home/kami/Programs/esp32-whisper-fine-tune/llm/`: + +| File | Phase | Purpose | +|---|---|---| +| `corpus_fetch.py` | 2.2 | pull open datasets → `data/cpt_raw/` | +| `corpus_synth.py` | 2.4 | router-generate synthetic RU → `data/cpt_raw/synth/` | +| `corpus_clean.py` | 2.3 | ftfy + langID + mixed-script + dedup → `data/cpt_clean/` | +| `corpus_stats.py` | 2.3 | per-bucket counts + sample dump (DONE-CHECKs) | +| `corpus_pack.py` | 2.6 | tokenize + pack 2048 blocks → `data/cpt_packed/` | +| `train_cpt.py` | 3 | continued pretraining, adafactor/DoRA | +| `eval_cyrillic.py` | 4 | Cyrillic % + PPL + EN-retention (extends `llama-eval-test.py`) | +| `data/cpt_packed/` | 2.6 | final corpus artifact (~300M tok) | +| `./Qwen3-1.7B-ru-cpt/` | 3 | CPT'd base checkpoint | + +### 8.1 One-time setup + +```bash +cd /home/kami/Programs/esp32-whisper-fine-tune/llm + +# python deps (torch/transformers already present from train_rocm.py) +pip install datasets ftfy datasketch fasttext openai peft accelerate + +# fastText language-id model → data/lid.176.bin (corpus_clean.py requires it) +wget -O data/lid.176.bin \ + https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin + +# HF login — needed to accept the CulturaX (gated) license +huggingface-cli login +``` + +If `fasttext` wheels fail to build, `pip install fasttext-wheel` is the prebuilt +fallback. `lid.176.bin` can also live elsewhere via `export FASTTEXT_LID=/path`. + +### 8.2 Runbook (full run order) + +```bash +cd /home/kami/Programs/esp32-whisper-fine-tune/llm + +# ── Phase 2: corpus ────────────────────────────────────────────── +python corpus_fetch.py all # 2.2 open datasets +ROUTER_BASE_URL="https://your-router/v1" \ + ROUTER_API_KEY="..." ROUTER_MODELS="m-a,m-b,m-c" \ + python corpus_synth.py # 2.4 synthetic (≤10%) +MAVEN_LOGS=/path/to/dialogue.jsonl python corpus_logs.py # 2.5 optional +python corpus_clean.py # 2.3 clean+dedup all +python corpus_stats.py # DONE-CHECK 2.3/2.4 +python corpus_pack.py # 2.6 → data/cpt_packed/ + +# ── Phase 3: continued pretraining (days; --resume to continue) ── +HSA_OVERRIDE_GFX_VERSION=11.0.0 python train_cpt.py 2>&1 | tee cpt_run.log +# OOM? → CPT_PATH=dora HSA_OVERRIDE_GFX_VERSION=11.0.0 python train_cpt.py + +# ── Phase 4: eval + gate (Phase 5) ────────────────────────────── +python eval_cyrillic.py --model Qwen/Qwen3-1.7B-Base # baseline +python eval_cyrillic.py --model ./Qwen3-1.7B-ru-cpt # must beat baseline + +# ── Phase 6-7: persona LoRA → merge → GGUF → Q8_0 → homesrv ────── +# train_rocm.py pointed at ./Qwen3-1.7B-ru-cpt ; then §7 deploy block. +``` + +Third-party deps: `datasets`, `ftfy`, `datasketch`, `fasttext` (+ `lid.176.bin`), +`openai`, `peft`, `transformers`, `accelerate`. + +--- + +## 9. Hard rules (do not violate) + +1. CPT corpus is **raw text**, no JSON, no ChatML. Persona `{response,mood}` data + is a **separate** dataset (Phase 6). +2. Synthetic ≤ **10%** of corpus. Bulk must be real human RU. +3. CPT lr = **1e-5**, **1 epoch**. Higher = catastrophic forgetting. +4. Base = **Qwen3-1.7B-*Base***, not Instruct. +5. **Never quantize before merge.** bf16 all the way through, quantize last. +6. Every corpus source passes the **same cleaning pipeline** (§2.3), synthetic included. +7. Don't advance a phase until its **DONE-CHECK** passes. Phase 5 gate is real — + if CPT loses to raw, stop. +8. Respect the two-machine split: train on the 7900 GRE, deploy CPU-only on homesrv ≤1.7B. diff --git a/docs/plans/2026-07-11-tts-piper-student.md b/docs/plans/2026-07-11-tts-piper-student.md new file mode 100644 index 0000000..0c3f94c --- /dev/null +++ b/docs/plans/2026-07-11-tts-piper-student.md @@ -0,0 +1,249 @@ +# Plan — Maven TTS: OmniVoice teacher → piper student (from scratch) + +> **Goal:** A tiny, CPU-real-time Russian TTS voice for Maven that replaces the +> piper `irina` floor. A heavy zero-shot teacher (**k2-fsa/OmniVoice**) clones/designs +> Maven's voice and mass-generates a labelled dataset; we train a **piper** voice +> **from scratch** on it. The whole quality game is **phoneme correctness (stress + +> homographs)**, not the model — the student aligns perfectly to whatever phonemes +> we feed it, so garbage stress in = garbage voice out. +> +> **Read the whole file before starting. Phases in order. Each has a DONE-CHECK.** + +--- + +## 0. Context an agent picking this up must know + +**Two machines (from `CLAUDE.md`):** + +| Role | Box | Constraint | +|---|---|---| +| **Generate + train** | workstation, RX 7900 GRE gfx1100, 16GB VRAM, ROCm 7.2 | teacher + piper-train both run here; GPU may be busy with the LLM CPT — TTS gen waits its turn, but the phoneme/dataset-prep scripts are CPU/file work and run anytime | +| **Deploy** | homesrv, Ryzen 5 5600U CPU, 13GB RAM swapping | piper `.onnx` runs real-time on CPU; replaces the irina floor | + +**Working dirs:** +- Teacher + dataset: `/home/kami/Programs/esp32-whisper-fine-tune/tts/` (this is the + SSHFS-remote-vs-local split; `/home/kami/Programs/` is **local** to the workstation). +- New piper student files: `tts/piper/` (created by this plan). +- Daemon that consumes the final voice: `/mnt/server/home/kami/apps/Maven/`. + +**What already exists in `tts/` (reuse, don't rebuild):** +- `find_voice.py` — audition teacher voice-designs → save `ref//maven_reference.{wav,txt}`. +- `generate_synthetic_voice.py` — clone the reference, synth per-mood → `dataset//NNNN.wav` + `dataset/metadata.csv`. +- `homograph_processor.py` — LLM rewrites sentences to remove stress homographs. +- `data/*-voice-dataset-list.txt` — the mood transcript lists (neutral/happy/thinking/confused/tired). +- `ruaccent` (RUAccent) + `plus_to_acute` — stress-marking, already imported in the gen script. +- A **large qwen-generated dataset already exists** in `dataset/` — but it was made by the + **old teacher (Qwen3-TTS)** and its text was **not** stress-marked (the `preprocess` call + is commented out). See Phase 1 decision: regenerate with OmniVoice + stressed text, or reuse. + +**Decisions locked this session:** +1. **Teacher = k2-fsa/OmniVoice** (0.6B on Qwen3-0.6B, 600+ langs incl RU, zero-shot + clone + voice-design, RTF 0.025, 24kHz out). Code Apache-2.0; **weights CC-BY-NC** + — fine, the teacher never ships, only the piper student deploys. +2. **Student = piper**, trained **from scratch** (no warm-start from irina or any + piper voice — GPU/patience is free, and warm-start's only benefit was fast + convergence we don't need). +3. **Phoneme correctness is the priority.** Stress-mark + homograph-resolve every + transcript so espeak-ng produces correct RU phonemes, and synth the audio from + the **same** stressed text so text↔audio↔phonemes stay locked. + +--- + +## 1. Pipeline overview + +``` +Phase 2 transcripts → stress-marked + homograph-clean mood lists +Phase 3 teacher swap → OmniVoice into find_voice.py / generate_synthetic_voice.py +Phase 4 generate → dataset//NNNN.wav + metadata.csv (24kHz) +Phase 5 build_dataset → piper LJSpeech dir: wav 22.05k + metadata (id|stressed_text) +Phase 6 check_phonemes → espeak-ng sanity — eyeball stress before training +Phase 7 train.sh → piper preprocess → train from scratch → export ONNX +Phase 8 deploy → scp .onnx+json to homesrv; point mavend.json; drop irina +``` + +**Run order = build order here** (unlike the CPT plan). Phase 2 (phonemes) gates +quality and is CPU work — do it while the LLM CPT still owns the GPU. + +--- + +## 2. Transcripts — stress + homographs (the quality lever) + +The `data/*-voice-dataset-list.txt` files are raw Russian, one sentence per line. +Two defects to fix **before** synthesis, because the audio must match the phonemes: + +1. **Homographs** — words whose stress (thus meaning) is ambiguous (`за́мок`/`замо́к`, + `сто́ят`/`стоя́т`). `homograph_processor.py` already rewrites these away via a local + LLM. Run it per mood list → `*-clean.txt`. +2. **Stress marks** — `ruaccent` (`RUAccent().process_all`) inserts `+` before every + stressed vowel. This is what makes espeak-ng stress correctly later. The gen script + already has `preprocess()` (accent → `plus_to_acute`) but it's **commented out** — + the decision is to run stress-marking on the transcripts and keep the `+`-marked + form as the canonical text (see Phase 5 for the two acute conventions). + +```bash +cd /home/kami/Programs/esp32-whisper-fine-tune/tts +# 1. de-homograph each mood list (LLM at localhost:10000 must be up) +python homograph_processor.py data/*-voice-dataset-list.txt --all +# 2. stress-mark → keep the +VOWEL form as canonical (Phase 5 converts per target) +``` + +**DONE-CHECK 2:** each mood has a cleaned, `+`-stress-marked list; a 30-line random +sample eyeballed — every multisyllable word has exactly one `+`, no homographs left. + +--- + +## 3. Teacher swap — OmniVoice into the two gen scripts + +`find_voice.py` and `generate_synthetic_voice.py` currently call `qwen_tts.Qwen3TTSModel` +(`generate_voice_design`, `generate_voice_clone`). Swap the teacher to OmniVoice via the +thin adapter `tts/piper/omnivoice_tts.py` (created by this plan) so only two call-sites +change and the rest of the resume/metadata logic is untouched. + +- `pip install omnivoice` (after torch), on the workstation, **GPU free** (post-CPT). +- **Verify the exact OmniVoice API** from its README / `pip show omnivoice` — the adapter + is written against the documented surface (ref-audio + transcription clone; attribute + voice-design) but the method names/signature **must be confirmed** before running. +- Keep synthesis text = the **stressed** transcripts from Phase 2 (re-enable `preprocess`, + or pass the pre-stressed lists). Teacher and student then share identical text. + +**DONE-CHECK 3:** `find_voice.py` produces an OmniVoice sample that plays; one clone +sample from `generate_synthetic_voice.py --dry-run`-then-real matches the reference timbre. + +--- + +## 4. Generate the dataset + +```bash +cd /home/kami/Programs/esp32-whisper-fine-tune/tts +python find_voice.py # audition → ref//maven_reference.{wav,txt} +python generate_synthetic_voice.py # all moods → dataset//NNNN.wav + metadata.csv +# resume-safe (.progress.json); --moods / --limit / --dry-run available +``` + +**Reuse-or-regenerate call:** the existing `dataset/` was made by Qwen3-TTS from +**unstressed** text. For phoneme-locked training you want OmniVoice audio from +**stressed** text → **regenerate**. (Keeping the old set is only OK if you accept the +text↔audio stress mismatch, which is the exact defect this plan exists to kill.) + +**DONE-CHECK 4:** `dataset/metadata.csv` rows all point to existing wavs; total +duration logged (aim ≥ ~2–3h across moods for a from-scratch single-speaker voice); +5 random wavs play and match their transcript. + +--- + +## 5. Build the piper dataset (`tts/piper/build_dataset.py`) + +Piper wants an **LJSpeech-format** dir: `wav/.wav` (its target sample rate) + +`metadata.csv` as `id|text`. Our `metadata.csv` is `file_path,text,mood,emotion_id` and +wavs are 24kHz. `build_dataset.py`: + +1. Read `dataset/metadata.csv`. +2. **Resample 24000 → 22050 Hz** (piper `medium` default), mono, into `piper/dataset/wav/`. +3. **Stress → espeak convention:** piper phonemizes via espeak-ng, which reads the + **combining acute U+0301 *after* the stressed vowel** (NOT the uppercase form + `plus_to_acute` makes for qwen). Convert the `+VOWEL` marks to `vowel+U+0301`. +4. Write `piper/dataset/metadata.csv` as `id|stressed_text` (single speaker). + +Self-check (`__main__`): assert `+а` → `а́`, id/text counts match wav count. + +**DONE-CHECK 5:** `piper/dataset/` has N wavs at 22050Hz and a metadata line each; +`soxi`/`soundfile` confirms sample rate. + +--- + +## 6. Phoneme sanity (`tts/piper/check_phonemes.py`) + +Before burning GPU-days, confirm espeak-ng stresses correctly. For a sample of +transcripts, print `espeak-ng -v ru --ipa` output and flag any multisyllable word +whose IPA carries no primary-stress mark `ˈ`. This is the cheapest catch for the +"sounds bad" failure — wrong stress shows here, not after training. + +Requires `espeak-ng` (`sudo apt install espeak-ng`). **ponytail:** thin wrapper over +the espeak CLI, ceiling = it only flags *missing* stress, not *wrong-position* stress +(that still needs an ear on Phase 4 audio). + +**DONE-CHECK 6:** sample run shows every content word carrying `ˈ`; obvious homograph +words stressed as intended. + +--- + +## 7. Train piper from scratch (`tts/piper/train.sh`) + +Install piper-train from the piper repo (`rhasspy/piper`, `src/python`): needs torch + +pytorch-lightning. ROCm env like `train_cpt.py` (`HSA_OVERRIDE_GFX_VERSION=11.0.0`). + +```bash +# preprocess: text → espeak-ng phonemes → training cache +python -m piper_train.preprocess \ + --language ru --input-dir piper/dataset --output-dir piper/train \ + --dataset-format ljspeech --single-speaker --sample-rate 22050 + +# train FROM SCRATCH (no --resume_from_checkpoint), medium quality +HSA_OVERRIDE_GFX_VERSION=11.0.0 python -m piper_train \ + --dataset-dir piper/train --accelerator gpu --devices 1 \ + --batch-size 16 --quality medium --precision 32 \ + --max_epochs 4000 --checkpoint-epochs 100 --validation-split 0.02 + +# export best checkpoint → ONNX +python -m piper_train.export_onnx piper/train/lightning_logs/version_0/checkpoints/last.ckpt \ + piper/maven.onnx +cp piper/train/config.json piper/maven.onnx.json +``` + +- **From scratch** = no warm-start (decision 2). Early checkpoints sound broken until + MAS alignment settles — expected; that's the patience cost we accepted. +- `--quality medium` (22.05k) is the CPU-real-time sweet spot; `high` only if homesrv + latency allows (measure). + +**DONE-CHECK 7:** training loss/mel decreases; a mid-run checkpoint synthesizes an +intelligible Russian sentence with **correct stress**; final ONNX exported + its `.json`. + +--- + +## 8. Deploy to homesrv + +```bash +scp piper/maven.onnx piper/maven.onnx.json kami@192.168.1.104: +# point deploy/mavend.json at maven.onnx; drop the irina floor +``` + +Measure CPU tok→audio latency with `--mlock`-class care (homesrv swaps). If too slow, +that's a quality/latency knob (medium already chosen), not a re-train. + +**DONE-CHECK 8:** Maven speaks on homesrv in the new voice, real-time enough for +conversation, correct Russian stress in live output. + +--- + +## 9. File manifest (what this plan creates in `tts/piper/`) + +| File | Phase | Purpose | +|---|---|---| +| `omnivoice_tts.py` | 3 | thin adapter isolating the OmniVoice API (design + clone) — the single swap point | +| `build_dataset.py` | 5 | our `metadata.csv` + 24k wav → piper LJSpeech dir (22.05k, espeak-acute stress) | +| `check_phonemes.py` | 6 | espeak-ng stress sanity on a transcript sample | +| `train.sh` | 7 | piper preprocess → train-from-scratch → export ONNX (runbook) | + +Reused, unchanged: `find_voice.py`, `generate_synthetic_voice.py` (two call-sites swapped +to the adapter), `homograph_processor.py`, `ruaccent`. + +--- + +## 10. Hard rules (do not violate) + +1. **Text↔audio↔phonemes locked:** synth the audio from the **same stressed text** + espeak-ng later phonemizes. Never train piper on audio whose transcript stress + differs from what you feed the preprocessor. +2. **Two acute conventions — don't confuse them:** `plus_to_acute` (uppercase vowel) + is for the **qwen/OmniVoice** teacher; espeak-ng wants **combining U+0301 after the + vowel**. `build_dataset.py` converts. +3. **From scratch** — no warm-start (decision 2). Do not `--resume_from_checkpoint` + off irina or any piper voice. +4. **Teacher weights are CC-BY-NC** — fine for the never-shipped teacher; the deployed + piper voice is your own weights. Don't redistribute the OmniVoice-generated dataset + as a product. +5. **Homographs out first** (Phase 2), then stress-mark. Order matters — rewriting a + sentence changes which words need stress. +6. Don't advance a phase until its DONE-CHECK passes. Phase 6 (phoneme sanity) is the + real gate — a bad phoneme table wastes the whole GPU-days train. +``` diff --git a/internal/delivery/dispatcher.go b/internal/delivery/dispatcher.go index fc7c3d2..df741e2 100644 --- a/internal/delivery/dispatcher.go +++ b/internal/delivery/dispatcher.go @@ -37,6 +37,7 @@ type PhrasedNudge struct { Candidate loop.Candidate Body string Summary string + Mood string } // PhrasedReminder — the phraser's output for a reminder. @@ -44,6 +45,7 @@ type PhrasedReminder struct { Decision loop.ReminderDecision Body string Summary string + Mood string } // Dispatch — record of one successful send. returned to the daemon for diff --git a/internal/llm/client.go b/internal/llm/client.go index ec19e2d..7760296 100644 --- a/internal/llm/client.go +++ b/internal/llm/client.go @@ -35,8 +35,10 @@ type Req struct { } type msg struct { - Role string `json:"role"` - Content string `json:"content"` + Role string `json:"role"` + Content string `json:"content"` + Reasoning string `json:"reasoning,omitempty"` + ReasoningContent string `json:"reasoning_content,omitempty"` } type body struct { Messages []msg `json:"messages"` @@ -54,7 +56,7 @@ type resp struct { func (c *Client) Complete(ctx context.Context, r Req) (string, error) { b, _ := json.Marshal(body{ - Messages: []msg{{"system", r.System}, {"user", r.User}}, + Messages: []msg{{Role: "system", Content: r.System}, {Role: "user", Content: r.User}}, MaxTokens: r.MaxTokens, Grammar: r.Grammar, Temp: 0, @@ -81,5 +83,9 @@ func (c *Client) Complete(ctx context.Context, r Req) (string, error) { if len(out.Choices) == 0 { return "", fmt.Errorf("llm: no choices") } - return out.Choices[0].Message.Content, nil + content := out.Choices[0].Message.Content + if content == "" { + content = out.Choices[0].Message.ReasoningContent + } + return content, nil } diff --git a/internal/phraser/llmphraser.go b/internal/phraser/llmphraser.go index c79d898..90879ab 100644 --- a/internal/phraser/llmphraser.go +++ b/internal/phraser/llmphraser.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "log" "net/http" "os/exec" "regexp" @@ -161,14 +162,18 @@ func (p *LLMPhraser) PhraseNudge(ctx context.Context, c loop.Candidate) (deliver if err != nil { return delivery.PhrasedNudge{}, err } - body, summary := parsePhrase(resp) + body, mood := parseResponseMood(resp) + if body == "" { + // fallback: try old body/summary format + body, _ = parsePhrase(resp) + } if body == "" { body = fmt.Sprintf("%s — %s", c.Rule.Name, sevLabel(c.Severity)) } - if summary == "" { - summary = c.Rule.Name + if mood == "" { + mood = "neutral" } - return delivery.PhrasedNudge{Candidate: c, Body: body, Summary: summary}, nil + return delivery.PhrasedNudge{Candidate: c, Body: body, Summary: body, Mood: mood}, nil } // PhraseQuery prompts the LLM with the user's utterance and matching notes to @@ -184,6 +189,9 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes [] if err != nil || resp == "" { return "не знаю.", nil } + if text, _ := parseResponseMood(resp); text != "" { + return text, nil + } return resp, nil } if len(notes) == 1 { @@ -201,6 +209,9 @@ func (p *LLMPhraser) PhraseQuery(ctx context.Context, utterance string, notes [] } return "вот что я нашла: " + strings.Join(notes, "; "), nil } + if text, _ := parseResponseMood(resp); text != "" { + return text, nil + } return resp, nil } @@ -212,20 +223,28 @@ func (p *LLMPhraser) PhraseChat(ctx context.Context, utterance string, history [ msgs := []chatMsg{ {Role: "system", Content: sys}, } - // Append dialogue history: user turns become "user" messages, and since we - // don't store assistant replies in the dialogue history, we reconstruct the - // pattern as alternating user messages. The model can infer maven's presence. + // Combine history and current utterance into one user message. + // Some model chat templates (Ministral, etc.) reject consecutive user turns. + var combined string for _, t := range history { - msgs = append(msgs, chatMsg{Role: "user", Content: t.Text}) + combined += t.Text + "\n" } - // Current utterance as the final user message. - msgs = append(msgs, chatMsg{Role: "user", Content: utterance}) + combined += utterance + msgs = append(msgs, chatMsg{Role: "user", Content: strings.TrimSpace(combined)}) resp, err := p.chatWithMessages(ctx, msgs, 512) if err != nil { + log.Printf("phraser: PhraseChat: %v", err) return "поговорили.", nil } - return resp, nil + if text, _ := parseResponseMood(resp); text != "" { + return text, nil + } + // fallback: plain text without JSON + if i := strings.IndexByte(resp, '\n'); i >= 0 { + resp = resp[:i] + } + return strings.TrimSpace(resp), nil } // chatSystemPrompt returns the system prompt for conversational chat. @@ -235,7 +254,7 @@ func chatSystemPrompt(persona string) string { Keep replies brief (1-3 sentences) and natural. You're helpful, curious, and a little warm. Respond in the user's language (Russian or English, matching their last message). Never roleplay emotions you don't have, but stay friendly. -Just answer directly — no JSON wrapper, no meta-commentary.` +Respond ONLY with valid JSON: {"response": "...", "mood": "neutral"}. "response" is your reply text; "mood" reflects your tone (neutral/happy/thinking/tired/confused).` if persona != "" { base = persona + "\n\n" + base } @@ -279,7 +298,12 @@ func (p *LLMPhraser) chatWithMessages(ctx context.Context, msgs []chatMsg, maxTo if len(cr.Choices) == 0 { return "", fmt.Errorf("llm: no choices in response") } - return cr.Choices[0].Message.Content, nil + content := cr.Choices[0].Message.Content + if content == "" { + content = cr.Choices[0].Message.ReasoningContent + } + log.Printf("llm raw content: %q", content) + return stripThink(content), nil } func (p *LLMPhraser) PhraseReminder(ctx context.Context, d loop.ReminderDecision) (delivery.PhrasedReminder, error) { @@ -289,24 +313,29 @@ func (p *LLMPhraser) PhraseReminder(ctx context.Context, d loop.ReminderDecision } prompt := fmt.Sprintf( - `The user set a reminder: "%s". Rephrase it briefly as a gentle nudge. Respond as JSON: {"body": "...", "summary": "..."}`, + `The user set a reminder: "%s". Rephrase it briefly as a gentle nudge. Respond as JSON: {"response": "...", "mood": "..."}`, text, ) resp, err := p.chat(ctx, prompt) if err != nil { return delivery.PhrasedReminder{}, err } - body, summary := parsePhrase(resp) + body, mood := parseResponseMood(resp) + if body == "" { + // fallback: try old body/summary format + body, _ = parsePhrase(resp) + } if body == "" { body = text } - if summary == "" { - summary = text - if len(summary) > 60 { - summary = summary[:57] + "..." - } + if mood == "" { + mood = "neutral" } - return delivery.PhrasedReminder{Decision: d, Body: body, Summary: summary}, nil + summary := body + if len(summary) > 60 { + summary = summary[:57] + "..." + } + return delivery.PhrasedReminder{Decision: d, Body: body, Summary: summary, Mood: mood}, nil } type chatMsg struct { @@ -324,13 +353,15 @@ type chatReq struct { type chatResp struct { Choices []struct { Message struct { - Content string `json:"content"` + Content string `json:"content"` + Reasoning string `json:"reasoning"` + ReasoningContent string `json:"reasoning_content"` } `json:"message"` } `json:"choices"` } func (p *LLMPhraser) chat(ctx context.Context, userPrompt string) (string, error) { - return p.chatWithSystem(ctx, p.systemPrompt(), userPrompt, 256) + return p.chatWithSystem(ctx, p.systemPrompt(), userPrompt, 512) } func (p *LLMPhraser) chatWithSystem(ctx context.Context, system, user string, maxTokens int) (string, error) { @@ -374,11 +405,15 @@ func (p *LLMPhraser) chatWithSystem(ctx context.Context, system, user string, ma if len(cr.Choices) == 0 { return "", fmt.Errorf("llm: no choices in response") } - return cr.Choices[0].Message.Content, nil + content := cr.Choices[0].Message.Content + if content == "" { + content = cr.Choices[0].Message.ReasoningContent + } + return stripThink(content), nil } func (p *LLMPhraser) systemPrompt() string { - base := `You are maven, a self-hosted personal assistant. Generate brief, natural nudge messages in the user's language (Russian or English). Respond ONLY with valid JSON: {"body": "full voice message", "summary": "brief away-channel version (<60 chars)"}. body is what the user hears on voice; summary is for push notifications (ntfy/telegram) — minimal, no exfil detail.` + base := `You are maven, a self-hosted personal assistant. Generate brief, natural nudge messages in the user's language (Russian or English). Respond ONLY with valid JSON: {"response": "full voice message", "mood": "neutral"}. "response" is what the user hears; "mood" reflects maven's tone (neutral/happy/thinking/tired/confused).` if p.cfg.Persona != "" { base = p.cfg.Persona + "\n\n" + base } @@ -388,7 +423,7 @@ func (p *LLMPhraser) systemPrompt() string { // querySystemPrompt returns the system prompt for PhraseQuery (notes + general // knowledge). Prepends the configured persona when set. func (p *LLMPhraser) querySystemPrompt() string { - base := "You are maven, a self-hosted personal assistant answering from your notes. Answer briefly and naturally in Russian starting with \"вот что я нашла: \". Respond with just the answer text, no JSON wrapper." + base := "You are maven, a self-hosted personal assistant answering from your notes. Answer briefly and naturally in Russian starting with \"вот что я нашла: \". Respond ONLY with valid JSON: {\"response\": \"...\", \"mood\": \"neutral\"}." if p.cfg.Persona != "" { base = p.cfg.Persona + "\n\n" + base } @@ -408,11 +443,33 @@ func buildNudgePrompt(c loop.Candidate) string { `Generate a nudge message. Context: %s -Respond as JSON: {"body": "...", "summary": "..."}`, +Respond as JSON: {"response": "...", "mood": "..."}`, strings.Join(ctxParts, "\n"), ) } +type responseMood struct { + Response string `json:"response"` + Mood string `json:"mood"` +} + +// parseResponseMood extracts {"response","mood"} from LLM output, tolerant +// of thinking tokens and extra text before/after the JSON block. Returns +// ("", "") when no valid JSON is found. +func parseResponseMood(raw string) (response, mood string) { + cleaned := strings.TrimSpace(raw) + start := strings.Index(cleaned, "{") + end := strings.LastIndex(cleaned, "}") + if start < 0 || end < 0 || end <= start { + return "", "" + } + var parsed responseMood + if err := json.Unmarshal([]byte(cleaned[start:end+1]), &parsed); err != nil { + return "", "" + } + return parsed.Response, parsed.Mood +} + func parsePhrase(raw string) (body, summary string) { cleaned := strings.TrimSpace(raw) start := strings.Index(cleaned, "{") @@ -437,3 +494,12 @@ func extractPort(listen string) string { } return port } + +// stripThink removes the block that Thinking-variant models emit +// before the actual response. No-op when no think block is present. +func stripThink(s string) string { + if i := strings.LastIndex(s, ""); i >= 0 { + s = strings.TrimSpace(s[i+8:]) + } + return s +} diff --git a/internal/router/knowledge.go b/internal/router/knowledge.go index 05773ff..6a16e4b 100644 --- a/internal/router/knowledge.go +++ b/internal/router/knowledge.go @@ -3,5 +3,5 @@ package router // KnowledgePrompt returns the system prompt for general knowledge questions // that the phraser uses when no notes match the query. func KnowledgePrompt() string { - return `Ты — Мавена, персональный ассистент. Ответь кратко из своих знаний. Если не знаешь — скажи "не знаю". Не выдумывай.` + return `Ты — Мавена, персональный ассистент. Ответь кратко из своих знаний. Если не знаешь — скажи "не знаю". Не выдумывай. Respond ONLY with valid JSON: {"response": "...", "mood": "neutral"}.` }