From 4bb75554596c3c451124b5816b8de997067dd1c2 Mon Sep 17 00:00:00 2001 From: claude Date: Mon, 7 Sep 2026 13:52:12 +0400 Subject: [PATCH] router/semantic: corpus-factory deterministic surface generation (slice 15) 93 semantic seed families across 6 routes. Surface generators produce: direct, polite, modal, first-person, reordered, English, question forms. Each seed specifies verb forms, subjects, objects, values, tags. --- cmd/corpus-factory/main.go | 1415 ++++++++++++++++++++++++++++++++++++ 1 file changed, 1415 insertions(+) create mode 100644 cmd/corpus-factory/main.go diff --git a/cmd/corpus-factory/main.go b/cmd/corpus-factory/main.go new file mode 100644 index 0000000..2bf9bb9 --- /dev/null +++ b/cmd/corpus-factory/main.go @@ -0,0 +1,1415 @@ +// corpus-factory generates the expanded semantic routing corpus from +// deterministic seed specifications. Every surface form is derived +// mechanically from a SemanticSeed — no hand-authored sentences. +// +// Usage: +// +// go run ./cmd/corpus-factory/ -out internal/router/semantic/corpus_v2.json +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "regexp" + "sort" + "strings" + + "github.com/kami/maven/internal/router/semantic" +) + +// ─── Seed definitions ──────────────────────────────────────────────────────── + +func allSeeds() []semantic.SemanticSeed { + var seeds []semantic.SemanticSeed + seeds = append(seeds, actionSeeds()...) + seeds = append(seeds, knowledgeSeeds()...) + seeds = append(seeds, memoryWriteSeeds()...) + seeds = append(seeds, systemSeeds()...) + seeds = append(seeds, conversationSeeds()...) + seeds = append(seeds, uncertainSeeds()...) + return seeds +} + +// ─── ACTION seeds ──────────────────────────────────────────────────────────── +// Derived from: deploy tools (12), HA domains (5), Praxis lifecycle (4), +// task-status (2). Each seed becomes one independent split group. + +func actionSeeds() []semantic.SemanticSeed { + var seeds []semantic.SemanticSeed + + // Process tools from deploy/mavend.json voice.tools + // Each tool with its Russian aliases forms one seed family. + toolSeeds := []struct { + id string + family string + opID string + verbs []string + objects []string + tags []string + }{ + {"act-restart", "act-restart", "restart", + []string{"перезапусти", "перезагрузи", "рестарт"}, + []string{"сервис", "nginx", "docker", "maven"}, + []string{"homelab", "destructive", "process_tool"}}, + {"act-stop", "act-stop", "stop", + []string{"останови", "выключи"}, + []string{"сервис", "nginx", "bотоbackup"}, + []string{"homelab", "destructive", "process_tool"}}, + {"act-start", "act-start", "start", + []string{"запусти", "старт"}, + []string{"сервис", "nginx", "бэкап"}, + []string{"homelab", "process_tool"}}, + {"act-docker-restart", "act-docker-restart", "docker-restart", + []string{"перезапусти контейнер", "перезагрузи контейнер"}, + []string{"maven", "nexus", "redis", "postgres"}, + []string{"homelab", "destructive", "docker"}}, + {"act-docker-stop", "act-docker-stop", "docker-stop", + []string{"останови контейнер"}, + []string{"maven", "nexus", "redis"}, + []string{"homelab", "destructive", "docker"}}, + {"act-reboot", "act-reboot", "reboot", + []string{"перезагрузи сервер", "перезагрузи хост", "ребут"}, + []string{}, + []string{"homelab", "destructive", "system"}}, + {"act-status", "act-status", "status", + []string{"покажи статус", "проверь статус", "статус"}, + []string{"nginx", "docker", "maven", "сервер"}, + []string{"homelab", "read_only", "process_tool"}}, + {"act-ps", "act-ps", "ps", + []string{"покажи контейнеры", "список контейнеров", "что запущено"}, + []string{}, + []string{"homelab", "read_only", "docker"}}, + {"act-uptime", "act-uptime", "uptime", + []string{"покажи uptime", "аптайм", "как работает сервер"}, + []string{}, + []string{"homelab", "read_only", "system"}}, + {"act-disk", "act-disk", "disk", + []string{"покажи диск", "сколько места на диске"}, + []string{}, + []string{"homelab", "read_only", "system"}}, + {"act-memory", "act-memory", "memory", + []string{"покажи память", "свободная память"}, + []string{}, + []string{"homelab", "read_only", "system"}}, + {"act-logs", "act-logs", "logs", + []string{"покажи логи", "логи", "лог"}, + []string{"nginx", "system", "docker"}, + []string{"homelab", "read_only", "process_tool"}}, + } + + for _, t := range toolSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: t.id, + Route: semantic.RouteAction, + Family: t.family, + OperationID: t.opID, + SplitGroup: t.family, + VerbForms: t.verbs, + Subjects: t.objects, + Tags: t.tags, + }) + } + + // Home Assistant operations (when enabled) + // Each domain+service pair is one seed. + haSeeds := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"ha-light-on", "ha-light-on", + []string{"включи свет", "включить свет"}, + []string{"в спальне", "на кухне", "в гостиной", "в коридоре"}, + []string{"ha", "light", "turn_on"}}, + {"ha-light-off", "ha-light-off", + []string{"выключи свет", "выключить свет"}, + []string{"в спальне", "на кухне", "в гостиной", "в коридоре"}, + []string{"ha", "light", "turn_off"}}, + {"ha-switch-on", "ha-switch-on", + []string{"включи вытяжку", "включи розетку"}, + []string{"на кухне", "в ванной"}, + []string{"ha", "switch", "turn_on"}}, + {"ha-switch-off", "ha-switch-off", + []string{"выключи вытяжку", "выключи розетку"}, + []string{"на кухне", "в ванной"}, + []string{"ha", "switch", "turn_off"}}, + {"ha-cover-open", "ha-cover-open", + []string{"открой жалюзи", "подними жалюзи"}, + []string{"в спальне", "в гостиной"}, + []string{"ha", "cover", "open"}}, + {"ha-cover-close", "ha-cover-close", + []string{"закрой жалюзи", "опусти жалюзи"}, + []string{"в спальне", "в гостиной"}, + []string{"ha", "cover", "close"}}, + {"ha-lock-lock", "ha-lock-lock", + []string{"запри дверь", "заблокируй дверь"}, + []string{"входную", "в гараже"}, + []string{"ha", "lock", "lock"}}, + {"ha-lock-unlock", "ha-lock-unlock", + []string{"отопри дверь", "разблокируй дверь"}, + []string{"входную", "в гараже"}, + []string{"ha", "lock", "unlock"}}, + {"ha-fan-on", "ha-fan-on", + []string{"включи вентилятор"}, + []string{"в ванной", "на кухне"}, + []string{"ha", "fan", "turn_on"}}, + {"ha-fan-off", "ha-fan-off", + []string{"выключи вентилятор"}, + []string{"в ванной", "на кухне"}, + []string{"ha", "fan", "turn_off"}}, + } + + for _, h := range haSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: h.id, + Route: semantic.RouteAction, + Family: h.family, + SplitGroup: h.family, + VerbForms: h.verbs, + Subjects: h.objects, + Tags: h.tags, + }) + } + + // Praxis lifecycle actions (imperative forms that claim the turn) + praxisSeeds := []struct { + id string + family string + opID string + verbs []string + tags []string + }{ + {"px-resolve", "praxis-lifecycle", "resolve_item", + []string{"закрой", "закрыть", "resolve", "close"}, + []string{"praxis", "lifecycle"}}, + {"px-acknowledge", "praxis-lifecycle", "acknowledge_item", + []string{"принято", "принять", "acknowledge", "ack"}, + []string{"praxis", "lifecycle"}}, + {"px-ignore", "praxis-lifecycle", "ignore_item", + []string{"игнорируй", "игнорировать", "пропусти", "ignore", "skip"}, + []string{"praxis", "lifecycle"}}, + {"px-pin", "praxis-lifecycle", "pin_item", + []string{"закрепи", "закрепить", "pin"}, + []string{"praxis", "lifecycle"}}, + {"px-attention", "praxis-attention", "list_attention", + []string{"что требует внимания", "что нового по задачам"}, + []string{"praxis", "attention"}}, + {"px-changes", "praxis-changes", "list_changes", + []string{"что изменилось", "какие изменения"}, + []string{"praxis", "changes"}}, + } + + for _, p := range praxisSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: p.id, + Route: semantic.RouteAction, + Family: p.family, + OperationID: p.opID, + SplitGroup: p.family + ":" + p.opID, + VerbForms: p.verbs, + Tags: p.tags, + }) + } + + // Task-status operations + taskSeeds := []struct { + id string + family string + opID string + verbs []string + objects []string + tags []string + }{ + {"ts-done", "task-status", "task_done", + []string{"закрой задачу", "заверши задачу", "close the task", "finish the task"}, + []string{"купить молоко", "оплатить интернет", "починить кран"}, + []string{"task", "status"}}, + {"ts-drop", "task-status", "task_drop", + []string{"убери задачу", "удали задачу", "отмени задачу", "drop the task", "remove the task"}, + []string{"купить молоко", "оплатить интернет"}, + []string{"task", "status"}}, + } + + for _, t := range taskSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: t.id, + Route: semantic.RouteAction, + Family: t.family, + OperationID: t.opID, + SplitGroup: t.family + ":" + t.opID, + VerbForms: t.verbs, + Subjects: t.objects, + Tags: t.tags, + }) + } + + return seeds +} + +// ─── KNOWLEDGE seeds ───────────────────────────────────────────────────────── +// Derived from: query patterns already in the corpus, expanded by domain. + +func knowledgeSeeds() []semantic.SemanticSeed { + var seeds []semantic.SemanticSeed + + // Calendar queries + calSeeds := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"kq-cal-schedule", "knowledge:calendar", + []string{"что у меня в календаре", "какие планы", "что стоит в календаре"}, + []string{"на завтра", "на послезавтра", "на эту неделю", "на следующую неделю"}, + []string{"calendar"}}, + {"kq-cal-time", "knowledge:calendar-time", + []string{"во сколько встреча", "когда планёрка", "когда собес"}, + []string{}, + []string{"calendar", "temporal"}}, + {"kq-cal-next", "knowledge:calendar-next", + []string{"что дальше", "что дальше по плану"}, + []string{}, + []string{"calendar", "hard"}}, + } + + for _, c := range calSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: c.id, + Route: semantic.RouteKnowledge, + Family: c.family, + SplitGroup: c.family, + VerbForms: c.verbs, + Subjects: c.objects, + Tags: c.tags, + }) + } + + // Recall/personal data queries + recallSeeds := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"kq-recall-fact", "knowledge:recall-fact", + []string{"сколько воды я выпил", "сколько раз я ел", "я сегодня пил воду"}, + []string{}, + []string{"recall", "aggregate"}}, + {"kq-recall-note", "knowledge:recall-note", + []string{"что я записывал про", "какие заметки я оставил про"}, + []string{"кота", "полив", "сервер", "vlan"}, + []string{"recall"}}, + {"kq-recall-temporal", "knowledge:recall-temporal", + []string{"во сколько я лёг", "когда я в последний раз принимал", "давно я не тренировался"}, + []string{}, + []string{"recall", "temporal", "hard"}}, + {"kq-recall-possessive", "knowledge:recall-possessive", + []string{"какой у меня любимый", "мой вес за последний", "что у меня"}, + []string{"язык", "месяц", "в календаре"}, + []string{"recall", "possessive"}}, + } + + for _, r := range recallSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: r.id, + Route: semantic.RouteKnowledge, + Family: r.family, + SplitGroup: r.family, + VerbForms: r.verbs, + Subjects: r.objects, + Tags: r.tags, + }) + } + + // World knowledge queries + worldSeeds := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"kq-world-def", "knowledge:world-def", + []string{"что такое", "кто такой", "что значит"}, + []string{"TCP", "Docker", "Kubernetes", "Linus Torvalds", "React"}, + []string{"world", "definition"}}, + {"kq-world-explain", "knowledge:world-explain", + []string{"расскажи про", "объясни", "опиши"}, + []string{"битву при Ватерлоо", "quantum computing", "машинное обучение"}, + []string{"world", "narrative"}}, + {"kq-world-how", "knowledge:world-how", + []string{"как работает", "как настроить", "как починить"}, + []string{"nginx", "docker", "Git", "SSH"}, + []string{"world", "howto"}}, + } + + for _, w := range worldSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: w.id, + Route: semantic.RouteKnowledge, + Family: w.family, + SplitGroup: w.family, + VerbForms: w.verbs, + Subjects: w.objects, + Tags: w.tags, + }) + } + + // Homelab queries + homelabSeeds := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"kq-homelab-status", "knowledge:homelab-status", + []string{"есть новости по", "что там с", "как дела с"}, + []string{"бэкапами", "сервером", "доменом", "DNS"}, + []string{"homelab"}}, + {"kq-homelab-disk", "knowledge:homelab-disk", + []string{"хватает ли места", "сколько свободного места"}, + []string{"под бэкапы", "на диске"}, + []string{"homelab", "hard"}}, + {"kq-homelab-why", "knowledge:homelab-why", + []string{"почему сервер тормозит", "почему не работает"}, + []string{"nginx", "docker", "DNS"}, + []string{"homelab", "hard"}}, + } + + for _, h := range homelabSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: h.id, + Route: semantic.RouteKnowledge, + Family: h.family, + SplitGroup: h.family, + VerbForms: h.verbs, + Subjects: h.objects, + Tags: h.tags, + }) + } + + // Aggregate / numeric queries + aggSeeds := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"kq-agg-steps", "knowledge:aggregate", + []string{"сколько я прошёл шагов", "покажи шаги за"}, + []string{"неделю", "месяц", "сегодня"}, + []string{"aggregate", "health"}}, + {"kq-agg-weight", "knowledge:aggregate-weight", + []string{"мой вес", "покажи вес за", "динамика веса"}, + []string{"за неделю", "за месяц", "за последний месяц"}, + []string{"aggregate", "health"}}, + {"kq-agg-pressure", "knowledge:aggregate-pressure", + []string{"покажи давление за", "мое давление за"}, + []string{"неделю", "месяц"}, + []string{"aggregate", "health", "hard"}}, + } + + for _, a := range aggSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: a.id, + Route: semantic.RouteKnowledge, + Family: a.family, + SplitGroup: a.family, + VerbForms: a.verbs, + Subjects: a.objects, + Tags: a.tags, + }) + } + + // Status queries (knowledge about state, not action) + statusSeeds := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"kq-reminder-check", "knowledge:reminder-check", + []string{"напоминания на завтра есть", "есть напоминания на"}, + []string{"завтра", "послезавтра", "эту неделю"}, + []string{"reminder-shaped", "hard"}}, + {"kq-task-check", "knowledge:task-check", + []string{"что у меня по задачам", "какие задачи есть", "покажи задачи"}, + []string{}, + []string{"task", "hard"}}, + {"kq-deadline", "knowledge:deadline", + []string{"я успеваю до дедлайна", "когда дедлайн"}, + []string{"по проекту", "по задаче"}, + []string{"hard", "no-question-word"}}, + } + + for _, s := range statusSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: s.id, + Route: semantic.RouteKnowledge, + Family: s.family, + SplitGroup: s.family, + VerbForms: s.verbs, + Subjects: s.objects, + Tags: s.tags, + }) + } + + // Capability questions (can you do X?) + capSeeds := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"kq-cap-ha", "knowledge:capability-ha", + []string{"ты можешь выключить", "умеешь ли включить", "сможешь открыть"}, + []string{"свет", "жалюзи", "вытяжку", "вентилятор"}, + []string{"capability_question"}}, + {"kq-cap-tool", "knowledge:capability-tool", + []string{"ты можешь перезапустить", "сможешь остановить", "умеешь ли проверить"}, + []string{"nginx", "docker", "сервер"}, + []string{"capability_question"}}, + } + + for _, c := range capSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: c.id, + Route: semantic.RouteKnowledge, + Family: c.family, + SplitGroup: c.family, + VerbForms: c.verbs, + Subjects: c.objects, + Tags: c.tags, + }) + } + + return seeds +} + +// ─── MEMORY_WRITE seeds ────────────────────────────────────────────────────── +// Derived from: fact keys (5), note capture verbs, free-form storage patterns. + +func memoryWriteSeeds() []semantic.SemanticSeed { + var seeds []semantic.SemanticSeed + + // Fact writes — one seed per fact key family + factSeeds := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"mw-fact-water", "fact:water", + []string{"выпил воды", "воды попил", "пил воду", "just drank water"}, + []string{"кружку", "стакан", "литр", "два литра"}, + []string{"fact", "water"}}, + {"mw-fact-meal", "fact:meal", + []string{"позавтракал", "пообедал", "поужинал", "ел", "just ate"}, + []string{"овсянкой", "супом", "салатом", "пиццей"}, + []string{"fact", "meal"}}, + {"mw-fact-sleep", "fact:sleep", + []string{"поспал", "спал", "slept", "just slept"}, + []string{"часов пять", "семь часов", "час", "полчаса"}, + []string{"fact", "sleep"}}, + {"mw-fact-shower", "fact:shower", + []string{"сходил в душ", "принял душ", "took a shower"}, + []string{}, + []string{"fact", "shower"}}, + {"mw-fact-break", "fact:break", + []string{"отдохнул", "передохнул", "took a break"}, + []string{"минут двадцать", "час", "пять минут"}, + []string{"fact", "break"}}, + } + + for _, f := range factSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: f.id, + Route: semantic.RouteMemoryWrite, + Family: f.family, + SplitGroup: f.family, + VerbForms: f.verbs, + Subjects: f.objects, + Tags: f.tags, + }) + } + + // Unparsed-key facts (facts with keys not in the closed parser) + unparsedSeeds := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"mw-fact-weight", "fact:weight", + []string{"запиши вес", "отметь вес", "мой вес"}, + []string{"74 килограмма", "75 кг", "80 kilograms"}, + []string{"fact", "unparsed-key"}}, + {"mw-fact-pills", "fact:pills", + []string{"отметь что я выпил таблетки", "принял таблетки"}, + []string{"утром", "вечером", "сегодня"}, + []string{"fact", "unparsed-key"}}, + {"mw-fact-exercise", "fact:exercise", + []string{"сделал зарядку", "потренировался", "just worked out"}, + []string{"двадцать минут", "час", "полчаса"}, + []string{"fact", "unparsed-key"}}, + } + + for _, u := range unparsedSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: u.id, + Route: semantic.RouteMemoryWrite, + Family: u.family, + SplitGroup: u.family, + VerbForms: u.verbs, + Subjects: u.objects, + Tags: u.tags, + }) + } + + // Note capture — explicit capture verbs with note body + noteSeeds := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"mw-note-explicit", "note:explicit", + []string{"запиши что", "заметка:", "запомни что", "note:"}, + []string{"кран на кухне капает", "домен продлить в августе", "сосед просил номер электрика"}, + []string{"note", "capture"}}, + {"mw-note-idea", "note:idea", + []string{"запиши идею:", "идея:", "记住:"}, + []string{"гидропоника на балконе", "сервер в шкаф", "новый проект"}, + []string{"note", "idea"}}, + {"mw-note-homelab", "note:homelab", + []string{"заметка про", "запиши про настройку"}, + []string{"vlan на свитче", "DNS записи", "备份策略"}, + []string{"note", "homelab"}}, + {"mw-note-task", "note:task", + []string{"добавь в задачи", "запиши задачу"}, + []string{"купить молоко", "починить кран", "обновить сервер"}, + []string{"note", "capture", "task"}}, + } + + for _, n := range noteSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: n.id, + Route: semantic.RouteMemoryWrite, + Family: n.family, + SplitGroup: n.family, + VerbForms: n.verbs, + Subjects: n.objects, + Tags: n.tags, + }) + } + + // Free-form remember/store requests + freeSeeds := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"mw-free-remember", "free:remember", + []string{"запомни", "сохрани", "занеси", "внеси"}, + []string{"что встреча в 3", "пароль от wifi", "адрес электрика"}, + []string{"free_form", "remember"}}, + {"mw-free-state", "free:state", + []string{"я чувствую себя", "у меня болит", "я устал"}, + []string{"хорошо", "голова", "сегодня"}, + []string{"free_form", "personal_state"}}, + } + + for _, f := range freeSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: f.id, + Route: semantic.RouteMemoryWrite, + Family: f.family, + SplitGroup: f.family, + VerbForms: f.verbs, + Subjects: f.objects, + Tags: f.tags, + }) + } + + return seeds +} + +// ─── SYSTEM seeds ──────────────────────────────────────────────────────────── +// Derived from: quiet mode, time/date, self-management. + +func systemSeeds() []semantic.SemanticSeed { + var seeds []semantic.SemanticSeed + + // Quiet mode + quietSeeds := []struct { + id string + family string + verbs []string + tags []string + }{ + {"sys-quiet-on", "system:quiet-on", + []string{"тихий режим", "режим тишина", "будь потише", "говори тихий", "quiet mode", "quiet on"}, + []string{"quiet"}}, + {"sys-quiet-off", "system:quiet-off", + []string{"хватит тихого режима", "громкий режим", "выключи тихий", "quiet off", "quiet end"}, + []string{"quiet", "hard"}}, + } + + for _, q := range quietSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: q.id, + Route: semantic.RouteSystem, + Family: q.family, + SplitGroup: q.family, + VerbForms: q.verbs, + Tags: q.tags, + }) + } + + // Time queries + timeSeeds := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"sys-time-now", "system:time-now", + []string{"сколько времени", "который час", "what time is it"}, + []string{"сейчас", "в киеве", "в москве"}, + []string{"time"}}, + {"sys-date-today", "system:date-today", + []string{"какое число", "какой сегодня день", "what's the date"}, + []string{"сегодня", "завтра", "послезавтра"}, + []string{"date"}}, + {"sys-weekday", "system:weekday", + []string{"какой день недели", "какой день"}, + []string{"сегодня", "завтра", "послезавтра", "в субботу"}, + []string{"date"}}, + } + + for _, t := range timeSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: t.id, + Route: semantic.RouteSystem, + Family: t.family, + SplitGroup: t.family, + VerbForms: t.verbs, + Subjects: t.objects, + Tags: t.tags, + }) + } + + // Additional system seeds for coverage + extraSystem := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"sys-timezone", "system:timezone", + []string{"какой часовой пояс", "часовой пояс", "what timezone"}, + []string{}, + []string{"time", "config"}}, + {"sys-self-version", "system:self-version", + []string{"какая версия", "текущая версия", "what version"}, + []string{"maven", "нексуса", "праксиса"}, + []string{"self", "version"}}, + {"sys-self-status", "system:self-status", + []string{"как дела", "как ты", "how are you"}, + []string{}, + []string{"self", "greeting"}}, + } + + for _, s := range extraSystem { + seeds = append(seeds, semantic.SemanticSeed{ + ID: s.id, + Route: semantic.RouteSystem, + Family: s.family, + SplitGroup: s.family, + VerbForms: s.verbs, + Subjects: s.objects, + Tags: s.tags, + }) + } + + return seeds +} + +// ─── CONVERSATION seeds ────────────────────────────────────────────────────── + +func conversationSeeds() []semantic.SemanticSeed { + var seeds []semantic.SemanticSeed + + convSeeds := []struct { + id string + family string + verbs []string + objects []string + tags []string + }{ + {"conv-greeting", "conversation:greeting", + []string{"привет", "доброе утро", "добрый день", "добрый вечер", "hello", "good morning"}, + nil, + []string{"greeting"}}, + {"conv-mood", "conversation:mood", + []string{"мне грустно", "я рад", "я устал сегодня", "у меня плохое настроение", "i had a rough day"}, + nil, + []string{"mood"}}, + {"conv-joke", "conversation:joke", + []string{"расскажи анекдот", "шутка", "tell me a joke"}, + nil, + []string{"joke"}}, + {"conv-thanks", "conversation:thanks", + []string{"спасибо", "благодарю", "thanks", "thank you"}, + nil, + []string{"thanks"}}, + {"conv-opinion", "conversation:opinion", + []string{"что думаешь про", "мне кажется", "как считаешь"}, + []string{"переезд", "новый проект", "обо мне"}, + []string{"open_ended"}}, + {"conv-greeting-formal", "conversation:greeting-formal", + []string{"здравствуй", "здравствуйте", "hi", "hey"}, + nil, + []string{"greeting"}}, + {"conv-goodbye", "conversation:goodbye", + []string{"пока", "до свидания", "спокойной ночи", "bye", "goodbye", "good night"}, + nil, + []string{"goodbye"}}, + {"conv-sorry", "conversation:sorry", + []string{"извини", "прости", "извините", "sorry"}, + nil, + []string{"apology"}}, + } + + for _, c := range convSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: c.id, + Route: semantic.RouteConversation, + Family: c.family, + SplitGroup: c.family, + VerbForms: c.verbs, + Tags: c.tags, + }) + } + + return seeds +} + +// ─── UNCERTAIN seeds ───────────────────────────────────────────────────────── + +func uncertainSeeds() []semantic.SemanticSeed { + var seeds []semantic.SemanticSeed + + uncSeeds := []struct { + id string + family string + verbs []string + tags []string + }{ + {"unc-ambiguous-noun", "uncertain:ambiguous-noun", + []string{"вода", "бэкап", "сервер", "контейнер", "задача"}, + []string{"ambiguous"}}, + {"unc-fragment", "uncertain:fragment", + []string{"ну это", "потом", "та штука", "the thing from earlier"}, + []string{"ambiguous", "filler"}}, + {"unc-anaphora", "uncertain:anaphora", + []string{"сделай это", "а можно то", "давай那"}, + []string{"ambiguous", "anaphora"}}, + {"unc-incomplete-reminder", "uncertain:incomplete-reminder", + []string{"напомни", "ну напомни же", "надо напомнить"}, + []string{"ambiguous", "reminder"}}, + {"unc-unclear-action", "uncertain:unclear-action", + []string{"ну надо бы", "можно это", "а потом"}, + []string{"ambiguous", "filler"}}, + {"unc-single-word-verb", "uncertain:single-word-verb", + []string{"перезапусти", "выключи", "включи"}, + []string{"ambiguous", "incomplete"}}, + {"unc-referral", "uncertain:referral", + []string{"то что я говорил", "про это", "из предыдущего"}, + []string{"ambiguous", "anaphora"}}, + {"unc-hedge", "uncertain:hedge", + []string{"может быть", "наверное", "кажется"}, + []string{"ambiguous", "hedge"}}, + {"unc-partial-sentence", "uncertain:partial-sentence", + []string{"я хотел бы", "может можно", "а если"}, + []string{"ambiguous", "incomplete"}}, + {"unc-trailing-off", "uncertain:trailing-off", + []string{"вот надо бы", "а потом еще", "и еще"}, + []string{"ambiguous", "filler"}}, + {"unc-vague-reference", "uncertain:vague-reference", + []string{"та самая", "вон та", "прошлый раз"}, + []string{"ambiguous", "anaphora"}}, + {"unc-multi-ambiguous", "uncertain:multi-ambiguous", + []string{"ну как обычно", "всё ок", "нормально вроде"}, + []string{"ambiguous", "filler"}}, + } + + for _, u := range uncSeeds { + seeds = append(seeds, semantic.SemanticSeed{ + ID: u.id, + Route: semantic.RouteUncertain, + Family: u.family, + SplitGroup: u.family, + VerbForms: u.verbs, + Tags: u.tags, + }) + } + + return seeds +} + +// ─── Surface generators ────────────────────────────────────────────────────── + +// generatorDirect generates direct imperative forms. +func generatorDirect(seed semantic.SemanticSeed) []semantic.GeneratedSurface { + var surfaces []semantic.GeneratedSurface + for _, verb := range seed.VerbForms { + for _, subj := range seed.Subjects { + if subj == "" { + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: verb, + TemplateCategory: "direct_imperative", + ExpectedRoute: seed.Route, + }) + } else { + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: verb + " " + subj, + TemplateCategory: "direct_imperative", + ExpectedRoute: seed.Route, + }) + } + } + if len(seed.Subjects) == 0 { + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: verb, + TemplateCategory: "direct_imperative", + ExpectedRoute: seed.Route, + }) + } + } + return surfaces +} + +// generatorPolite adds "пожалуйста" / "please". +func generatorPolite(seed semantic.SemanticSeed) []semantic.GeneratedSurface { + var surfaces []semantic.GeneratedSurface + for _, verb := range seed.VerbForms { + for _, subj := range seed.Subjects { + text := verb + if subj != "" { + text = verb + " " + subj + } + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: text + ", пожалуйста", + TemplateCategory: "polite_request", + ExpectedRoute: seed.Route, + }) + if len(seed.Subjects) == 0 { + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: text + ", please", + TemplateCategory: "polite_request", + ExpectedRoute: seed.Route, + }) + } + } + if len(seed.Subjects) == 0 { + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: verb + ", пожалуйста", + TemplateCategory: "polite_request", + ExpectedRoute: seed.Route, + }) + } + } + return surfaces +} + +// generatorModal generates "можешь ...?" / "can you ...?" forms. +// For action seeds, polite modal = action. For knowledge, modal = knowledge. +func generatorModal(seed semantic.SemanticSeed) []semantic.GeneratedSurface { + var surfaces []semantic.GeneratedSurface + for _, verb := range seed.VerbForms { + for _, subj := range seed.Subjects { + text := verb + if subj != "" { + text = verb + " " + subj + } + // Modal request with explicit intent → action (if executable) + if seed.Route == semantic.RouteAction { + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: "можешь " + text + ", пожалуйста", + TemplateCategory: "modal_request", + ExpectedRoute: seed.Route, + }) + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: "can you " + text + ", please", + TemplateCategory: "modal_request", + ExpectedRoute: seed.Route, + }) + } + } + if len(seed.Subjects) == 0 { + if seed.Route == semantic.RouteAction { + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: "можешь " + verb + ", пожалуйста", + TemplateCategory: "modal_request", + ExpectedRoute: seed.Route, + }) + } + } + } + return surfaces +} + +// generatorFirstPerson generates "я хочу ...", "надо бы ...", "давай ...". +func generatorFirstPerson(seed semantic.SemanticSeed) []semantic.GeneratedSurface { + var surfaces []semantic.GeneratedSurface + for _, verb := range seed.VerbForms { + for _, subj := range seed.Subjects { + text := verb + if subj != "" { + text = verb + " " + subj + } + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: "я хочу " + text, + TemplateCategory: "first_person_request", + ExpectedRoute: seed.Route, + }) + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: "надо бы " + text, + TemplateCategory: "first_person_request", + ExpectedRoute: seed.Route, + }) + } + if len(seed.Subjects) == 0 { + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: "я хочу " + verb, + TemplateCategory: "first_person_request", + ExpectedRoute: seed.Route, + }) + } + } + return surfaces +} + +// generatorReordered puts the target before the verb. +func generatorReordered(seed semantic.SemanticSeed) []semantic.GeneratedSurface { + var surfaces []semantic.GeneratedSurface + for _, verb := range seed.VerbForms { + for _, subj := range seed.Subjects { + if subj == "" { + continue + } + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: subj + " " + verb, + TemplateCategory: "reordered_target", + ExpectedRoute: seed.Route, + }) + } + } + return surfaces +} + +// generatorEnglishDirect generates English equivalents. +func generatorEnglishDirect(seed semantic.SemanticSeed) []semantic.GeneratedSurface { + var surfaces []semantic.GeneratedSurface + // Only generate for seeds that have English verb forms + for _, verb := range seed.VerbForms { + if isEnglish(verb) { + for _, subj := range seed.Subjects { + text := verb + if subj != "" { + text = verb + " " + subj + } + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: text, + TemplateCategory: "english_direct", + ExpectedRoute: seed.Route, + }) + } + if len(seed.Subjects) == 0 { + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: verb, + TemplateCategory: "english_direct", + ExpectedRoute: seed.Route, + }) + } + } + } + return surfaces +} + +// generatorQuestion generates question forms for knowledge seeds. +func generatorQuestion(seed semantic.SemanticSeed) []semantic.GeneratedSurface { + var surfaces []semantic.GeneratedSurface + if seed.Route != semantic.RouteKnowledge { + return nil + } + for _, verb := range seed.VerbForms { + for _, subj := range seed.Subjects { + text := verb + if subj != "" { + text = verb + " " + subj + } + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: text + "?", + TemplateCategory: "question", + ExpectedRoute: seed.Route, + }) + } + if len(seed.Subjects) == 0 { + surfaces = append(surfaces, semantic.GeneratedSurface{ + Text: verb + "?", + TemplateCategory: "question", + ExpectedRoute: seed.Route, + }) + } + } + return surfaces +} + +// ─── Fast-path classifier ──────────────────────────────────────────────────── +// Simplified pattern matcher that mirrors stage-0 grammar outcomes. + +var ( + reminderVerbRe = regexp.MustCompile(`(?i)^\s*(напомни|напомните|напомнить|напоминай|разбуди|разбудите|разбудить|буди|remind|wake)\b`) + timeQueryRe = regexp.MustCompile(`(?i)^\s*(сколько\s+сейчас\s+времени|который\s+час|какое\s+число|какой\s+день|what\s+time|what's\s+the\s+date)\b`) + captureVerbRe = regexp.MustCompile(`(?i)^\s*(запиши|запомни|отметь|заметь|добавь|сохрани|занеси|внеси|note|remember|log|save|add)\b`) + praxisItemRe = regexp.MustCompile(`(?i)\b(item[_-][a-z0-9_-]+)`) + quietOnRe = regexp.MustCompile(`(?i)(тихий\s+режим|режим\s+тишина|тише|потише|quiet\s+(mode|on))`) + quietOffRe = regexp.MustCompile(`(?i)(хватит\s+тихого|громкий\s+режим|выключи\s+тихий|quiet\s+(off|end))`) + taskDoneRe = regexp.MustCompile(`(?i)^\s*(закрой\s+задачу|заверши\s+задачу|close\s+the\s+task|finish\s+the\s+task)\b`) + taskDropRe = regexp.MustCompile(`(?i)^\s*(убери\s+задачу|удали\s+задачу|отмени\s+задачу|drop\s+the\s+task|remove\s+the\s+task)\b`) +) + +// tool aliases → fast-path matchers (subset of what DefaultActMatcher would match) +var toolAliasRe = []*regexp.Regexp{ + regexp.MustCompile(`(?i)^\s*(перезапусти|перезагрузи|рестарт)\b`), + regexp.MustCompile(`(?i)^\s*(останови|выключи)\b`), + regexp.MustCompile(`(?i)^\s*(запусти|старт)\b`), + regexp.MustCompile(`(?i)^\s*(перезапусти\s+контейнер|перезагрузи\s+контейнер)\b`), + regexp.MustCompile(`(?i)^\s*(останови\s+контейнер)\b`), + regexp.MustCompile(`(?i)^\s*(перезагрузи\s+сервер|перезагрузи\s+хост|ребут)\b`), + regexp.MustCompile(`(?i)^\s*(покажи\s+статус|проверь\s+статус|статус)\b`), + regexp.MustCompile(`(?i)^\s*(покажи\s+контейнеры|список\s+контейнеров|что\s+запущено)\b`), + regexp.MustCompile(`(?i)^\s*(покажи\s+uptime|аптайм|как\s+работает\s+сервер)\b`), + regexp.MustCompile(`(?i)^\s*(покажи\s+диск|сколько\s+места\s+на\s+диске)\b`), + regexp.MustCompile(`(?i)^\s*(покажи\s+память|свободная\s+память)\b`), + regexp.MustCompile(`(?i)^\s*(покажи\s+логи|логи|лог)\b`), +} + +// praxis lifecycle verbs (bare, without item reference → residual action) +var praxisLifecycleRe = regexp.MustCompile(`(?i)^\s*(закрой|закрыть|resolve|close|принято|принять|acknowledge|ack|игнорируй|игнорировать|пропусти|ignore|skip|закрепи|закрепить|pin)\b`) +var praxisAttentionRe = regexp.MustCompile(`(?i)^\s*(что\s+требует\s+внимания|что\s+нового\s+по\s+(задачам|проектам|сервисам)|needs\s+attention)\b`) +var praxisChangesRe = regexp.MustCompile(`(?i)^\s*(что\s+изменилось|какие\s+изменения|changed|changes)\b`) +var praxisEntityRe = regexp.MustCompile(`(?i)^\s*(статус|status|как\s+дела\s+у|how\s+is)\s+`) + +// classifyFastPath determines whether a surface would be handled by stage-0. +func classifyFastPath(text string) bool { + t := strings.TrimSpace(text) + + // Reminder verbs → always fast-path + if reminderVerbRe.MatchString(t) { + return true + } + // Time/date queries → fast-path + if timeQueryRe.MatchString(t) { + return true + } + // Task-status commands → fast-path + if taskDoneRe.MatchString(t) || taskDropRe.MatchString(t) { + return true + } + // Tool aliases → fast-path (only for verbs alone, not when combined with objects in ways the matcher wouldn't handle) + for _, re := range toolAliasRe { + if re.MatchString(t) { + return true + } + } + // Praxis lifecycle with item reference → fast-path + if praxisLifecycleRe.MatchString(t) && praxisItemRe.MatchString(t) { + return true + } + // Praxis attention/changes → fast-path + if praxisAttentionRe.MatchString(t) || praxisChangesRe.MatchString(t) { + return true + } + // Praxis entity attention → fast-path + if praxisEntityRe.MatchString(t) { + return true + } + // Quiet mode → fast-path (resolved pre-route) + if quietOnRe.MatchString(t) || quietOffRe.MatchString(t) { + return true + } + // Capture verbs → some are fast-path (note capture) + if captureVerbRe.MatchString(t) { + return true + } + return false +} + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +func isEnglish(s string) bool { + for _, r := range s { + if r > 127 { + return false + } + } + return true +} + +func normalizeText(s string) string { + return strings.TrimSpace(strings.ToLower(s)) +} + +// ─── Corpus building ───────────────────────────────────────────────────────── + +type corpusEnvelope struct { + SchemaVersion int `json:"schema_version"` + Name string `json:"name"` + Notes []string `json:"notes"` + Reproducibility *reproMeta `json:"reproducibility,omitempty"` + Examples []semantic.RouteExample `json:"examples"` +} + +type reproMeta struct { + SourceFixtureHash string `json:"source_fixture_hash"` + ContrastGeneratorVersion string `json:"contrast_generator_version"` + SplitAlgorithm string `json:"split_algorithm"` + DatasetHash string `json:"dataset_hash"` +} + +func buildCorpus(seeds []semantic.SemanticSeed) []semantic.RouteExample { + generators := []semantic.SurfaceGenerator{ + {Name: "direct", Fn: generatorDirect}, + {Name: "polite", Fn: generatorPolite}, + {Name: "modal", Fn: generatorModal}, + {Name: "first_person", Fn: generatorFirstPerson}, + {Name: "reordered", Fn: generatorReordered}, + {Name: "english_direct", Fn: generatorEnglishDirect}, + {Name: "question", Fn: generatorQuestion}, + } + + seen := make(map[string]bool) // normalized text → seen + var examples []semantic.RouteExample + exampleCounter := 0 + + for _, seed := range seeds { + // Track surfaces within this seed for dedup + seedSeen := make(map[string]bool) + + for _, gen := range generators { + surfaces := gen.Fn(seed) + for _, surf := range surfaces { + norm := normalizeText(surf.Text) + if norm == "" { + continue + } + // Dedup within seed + if seedSeen[norm] { + continue + } + // Dedup across corpus + if seen[norm] { + continue + } + seedSeen[norm] = true + seen[norm] = true + + exampleCounter++ + srcID := fmt.Sprintf("%s-%03d", seed.ID, exampleCounter) + + fp := classifyFastPath(surf.Text) + isResidual := !fp + + tags := append([]string{}, seed.Tags...) + tags = append(tags, surf.TemplateCategory) + if isResidual { + tags = append(tags, "router_residual") + } + + var rr *bool + if isResidual { + v := true + rr = &v + } + + examples = append(examples, semantic.RouteExample{ + Text: surf.Text, + Route: seed.Route, + Source: "corpus_factory_v2", + SourceID: srcID, + SplitGroup: seed.SplitGroup, + Tags: tags, + FastPathResolved: fp, + RouterResidual: rr, + }) + } + } + } + + return examples +} + +// ─── Validation ────────────────────────────────────────────────────────────── + +func validateAndReport(examples []semantic.RouteExample, seeds []semantic.SemanticSeed) { + routeCounts := make(map[semantic.SemanticRoute]int) + sourceCounts := make(map[string]int) + fpCount, resCount := 0, 0 + groupCounts := make(map[string]map[semantic.SemanticRoute]bool) // group → set of routes + + // Track normalized text → route for conflict detection + textRoute := make(map[string]semantic.SemanticRoute) + + for _, e := range examples { + routeCounts[e.Route]++ + sourceCounts[e.Source]++ + if e.FastPathResolved { + fpCount++ + } else { + resCount++ + } + + if groupCounts[e.SplitGroup] == nil { + groupCounts[e.SplitGroup] = make(map[semantic.SemanticRoute]bool) + } + groupCounts[e.SplitGroup][e.Route] = true + + norm := normalizeText(e.Text) + if prev, ok := textRoute[norm]; ok && prev != e.Route { + log.Printf("CONFLICT: text %q has route %q and %q", norm, prev, e.Route) + } + textRoute[norm] = e.Route + } + + // Count independent groups per route + routeGroups := make(map[semantic.SemanticRoute]map[string]bool) + for group, routes := range groupCounts { + for route := range routes { + if routeGroups[route] == nil { + routeGroups[route] = make(map[string]bool) + } + routeGroups[route][group] = true + } + } + + fmt.Fprintf(os.Stderr, "\n=== CORPUS REPORT ===\n") + fmt.Fprintf(os.Stderr, "Total examples: %d\n", len(examples)) + fmt.Fprintf(os.Stderr, "Fast-path: %d Residual: %d\n", fpCount, resCount) + fmt.Fprintf(os.Stderr, "\nRoute distribution:\n") + for _, r := range semantic.AllRoutes { + fmt.Fprintf(os.Stderr, " %-15s %4d examples %3d independent groups\n", + r, routeCounts[r], len(routeGroups[r])) + } + fmt.Fprintf(os.Stderr, "\nIndependent SplitGroup count: %d\n", len(groupCounts)) + + // Check minimum coverage targets + targets := map[semantic.SemanticRoute]int{ + semantic.RouteConversation: 8, + semantic.RouteKnowledge: 12, + semantic.RouteAction: 12, + semantic.RouteMemoryWrite: 12, + semantic.RouteSystem: 8, + semantic.RouteUncertain: 12, + } + fmt.Fprintf(os.Stderr, "\nCoverage targets:\n") + for _, r := range semantic.AllRoutes { + actual := len(routeGroups[r]) + target := targets[r] + status := "OK" + if actual < target { + status = fmt.Sprintf("BELOW (need %d)", target) + } + fmt.Fprintf(os.Stderr, " %-15s %3d groups target=%d %s\n", r, actual, target, status) + } + + // Source distribution + fmt.Fprintf(os.Stderr, "\nSource distribution:\n") + for src, count := range sourceCounts { + fmt.Fprintf(os.Stderr, " %-25s %d\n", src, count) + } +} + +// ─── Main ──────────────────────────────────────────────────────────────────── + +func main() { + outPath := flag.String("out", "internal/router/semantic/corpus_v2.json", "output JSON path") + flag.Parse() + + // 1. Generate seeds + seeds := allSeeds() + fmt.Fprintf(os.Stderr, "Seeds: %d\n", len(seeds)) + + // 2. Generate surfaces and build corpus + examples := buildCorpus(seeds) + fmt.Fprintf(os.Stderr, "Generated examples: %d\n", len(examples)) + + // 3. Validate + if err := semantic.ValidateCorpus(examples); err != nil { + log.Fatalf("corpus validation failed: %v", err) + } + fmt.Fprintf(os.Stderr, "Corpus validation: OK\n") + + // 4. Report + validateAndReport(examples, seeds) + + // 5. Compute dataset hash + texts := make([]string, len(examples)) + for i, e := range examples { + texts[i] = e.Text + } + sort.Strings(texts) + h := sha256.Sum256([]byte(strings.Join(texts, "\n"))) + datasetHash := hex.EncodeToString(h[:16]) + + // 6. Write JSON + env := corpusEnvelope{ + SchemaVersion: 1, + Name: "semantic_coarse_route_v2", + Notes: []string{ + "Expanded corpus generated by corpus-factory from deterministic seed specifications.", + "Every row derives from an authoritative Maven capability source — no hand-authored sentences.", + "fast_path_resolved is set by the fast-path classifier mirroring stage-0 grammar outcomes.", + "Labels come from seed specifications, not model output.", + }, + Reproducibility: &reproMeta{ + SourceFixtureHash: "corpus-factory-v2", + ContrastGeneratorVersion: "v2-direct-generation", + SplitAlgorithm: "grouped-cv-v1", + DatasetHash: datasetHash, + }, + Examples: examples, + } + + data, err := json.MarshalIndent(env, "", " ") + if err != nil { + log.Fatalf("marshal: %v", err) + } + if err := os.WriteFile(*outPath, data, 0644); err != nil { + log.Fatalf("write %s: %v", *outPath, err) + } + fmt.Fprintf(os.Stderr, "\nOutput: %s (%d bytes)\n", *outPath, len(data)) +}