// corpus-factory generates the expanded semantic routing corpus from // deterministic seed specifications. Every surface form is derived // mechanically from a SemanticSeed — no hand-authored sentences. // // Usage: // // go run ./cmd/corpus-factory/ -out internal/router/semantic/corpus_v2.json package main import ( "crypto/sha256" "encoding/hex" "encoding/json" "flag" "fmt" "log" "os" "regexp" "sort" "strings" "github.com/kami/maven/internal/router/semantic" ) // ─── Seed definitions ──────────────────────────────────────────────────────── func allSeeds() []semantic.SemanticSeed { var seeds []semantic.SemanticSeed seeds = append(seeds, actionSeeds()...) seeds = append(seeds, knowledgeSeeds()...) seeds = append(seeds, memoryWriteSeeds()...) seeds = append(seeds, systemSeeds()...) seeds = append(seeds, conversationSeeds()...) seeds = append(seeds, uncertainSeeds()...) return seeds } // ─── ACTION seeds ──────────────────────────────────────────────────────────── // Derived from: deploy tools (12), HA domains (5), Praxis lifecycle (4), // task-status (2). Each seed becomes one independent split group. func actionSeeds() []semantic.SemanticSeed { var seeds []semantic.SemanticSeed // Process tools from deploy/mavend.json voice.tools // Each tool with its Russian aliases forms one seed family. toolSeeds := []struct { id string family string opID string verbs []string objects []string tags []string }{ {"act-restart", "act-restart", "restart", []string{"перезапусти", "перезагрузи", "рестарт"}, []string{"сервис", "nginx", "docker", "maven"}, []string{"homelab", "destructive", "process_tool"}}, {"act-stop", "act-stop", "stop", []string{"останови", "выключи"}, []string{"сервис", "nginx", "bотоbackup"}, []string{"homelab", "destructive", "process_tool"}}, {"act-start", "act-start", "start", []string{"запусти", "старт"}, []string{"сервис", "nginx", "бэкап"}, []string{"homelab", "process_tool"}}, {"act-docker-restart", "act-docker-restart", "docker-restart", []string{"перезапусти контейнер", "перезагрузи контейнер"}, []string{"maven", "nexus", "redis", "postgres"}, []string{"homelab", "destructive", "docker"}}, {"act-docker-stop", "act-docker-stop", "docker-stop", []string{"останови контейнер"}, []string{"maven", "nexus", "redis"}, []string{"homelab", "destructive", "docker"}}, {"act-reboot", "act-reboot", "reboot", []string{"перезагрузи сервер", "перезагрузи хост", "ребут"}, []string{}, []string{"homelab", "destructive", "system"}}, {"act-status", "act-status", "status", []string{"покажи статус", "проверь статус", "статус"}, []string{"nginx", "docker", "maven", "сервер"}, []string{"homelab", "read_only", "process_tool"}}, {"act-ps", "act-ps", "ps", []string{"покажи контейнеры", "список контейнеров", "что запущено"}, []string{}, []string{"homelab", "read_only", "docker"}}, {"act-uptime", "act-uptime", "uptime", []string{"покажи uptime", "аптайм", "как работает сервер"}, []string{}, []string{"homelab", "read_only", "system"}}, {"act-disk", "act-disk", "disk", []string{"покажи диск", "сколько места на диске"}, []string{}, []string{"homelab", "read_only", "system"}}, {"act-memory", "act-memory", "memory", []string{"покажи память", "свободная память"}, []string{}, []string{"homelab", "read_only", "system"}}, {"act-logs", "act-logs", "logs", []string{"покажи логи", "логи", "лог"}, []string{"nginx", "system", "docker"}, []string{"homelab", "read_only", "process_tool"}}, } for _, t := range toolSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: t.id, Route: semantic.RouteAction, Family: t.family, OperationID: t.opID, SplitGroup: t.family, VerbForms: t.verbs, Subjects: t.objects, Tags: t.tags, }) } // Home Assistant operations (when enabled) // Each domain+service pair is one seed. haSeeds := []struct { id string family string verbs []string objects []string tags []string }{ {"ha-light-on", "ha-light-on", []string{"включи свет", "включить свет"}, []string{"в спальне", "на кухне", "в гостиной", "в коридоре"}, []string{"ha", "light", "turn_on"}}, {"ha-light-off", "ha-light-off", []string{"выключи свет", "выключить свет"}, []string{"в спальне", "на кухне", "в гостиной", "в коридоре"}, []string{"ha", "light", "turn_off"}}, {"ha-switch-on", "ha-switch-on", []string{"включи вытяжку", "включи розетку"}, []string{"на кухне", "в ванной"}, []string{"ha", "switch", "turn_on"}}, {"ha-switch-off", "ha-switch-off", []string{"выключи вытяжку", "выключи розетку"}, []string{"на кухне", "в ванной"}, []string{"ha", "switch", "turn_off"}}, {"ha-cover-open", "ha-cover-open", []string{"открой жалюзи", "подними жалюзи"}, []string{"в спальне", "в гостиной"}, []string{"ha", "cover", "open"}}, {"ha-cover-close", "ha-cover-close", []string{"закрой жалюзи", "опусти жалюзи"}, []string{"в спальне", "в гостиной"}, []string{"ha", "cover", "close"}}, {"ha-lock-lock", "ha-lock-lock", []string{"запри дверь", "заблокируй дверь"}, []string{"входную", "в гараже"}, []string{"ha", "lock", "lock"}}, {"ha-lock-unlock", "ha-lock-unlock", []string{"отопри дверь", "разблокируй дверь"}, []string{"входную", "в гараже"}, []string{"ha", "lock", "unlock"}}, {"ha-fan-on", "ha-fan-on", []string{"включи вентилятор"}, []string{"в ванной", "на кухне"}, []string{"ha", "fan", "turn_on"}}, {"ha-fan-off", "ha-fan-off", []string{"выключи вентилятор"}, []string{"в ванной", "на кухне"}, []string{"ha", "fan", "turn_off"}}, } for _, h := range haSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: h.id, Route: semantic.RouteAction, Family: h.family, SplitGroup: h.family, VerbForms: h.verbs, Subjects: h.objects, Tags: h.tags, }) } // Praxis lifecycle actions (imperative forms that claim the turn) praxisSeeds := []struct { id string family string opID string verbs []string tags []string }{ {"px-resolve", "praxis-lifecycle", "resolve_item", []string{"закрой", "закрыть", "resolve", "close"}, []string{"praxis", "lifecycle"}}, {"px-acknowledge", "praxis-lifecycle", "acknowledge_item", []string{"принято", "принять", "acknowledge", "ack"}, []string{"praxis", "lifecycle"}}, {"px-ignore", "praxis-lifecycle", "ignore_item", []string{"игнорируй", "игнорировать", "пропусти", "ignore", "skip"}, []string{"praxis", "lifecycle"}}, {"px-pin", "praxis-lifecycle", "pin_item", []string{"закрепи", "закрепить", "pin"}, []string{"praxis", "lifecycle"}}, {"px-attention", "praxis-attention", "list_attention", []string{"что требует внимания", "что нового по задачам"}, []string{"praxis", "attention"}}, {"px-changes", "praxis-changes", "list_changes", []string{"что изменилось", "какие изменения"}, []string{"praxis", "changes"}}, } for _, p := range praxisSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: p.id, Route: semantic.RouteAction, Family: p.family, OperationID: p.opID, SplitGroup: p.family + ":" + p.opID, VerbForms: p.verbs, Tags: p.tags, }) } // Task-status operations taskSeeds := []struct { id string family string opID string verbs []string objects []string tags []string }{ {"ts-done", "task-status", "task_done", []string{"закрой задачу", "заверши задачу", "close the task", "finish the task"}, []string{"купить молоко", "оплатить интернет", "починить кран"}, []string{"task", "status"}}, {"ts-drop", "task-status", "task_drop", []string{"убери задачу", "удали задачу", "отмени задачу", "drop the task", "remove the task"}, []string{"купить молоко", "оплатить интернет"}, []string{"task", "status"}}, } for _, t := range taskSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: t.id, Route: semantic.RouteAction, Family: t.family, OperationID: t.opID, SplitGroup: t.family + ":" + t.opID, VerbForms: t.verbs, Subjects: t.objects, Tags: t.tags, }) } return seeds } // ─── KNOWLEDGE seeds ───────────────────────────────────────────────────────── // Derived from: query patterns already in the corpus, expanded by domain. func knowledgeSeeds() []semantic.SemanticSeed { var seeds []semantic.SemanticSeed // Calendar queries calSeeds := []struct { id string family string verbs []string objects []string tags []string }{ {"kq-cal-schedule", "knowledge:calendar", []string{"что у меня в календаре", "какие планы", "что стоит в календаре"}, []string{"на завтра", "на послезавтра", "на эту неделю", "на следующую неделю"}, []string{"calendar"}}, {"kq-cal-time", "knowledge:calendar-time", []string{"во сколько встреча", "когда планёрка", "когда собес"}, []string{}, []string{"calendar", "temporal"}}, {"kq-cal-next", "knowledge:calendar-next", []string{"что дальше", "что дальше по плану"}, []string{}, []string{"calendar", "hard"}}, } for _, c := range calSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: c.id, Route: semantic.RouteKnowledge, Family: c.family, SplitGroup: c.family, VerbForms: c.verbs, Subjects: c.objects, Tags: c.tags, }) } // Recall/personal data queries recallSeeds := []struct { id string family string verbs []string objects []string tags []string }{ {"kq-recall-fact", "knowledge:recall-fact", []string{"сколько воды я выпил", "сколько раз я ел", "я сегодня пил воду"}, []string{}, []string{"recall", "aggregate"}}, {"kq-recall-note", "knowledge:recall-note", []string{"что я записывал про", "какие заметки я оставил про"}, []string{"кота", "полив", "сервер", "vlan"}, []string{"recall"}}, {"kq-recall-temporal", "knowledge:recall-temporal", []string{"во сколько я лёг", "когда я в последний раз принимал", "давно я не тренировался"}, []string{}, []string{"recall", "temporal", "hard"}}, {"kq-recall-possessive", "knowledge:recall-possessive", []string{"какой у меня любимый", "мой вес за последний", "что у меня"}, []string{"язык", "месяц", "в календаре"}, []string{"recall", "possessive"}}, } for _, r := range recallSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: r.id, Route: semantic.RouteKnowledge, Family: r.family, SplitGroup: r.family, VerbForms: r.verbs, Subjects: r.objects, Tags: r.tags, }) } // World knowledge queries worldSeeds := []struct { id string family string verbs []string objects []string tags []string }{ {"kq-world-def", "knowledge:world-def", []string{"что такое", "кто такой", "что значит"}, []string{"TCP", "Docker", "Kubernetes", "Linus Torvalds", "React"}, []string{"world", "definition"}}, {"kq-world-explain", "knowledge:world-explain", []string{"расскажи про", "объясни", "опиши"}, []string{"битву при Ватерлоо", "quantum computing", "машинное обучение"}, []string{"world", "narrative"}}, {"kq-world-how", "knowledge:world-how", []string{"как работает", "как настроить", "как починить"}, []string{"nginx", "docker", "Git", "SSH"}, []string{"world", "howto"}}, } for _, w := range worldSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: w.id, Route: semantic.RouteKnowledge, Family: w.family, SplitGroup: w.family, VerbForms: w.verbs, Subjects: w.objects, Tags: w.tags, }) } // Homelab queries homelabSeeds := []struct { id string family string verbs []string objects []string tags []string }{ {"kq-homelab-status", "knowledge:homelab-status", []string{"есть новости по", "что там с", "как дела с"}, []string{"бэкапами", "сервером", "доменом", "DNS"}, []string{"homelab"}}, {"kq-homelab-disk", "knowledge:homelab-disk", []string{"хватает ли места", "сколько свободного места"}, []string{"под бэкапы", "на диске"}, []string{"homelab", "hard"}}, {"kq-homelab-why", "knowledge:homelab-why", []string{"почему сервер тормозит", "почему не работает"}, []string{"nginx", "docker", "DNS"}, []string{"homelab", "hard"}}, } for _, h := range homelabSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: h.id, Route: semantic.RouteKnowledge, Family: h.family, SplitGroup: h.family, VerbForms: h.verbs, Subjects: h.objects, Tags: h.tags, }) } // Aggregate / numeric queries aggSeeds := []struct { id string family string verbs []string objects []string tags []string }{ {"kq-agg-steps", "knowledge:aggregate", []string{"сколько я прошёл шагов", "покажи шаги за"}, []string{"неделю", "месяц", "сегодня"}, []string{"aggregate", "health"}}, {"kq-agg-weight", "knowledge:aggregate-weight", []string{"мой вес", "покажи вес за", "динамика веса"}, []string{"за неделю", "за месяц", "за последний месяц"}, []string{"aggregate", "health"}}, {"kq-agg-pressure", "knowledge:aggregate-pressure", []string{"покажи давление за", "мое давление за"}, []string{"неделю", "месяц"}, []string{"aggregate", "health", "hard"}}, } for _, a := range aggSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: a.id, Route: semantic.RouteKnowledge, Family: a.family, SplitGroup: a.family, VerbForms: a.verbs, Subjects: a.objects, Tags: a.tags, }) } // Status queries (knowledge about state, not action) statusSeeds := []struct { id string family string verbs []string objects []string tags []string }{ {"kq-reminder-check", "knowledge:reminder-check", []string{"напоминания на завтра есть", "есть напоминания на"}, []string{"завтра", "послезавтра", "эту неделю"}, []string{"reminder-shaped", "hard"}}, {"kq-task-check", "knowledge:task-check", []string{"что у меня по задачам", "какие задачи есть", "покажи задачи"}, []string{}, []string{"task", "hard"}}, {"kq-deadline", "knowledge:deadline", []string{"я успеваю до дедлайна", "когда дедлайн"}, []string{"по проекту", "по задаче"}, []string{"hard", "no-question-word"}}, } for _, s := range statusSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: s.id, Route: semantic.RouteKnowledge, Family: s.family, SplitGroup: s.family, VerbForms: s.verbs, Subjects: s.objects, Tags: s.tags, }) } // Capability questions (can you do X?) capSeeds := []struct { id string family string verbs []string objects []string tags []string }{ {"kq-cap-ha", "knowledge:capability-ha", []string{"ты можешь выключить", "умеешь ли включить", "сможешь открыть"}, []string{"свет", "жалюзи", "вытяжку", "вентилятор"}, []string{"capability_question"}}, {"kq-cap-tool", "knowledge:capability-tool", []string{"ты можешь перезапустить", "сможешь остановить", "умеешь ли проверить"}, []string{"nginx", "docker", "сервер"}, []string{"capability_question"}}, } for _, c := range capSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: c.id, Route: semantic.RouteKnowledge, Family: c.family, SplitGroup: c.family, VerbForms: c.verbs, Subjects: c.objects, Tags: c.tags, }) } return seeds } // ─── MEMORY_WRITE seeds ────────────────────────────────────────────────────── // Derived from: fact keys (5), note capture verbs, free-form storage patterns. func memoryWriteSeeds() []semantic.SemanticSeed { var seeds []semantic.SemanticSeed // Fact writes — one seed per fact key family factSeeds := []struct { id string family string verbs []string objects []string tags []string }{ {"mw-fact-water", "fact:water", []string{"выпил воды", "воды попил", "пил воду", "just drank water"}, []string{"кружку", "стакан", "литр", "два литра"}, []string{"fact", "water"}}, {"mw-fact-meal", "fact:meal", []string{"позавтракал", "пообедал", "поужинал", "ел", "just ate"}, []string{"овсянкой", "супом", "салатом", "пиццей"}, []string{"fact", "meal"}}, {"mw-fact-sleep", "fact:sleep", []string{"поспал", "спал", "slept", "just slept"}, []string{"часов пять", "семь часов", "час", "полчаса"}, []string{"fact", "sleep"}}, {"mw-fact-shower", "fact:shower", []string{"сходил в душ", "принял душ", "took a shower"}, []string{}, []string{"fact", "shower"}}, {"mw-fact-break", "fact:break", []string{"отдохнул", "передохнул", "took a break"}, []string{"минут двадцать", "час", "пять минут"}, []string{"fact", "break"}}, } for _, f := range factSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: f.id, Route: semantic.RouteMemoryWrite, Family: f.family, SplitGroup: f.family, VerbForms: f.verbs, Subjects: f.objects, Tags: f.tags, }) } // Unparsed-key facts (facts with keys not in the closed parser) unparsedSeeds := []struct { id string family string verbs []string objects []string tags []string }{ {"mw-fact-weight", "fact:weight", []string{"запиши вес", "отметь вес", "мой вес"}, []string{"74 килограмма", "75 кг", "80 kilograms"}, []string{"fact", "unparsed-key"}}, {"mw-fact-pills", "fact:pills", []string{"отметь что я выпил таблетки", "принял таблетки"}, []string{"утром", "вечером", "сегодня"}, []string{"fact", "unparsed-key"}}, {"mw-fact-exercise", "fact:exercise", []string{"сделал зарядку", "потренировался", "just worked out"}, []string{"двадцать минут", "час", "полчаса"}, []string{"fact", "unparsed-key"}}, } for _, u := range unparsedSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: u.id, Route: semantic.RouteMemoryWrite, Family: u.family, SplitGroup: u.family, VerbForms: u.verbs, Subjects: u.objects, Tags: u.tags, }) } // Note capture — explicit capture verbs with note body noteSeeds := []struct { id string family string verbs []string objects []string tags []string }{ {"mw-note-explicit", "note:explicit", []string{"запиши что", "заметка:", "запомни что", "note:"}, []string{"кран на кухне капает", "домен продлить в августе", "сосед просил номер электрика"}, []string{"note", "capture"}}, {"mw-note-idea", "note:idea", []string{"запиши идею:", "идея:", "记住:"}, []string{"гидропоника на балконе", "сервер в шкаф", "новый проект"}, []string{"note", "idea"}}, {"mw-note-homelab", "note:homelab", []string{"заметка про", "запиши про настройку"}, []string{"vlan на свитче", "DNS записи", "备份策略"}, []string{"note", "homelab"}}, {"mw-note-task", "note:task", []string{"добавь в задачи", "запиши задачу"}, []string{"купить молоко", "починить кран", "обновить сервер"}, []string{"note", "capture", "task"}}, } for _, n := range noteSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: n.id, Route: semantic.RouteMemoryWrite, Family: n.family, SplitGroup: n.family, VerbForms: n.verbs, Subjects: n.objects, Tags: n.tags, }) } // Free-form remember/store requests freeSeeds := []struct { id string family string verbs []string objects []string tags []string }{ {"mw-free-remember", "free:remember", []string{"запомни", "сохрани", "занеси", "внеси"}, []string{"что встреча в 3", "пароль от wifi", "адрес электрика"}, []string{"free_form", "remember"}}, {"mw-free-state", "free:state", []string{"я чувствую себя", "у меня болит", "я устал"}, []string{"хорошо", "голова", "сегодня"}, []string{"free_form", "personal_state"}}, } for _, f := range freeSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: f.id, Route: semantic.RouteMemoryWrite, Family: f.family, SplitGroup: f.family, VerbForms: f.verbs, Subjects: f.objects, Tags: f.tags, }) } return seeds } // ─── SYSTEM seeds ──────────────────────────────────────────────────────────── // Derived from: quiet mode, time/date, self-management. func systemSeeds() []semantic.SemanticSeed { var seeds []semantic.SemanticSeed // Quiet mode quietSeeds := []struct { id string family string verbs []string tags []string }{ {"sys-quiet-on", "system:quiet-on", []string{"тихий режим", "режим тишина", "будь потише", "говори тихий", "quiet mode", "quiet on"}, []string{"quiet"}}, {"sys-quiet-off", "system:quiet-off", []string{"хватит тихого режима", "громкий режим", "выключи тихий", "quiet off", "quiet end"}, []string{"quiet", "hard"}}, } for _, q := range quietSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: q.id, Route: semantic.RouteSystem, Family: q.family, SplitGroup: q.family, VerbForms: q.verbs, Tags: q.tags, }) } // Time queries timeSeeds := []struct { id string family string verbs []string objects []string tags []string }{ {"sys-time-now", "system:time-now", []string{"сколько времени", "который час", "what time is it"}, []string{"сейчас", "в киеве", "в москве"}, []string{"time"}}, {"sys-date-today", "system:date-today", []string{"какое число", "какой сегодня день", "what's the date"}, []string{"сегодня", "завтра", "послезавтра"}, []string{"date"}}, {"sys-weekday", "system:weekday", []string{"какой день недели", "какой день"}, []string{"сегодня", "завтра", "послезавтра", "в субботу"}, []string{"date"}}, } for _, t := range timeSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: t.id, Route: semantic.RouteSystem, Family: t.family, SplitGroup: t.family, VerbForms: t.verbs, Subjects: t.objects, Tags: t.tags, }) } // Additional system seeds for coverage extraSystem := []struct { id string family string verbs []string objects []string tags []string }{ {"sys-timezone", "system:timezone", []string{"какой часовой пояс", "часовой пояс", "what timezone"}, []string{}, []string{"time", "config"}}, {"sys-self-version", "system:self-version", []string{"какая версия", "текущая версия", "what version"}, []string{"maven", "нексуса", "праксиса"}, []string{"self", "version"}}, {"sys-self-status", "system:self-status", []string{"как дела", "как ты", "how are you"}, []string{}, []string{"self", "greeting"}}, } for _, s := range extraSystem { seeds = append(seeds, semantic.SemanticSeed{ ID: s.id, Route: semantic.RouteSystem, Family: s.family, SplitGroup: s.family, VerbForms: s.verbs, Subjects: s.objects, Tags: s.tags, }) } return seeds } // ─── CONVERSATION seeds ────────────────────────────────────────────────────── func conversationSeeds() []semantic.SemanticSeed { var seeds []semantic.SemanticSeed convSeeds := []struct { id string family string verbs []string objects []string tags []string }{ {"conv-greeting", "conversation:greeting", []string{"привет", "доброе утро", "добрый день", "добрый вечер", "hello", "good morning"}, nil, []string{"greeting"}}, {"conv-mood", "conversation:mood", []string{"мне грустно", "я рад", "я устал сегодня", "у меня плохое настроение", "i had a rough day"}, nil, []string{"mood"}}, {"conv-joke", "conversation:joke", []string{"расскажи анекдот", "шутка", "tell me a joke"}, nil, []string{"joke"}}, {"conv-thanks", "conversation:thanks", []string{"спасибо", "благодарю", "thanks", "thank you"}, nil, []string{"thanks"}}, {"conv-opinion", "conversation:opinion", []string{"что думаешь про", "мне кажется", "как считаешь"}, []string{"переезд", "новый проект", "обо мне"}, []string{"open_ended"}}, {"conv-greeting-formal", "conversation:greeting-formal", []string{"здравствуй", "здравствуйте", "hi", "hey"}, nil, []string{"greeting"}}, {"conv-goodbye", "conversation:goodbye", []string{"пока", "до свидания", "спокойной ночи", "bye", "goodbye", "good night"}, nil, []string{"goodbye"}}, {"conv-sorry", "conversation:sorry", []string{"извини", "прости", "извините", "sorry"}, nil, []string{"apology"}}, } for _, c := range convSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: c.id, Route: semantic.RouteConversation, Family: c.family, SplitGroup: c.family, VerbForms: c.verbs, Tags: c.tags, }) } return seeds } // ─── UNCERTAIN seeds ───────────────────────────────────────────────────────── func uncertainSeeds() []semantic.SemanticSeed { var seeds []semantic.SemanticSeed uncSeeds := []struct { id string family string verbs []string tags []string }{ {"unc-ambiguous-noun", "uncertain:ambiguous-noun", []string{"вода", "бэкап", "сервер", "контейнер", "задача"}, []string{"ambiguous"}}, {"unc-fragment", "uncertain:fragment", []string{"ну это", "потом", "та штука", "the thing from earlier"}, []string{"ambiguous", "filler"}}, {"unc-anaphora", "uncertain:anaphora", []string{"сделай это", "а можно то", "давай那"}, []string{"ambiguous", "anaphora"}}, {"unc-incomplete-reminder", "uncertain:incomplete-reminder", []string{"напомни", "ну напомни же", "надо напомнить"}, []string{"ambiguous", "reminder"}}, {"unc-unclear-action", "uncertain:unclear-action", []string{"ну надо бы", "можно это", "а потом"}, []string{"ambiguous", "filler"}}, {"unc-single-word-verb", "uncertain:single-word-verb", []string{"перезапусти", "выключи", "включи"}, []string{"ambiguous", "incomplete"}}, {"unc-referral", "uncertain:referral", []string{"то что я говорил", "про это", "из предыдущего"}, []string{"ambiguous", "anaphora"}}, {"unc-hedge", "uncertain:hedge", []string{"может быть", "наверное", "кажется"}, []string{"ambiguous", "hedge"}}, {"unc-partial-sentence", "uncertain:partial-sentence", []string{"я хотел бы", "может можно", "а если"}, []string{"ambiguous", "incomplete"}}, {"unc-trailing-off", "uncertain:trailing-off", []string{"вот надо бы", "а потом еще", "и еще"}, []string{"ambiguous", "filler"}}, {"unc-vague-reference", "uncertain:vague-reference", []string{"та самая", "вон та", "прошлый раз"}, []string{"ambiguous", "anaphora"}}, {"unc-multi-ambiguous", "uncertain:multi-ambiguous", []string{"ну как обычно", "всё ок", "нормально вроде"}, []string{"ambiguous", "filler"}}, } for _, u := range uncSeeds { seeds = append(seeds, semantic.SemanticSeed{ ID: u.id, Route: semantic.RouteUncertain, Family: u.family, SplitGroup: u.family, VerbForms: u.verbs, Tags: u.tags, }) } return seeds } // ─── Surface generators ────────────────────────────────────────────────────── // generatorDirect generates direct imperative forms. func generatorDirect(seed semantic.SemanticSeed) []semantic.GeneratedSurface { var surfaces []semantic.GeneratedSurface for _, verb := range seed.VerbForms { for _, subj := range seed.Subjects { if subj == "" { surfaces = append(surfaces, semantic.GeneratedSurface{ Text: verb, TemplateCategory: "direct_imperative", ExpectedRoute: seed.Route, }) } else { surfaces = append(surfaces, semantic.GeneratedSurface{ Text: verb + " " + subj, TemplateCategory: "direct_imperative", ExpectedRoute: seed.Route, }) } } if len(seed.Subjects) == 0 { surfaces = append(surfaces, semantic.GeneratedSurface{ Text: verb, TemplateCategory: "direct_imperative", ExpectedRoute: seed.Route, }) } } return surfaces } // generatorPolite adds "пожалуйста" / "please". func generatorPolite(seed semantic.SemanticSeed) []semantic.GeneratedSurface { var surfaces []semantic.GeneratedSurface for _, verb := range seed.VerbForms { for _, subj := range seed.Subjects { text := verb if subj != "" { text = verb + " " + subj } surfaces = append(surfaces, semantic.GeneratedSurface{ Text: text + ", пожалуйста", TemplateCategory: "polite_request", ExpectedRoute: seed.Route, }) if len(seed.Subjects) == 0 { surfaces = append(surfaces, semantic.GeneratedSurface{ Text: text + ", please", TemplateCategory: "polite_request", ExpectedRoute: seed.Route, }) } } if len(seed.Subjects) == 0 { surfaces = append(surfaces, semantic.GeneratedSurface{ Text: verb + ", пожалуйста", TemplateCategory: "polite_request", ExpectedRoute: seed.Route, }) } } return surfaces } // generatorModal generates "можешь ...?" / "can you ...?" forms. // For action seeds, polite modal = action. For knowledge, modal = knowledge. func generatorModal(seed semantic.SemanticSeed) []semantic.GeneratedSurface { var surfaces []semantic.GeneratedSurface for _, verb := range seed.VerbForms { for _, subj := range seed.Subjects { text := verb if subj != "" { text = verb + " " + subj } // Modal request with explicit intent → action (if executable) if seed.Route == semantic.RouteAction { surfaces = append(surfaces, semantic.GeneratedSurface{ Text: "можешь " + text + ", пожалуйста", TemplateCategory: "modal_request", ExpectedRoute: seed.Route, }) surfaces = append(surfaces, semantic.GeneratedSurface{ Text: "can you " + text + ", please", TemplateCategory: "modal_request", ExpectedRoute: seed.Route, }) } } if len(seed.Subjects) == 0 { if seed.Route == semantic.RouteAction { surfaces = append(surfaces, semantic.GeneratedSurface{ Text: "можешь " + verb + ", пожалуйста", TemplateCategory: "modal_request", ExpectedRoute: seed.Route, }) } } } return surfaces } // generatorFirstPerson generates "я хочу ...", "надо бы ...", "давай ...". func generatorFirstPerson(seed semantic.SemanticSeed) []semantic.GeneratedSurface { var surfaces []semantic.GeneratedSurface for _, verb := range seed.VerbForms { for _, subj := range seed.Subjects { text := verb if subj != "" { text = verb + " " + subj } surfaces = append(surfaces, semantic.GeneratedSurface{ Text: "я хочу " + text, TemplateCategory: "first_person_request", ExpectedRoute: seed.Route, }) surfaces = append(surfaces, semantic.GeneratedSurface{ Text: "надо бы " + text, TemplateCategory: "first_person_request", ExpectedRoute: seed.Route, }) } if len(seed.Subjects) == 0 { surfaces = append(surfaces, semantic.GeneratedSurface{ Text: "я хочу " + verb, TemplateCategory: "first_person_request", ExpectedRoute: seed.Route, }) } } return surfaces } // generatorReordered puts the target before the verb. func generatorReordered(seed semantic.SemanticSeed) []semantic.GeneratedSurface { var surfaces []semantic.GeneratedSurface for _, verb := range seed.VerbForms { for _, subj := range seed.Subjects { if subj == "" { continue } surfaces = append(surfaces, semantic.GeneratedSurface{ Text: subj + " " + verb, TemplateCategory: "reordered_target", ExpectedRoute: seed.Route, }) } } return surfaces } // generatorEnglishDirect generates English equivalents. func generatorEnglishDirect(seed semantic.SemanticSeed) []semantic.GeneratedSurface { var surfaces []semantic.GeneratedSurface // Only generate for seeds that have English verb forms for _, verb := range seed.VerbForms { if isEnglish(verb) { for _, subj := range seed.Subjects { text := verb if subj != "" { text = verb + " " + subj } surfaces = append(surfaces, semantic.GeneratedSurface{ Text: text, TemplateCategory: "english_direct", ExpectedRoute: seed.Route, }) } if len(seed.Subjects) == 0 { surfaces = append(surfaces, semantic.GeneratedSurface{ Text: verb, TemplateCategory: "english_direct", ExpectedRoute: seed.Route, }) } } } return surfaces } // generatorQuestion generates question forms for knowledge seeds. func generatorQuestion(seed semantic.SemanticSeed) []semantic.GeneratedSurface { var surfaces []semantic.GeneratedSurface if seed.Route != semantic.RouteKnowledge { return nil } for _, verb := range seed.VerbForms { for _, subj := range seed.Subjects { text := verb if subj != "" { text = verb + " " + subj } surfaces = append(surfaces, semantic.GeneratedSurface{ Text: text + "?", TemplateCategory: "question", ExpectedRoute: seed.Route, }) } if len(seed.Subjects) == 0 { surfaces = append(surfaces, semantic.GeneratedSurface{ Text: verb + "?", TemplateCategory: "question", ExpectedRoute: seed.Route, }) } } return surfaces } // ─── Fast-path classifier ──────────────────────────────────────────────────── // Simplified pattern matcher that mirrors stage-0 grammar outcomes. var ( reminderVerbRe = regexp.MustCompile(`(?i)^\s*(напомни|напомните|напомнить|напоминай|разбуди|разбудите|разбудить|буди|remind|wake)\b`) timeQueryRe = regexp.MustCompile(`(?i)^\s*(сколько\s+сейчас\s+времени|который\s+час|какое\s+число|какой\s+день|what\s+time|what's\s+the\s+date)\b`) captureVerbRe = regexp.MustCompile(`(?i)^\s*(запиши|запомни|отметь|заметь|добавь|сохрани|занеси|внеси|note|remember|log|save|add)\b`) praxisItemRe = regexp.MustCompile(`(?i)\b(item[_-][a-z0-9_-]+)`) quietOnRe = regexp.MustCompile(`(?i)(тихий\s+режим|режим\s+тишина|тише|потише|quiet\s+(mode|on))`) quietOffRe = regexp.MustCompile(`(?i)(хватит\s+тихого|громкий\s+режим|выключи\s+тихий|quiet\s+(off|end))`) taskDoneRe = regexp.MustCompile(`(?i)^\s*(закрой\s+задачу|заверши\s+задачу|close\s+the\s+task|finish\s+the\s+task)\b`) taskDropRe = regexp.MustCompile(`(?i)^\s*(убери\s+задачу|удали\s+задачу|отмени\s+задачу|drop\s+the\s+task|remove\s+the\s+task)\b`) ) // tool aliases → fast-path matchers (subset of what DefaultActMatcher would match) var toolAliasRe = []*regexp.Regexp{ regexp.MustCompile(`(?i)^\s*(перезапусти|перезагрузи|рестарт)\b`), regexp.MustCompile(`(?i)^\s*(останови|выключи)\b`), regexp.MustCompile(`(?i)^\s*(запусти|старт)\b`), regexp.MustCompile(`(?i)^\s*(перезапусти\s+контейнер|перезагрузи\s+контейнер)\b`), regexp.MustCompile(`(?i)^\s*(останови\s+контейнер)\b`), regexp.MustCompile(`(?i)^\s*(перезагрузи\s+сервер|перезагрузи\s+хост|ребут)\b`), regexp.MustCompile(`(?i)^\s*(покажи\s+статус|проверь\s+статус|статус)\b`), regexp.MustCompile(`(?i)^\s*(покажи\s+контейнеры|список\s+контейнеров|что\s+запущено)\b`), regexp.MustCompile(`(?i)^\s*(покажи\s+uptime|аптайм|как\s+работает\s+сервер)\b`), regexp.MustCompile(`(?i)^\s*(покажи\s+диск|сколько\s+места\s+на\s+диске)\b`), regexp.MustCompile(`(?i)^\s*(покажи\s+память|свободная\s+память)\b`), regexp.MustCompile(`(?i)^\s*(покажи\s+логи|логи|лог)\b`), } // praxis lifecycle verbs (bare, without item reference → residual action) var praxisLifecycleRe = regexp.MustCompile(`(?i)^\s*(закрой|закрыть|resolve|close|принято|принять|acknowledge|ack|игнорируй|игнорировать|пропусти|ignore|skip|закрепи|закрепить|pin)\b`) var praxisAttentionRe = regexp.MustCompile(`(?i)^\s*(что\s+требует\s+внимания|что\s+нового\s+по\s+(задачам|проектам|сервисам)|needs\s+attention)\b`) var praxisChangesRe = regexp.MustCompile(`(?i)^\s*(что\s+изменилось|какие\s+изменения|changed|changes)\b`) var praxisEntityRe = regexp.MustCompile(`(?i)^\s*(статус|status|как\s+дела\s+у|how\s+is)\s+`) // classifyFastPath determines whether a surface would be handled by stage-0. func classifyFastPath(text string) bool { t := strings.TrimSpace(text) // Reminder verbs → always fast-path if reminderVerbRe.MatchString(t) { return true } // Time/date queries → fast-path if timeQueryRe.MatchString(t) { return true } // Task-status commands → fast-path if taskDoneRe.MatchString(t) || taskDropRe.MatchString(t) { return true } // Tool aliases → fast-path (only for verbs alone, not when combined with objects in ways the matcher wouldn't handle) for _, re := range toolAliasRe { if re.MatchString(t) { return true } } // Praxis lifecycle with item reference → fast-path if praxisLifecycleRe.MatchString(t) && praxisItemRe.MatchString(t) { return true } // Praxis attention/changes → fast-path if praxisAttentionRe.MatchString(t) || praxisChangesRe.MatchString(t) { return true } // Praxis entity attention → fast-path if praxisEntityRe.MatchString(t) { return true } // Quiet mode → fast-path (resolved pre-route) if quietOnRe.MatchString(t) || quietOffRe.MatchString(t) { return true } // Capture verbs → some are fast-path (note capture) if captureVerbRe.MatchString(t) { return true } return false } // ─── Helpers ───────────────────────────────────────────────────────────────── func isEnglish(s string) bool { for _, r := range s { if r > 127 { return false } } return true } func normalizeText(s string) string { return strings.TrimSpace(strings.ToLower(s)) } // ─── Corpus building ───────────────────────────────────────────────────────── type corpusEnvelope struct { SchemaVersion int `json:"schema_version"` Name string `json:"name"` Notes []string `json:"notes"` Reproducibility *reproMeta `json:"reproducibility,omitempty"` Examples []semantic.RouteExample `json:"examples"` } type reproMeta struct { SourceFixtureHash string `json:"source_fixture_hash"` ContrastGeneratorVersion string `json:"contrast_generator_version"` SplitAlgorithm string `json:"split_algorithm"` DatasetHash string `json:"dataset_hash"` } func buildCorpus(seeds []semantic.SemanticSeed) []semantic.RouteExample { generators := []semantic.SurfaceGenerator{ {Name: "direct", Fn: generatorDirect}, {Name: "polite", Fn: generatorPolite}, {Name: "modal", Fn: generatorModal}, {Name: "first_person", Fn: generatorFirstPerson}, {Name: "reordered", Fn: generatorReordered}, {Name: "english_direct", Fn: generatorEnglishDirect}, {Name: "question", Fn: generatorQuestion}, } seen := make(map[string]bool) // normalized text → seen var examples []semantic.RouteExample exampleCounter := 0 for _, seed := range seeds { // Track surfaces within this seed for dedup seedSeen := make(map[string]bool) for _, gen := range generators { surfaces := gen.Fn(seed) for _, surf := range surfaces { norm := normalizeText(surf.Text) if norm == "" { continue } // Dedup within seed if seedSeen[norm] { continue } // Dedup across corpus if seen[norm] { continue } seedSeen[norm] = true seen[norm] = true exampleCounter++ srcID := fmt.Sprintf("%s-%03d", seed.ID, exampleCounter) fp := classifyFastPath(surf.Text) isResidual := !fp tags := append([]string{}, seed.Tags...) tags = append(tags, surf.TemplateCategory) if isResidual { tags = append(tags, "router_residual") } var rr *bool if isResidual { v := true rr = &v } examples = append(examples, semantic.RouteExample{ Text: surf.Text, Route: seed.Route, Source: "corpus_factory_v2", SourceID: srcID, SplitGroup: seed.SplitGroup, Tags: tags, FastPathResolved: fp, RouterResidual: rr, }) } } } return examples } // ─── Validation ────────────────────────────────────────────────────────────── func validateAndReport(examples []semantic.RouteExample, seeds []semantic.SemanticSeed) { routeCounts := make(map[semantic.SemanticRoute]int) sourceCounts := make(map[string]int) fpCount, resCount := 0, 0 groupCounts := make(map[string]map[semantic.SemanticRoute]bool) // group → set of routes // Track normalized text → route for conflict detection textRoute := make(map[string]semantic.SemanticRoute) for _, e := range examples { routeCounts[e.Route]++ sourceCounts[e.Source]++ if e.FastPathResolved { fpCount++ } else { resCount++ } if groupCounts[e.SplitGroup] == nil { groupCounts[e.SplitGroup] = make(map[semantic.SemanticRoute]bool) } groupCounts[e.SplitGroup][e.Route] = true norm := normalizeText(e.Text) if prev, ok := textRoute[norm]; ok && prev != e.Route { log.Printf("CONFLICT: text %q has route %q and %q", norm, prev, e.Route) } textRoute[norm] = e.Route } // Count independent groups per route routeGroups := make(map[semantic.SemanticRoute]map[string]bool) for group, routes := range groupCounts { for route := range routes { if routeGroups[route] == nil { routeGroups[route] = make(map[string]bool) } routeGroups[route][group] = true } } fmt.Fprintf(os.Stderr, "\n=== CORPUS REPORT ===\n") fmt.Fprintf(os.Stderr, "Total examples: %d\n", len(examples)) fmt.Fprintf(os.Stderr, "Fast-path: %d Residual: %d\n", fpCount, resCount) fmt.Fprintf(os.Stderr, "\nRoute distribution:\n") for _, r := range semantic.AllRoutes { fmt.Fprintf(os.Stderr, " %-15s %4d examples %3d independent groups\n", r, routeCounts[r], len(routeGroups[r])) } fmt.Fprintf(os.Stderr, "\nIndependent SplitGroup count: %d\n", len(groupCounts)) // Check minimum coverage targets targets := map[semantic.SemanticRoute]int{ semantic.RouteConversation: 8, semantic.RouteKnowledge: 12, semantic.RouteAction: 12, semantic.RouteMemoryWrite: 12, semantic.RouteSystem: 8, semantic.RouteUncertain: 12, } fmt.Fprintf(os.Stderr, "\nCoverage targets:\n") for _, r := range semantic.AllRoutes { actual := len(routeGroups[r]) target := targets[r] status := "OK" if actual < target { status = fmt.Sprintf("BELOW (need %d)", target) } fmt.Fprintf(os.Stderr, " %-15s %3d groups target=%d %s\n", r, actual, target, status) } // Source distribution fmt.Fprintf(os.Stderr, "\nSource distribution:\n") for src, count := range sourceCounts { fmt.Fprintf(os.Stderr, " %-25s %d\n", src, count) } } // ─── Main ──────────────────────────────────────────────────────────────────── func main() { outPath := flag.String("out", "internal/router/semantic/corpus_v2.json", "output JSON path") flag.Parse() // 1. Generate seeds seeds := allSeeds() fmt.Fprintf(os.Stderr, "Seeds: %d\n", len(seeds)) // 2. Generate surfaces and build corpus examples := buildCorpus(seeds) fmt.Fprintf(os.Stderr, "Generated examples: %d\n", len(examples)) // 3. Validate if err := semantic.ValidateCorpus(examples); err != nil { log.Fatalf("corpus validation failed: %v", err) } fmt.Fprintf(os.Stderr, "Corpus validation: OK\n") // 4. Report validateAndReport(examples, seeds) // 5. Compute dataset hash texts := make([]string, len(examples)) for i, e := range examples { texts[i] = e.Text } sort.Strings(texts) h := sha256.Sum256([]byte(strings.Join(texts, "\n"))) datasetHash := hex.EncodeToString(h[:16]) // 6. Write JSON env := corpusEnvelope{ SchemaVersion: 1, Name: "semantic_coarse_route_v2", Notes: []string{ "Expanded corpus generated by corpus-factory from deterministic seed specifications.", "Every row derives from an authoritative Maven capability source — no hand-authored sentences.", "fast_path_resolved is set by the fast-path classifier mirroring stage-0 grammar outcomes.", "Labels come from seed specifications, not model output.", }, Reproducibility: &reproMeta{ SourceFixtureHash: "corpus-factory-v2", ContrastGeneratorVersion: "v2-direct-generation", SplitAlgorithm: "grouped-cv-v1", DatasetHash: datasetHash, }, Examples: examples, } data, err := json.MarshalIndent(env, "", " ") if err != nil { log.Fatalf("marshal: %v", err) } if err := os.WriteFile(*outPath, data, 0644); err != nil { log.Fatalf("write %s: %v", *outPath, err) } fmt.Fprintf(os.Stderr, "\nOutput: %s (%d bytes)\n", *outPath, len(data)) }