Maven/cmd/semantic-router-experiment/slice22/legacy_main.go

package main

import (
	"bufio"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/kami/maven/internal/router"
)

// Legacy-baseline runner for slice 22: run the actual router cascade (stage 0
// grammars → hash-embedder nearest-centroid classifier → 0.55 confidence gate)
// over the frozen residual non-action dev pool and project each decision into
// the five-way non-action semantic space.
//
// Projection rules (the daemon's behaviour, not just ScoreLegacy's):
//   - route error            → uncertain
//   - Clarify=true (stage 3)  → uncertain: the daemon asks, it does not commit
//     to a semantic bucket
//   - chat/query/fact+note/system → conversation/knowledge/memory_write/system
//   - act/reminder on a trusted non-action row → class "action" recorded
//     VERBATIM with illegal_action_prediction=true; never mapped to uncertain
//   - anything else           → uncertain
//
// Reads /tmp/mvn-s22/pool.json (emit step) and writes /tmp/mvn-s22/legacy.json
// with both the raw decision fields and the projected class, plus a summary
// printout. No embedding is recomputed and no label is changed.

type poolRow struct {
	IDX        int      `json:"idx"`
	Text       string   `json:"text"`
	NText      string   `json:"n_text"`
	Route      string   `json:"route"`
	Tags       []string `json:"tags"`
	CVFold     int      `json:"cv_fold"`
	SplitGroup string   `json:"split_group"`
	FamilyID   string   `json:"family_id"`
	SourceID   string   `json:"source_id"`
}

type legacyRow struct {
	IDX        int     `json:"idx"`
	Text       string  `json:"text"`
	Route      string  `json:"route"`
	Intent     string  `json:"intent"`
	Class      string  `json:"class"`
	Illegal    bool    `json:"illegal_action_prediction"`
	Confidence float64 `json:"confidence"`
	Stage      int     `json:"stage"`
	Clarify    bool    `json:"clarify"`
	Producer   string  `json:"producer"`
	Error      string  `json:"error,omitempty"`
	SourceID   string  `json:"source_id"`
}

func main() {
	var mode, poolPath, outPath string
	flag.StringVar(&mode, "mode", "legacy", "baseline mode: legacy (hash classifier) or heads (ONNX cascade minus LLM)")
	flag.StringVar(&poolPath, "pool", "/tmp/mvn-s22/pool.json", "emit-step pool.json")
	flag.StringVar(&outPath, "out", "/tmp/mvn-s22/legacy.json", "output path")
	flag.Parse()
	switch mode {
	case "legacy":
		legacyMain(poolPath, outPath)
	case "heads":
		headsMain(poolPath, outPath)
	default:
		fmt.Fprintf(os.Stderr, "unknown -mode %q\n", mode)
		os.Exit(2)
	}
}

func legacyMain(poolPath, outPath string) {
	runOverPool(buildMinimalRouter(), poolPath, outPath)
}

func runOverPool(r *router.Router, poolPath, outPath string) {
	raw, err := os.ReadFile(poolPath)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	var rows []poolRow
	if err := json.Unmarshal(raw, &rows); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	ctx := context.Background()
	now := time.Now()

	out := make([]legacyRow, 0, len(rows))
	classCount := map[string]int{}
	for _, pr := range rows {
		d, err := r.Route(ctx, router.NormalizedInput{Text: pr.Text}, now)
		lr := legacyRow{
			IDX:      pr.IDX,
			Text:     pr.Text,
			Route:    pr.Route,
			SourceID: pr.SourceID,
		}
		if err != nil {
			lr.Class = "uncertain"
			lr.Error = err.Error()
		} else {
			lr.Intent = string(d.Intent)
			lr.Confidence = d.Confidence
			lr.Stage = d.Stage
			lr.Clarify = d.Clarify
			lr.Producer = string(d.Producer)
		}
		lr.Class, lr.Illegal = project(d, err)
		classCount[lr.Class]++
		out = append(out, lr)
	}

	if err := writeJSON(outPath, out); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	fmt.Printf("legacy baseline over %d residual non-action rows:\n", len(out))
	for _, c := range []string{"conversation", "knowledge", "memory_write", "system", "uncertain", "action"} {
		fmt.Printf("  %-14s %d (%.1f%%)\n", c, classCount[c], 100*float64(classCount[c])/float64(len(out)))
	}
	fmt.Printf("  illegal_action_prediction: %d\n", classCount["action"])
	// Grammar hits inside a corpus-residual population would be a
	// corpus/harness disagreement worth telling the report about: the corpus
	// marked each row not-fast-path-resolved, so a current stage-0 rule
	// resolving it means the corpus's fast-path mirror is stale or a grammar
	// landed after the corpus froze.
	gh := 0
	ghByRoute := map[string]int{}
	ghByIntent := map[string]int{}
	for _, lr := range out {
		if lr.Producer == string(router.RouteProducerGrammar) {
			gh++
			ghByRoute[lr.Route]++
			ghByIntent[lr.Intent]++
		}
	}
	fmt.Printf("  stage-0 grammar hits: %d\n", gh)
	if gh > 0 {
		fmt.Printf("    by ground-truth route: %v\n", ghByRoute)
		fmt.Printf("    by grammar intent: %v\n", ghByIntent)
	}
}

// project maps the router's authoritative output into the five-way non-action
// space, or to the "action" bucket verbatim when the router calls an act or a
// reminder on a non-action row.
func project(d router.Decision, err error) (string, bool) {
	if err != nil {
		return "uncertain", false
	}
	if d.Clarify {
		return "uncertain", false
	}
	switch d.Intent {
	case router.IntentChat:
		return "conversation", false
	case router.IntentQuery:
		return "knowledge", false
	case router.IntentFact, router.IntentNote:
		return "memory_write", false
	case router.IntentSystem:
		return "system", false
	case router.IntentAct, router.IntentReminder:
		return "action", true
	default:
		return "uncertain", false
	}
}

func writeJSON(path string, v any) error {
	fh, err := os.Create(path)
	if err != nil {
		return err
	}
	defer fh.Close()
	w := bufio.NewWriter(fh)
	enc := json.NewEncoder(w)
	enc.SetIndent("", "  ")
	if err := enc.Encode(v); err != nil {
		return err
	}
	return w.Flush()
}