router/semantic: merge-corpus tool for frozen holdout preservation (slice 15)

Merges frozen holdout from v1 with v2 development pool, deduplicates,
preserves frozen entries, validates, writes merged corpus.

cmd/merge-corpus/main.go

@@ -0,0 +1,154 @@
+// merge-corpus merges the frozen holdout from corpus_v1.json with the
+// expanded v2 development pool, writing the result back to corpus_v1.json.
+//
+// Usage:
+//
+//	go run ./cmd/merge-corpus/ -v1 internal/router/semantic/corpus_v1.json \
+//	  -v2 /tmp/corpus_v2.json -out internal/router/semantic/corpus_v1.json
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/kami/maven/internal/router/semantic"
+)
+
+func main() {
+	v1Path := flag.String("v1", "internal/router/semantic/corpus_v1.json", "v1 corpus path")
+	v2Path := flag.String("v2", "/tmp/corpus_v2.json", "v2 factory output path")
+	outPath := flag.String("out", "internal/router/semantic/corpus_v1.json", "output path")
+	flag.Parse()
+
+	// 1. Load v1
+	v1Data, err := os.ReadFile(*v1Path)
+	if err != nil {
+		log.Fatalf("read v1: %v", err)
+	}
+	var v1Env semantic.CorpusEnvelope
+	if err := json.Unmarshal(v1Data, &v1Env); err != nil {
+		log.Fatalf("parse v1: %v", err)
+	}
+	fmt.Fprintf(os.Stderr, "v1: %d examples\n", len(v1Env.Examples))
+
+	// 2. Identify frozen holdout from v1
+	frozen, _, _ := semantic.FrozenHoldoutSplit(v1Env.Examples)
+	frozenTexts := make(map[string]bool)
+	for _, e := range frozen {
+		frozenTexts[strings.TrimSpace(e.Text)] = true
+	}
+	fmt.Fprintf(os.Stderr, "v1 frozen holdout: %d examples\n", len(frozen))
+
+	// 3. Load v2
+	v2Data, err := os.ReadFile(*v2Path)
+	if err != nil {
+		log.Fatalf("read v2: %v", err)
+	}
+	var v2Env semantic.CorpusEnvelope
+	if err := json.Unmarshal(v2Data, &v2Env); err != nil {
+		log.Fatalf("parse v2: %v", err)
+	}
+	fmt.Fprintf(os.Stderr, "v2: %d examples\n", len(v2Env.Examples))
+
+	// 4. Merge: frozen from v1 + all from v2
+	// Dedup by normalized text
+	seen := make(map[string]bool)
+	var merged []semantic.RouteExample
+
+	// Frozen holdout first
+	for _, e := range frozen {
+		norm := strings.TrimSpace(e.Text)
+		if seen[norm] {
+			fmt.Fprintf(os.Stderr, "SKIP v1 frozen dup: %q\n", norm)
+			continue
+		}
+		seen[norm] = true
+		merged = append(merged, e)
+	}
+
+	// V2 examples
+	skipped := 0
+	for _, e := range v2Env.Examples {
+		norm := strings.TrimSpace(e.Text)
+		if seen[norm] {
+			skipped++
+			continue
+		}
+		// Check if this text conflicts with a frozen holdout entry
+		if frozenTexts[norm] {
+			// Text exists in frozen — skip v2 version to preserve frozen
+			skipped++
+			continue
+		}
+		seen[norm] = true
+		merged = append(merged, e)
+	}
+	fmt.Fprintf(os.Stderr, "merged: %d examples (skipped %d duplicates)\n", len(merged), skipped)
+
+	// 5. Validate
+	if err := semantic.ValidateCorpus(merged); err != nil {
+		log.Fatalf("validation failed: %v", err)
+	}
+	fmt.Fprintf(os.Stderr, "validation: OK\n")
+
+	// 6. Compute dataset hash
+	texts := make([]string, len(merged))
+	for i, e := range merged {
+		texts[i] = e.Text
+	}
+	sort.Strings(texts)
+	h := sha256.Sum256([]byte(strings.Join(texts, "\n")))
+	datasetHash := hex.EncodeToString(h[:16])
+
+	// 7. Stats
+	routeCounts := make(map[semantic.SemanticRoute]int)
+	fpCount, resCount := 0, 0
+	for _, e := range merged {
+		routeCounts[e.Route]++
+		if e.FastPathResolved {
+			fpCount++
+		} else {
+			resCount++
+		}
+	}
+	fmt.Fprintf(os.Stderr, "\nRoute distribution:\n")
+	for _, r := range semantic.AllRoutes {
+		fmt.Fprintf(os.Stderr, "  %-15s %d\n", r, routeCounts[r])
+	}
+	fmt.Fprintf(os.Stderr, "fast-path: %d  residual: %d\n", fpCount, resCount)
+
+	// 8. Write merged corpus
+	outEnv := semantic.CorpusEnvelope{
+		SchemaVersion: 1,
+		Name:          "semantic_coarse_route_v1",
+		Notes: []string{
+			"Coarse semantic route corpus for the learned-router experiment.",
+			"Frozen holdout preserved from v1. Development pool expanded by corpus-factory v2.",
+			"Every row carries provenance (source + source_id) and a split_group.",
+			"Labels come from explicit mapping rules, not model output.",
+		},
+		Reproducibility: &semantic.ReproducibilityMeta{
+			SourceFixtureHash:      "v1-frozen + corpus-factory-v2",
+			ContrastGeneratorVersion: "v2-direct-generation",
+			SplitAlgorithm:         "grouped-cv-v1",
+			DatasetHash:            datasetHash,
+		},
+		Examples: merged,
+	}
+
+	data, err := json.MarshalIndent(outEnv, "", "  ")
+	if err != nil {
+		log.Fatalf("marshal: %v", err)
+	}
+	if err := os.WriteFile(*outPath, data, 0644); err != nil {
+		log.Fatalf("write %s: %v", *outPath, err)
+	}
+	fmt.Fprintf(os.Stderr, "\nOutput: %s (%d bytes)\n", *outPath, len(data))
+}