Maven/cmd/merge-corpus/main.go

// merge-corpus merges the frozen holdout from corpus_v1.json with the
// expanded v2 development pool, writing the result back to corpus_v1.json.
//
// Usage:
//
//	go run ./cmd/merge-corpus/ -v1 internal/router/semantic/corpus_v1.json \
//	  -v2 /tmp/corpus_v2.json -out internal/router/semantic/corpus_v1.json
package main

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
	"strings"

	"github.com/kami/maven/internal/router/semantic"
)

func main() {
	v1Path := flag.String("v1", "internal/router/semantic/corpus_v1.json", "v1 corpus path")
	v2Path := flag.String("v2", "/tmp/corpus_v2.json", "v2 factory output path")
	outPath := flag.String("out", "internal/router/semantic/corpus_v1.json", "output path")
	flag.Parse()

	// 1. Load v1
	v1Data, err := os.ReadFile(*v1Path)
	if err != nil {
		log.Fatalf("read v1: %v", err)
	}
	var v1Env semantic.CorpusEnvelope
	if err := json.Unmarshal(v1Data, &v1Env); err != nil {
		log.Fatalf("parse v1: %v", err)
	}
	fmt.Fprintf(os.Stderr, "v1: %d examples\n", len(v1Env.Examples))

	// 2. Identify frozen holdout from v1
	frozen, _, _ := semantic.FrozenHoldoutSplit(v1Env.Examples)
	frozenTexts := make(map[string]bool)
	for _, e := range frozen {
		frozenTexts[strings.TrimSpace(e.Text)] = true
	}
	fmt.Fprintf(os.Stderr, "v1 frozen holdout: %d examples\n", len(frozen))

	// 3. Load v2
	v2Data, err := os.ReadFile(*v2Path)
	if err != nil {
		log.Fatalf("read v2: %v", err)
	}
	var v2Env semantic.CorpusEnvelope
	if err := json.Unmarshal(v2Data, &v2Env); err != nil {
		log.Fatalf("parse v2: %v", err)
	}
	fmt.Fprintf(os.Stderr, "v2: %d examples\n", len(v2Env.Examples))

	// 4. Merge: frozen from v1 + all from v2
	// Dedup by normalized text
	seen := make(map[string]bool)
	var merged []semantic.RouteExample

	// Frozen holdout first
	for _, e := range frozen {
		norm := strings.TrimSpace(e.Text)
		if seen[norm] {
			fmt.Fprintf(os.Stderr, "SKIP v1 frozen dup: %q\n", norm)
			continue
		}
		seen[norm] = true
		merged = append(merged, e)
	}

	// V2 examples
	skipped := 0
	for _, e := range v2Env.Examples {
		norm := strings.TrimSpace(e.Text)
		if seen[norm] {
			skipped++
			continue
		}
		// Check if this text conflicts with a frozen holdout entry
		if frozenTexts[norm] {
			// Text exists in frozen — skip v2 version to preserve frozen
			skipped++
			continue
		}
		seen[norm] = true
		merged = append(merged, e)
	}
	fmt.Fprintf(os.Stderr, "merged: %d examples (skipped %d duplicates)\n", len(merged), skipped)

	// 5. Validate
	if err := semantic.ValidateCorpus(merged); err != nil {
		log.Fatalf("validation failed: %v", err)
	}
	fmt.Fprintf(os.Stderr, "validation: OK\n")

	// 6. Compute dataset hash
	texts := make([]string, len(merged))
	for i, e := range merged {
		texts[i] = e.Text
	}
	sort.Strings(texts)
	h := sha256.Sum256([]byte(strings.Join(texts, "\n")))
	datasetHash := hex.EncodeToString(h[:16])

	// 7. Stats
	routeCounts := make(map[semantic.SemanticRoute]int)
	fpCount, resCount := 0, 0
	for _, e := range merged {
		routeCounts[e.Route]++
		if e.FastPathResolved {
			fpCount++
		} else {
			resCount++
		}
	}
	fmt.Fprintf(os.Stderr, "\nRoute distribution:\n")
	for _, r := range semantic.AllRoutes {
		fmt.Fprintf(os.Stderr, "  %-15s %d\n", r, routeCounts[r])
	}
	fmt.Fprintf(os.Stderr, "fast-path: %d  residual: %d\n", fpCount, resCount)

	// 8. Write merged corpus
	outEnv := semantic.CorpusEnvelope{
		SchemaVersion: 1,
		Name:          "semantic_coarse_route_v1",
		Notes: []string{
			"Coarse semantic route corpus for the learned-router experiment.",
			"Frozen holdout preserved from v1. Development pool expanded by corpus-factory v2.",
			"Every row carries provenance (source + source_id) and a split_group.",
			"Labels come from explicit mapping rules, not model output.",
		},
		Reproducibility: &semantic.ReproducibilityMeta{
			SourceFixtureHash:      "v1-frozen + corpus-factory-v2",
			ContrastGeneratorVersion: "v2-direct-generation",
			SplitAlgorithm:         "grouped-cv-v1",
			DatasetHash:            datasetHash,
		},
		Examples: merged,
	}

	data, err := json.MarshalIndent(outEnv, "", "  ")
	if err != nil {
		log.Fatalf("marshal: %v", err)
	}
	if err := os.WriteFile(*outPath, data, 0644); err != nil {
		log.Fatalf("write %s: %v", *outPath, err)
	}
	fmt.Fprintf(os.Stderr, "\nOutput: %s (%d bytes)\n", *outPath, len(data))
}