package com.correx.core.context.compression

import com.correx.core.context.model.ContextEntry

/**
 * Stage 8 (TOME_MERGE): merge near-duplicate freeform turns into one representative instead of
 * dropping them outright, preserving partial signal (pipeline §1, "Token merging"). Deterministic:
 * similarity is Jaccard overlap of word shingles, so no model call. Adjacent-in-relevance entries
 * whose overlap exceeds [threshold] collapse to the newest (highest ordinal), which carries the
 * merged content forward; the older duplicates are dropped.
 */
class ToMeMerger(private val threshold: Double = DEFAULT_THRESHOLD) {

    fun merge(entries: List<ContextEntry>): List<ContextEntry> {
        if (entries.size < 2) return entries
        val kept = mutableListOf<ContextEntry>()
        // Process newest-first so the survivor of a near-duplicate pair is the most recent turn.
        for (entry in entries.sortedByDescending { it.ordinal }) {
            val dup = kept.any { jaccard(shingles(it.content), shingles(entry.content)) >= threshold }
            if (!dup) kept += entry
        }
        return kept.sortedBy { it.ordinal }
    }

    private fun shingles(text: String): Set<String> =
        text.lowercase().split(WHITESPACE).filter { it.isNotBlank() }.toSet()

    private fun jaccard(a: Set<String>, b: Set<String>): Double {
        if (a.isEmpty() && b.isEmpty()) return 1.0
        val inter = a.intersect(b).size.toDouble()
        val union = a.union(b).size.toDouble()
        return if (union == 0.0) 0.0 else inter / union
    }

    private companion object {
        const val DEFAULT_THRESHOLD = 0.9
        val WHITESPACE = Regex("\\s+")
    }
}
