From b50688df1dd3f98c4a37fb9c2c5f76b2dfbec75b Mon Sep 17 00:00:00 2001 From: kami Date: Wed, 1 Jul 2026 13:56:09 +0400 Subject: [PATCH] =?UTF-8?q?feat(context):=20compression=20pipeline=20stage?= =?UTF-8?q?=202=20=E2=80=94=20fact=20sheet=20+=20strict=20allocation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FactSheet derives load-bearing spans (newest-first, capped) and pins them as an L0 SYSTEM entry never dropped by the compressor (level 7). Event-derived each assembly, not a persisted file, per invariant #1. Strict budget allocation (level 9) compresses structured groups before freeform so conversation can't starve tool logs / artifacts. --- .../builder/DefaultContextPackBuilder.kt | 44 +++++++++++++++++-- .../core/context/compression/FactSheet.kt | 39 ++++++++++++++++ .../kotlin/CompressionPipelineStagesTest.kt | 33 ++++++++++++++ 3 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 core/context/src/main/kotlin/com/correx/core/context/compression/FactSheet.kt diff --git a/core/context/src/main/kotlin/com/correx/core/context/builder/DefaultContextPackBuilder.kt b/core/context/src/main/kotlin/com/correx/core/context/builder/DefaultContextPackBuilder.kt index 2ca2eb0c..e0311a01 100644 --- a/core/context/src/main/kotlin/com/correx/core/context/builder/DefaultContextPackBuilder.kt +++ b/core/context/src/main/kotlin/com/correx/core/context/builder/DefaultContextPackBuilder.kt @@ -6,11 +6,16 @@ import com.correx.core.context.compression.CompressionStrategy import com.correx.core.context.compression.ContextClass import com.correx.core.context.compression.ContextClassifier import com.correx.core.context.compression.ContextCompressor +import com.correx.core.context.compression.FactSheet import com.correx.core.context.compression.FormatCompressor +import com.correx.core.context.compression.ProtectedSpanTagger import com.correx.core.context.model.CompressionMetadata import com.correx.core.context.model.ContextEntry +import com.correx.core.context.model.ContextLayer import com.correx.core.context.model.ContextPack +import com.correx.core.context.model.EntryRole import com.correx.core.context.model.TokenBudget +import com.correx.core.events.types.ContextEntryId import com.correx.core.events.types.ContextPackId import com.correx.core.events.types.SessionId import com.correx.core.events.types.StageId @@ -20,12 +25,14 @@ class DefaultContextPackBuilder( private val policy: CompressionPolicy = CompressionPolicy(), private val classifier: ContextClassifier = ContextClassifier(), private val formatCompressor: FormatCompressor = FormatCompressor(), + private val factSheet: FactSheet = FactSheet(ProtectedSpanTagger()), ) : ContextPackBuilder { - private val neverDropSourceTypes = setOf("steeringNote", "eventHistory", "decisionJournal") + private val neverDropSourceTypes = setOf("steeringNote", "eventHistory", "decisionJournal", "factSheet") private companion object { const val CHARS_PER_TOKEN = 4 + const val FACT_SHEET_ORDINAL = -1 } override fun build( @@ -52,15 +59,31 @@ class DefaultContextPackBuilder( } else entry } } else stamped - val (pinned, compressible) = ordered.partition { it.sourceType in neverDropSourceTypes } + // Stage 7 (level 7+): derive the fact sheet from load-bearing spans and pin it as an + // L0 SYSTEM entry (folds into the leading system message, never dropped). Accuracy + // insurance — the most compressed tier still can't lose an ID/number/path. + val withFactSheet = if (policy.enabled(CompressionStage.FACT_SHEET)) { + val facts = factSheet.extract(ordered) + if (facts.isEmpty()) ordered else ordered + factSheetEntry(factSheet.render(facts)) + } else ordered + val (pinned, compressible) = withFactSheet.partition { it.sourceType in neverDropSourceTypes } val pinnedTokens = pinned.sumOf { it.tokenEstimate } var remainingTokens = (budget.limit - pinnedTokens).coerceAtLeast(0) // Dispatch compression strategy by entry sourceType — applying Conversation // strategy uniformly mangles tool logs and artifacts (their shape isn't conversational). + // Stage 9 (level 9+): strict allocation reserves budget for structured content by + // compressing structured-class groups before freeform, so a long conversation can't + // starve tool logs / artifacts out of the window. Below level 9 keep encounter order. + val strict = policy.enabled(CompressionStage.BUDGET_ALLOCATOR_STRICT) val strategiesUsed = linkedSetOf() - val compressed = compressible - .groupBy { it.sourceType } + val grouped = compressible.groupBy { it.sourceType }.entries + val orderedGroups = if (strict) { + grouped.sortedBy { (_, group) -> + if (classifier.classify(group.first()) == ContextClass.FREEFORM) 1 else 0 + } + } else grouped.toList() + val compressed = orderedGroups .flatMap { (sourceType, group) -> val strategy = strategyFor(sourceType) strategiesUsed += strategy::class.simpleName ?: "Unknown" @@ -101,6 +124,19 @@ class DefaultContextPackBuilder( // math stays consistent after a format-compressed entry shrinks. private fun estimateTokens(text: String): Int = (text.length + (CHARS_PER_TOKEN - 1)) / CHARS_PER_TOKEN + // Negative ordinal so it sorts ahead of every turn; L0 SYSTEM so PromptRenderer folds it + // into the leading system message rather than emitting an illegal mid-stream system turn. + private fun factSheetEntry(content: String) = ContextEntry( + id = ContextEntryId("factsheet"), + layer = ContextLayer.L0, + content = content, + sourceType = "factSheet", + sourceId = "factSheet", + tokenEstimate = estimateTokens(content), + role = EntryRole.SYSTEM, + ordinal = FACT_SHEET_ORDINAL, + ) + private fun strategyFor(sourceType: String): CompressionStrategy = when (sourceType) { "toolLog" -> CompressionStrategy.ToolLog "artifact" -> CompressionStrategy.Artifact diff --git a/core/context/src/main/kotlin/com/correx/core/context/compression/FactSheet.kt b/core/context/src/main/kotlin/com/correx/core/context/compression/FactSheet.kt new file mode 100644 index 00000000..8a00fae9 --- /dev/null +++ b/core/context/src/main/kotlin/com/correx/core/context/compression/FactSheet.kt @@ -0,0 +1,39 @@ +package com.correx.core.context.compression + +import com.correx.core.context.model.ContextEntry + +/** + * Standing block of load-bearing entities (IDs, numbers, paths, protected terms) pulled out + * of the lossy path entirely, so even the most compressed tier can't drop them (pipeline §2). + * + * Correx is event-sourced, so — unlike the plan's `state/factsheet.toml` — this is NOT a + * persisted file (that would violate invariant #1). It is derived each assembly from the + * entries being packed and pinned as a single L0 entry the compressor never drops. + * + * Staleness/eviction (pipeline §6): facts are taken newest-first (highest ordinal) and capped + * at [maxFacts], so least-recently-referenced entities age out instead of growing unbounded. + */ +class FactSheet( + private val tagger: ProtectedSpanTagger, + private val maxFacts: Int = DEFAULT_MAX_FACTS, +) { + + fun extract(entries: List): List { + val ordered = LinkedHashSet() + entries.sortedByDescending { it.ordinal }.forEach { entry -> + tagger.protectedSpans(entry.content).forEach { span -> + if (ordered.size < maxFacts) ordered += span.trim() + } + } + return ordered.toList() + } + + fun render(facts: List): String = + "## Facts (load-bearing — do not contradict or omit)\n" + facts.joinToString(", ") + + fun isEmpty(facts: List): Boolean = facts.isEmpty() + + private companion object { + const val DEFAULT_MAX_FACTS = 40 + } +} diff --git a/core/context/src/test/kotlin/CompressionPipelineStagesTest.kt b/core/context/src/test/kotlin/CompressionPipelineStagesTest.kt index 5593e550..b47fba64 100644 --- a/core/context/src/test/kotlin/CompressionPipelineStagesTest.kt +++ b/core/context/src/test/kotlin/CompressionPipelineStagesTest.kt @@ -2,6 +2,7 @@ import com.correx.core.context.compression.CompressionPolicy import com.correx.core.context.compression.CompressionStage import com.correx.core.context.compression.ContextClass import com.correx.core.context.compression.ContextClassifier +import com.correx.core.context.compression.FactSheet import com.correx.core.context.compression.FormatCompressor import com.correx.core.context.compression.ProtectedSpanTagger import com.correx.core.context.model.ContextEntry @@ -67,6 +68,38 @@ class CompressionPipelineStagesTest { assertEquals("log line\nlog line\nother", out) } + @Test + fun `fact sheet extracts newest-first and caps size`() { + val fs = FactSheet(ProtectedSpanTagger(), maxFacts = 2) + val entries = listOf( + entry("chat", ContextLayer.L2, EntryRole.USER, "old 1.1.1.1").copy(ordinal = 0), + entry("chat", ContextLayer.L2, EntryRole.USER, "new 2.2.2.2 and 3.3.3.3").copy(ordinal = 1), + ) + val facts = fs.extract(entries) + // newest entry's spans come first; cap of 2 evicts the oldest entry's IP + assertEquals(listOf("2.2.2.2", "3.3.3.3"), facts) + } + + @Test + fun `builder pins a fact sheet at level 7 and omits it below`() { + val entries = listOf(entry("chat", ContextLayer.L1, EntryRole.USER, "deploy to 10.0.0.1")) + val compressor = com.correx.core.context.compression.DefaultContextCompressor() + + fun buildAt(level: Int) = com.correx.core.context.builder.DefaultContextPackBuilder( + compressor, CompressionPolicy(level) + ).build( + com.correx.core.events.types.ContextPackId("p"), + com.correx.core.events.types.SessionId("s"), + com.correx.core.events.types.StageId("st"), + entries, + com.correx.core.context.model.TokenBudget(limit = 4000), + ) + + val atSeven = buildAt(7).layers.values.flatten() + assertTrue(atSeven.any { it.sourceType == "factSheet" && it.content.contains("10.0.0.1") }) + assertFalse(buildAt(6).layers.values.flatten().any { it.sourceType == "factSheet" }) + } + @Test fun `classifier separates static structured freeform`() { val c = ContextClassifier()