feat(context): compression pipeline stage 2 — fact sheet + strict allocation

FactSheet derives load-bearing spans (newest-first, capped) and pins them as an
L0 SYSTEM entry never dropped by the compressor (level 7). Event-derived each
assembly, not a persisted file, per invariant #1. Strict budget allocation
(level 9) compresses structured groups before freeform so conversation can't
starve tool logs / artifacts.
This commit is contained in:
2026-07-01 13:56:09 +04:00
parent 61be90070f
commit b50688df1d
3 changed files with 112 additions and 4 deletions
@@ -6,11 +6,16 @@ import com.correx.core.context.compression.CompressionStrategy
import com.correx.core.context.compression.ContextClass import com.correx.core.context.compression.ContextClass
import com.correx.core.context.compression.ContextClassifier import com.correx.core.context.compression.ContextClassifier
import com.correx.core.context.compression.ContextCompressor import com.correx.core.context.compression.ContextCompressor
import com.correx.core.context.compression.FactSheet
import com.correx.core.context.compression.FormatCompressor import com.correx.core.context.compression.FormatCompressor
import com.correx.core.context.compression.ProtectedSpanTagger
import com.correx.core.context.model.CompressionMetadata import com.correx.core.context.model.CompressionMetadata
import com.correx.core.context.model.ContextEntry import com.correx.core.context.model.ContextEntry
import com.correx.core.context.model.ContextLayer
import com.correx.core.context.model.ContextPack import com.correx.core.context.model.ContextPack
import com.correx.core.context.model.EntryRole
import com.correx.core.context.model.TokenBudget import com.correx.core.context.model.TokenBudget
import com.correx.core.events.types.ContextEntryId
import com.correx.core.events.types.ContextPackId import com.correx.core.events.types.ContextPackId
import com.correx.core.events.types.SessionId import com.correx.core.events.types.SessionId
import com.correx.core.events.types.StageId import com.correx.core.events.types.StageId
@@ -20,12 +25,14 @@ class DefaultContextPackBuilder(
private val policy: CompressionPolicy = CompressionPolicy(), private val policy: CompressionPolicy = CompressionPolicy(),
private val classifier: ContextClassifier = ContextClassifier(), private val classifier: ContextClassifier = ContextClassifier(),
private val formatCompressor: FormatCompressor = FormatCompressor(), private val formatCompressor: FormatCompressor = FormatCompressor(),
private val factSheet: FactSheet = FactSheet(ProtectedSpanTagger()),
) : ContextPackBuilder { ) : ContextPackBuilder {
private val neverDropSourceTypes = setOf("steeringNote", "eventHistory", "decisionJournal") private val neverDropSourceTypes = setOf("steeringNote", "eventHistory", "decisionJournal", "factSheet")
private companion object { private companion object {
const val CHARS_PER_TOKEN = 4 const val CHARS_PER_TOKEN = 4
const val FACT_SHEET_ORDINAL = -1
} }
override fun build( override fun build(
@@ -52,15 +59,31 @@ class DefaultContextPackBuilder(
} else entry } else entry
} }
} else stamped } else stamped
val (pinned, compressible) = ordered.partition { it.sourceType in neverDropSourceTypes } // Stage 7 (level 7+): derive the fact sheet from load-bearing spans and pin it as an
// L0 SYSTEM entry (folds into the leading system message, never dropped). Accuracy
// insurance — the most compressed tier still can't lose an ID/number/path.
val withFactSheet = if (policy.enabled(CompressionStage.FACT_SHEET)) {
val facts = factSheet.extract(ordered)
if (facts.isEmpty()) ordered else ordered + factSheetEntry(factSheet.render(facts))
} else ordered
val (pinned, compressible) = withFactSheet.partition { it.sourceType in neverDropSourceTypes }
val pinnedTokens = pinned.sumOf { it.tokenEstimate } val pinnedTokens = pinned.sumOf { it.tokenEstimate }
var remainingTokens = (budget.limit - pinnedTokens).coerceAtLeast(0) var remainingTokens = (budget.limit - pinnedTokens).coerceAtLeast(0)
// Dispatch compression strategy by entry sourceType — applying Conversation // Dispatch compression strategy by entry sourceType — applying Conversation
// strategy uniformly mangles tool logs and artifacts (their shape isn't conversational). // strategy uniformly mangles tool logs and artifacts (their shape isn't conversational).
// Stage 9 (level 9+): strict allocation reserves budget for structured content by
// compressing structured-class groups before freeform, so a long conversation can't
// starve tool logs / artifacts out of the window. Below level 9 keep encounter order.
val strict = policy.enabled(CompressionStage.BUDGET_ALLOCATOR_STRICT)
val strategiesUsed = linkedSetOf<String>() val strategiesUsed = linkedSetOf<String>()
val compressed = compressible val grouped = compressible.groupBy { it.sourceType }.entries
.groupBy { it.sourceType } val orderedGroups = if (strict) {
grouped.sortedBy { (_, group) ->
if (classifier.classify(group.first()) == ContextClass.FREEFORM) 1 else 0
}
} else grouped.toList()
val compressed = orderedGroups
.flatMap { (sourceType, group) -> .flatMap { (sourceType, group) ->
val strategy = strategyFor(sourceType) val strategy = strategyFor(sourceType)
strategiesUsed += strategy::class.simpleName ?: "Unknown" strategiesUsed += strategy::class.simpleName ?: "Unknown"
@@ -101,6 +124,19 @@ class DefaultContextPackBuilder(
// math stays consistent after a format-compressed entry shrinks. // math stays consistent after a format-compressed entry shrinks.
private fun estimateTokens(text: String): Int = (text.length + (CHARS_PER_TOKEN - 1)) / CHARS_PER_TOKEN private fun estimateTokens(text: String): Int = (text.length + (CHARS_PER_TOKEN - 1)) / CHARS_PER_TOKEN
// Negative ordinal so it sorts ahead of every turn; L0 SYSTEM so PromptRenderer folds it
// into the leading system message rather than emitting an illegal mid-stream system turn.
private fun factSheetEntry(content: String) = ContextEntry(
id = ContextEntryId("factsheet"),
layer = ContextLayer.L0,
content = content,
sourceType = "factSheet",
sourceId = "factSheet",
tokenEstimate = estimateTokens(content),
role = EntryRole.SYSTEM,
ordinal = FACT_SHEET_ORDINAL,
)
private fun strategyFor(sourceType: String): CompressionStrategy = when (sourceType) { private fun strategyFor(sourceType: String): CompressionStrategy = when (sourceType) {
"toolLog" -> CompressionStrategy.ToolLog "toolLog" -> CompressionStrategy.ToolLog
"artifact" -> CompressionStrategy.Artifact "artifact" -> CompressionStrategy.Artifact
@@ -0,0 +1,39 @@
package com.correx.core.context.compression
import com.correx.core.context.model.ContextEntry
/**
* Standing block of load-bearing entities (IDs, numbers, paths, protected terms) pulled out
* of the lossy path entirely, so even the most compressed tier can't drop them (pipeline §2).
*
* Correx is event-sourced, so — unlike the plan's `state/factsheet.toml` — this is NOT a
* persisted file (that would violate invariant #1). It is derived each assembly from the
* entries being packed and pinned as a single L0 entry the compressor never drops.
*
* Staleness/eviction (pipeline §6): facts are taken newest-first (highest ordinal) and capped
* at [maxFacts], so least-recently-referenced entities age out instead of growing unbounded.
*/
class FactSheet(
private val tagger: ProtectedSpanTagger,
private val maxFacts: Int = DEFAULT_MAX_FACTS,
) {
fun extract(entries: List<ContextEntry>): List<String> {
val ordered = LinkedHashSet<String>()
entries.sortedByDescending { it.ordinal }.forEach { entry ->
tagger.protectedSpans(entry.content).forEach { span ->
if (ordered.size < maxFacts) ordered += span.trim()
}
}
return ordered.toList()
}
fun render(facts: List<String>): String =
"## Facts (load-bearing — do not contradict or omit)\n" + facts.joinToString(", ")
fun isEmpty(facts: List<String>): Boolean = facts.isEmpty()
private companion object {
const val DEFAULT_MAX_FACTS = 40
}
}
@@ -2,6 +2,7 @@ import com.correx.core.context.compression.CompressionPolicy
import com.correx.core.context.compression.CompressionStage import com.correx.core.context.compression.CompressionStage
import com.correx.core.context.compression.ContextClass import com.correx.core.context.compression.ContextClass
import com.correx.core.context.compression.ContextClassifier import com.correx.core.context.compression.ContextClassifier
import com.correx.core.context.compression.FactSheet
import com.correx.core.context.compression.FormatCompressor import com.correx.core.context.compression.FormatCompressor
import com.correx.core.context.compression.ProtectedSpanTagger import com.correx.core.context.compression.ProtectedSpanTagger
import com.correx.core.context.model.ContextEntry import com.correx.core.context.model.ContextEntry
@@ -67,6 +68,38 @@ class CompressionPipelineStagesTest {
assertEquals("log line\nlog line\nother", out) assertEquals("log line\nlog line\nother", out)
} }
@Test
fun `fact sheet extracts newest-first and caps size`() {
val fs = FactSheet(ProtectedSpanTagger(), maxFacts = 2)
val entries = listOf(
entry("chat", ContextLayer.L2, EntryRole.USER, "old 1.1.1.1").copy(ordinal = 0),
entry("chat", ContextLayer.L2, EntryRole.USER, "new 2.2.2.2 and 3.3.3.3").copy(ordinal = 1),
)
val facts = fs.extract(entries)
// newest entry's spans come first; cap of 2 evicts the oldest entry's IP
assertEquals(listOf("2.2.2.2", "3.3.3.3"), facts)
}
@Test
fun `builder pins a fact sheet at level 7 and omits it below`() {
val entries = listOf(entry("chat", ContextLayer.L1, EntryRole.USER, "deploy to 10.0.0.1"))
val compressor = com.correx.core.context.compression.DefaultContextCompressor()
fun buildAt(level: Int) = com.correx.core.context.builder.DefaultContextPackBuilder(
compressor, CompressionPolicy(level)
).build(
com.correx.core.events.types.ContextPackId("p"),
com.correx.core.events.types.SessionId("s"),
com.correx.core.events.types.StageId("st"),
entries,
com.correx.core.context.model.TokenBudget(limit = 4000),
)
val atSeven = buildAt(7).layers.values.flatten()
assertTrue(atSeven.any { it.sourceType == "factSheet" && it.content.contains("10.0.0.1") })
assertFalse(buildAt(6).layers.values.flatten().any { it.sourceType == "factSheet" })
}
@Test @Test
fun `classifier separates static structured freeform`() { fun `classifier separates static structured freeform`() {
val c = ContextClassifier() val c = ContextClassifier()