kami/correx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(context): compression pipeline stage 2 — fact sheet + strict allocation

Browse Source 
FactSheet derives load-bearing spans (newest-first, capped) and pins them as an
L0 SYSTEM entry never dropped by the compressor (level 7). Event-derived each
assembly, not a persisted file, per invariant #1. Strict budget allocation
(level 9) compresses structured groups before freeform so conversation can't
starve tool logs / artifacts.
This commit is contained in:
kami
2026-07-01 13:56:09 +04:00
parent 61be90070f
commit b50688df1d
3 changed files with 112 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/context/src/main/kotlin/com/correx/core/context/builder/DefaultContextPackBuilder.kt
+40 -4
View File
@@ -6,11 +6,16 @@ import com.correx.core.context.compression.CompressionStrategy
import com.correx.core.context.compression.ContextClass
import com.correx.core.context.compression.ContextClassifier
import com.correx.core.context.compression.ContextCompressor
import com.correx.core.context.compression.FactSheet
import com.correx.core.context.compression.FormatCompressor
import com.correx.core.context.compression.ProtectedSpanTagger
import com.correx.core.context.model.CompressionMetadata
import com.correx.core.context.model.ContextEntry
import com.correx.core.context.model.ContextLayer
import com.correx.core.context.model.ContextPack
import com.correx.core.context.model.EntryRole
import com.correx.core.context.model.TokenBudget
import com.correx.core.events.types.ContextEntryId
import com.correx.core.events.types.ContextPackId
import com.correx.core.events.types.SessionId
import com.correx.core.events.types.StageId
@@ -20,12 +25,14 @@ class DefaultContextPackBuilder(
private val policy: CompressionPolicy = CompressionPolicy(),
private val classifier: ContextClassifier = ContextClassifier(),
private val formatCompressor: FormatCompressor = FormatCompressor(),
private val factSheet: FactSheet = FactSheet(ProtectedSpanTagger()),
) : ContextPackBuilder {
private val neverDropSourceTypes = setOf("steeringNote", "eventHistory", "decisionJournal")
private val neverDropSourceTypes = setOf("steeringNote", "eventHistory", "decisionJournal", "factSheet")
private companion object {
const val CHARS_PER_TOKEN = 4
const val FACT_SHEET_ORDINAL = -1
}
override fun build(
@@ -52,15 +59,31 @@ class DefaultContextPackBuilder(
} else entry
}
} else stamped
val (pinned, compressible) = ordered.partition { it.sourceType in neverDropSourceTypes }
// Stage 7 (level 7+): derive the fact sheet from load-bearing spans and pin it as an
// L0 SYSTEM entry (folds into the leading system message, never dropped). Accuracy
// insurance — the most compressed tier still can't lose an ID/number/path.
val withFactSheet = if (policy.enabled(CompressionStage.FACT_SHEET)) {
val facts = factSheet.extract(ordered)
if (facts.isEmpty()) ordered else ordered + factSheetEntry(factSheet.render(facts))
} else ordered
val (pinned, compressible) = withFactSheet.partition { it.sourceType in neverDropSourceTypes }
val pinnedTokens = pinned.sumOf { it.tokenEstimate }
var remainingTokens = (budget.limit - pinnedTokens).coerceAtLeast(0)
// Dispatch compression strategy by entry sourceType — applying Conversation
// strategy uniformly mangles tool logs and artifacts (their shape isn't conversational).
// Stage 9 (level 9+): strict allocation reserves budget for structured content by
// compressing structured-class groups before freeform, so a long conversation can't
// starve tool logs / artifacts out of the window. Below level 9 keep encounter order.
val strict = policy.enabled(CompressionStage.BUDGET_ALLOCATOR_STRICT)
val strategiesUsed = linkedSetOf<String>()
val compressed = compressible
.groupBy { it.sourceType }
val grouped = compressible.groupBy { it.sourceType }.entries
val orderedGroups = if (strict) {
grouped.sortedBy { (_, group) ->
if (classifier.classify(group.first()) == ContextClass.FREEFORM) 1 else 0
}
} else grouped.toList()
val compressed = orderedGroups
.flatMap { (sourceType, group) ->
val strategy = strategyFor(sourceType)
strategiesUsed += strategy::class.simpleName ?: "Unknown"
@@ -101,6 +124,19 @@ class DefaultContextPackBuilder(
// math stays consistent after a format-compressed entry shrinks.
private fun estimateTokens(text: String): Int = (text.length + (CHARS_PER_TOKEN - 1)) / CHARS_PER_TOKEN
// Negative ordinal so it sorts ahead of every turn; L0 SYSTEM so PromptRenderer folds it
// into the leading system message rather than emitting an illegal mid-stream system turn.
private fun factSheetEntry(content: String) = ContextEntry(
id = ContextEntryId("factsheet"),
layer = ContextLayer.L0,
content = content,
sourceType = "factSheet",
sourceId = "factSheet",
tokenEstimate = estimateTokens(content),
role = EntryRole.SYSTEM,
ordinal = FACT_SHEET_ORDINAL,
)
private fun strategyFor(sourceType: String): CompressionStrategy = when (sourceType) {
"toolLog" -> CompressionStrategy.ToolLog
"artifact" -> CompressionStrategy.Artifact
core/context/src/main/kotlin/com/correx/core/context/compression/FactSheet.kt
+39
View File
@@ -0,0 +1,39 @@
package com.correx.core.context.compression
import com.correx.core.context.model.ContextEntry
/**
* Standing block of load-bearing entities (IDs, numbers, paths, protected terms) pulled out
* of the lossy path entirely, so even the most compressed tier can't drop them (pipeline §2).
*
* Correx is event-sourced, so — unlike the plan's `state/factsheet.toml` — this is NOT a
* persisted file (that would violate invariant #1). It is derived each assembly from the
* entries being packed and pinned as a single L0 entry the compressor never drops.
*
* Staleness/eviction (pipeline §6): facts are taken newest-first (highest ordinal) and capped
* at [maxFacts], so least-recently-referenced entities age out instead of growing unbounded.
*/
class FactSheet(
private val tagger: ProtectedSpanTagger,
private val maxFacts: Int = DEFAULT_MAX_FACTS,
) {
fun extract(entries: List<ContextEntry>): List<String> {
val ordered = LinkedHashSet<String>()
entries.sortedByDescending { it.ordinal }.forEach { entry ->
tagger.protectedSpans(entry.content).forEach { span ->
if (ordered.size < maxFacts) ordered += span.trim()
}
}
return ordered.toList()
}
fun render(facts: List<String>): String =
"## Facts (load-bearing — do not contradict or omit)\n" + facts.joinToString(", ")
fun isEmpty(facts: List<String>): Boolean = facts.isEmpty()
private companion object {
const val DEFAULT_MAX_FACTS = 40
}
}
core/context/src/test/kotlin/CompressionPipelineStagesTest.kt
+33
View File
@@ -2,6 +2,7 @@ import com.correx.core.context.compression.CompressionPolicy
import com.correx.core.context.compression.CompressionStage
import com.correx.core.context.compression.ContextClass
import com.correx.core.context.compression.ContextClassifier
import com.correx.core.context.compression.FactSheet
import com.correx.core.context.compression.FormatCompressor
import com.correx.core.context.compression.ProtectedSpanTagger
import com.correx.core.context.model.ContextEntry
@@ -67,6 +68,38 @@ class CompressionPipelineStagesTest {
assertEquals("log line\nlog line\nother", out)
}
@Test
fun `fact sheet extracts newest-first and caps size`() {
val fs = FactSheet(ProtectedSpanTagger(), maxFacts = 2)
val entries = listOf(
entry("chat", ContextLayer.L2, EntryRole.USER, "old 1.1.1.1").copy(ordinal = 0),
entry("chat", ContextLayer.L2, EntryRole.USER, "new 2.2.2.2 and 3.3.3.3").copy(ordinal = 1),
)
val facts = fs.extract(entries)
// newest entry's spans come first; cap of 2 evicts the oldest entry's IP
assertEquals(listOf("2.2.2.2", "3.3.3.3"), facts)
}
@Test
fun `builder pins a fact sheet at level 7 and omits it below`() {
val entries = listOf(entry("chat", ContextLayer.L1, EntryRole.USER, "deploy to 10.0.0.1"))
val compressor = com.correx.core.context.compression.DefaultContextCompressor()
fun buildAt(level: Int) = com.correx.core.context.builder.DefaultContextPackBuilder(
compressor, CompressionPolicy(level)
).build(
com.correx.core.events.types.ContextPackId("p"),
com.correx.core.events.types.SessionId("s"),
com.correx.core.events.types.StageId("st"),
entries,
com.correx.core.context.model.TokenBudget(limit = 4000),
)
val atSeven = buildAt(7).layers.values.flatten()
assertTrue(atSeven.any { it.sourceType == "factSheet" && it.content.contains("10.0.0.1") })
assertFalse(buildAt(6).layers.values.flatten().any { it.sourceType == "factSheet" })
}
@Test
fun `classifier separates static structured freeform`() {
val c = ContextClassifier()