feat(context): compression pipeline stage 2 — fact sheet + strict allocation
FactSheet derives load-bearing spans (newest-first, capped) and pins them as an L0 SYSTEM entry never dropped by the compressor (level 7). Event-derived each assembly, not a persisted file, per invariant #1. Strict budget allocation (level 9) compresses structured groups before freeform so conversation can't starve tool logs / artifacts.
This commit is contained in:
+40
-4
@@ -6,11 +6,16 @@ import com.correx.core.context.compression.CompressionStrategy
|
|||||||
import com.correx.core.context.compression.ContextClass
|
import com.correx.core.context.compression.ContextClass
|
||||||
import com.correx.core.context.compression.ContextClassifier
|
import com.correx.core.context.compression.ContextClassifier
|
||||||
import com.correx.core.context.compression.ContextCompressor
|
import com.correx.core.context.compression.ContextCompressor
|
||||||
|
import com.correx.core.context.compression.FactSheet
|
||||||
import com.correx.core.context.compression.FormatCompressor
|
import com.correx.core.context.compression.FormatCompressor
|
||||||
|
import com.correx.core.context.compression.ProtectedSpanTagger
|
||||||
import com.correx.core.context.model.CompressionMetadata
|
import com.correx.core.context.model.CompressionMetadata
|
||||||
import com.correx.core.context.model.ContextEntry
|
import com.correx.core.context.model.ContextEntry
|
||||||
|
import com.correx.core.context.model.ContextLayer
|
||||||
import com.correx.core.context.model.ContextPack
|
import com.correx.core.context.model.ContextPack
|
||||||
|
import com.correx.core.context.model.EntryRole
|
||||||
import com.correx.core.context.model.TokenBudget
|
import com.correx.core.context.model.TokenBudget
|
||||||
|
import com.correx.core.events.types.ContextEntryId
|
||||||
import com.correx.core.events.types.ContextPackId
|
import com.correx.core.events.types.ContextPackId
|
||||||
import com.correx.core.events.types.SessionId
|
import com.correx.core.events.types.SessionId
|
||||||
import com.correx.core.events.types.StageId
|
import com.correx.core.events.types.StageId
|
||||||
@@ -20,12 +25,14 @@ class DefaultContextPackBuilder(
|
|||||||
private val policy: CompressionPolicy = CompressionPolicy(),
|
private val policy: CompressionPolicy = CompressionPolicy(),
|
||||||
private val classifier: ContextClassifier = ContextClassifier(),
|
private val classifier: ContextClassifier = ContextClassifier(),
|
||||||
private val formatCompressor: FormatCompressor = FormatCompressor(),
|
private val formatCompressor: FormatCompressor = FormatCompressor(),
|
||||||
|
private val factSheet: FactSheet = FactSheet(ProtectedSpanTagger()),
|
||||||
) : ContextPackBuilder {
|
) : ContextPackBuilder {
|
||||||
|
|
||||||
private val neverDropSourceTypes = setOf("steeringNote", "eventHistory", "decisionJournal")
|
private val neverDropSourceTypes = setOf("steeringNote", "eventHistory", "decisionJournal", "factSheet")
|
||||||
|
|
||||||
private companion object {
|
private companion object {
|
||||||
const val CHARS_PER_TOKEN = 4
|
const val CHARS_PER_TOKEN = 4
|
||||||
|
const val FACT_SHEET_ORDINAL = -1
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun build(
|
override fun build(
|
||||||
@@ -52,15 +59,31 @@ class DefaultContextPackBuilder(
|
|||||||
} else entry
|
} else entry
|
||||||
}
|
}
|
||||||
} else stamped
|
} else stamped
|
||||||
val (pinned, compressible) = ordered.partition { it.sourceType in neverDropSourceTypes }
|
// Stage 7 (level 7+): derive the fact sheet from load-bearing spans and pin it as an
|
||||||
|
// L0 SYSTEM entry (folds into the leading system message, never dropped). Accuracy
|
||||||
|
// insurance — the most compressed tier still can't lose an ID/number/path.
|
||||||
|
val withFactSheet = if (policy.enabled(CompressionStage.FACT_SHEET)) {
|
||||||
|
val facts = factSheet.extract(ordered)
|
||||||
|
if (facts.isEmpty()) ordered else ordered + factSheetEntry(factSheet.render(facts))
|
||||||
|
} else ordered
|
||||||
|
val (pinned, compressible) = withFactSheet.partition { it.sourceType in neverDropSourceTypes }
|
||||||
val pinnedTokens = pinned.sumOf { it.tokenEstimate }
|
val pinnedTokens = pinned.sumOf { it.tokenEstimate }
|
||||||
var remainingTokens = (budget.limit - pinnedTokens).coerceAtLeast(0)
|
var remainingTokens = (budget.limit - pinnedTokens).coerceAtLeast(0)
|
||||||
|
|
||||||
// Dispatch compression strategy by entry sourceType — applying Conversation
|
// Dispatch compression strategy by entry sourceType — applying Conversation
|
||||||
// strategy uniformly mangles tool logs and artifacts (their shape isn't conversational).
|
// strategy uniformly mangles tool logs and artifacts (their shape isn't conversational).
|
||||||
|
// Stage 9 (level 9+): strict allocation reserves budget for structured content by
|
||||||
|
// compressing structured-class groups before freeform, so a long conversation can't
|
||||||
|
// starve tool logs / artifacts out of the window. Below level 9 keep encounter order.
|
||||||
|
val strict = policy.enabled(CompressionStage.BUDGET_ALLOCATOR_STRICT)
|
||||||
val strategiesUsed = linkedSetOf<String>()
|
val strategiesUsed = linkedSetOf<String>()
|
||||||
val compressed = compressible
|
val grouped = compressible.groupBy { it.sourceType }.entries
|
||||||
.groupBy { it.sourceType }
|
val orderedGroups = if (strict) {
|
||||||
|
grouped.sortedBy { (_, group) ->
|
||||||
|
if (classifier.classify(group.first()) == ContextClass.FREEFORM) 1 else 0
|
||||||
|
}
|
||||||
|
} else grouped.toList()
|
||||||
|
val compressed = orderedGroups
|
||||||
.flatMap { (sourceType, group) ->
|
.flatMap { (sourceType, group) ->
|
||||||
val strategy = strategyFor(sourceType)
|
val strategy = strategyFor(sourceType)
|
||||||
strategiesUsed += strategy::class.simpleName ?: "Unknown"
|
strategiesUsed += strategy::class.simpleName ?: "Unknown"
|
||||||
@@ -101,6 +124,19 @@ class DefaultContextPackBuilder(
|
|||||||
// math stays consistent after a format-compressed entry shrinks.
|
// math stays consistent after a format-compressed entry shrinks.
|
||||||
private fun estimateTokens(text: String): Int = (text.length + (CHARS_PER_TOKEN - 1)) / CHARS_PER_TOKEN
|
private fun estimateTokens(text: String): Int = (text.length + (CHARS_PER_TOKEN - 1)) / CHARS_PER_TOKEN
|
||||||
|
|
||||||
|
// Negative ordinal so it sorts ahead of every turn; L0 SYSTEM so PromptRenderer folds it
|
||||||
|
// into the leading system message rather than emitting an illegal mid-stream system turn.
|
||||||
|
private fun factSheetEntry(content: String) = ContextEntry(
|
||||||
|
id = ContextEntryId("factsheet"),
|
||||||
|
layer = ContextLayer.L0,
|
||||||
|
content = content,
|
||||||
|
sourceType = "factSheet",
|
||||||
|
sourceId = "factSheet",
|
||||||
|
tokenEstimate = estimateTokens(content),
|
||||||
|
role = EntryRole.SYSTEM,
|
||||||
|
ordinal = FACT_SHEET_ORDINAL,
|
||||||
|
)
|
||||||
|
|
||||||
private fun strategyFor(sourceType: String): CompressionStrategy = when (sourceType) {
|
private fun strategyFor(sourceType: String): CompressionStrategy = when (sourceType) {
|
||||||
"toolLog" -> CompressionStrategy.ToolLog
|
"toolLog" -> CompressionStrategy.ToolLog
|
||||||
"artifact" -> CompressionStrategy.Artifact
|
"artifact" -> CompressionStrategy.Artifact
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package com.correx.core.context.compression
|
||||||
|
|
||||||
|
import com.correx.core.context.model.ContextEntry
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Standing block of load-bearing entities (IDs, numbers, paths, protected terms) pulled out
|
||||||
|
* of the lossy path entirely, so even the most compressed tier can't drop them (pipeline §2).
|
||||||
|
*
|
||||||
|
* Correx is event-sourced, so — unlike the plan's `state/factsheet.toml` — this is NOT a
|
||||||
|
* persisted file (that would violate invariant #1). It is derived each assembly from the
|
||||||
|
* entries being packed and pinned as a single L0 entry the compressor never drops.
|
||||||
|
*
|
||||||
|
* Staleness/eviction (pipeline §6): facts are taken newest-first (highest ordinal) and capped
|
||||||
|
* at [maxFacts], so least-recently-referenced entities age out instead of growing unbounded.
|
||||||
|
*/
|
||||||
|
class FactSheet(
|
||||||
|
private val tagger: ProtectedSpanTagger,
|
||||||
|
private val maxFacts: Int = DEFAULT_MAX_FACTS,
|
||||||
|
) {
|
||||||
|
|
||||||
|
fun extract(entries: List<ContextEntry>): List<String> {
|
||||||
|
val ordered = LinkedHashSet<String>()
|
||||||
|
entries.sortedByDescending { it.ordinal }.forEach { entry ->
|
||||||
|
tagger.protectedSpans(entry.content).forEach { span ->
|
||||||
|
if (ordered.size < maxFacts) ordered += span.trim()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ordered.toList()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun render(facts: List<String>): String =
|
||||||
|
"## Facts (load-bearing — do not contradict or omit)\n" + facts.joinToString(", ")
|
||||||
|
|
||||||
|
fun isEmpty(facts: List<String>): Boolean = facts.isEmpty()
|
||||||
|
|
||||||
|
private companion object {
|
||||||
|
const val DEFAULT_MAX_FACTS = 40
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,6 +2,7 @@ import com.correx.core.context.compression.CompressionPolicy
|
|||||||
import com.correx.core.context.compression.CompressionStage
|
import com.correx.core.context.compression.CompressionStage
|
||||||
import com.correx.core.context.compression.ContextClass
|
import com.correx.core.context.compression.ContextClass
|
||||||
import com.correx.core.context.compression.ContextClassifier
|
import com.correx.core.context.compression.ContextClassifier
|
||||||
|
import com.correx.core.context.compression.FactSheet
|
||||||
import com.correx.core.context.compression.FormatCompressor
|
import com.correx.core.context.compression.FormatCompressor
|
||||||
import com.correx.core.context.compression.ProtectedSpanTagger
|
import com.correx.core.context.compression.ProtectedSpanTagger
|
||||||
import com.correx.core.context.model.ContextEntry
|
import com.correx.core.context.model.ContextEntry
|
||||||
@@ -67,6 +68,38 @@ class CompressionPipelineStagesTest {
|
|||||||
assertEquals("log line\nlog line\nother", out)
|
assertEquals("log line\nlog line\nother", out)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `fact sheet extracts newest-first and caps size`() {
|
||||||
|
val fs = FactSheet(ProtectedSpanTagger(), maxFacts = 2)
|
||||||
|
val entries = listOf(
|
||||||
|
entry("chat", ContextLayer.L2, EntryRole.USER, "old 1.1.1.1").copy(ordinal = 0),
|
||||||
|
entry("chat", ContextLayer.L2, EntryRole.USER, "new 2.2.2.2 and 3.3.3.3").copy(ordinal = 1),
|
||||||
|
)
|
||||||
|
val facts = fs.extract(entries)
|
||||||
|
// newest entry's spans come first; cap of 2 evicts the oldest entry's IP
|
||||||
|
assertEquals(listOf("2.2.2.2", "3.3.3.3"), facts)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `builder pins a fact sheet at level 7 and omits it below`() {
|
||||||
|
val entries = listOf(entry("chat", ContextLayer.L1, EntryRole.USER, "deploy to 10.0.0.1"))
|
||||||
|
val compressor = com.correx.core.context.compression.DefaultContextCompressor()
|
||||||
|
|
||||||
|
fun buildAt(level: Int) = com.correx.core.context.builder.DefaultContextPackBuilder(
|
||||||
|
compressor, CompressionPolicy(level)
|
||||||
|
).build(
|
||||||
|
com.correx.core.events.types.ContextPackId("p"),
|
||||||
|
com.correx.core.events.types.SessionId("s"),
|
||||||
|
com.correx.core.events.types.StageId("st"),
|
||||||
|
entries,
|
||||||
|
com.correx.core.context.model.TokenBudget(limit = 4000),
|
||||||
|
)
|
||||||
|
|
||||||
|
val atSeven = buildAt(7).layers.values.flatten()
|
||||||
|
assertTrue(atSeven.any { it.sourceType == "factSheet" && it.content.contains("10.0.0.1") })
|
||||||
|
assertFalse(buildAt(6).layers.values.flatten().any { it.sourceType == "factSheet" })
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun `classifier separates static structured freeform`() {
|
fun `classifier separates static structured freeform`() {
|
||||||
val c = ContextClassifier()
|
val c = ContextClassifier()
|
||||||
|
|||||||
Reference in New Issue
Block a user