feat(context): required/optional bucket split, budget ceilings, grounded handoffs

Context rework from the 2026-07-05 context-poisoning diagnosis (session
6ca236c4 read-loop):

- ContextBucket on ContextEntry: REQUIRED (task, constraints, needed
  artifacts, failure feedback) is never pruned/compressed/dropped; the
  build fails fast (RequiredContextOverflowException -> non-retryable
  stage failure) if required alone exceeds the stage budget.
- OptionalCeilings: per-sourceType token caps for the optional block
  (repoMap 1200, relevantFiles 1600, journal 800, profiles 400/600,
  vocabulary 400), tuned for the 9-26B tier at 16k; enforced by
  truncation (journal keeps its tail, others their head).
- Repo map is now a structural 'what exists' listing (alphabetical,
  per-directory, no recency ranking, no symbols); .md/docs paths are
  gated behind an explicit docs mention in the stage prompt — retrieval
  hits remain the only other path for docs to enter context.
- file_written needs render as a manifest of ALL files the producing
  stage wrote (projected from ArtifactContentStored -> ToolInvocation
  -> FileWritten events), fixing the last-write-wins collapse; content
  stays lazy behind file_read.
- Decision journal folds a stage's RETRY records into one summary line
  once a later transition leaves that stage (render-time only).
- Dedupe retrieved-file lines in buildRelevantFilesEntry.

core/context/src/main/kotlin/com/correx/core/context/builder/DefaultContextPackBuilder.kt

@@ -15,10 +15,12 @@ import com.correx.core.context.compression.ToMeMerger
 import com.correx.core.context.compression.TokenPruner
 import com.correx.core.context.compression.UniformRelevanceScorer
 import com.correx.core.context.model.CompressionMetadata
+import com.correx.core.context.model.ContextBucket
 import com.correx.core.context.model.ContextEntry
 import com.correx.core.context.model.ContextLayer
 import com.correx.core.context.model.ContextPack
 import com.correx.core.context.model.EntryRole
+import com.correx.core.context.model.OptionalCeilings
 import com.correx.core.context.model.TokenBudget
 import com.correx.core.events.types.ContextEntryId
 import com.correx.core.events.types.ContextPackId
@@ -43,9 +45,13 @@ class DefaultContextPackBuilder(
     private val tokenPruner: TokenPruner = NoOpTokenPruner,
     private val toMeMerger: ToMeMerger = ToMeMerger(),
     private val relevanceScorer: RelevanceScorer = UniformRelevanceScorer,
+    private val ceilings: OptionalCeilings = OptionalCeilings(),
 ) : ContextPackBuilder {
 
-    private val neverDropSourceTypes = setOf("steeringNote", "eventHistory", "decisionJournal", "factSheet")
+    // decisionJournal is deliberately NOT here anymore: the journal is OPTIONAL-bucket, capped
+    // by OptionalCeilings. Pinning is otherwise bucket-driven (ContextBucket.REQUIRED); this set
+    // only covers entries built outside the orchestrator that never stamp buckets.
+    private val neverDropSourceTypes = setOf("steeringNote", "eventHistory", "factSheet")
 
     private companion object {
         const val CHARS_PER_TOKEN = 4
@@ -95,15 +101,34 @@ class DefaultContextPackBuilder(
             if (facts.isEmpty()) merged else merged + factSheetEntry(factSheet.render(facts))
         } else merged
 
-        val (pinned, compressible) = withFactSheet.partition { it.sourceType in neverDropSourceTypes }
+        val (pinned, compressible) = withFactSheet.partition {
+            it.bucket == ContextBucket.REQUIRED || it.sourceType in neverDropSourceTypes
+        }
+        // Required entries are never trimmed: if they alone blow the budget, fail fast rather
+        // than ship a silently truncated instruction set.
+        val required = withFactSheet.filter { it.bucket == ContextBucket.REQUIRED }
+        val requiredTokens = required.sumOf { it.tokenEstimate }
+        if (requiredTokens > budget.limit) {
+            throw RequiredContextOverflowException(
+                requiredTokens = requiredTokens,
+                budgetLimit = budget.limit,
+                breakdown = required.groupBy { it.sourceType }
+                    .mapValues { (_, group) -> group.sumOf { it.tokenEstimate } },
+            )
+        }
         val pinnedTokens = pinned.sumOf { it.tokenEstimate }
         var remainingTokens = (budget.limit - pinnedTokens).coerceAtLeast(0)
 
+        // Optional per-type ceilings are enforced by truncation here, not left to the group
+        // compressor: several strategies (EventHistory) pass entries through untouched, and these
+        // feeds are single large entries — dropping whole entries would lose them entirely.
+        val ceilinged = compressible.map(::applyCeiling)
+
         // Query-conditioned selection: when a real relevance scorer is wired, reorder each freeform
         // group so the least-relevant turns sit at the front (dropped first) and the most-relevant
         // land at the tail the Conversation strategy keeps. Final layering re-sorts by ordinal, so
         // chronological render order is unaffected — only *what survives* the budget changes.
-        val relevanceRanked = rankByRelevance(compressible, currentQuery(withFactSheet))
+        val relevanceRanked = rankByRelevance(ceilinged, currentQuery(withFactSheet))
 
         // Stage 9 (BUDGET_ALLOCATOR_STRICT): compress structured groups before freeform so a long
         // conversation can't starve tool logs / artifacts out of the window.
@@ -116,7 +141,12 @@ class DefaultContextPackBuilder(
         val compressed = orderedGroups.flatMap { (sourceType, group) ->
             val strategy = strategyFor(sourceType)
             strategiesUsed += strategy::class.simpleName ?: "Unknown"
-            val result = compressor.compress(group, TokenBudget(limit = remainingTokens.coerceAtLeast(0)), strategy)
+            // Optional groups run under their per-type ceiling so no single feed (repo map,
+            // journal, profile) can crowd out the rest of the optional block.
+            val groupLimit = ceilings.capFor(sourceType)
+                ?.coerceAtMost(remainingTokens.coerceAtLeast(0))
+                ?: remainingTokens.coerceAtLeast(0)
+            val result = compressor.compress(group, TokenBudget(limit = groupLimit), strategy)
             remainingTokens -= result.sumOf { it.tokenEstimate }
             result
         }
@@ -151,6 +181,8 @@ class DefaultContextPackBuilder(
             .map { it.ordinal }.sortedDescending()
         val tier0 = if (tierSplit) freeformOrdinals.take(TIER0_TURNS).toSet() else emptySet()
         return entries.map { entry ->
+            // Required entries are exempt from lossy pruning by definition of the bucket.
+            if (entry.bucket == ContextBucket.REQUIRED) return@map entry
             val isFreeform = classifier.classify(entry) == ContextClass.FREEFORM && entry.ordinal !in tier0
             val isLargeDoc = entry.sourceType in DOC_SOURCE_TYPES && entry.content.length >= DOC_MIN_CHARS
             when {
@@ -162,10 +194,30 @@ class DefaultContextPackBuilder(
         }
     }
 
+    /**
+     * Truncates an OPTIONAL entry to its per-sourceType token ceiling. The decision journal is
+     * chronological (newest decisions at the tail), so it keeps its tail; every other feed keeps
+     * its head (headers/most-structural lines first).
+     */
+    private fun applyCeiling(entry: ContextEntry): ContextEntry {
+        val cap = ceilings.capFor(entry.sourceType) ?: return entry
+        if (entry.tokenEstimate <= cap) return entry
+        val trimmedTailMarker = "…(older entries trimmed)\n"
+        val trimmedHeadMarker = "\n…(trimmed to budget)"
+        val content = if (entry.sourceType == "decisionJournal") {
+            trimmedTailMarker + entry.content.takeLast(cap * CHARS_PER_TOKEN - trimmedTailMarker.length)
+        } else {
+            entry.content.take(cap * CHARS_PER_TOKEN - trimmedHeadMarker.length) + trimmedHeadMarker
+        }
+        return entry.copy(content = content, tokenEstimate = estimateTokens(content))
+    }
+
     private fun spansOf(entry: ContextEntry) = tagger.protectedSpans(entry.content)
 
     private fun applyToMe(entries: List<ContextEntry>): List<ContextEntry> {
-        val (freeform, rest) = entries.partition { classifier.classify(it) == ContextClass.FREEFORM }
+        val (freeform, rest) = entries.partition {
+            it.bucket != ContextBucket.REQUIRED && classifier.classify(it) == ContextClass.FREEFORM
+        }
         return (rest + toMeMerger.merge(freeform)).sortedBy { it.ordinal }
     }
 

core/context/src/main/kotlin/com/correx/core/context/builder/RequiredContextOverflowException.kt

@@ -0,0 +1,15 @@
+package com.correx.core.context.builder
+
+/**
+ * The REQUIRED bucket alone (task, constraints, loaded inputs, failures) does not fit the
+ * stage's token budget. Required context is never trimmed, so the only correct outcome is
+ * failing the build loudly — a silently truncated instruction is worse than no run.
+ */
+class RequiredContextOverflowException(
+    val requiredTokens: Int,
+    val budgetLimit: Int,
+    val breakdown: Map<String, Int>,
+) : IllegalStateException(
+    "Required context ($requiredTokens tokens) exceeds the stage budget ($budgetLimit); " +
+        "by sourceType: " + breakdown.entries.joinToString(", ") { "${it.key}=${it.value}" },
+)

core/context/src/main/kotlin/com/correx/core/context/model/ContextEntry.kt

@@ -5,6 +5,15 @@ import kotlinx.serialization.Serializable
 
 enum class EntryRole { SYSTEM, USER, ASSISTANT, TOOL }
 
+/**
+ * Budget class of an entry. REQUIRED entries (task, constraints, loaded inputs, failures) are
+ * never lossily compressed or dropped — if they alone exceed the budget the build fails fast
+ * instead of silently trimming. OPTIONAL entries (repo map, retrieval, journal, profiles) share
+ * whatever budget remains, each source type under its own ceiling.
+ */
+@Serializable
+enum class ContextBucket { REQUIRED, OPTIONAL }
+
 @Serializable
 data class ContextEntry(
     val id: ContextEntryId,
@@ -20,4 +29,7 @@ data class ContextEntry(
     // Entries built outside that builder (e.g. router chat) leave this at 0 and fall
     // back to layer-priority ordering in PromptRenderer.
     val ordinal: Int = 0,
+    // Default OPTIONAL keeps prior behavior for callers that don't stamp buckets (router chat);
+    // the orchestrator stamps REQUIRED on task/constraint/input/failure entries.
+    val bucket: ContextBucket = ContextBucket.OPTIONAL,
 )

core/context/src/main/kotlin/com/correx/core/context/model/OptionalCeilings.kt

@@ -0,0 +1,30 @@
+package com.correx.core.context.model
+
+import kotlinx.serialization.Serializable
+
+/**
+ * Per-sourceType token ceilings for OPTIONAL bucket entries. Defaults are tuned for the
+ * 9–26B local-model tier running a 16k prompt budget: the optional block as a whole stays
+ * under ~5k tokens so the required block (task, constraints, loaded inputs, failures) and
+ * the running transcript always dominate the window.
+ *
+ * A source type absent from [caps] has no per-type ceiling — it is still bounded by the
+ * remaining post-required budget like everything else.
+ */
+@Serializable
+data class OptionalCeilings(
+    val caps: Map<String, Int> = DEFAULT_CAPS,
+) {
+    fun capFor(sourceType: String): Int? = caps[sourceType]
+
+    companion object {
+        val DEFAULT_CAPS: Map<String, Int> = mapOf(
+            "repoMap" to 1200,
+            "relevantFiles" to 1600,
+            "decisionJournal" to 800,
+            "operatorProfile" to 400,
+            "projectProfile" to 600,
+            "artifactKindVocabulary" to 400,
+        )
+    }
+}