fix(kernel,server,workflow): per-stage retry budget, sane freestyle budget, tail /events

Three defects surfaced by a live headless freestyle_planning run (all made a
long execution workflow fail or stall in ways invisible to a REST operator):

1. retryCount was session-global (DefaultOrchestrationReducer): TransitionExecuted
   updated currentStageId but never reset retryCount, while shouldRetry gates on
   it per-stage. An early stage that burned its 3 retries starved every later
   stage of its own — the next stage's first *retryable* failure went terminal.
   Reset retryCount on stage entry; back-edge loops stay bounded by the separate
   refinementIterations counter. Regression test added.

2. Freestyle stages ran at StageConfig's 4096-token default (ExecutionPlanCompiler
   never set a budget) vs 16384-32768 for static role_pipeline stages. A tool-heavy
   stage truncated its context every round, evicting the file_read results it just
   gathered, so it re-read forever and never accumulated enough to write. Set 16384.

3. GET /events returned the OLDEST `limit` events (readFrom(0) then take): for a
   session past `limit` events the tail — including a pending ApprovalRequested —
   was invisible to a headless/REST poller, the exact caller POST /approve serves.
   Default now tails; explicit fromSeq still paginates forward.
This commit is contained in:
2026-07-01 13:41:06 +04:00
parent c8c2521fa2
commit 4be8f292ae
4 changed files with 43 additions and 4 deletions
@@ -199,10 +199,11 @@ private fun Route.getEventsRoute(module: ServerModule) {
get("/events") {
val id = call.parameters["id"]
?: return@get call.respond(HttpStatusCode.BadRequest, "Missing session id")
val fromSeq = call.request.queryParameters["fromSeq"]?.toLongOrNull() ?: 0L
val fromSeqParam = call.request.queryParameters["fromSeq"]?.toLongOrNull()
val fromSeq = fromSeqParam ?: 0L
val limit = call.request.queryParameters["limit"]?.toIntOrNull() ?: DEFAULT_EVENT_LIMIT
val typeFilter = call.request.queryParameters["type"]
val rows = module.eventStore.readFrom(TypeId(id), fromSeq)
val mapped = module.eventStore.readFrom(TypeId(id), fromSeq)
.asSequence()
.map { stored ->
val payloadJson = eventJson.encodeToJsonElement(EventPayload.serializer(), stored.payload)
@@ -218,8 +219,12 @@ private fun Route.getEventsRoute(module: ServerModule) {
)
}
.filter { typeFilter == null || it.type == typeFilter }
.take(limit)
.toList()
// No explicit fromSeq = a "what's happening now" tail: return the most recent `limit` events
// (a pending ApprovalRequested lives at the tail, so the oldest-200 default hid it from a
// headless/REST operator — the very caller POST /approve exists to serve). Explicit fromSeq
// keeps forward, ascending pagination from that point.
val rows = if (fromSeqParam == null) mapped.takeLast(limit) else mapped.take(limit)
call.respond(rows)
}
}
@@ -53,7 +53,13 @@ class DefaultOrchestrationReducer : OrchestrationReducer {
pauseReason = null,
)
is TransitionExecutedEvent -> state.copy(currentStageId = p.to)
// Entering a new stage resets the retry budget: retryCount is per-stage (RetryAttemptedEvent
// is stage-scoped, maxAttempts is a per-stage cap), so without this reset the budget is shared
// session-wide — an early stage that exhausts its retries starves every later stage of its own,
// turning the next stage's first retryable failure into a terminal WorkflowFailed (live repro:
// implement burned 3/3, then review's retryable JSON-decode error got 0 retries). Back-edge
// loops stay bounded by the separate refinementIterations counter, so this can't loop forever.
is TransitionExecutedEvent -> state.copy(currentStageId = p.to, retryCount = 0)
is RetryAttemptedEvent -> state.copy(
status = OrchestrationStatus.RUNNING,
@@ -15,6 +15,14 @@ import com.fasterxml.jackson.module.kotlin.readValue
private const val TERMINAL = "done"
// Freestyle plans don't specify a per-stage token budget, so without this every compiled stage
// inherits StageConfig's 4096 default — 1/8th of what the static role_pipeline execution stages
// use (1638432768). At 4096 a tool-heavy stage (scaffold/implement) truncates its context every
// round, evicting the file_read results the model just gathered, so it re-reads forever and never
// accumulates enough to write. Match the static execution baseline. (gemma ctx is 32768, so this
// leaves ample headroom for completion.)
private const val DEFAULT_STAGE_TOKEN_BUDGET = 16384
class ExecutionPlanCompiler(
private val registry: ArtifactKindRegistry,
// Names of every registered tool. A stage that references a tool the runtime can't resolve
@@ -55,6 +63,7 @@ class ExecutionPlanCompiler(
needs = s.needs.map { ArtifactId(it) }.toSet(),
allowedTools = s.tools.toSet(),
writeManifest = writeManifest,
tokenBudget = DEFAULT_STAGE_TOKEN_BUDGET,
metadata = mapOf("promptInline" to s.prompt),
)
}
@@ -1,6 +1,7 @@
import com.correx.core.events.events.OrchestrationPausedEvent
import com.correx.core.events.events.OrchestrationResumedEvent
import com.correx.core.events.events.RetryAttemptedEvent
import com.correx.core.events.events.TransitionExecutedEvent
import com.correx.core.events.events.WorkflowCompletedEvent
import com.correx.core.events.events.WorkflowFailedEvent
import com.correx.core.events.events.WorkflowStartedEvent
@@ -8,6 +9,7 @@ import com.correx.core.events.orchestration.OrchestrationState
import com.correx.core.events.orchestration.OrchestrationStatus
import com.correx.core.events.types.SessionId
import com.correx.core.events.types.StageId
import com.correx.core.events.types.TransitionId
import com.correx.core.kernel.orchestration.DefaultOrchestrationReducer
import com.correx.testing.fixtures.EventFixtures.stored
import org.junit.jupiter.api.Assertions.assertEquals
@@ -133,6 +135,23 @@ class OrchestrationReducerTest {
assertNull(retrying1.failureReason)
}
@Test
fun `TransitionExecutedEvent resets retryCount so each stage gets its own retry budget`() {
val exhausted = reducer.reduce(
state,
stored(payload = RetryAttemptedEvent(sessionId, stageId, 3, 3, "boom")),
)
assertEquals(3, exhausted.retryCount)
val nextStage = StageId("stage-2")
val advanced = reducer.reduce(
exhausted,
stored(payload = TransitionExecutedEvent(sessionId, stageId, nextStage, TransitionId("t1"))),
)
assertEquals(nextStage, advanced.currentStageId)
assertEquals(0, advanced.retryCount)
}
@Test
fun `unrelated event does nothing`() {