feat(retry): per-gate retry budgets + progress-aware charging + hybrid salvage
Replaces the single shared per-stage retryCount (reset on TransitionExecuted, shared across all post-stage gates) with a per-gate budget. Motivated by run 2e468f9f, where a promising freestyle run was terminally FAILED because the contract gate burned all 3 shared retries on one trivial miss before the semantic-review gate ever ran. - StageExecutionResult.Failure gains a `gate` id; each gate tags its failures. - OrchestrationState gains gateRetryBudgets / gateFailureFingerprints / gateSalvageUsed (all rebuilt from events, reset per-stage on TransitionExecuted). - FailureFingerprint normalizes+hashes the failure reason; RetryCoordinator.decide charges a gate's budget only when the fingerprint is unchanged (no progress), so a moving run is never penalised. Returns Retry | Exhausted. - Hybrid exhaustion: deterministic gates fail; the review gate consults a SalvageJudge (LLM, or a deterministic allow-one-reset fallback), recorded as RetrySalvageDecidedEvent (invariant #9). CONTINUE resets the gate budget once; a second exhaustion is terminal. - Review gate is now the sole authority on review-retry termination; the old REVIEW_BLOCK_RETRY_CAP is repurposed as a high absolute backstop only. - ServerModule escaped-exception path now records a truthful terminal WorkflowFailed (real stage/reason/retryExhausted) via recordUnhandledFailure. Tests: DefaultRetryCoordinatorTest (fingerprint charging) + GateRetryBudgetExhaustionTest (deterministic exhaust / review CONTINUE-then- terminal / review FAIL / null-judge fallback) + reducer coverage.
This commit is contained in:
+24
-1
@@ -5,6 +5,8 @@ import com.correx.core.events.events.OrchestrationPausedEvent
|
||||
import com.correx.core.events.events.OrchestrationResumedEvent
|
||||
import com.correx.core.events.events.RefinementIterationEvent
|
||||
import com.correx.core.events.events.RetryAttemptedEvent
|
||||
import com.correx.core.events.events.RetrySalvageDecidedEvent
|
||||
import com.correx.core.events.events.SalvageDecision
|
||||
import com.correx.core.events.events.StoredEvent
|
||||
import com.correx.core.events.events.TransitionExecutedEvent
|
||||
import com.correx.core.events.events.WorkflowCompletedEvent
|
||||
@@ -59,14 +61,35 @@ class DefaultOrchestrationReducer : OrchestrationReducer {
|
||||
// turning the next stage's first retryable failure into a terminal WorkflowFailed (live repro:
|
||||
// implement burned 3/3, then review's retryable JSON-decode error got 0 retries). Back-edge
|
||||
// loops stay bounded by the separate refinementIterations counter, so this can't loop forever.
|
||||
is TransitionExecutedEvent -> state.copy(currentStageId = p.to, retryCount = 0)
|
||||
is TransitionExecutedEvent -> state.copy(
|
||||
currentStageId = p.to,
|
||||
retryCount = 0,
|
||||
gateRetryBudgets = emptyMap(),
|
||||
gateFailureFingerprints = emptyMap(),
|
||||
gateSalvageUsed = emptySet(),
|
||||
)
|
||||
|
||||
is RetryAttemptedEvent -> state.copy(
|
||||
status = OrchestrationStatus.RUNNING,
|
||||
retryCount = p.attemptNumber,
|
||||
failureReason = null,
|
||||
gateRetryBudgets = if (p.charged) {
|
||||
state.gateRetryBudgets + (p.gate to ((state.gateRetryBudgets[p.gate] ?: 0) + 1))
|
||||
} else {
|
||||
state.gateRetryBudgets
|
||||
},
|
||||
gateFailureFingerprints = state.gateFailureFingerprints + (p.gate to p.fingerprint),
|
||||
)
|
||||
|
||||
is RetrySalvageDecidedEvent -> if (p.decision == SalvageDecision.CONTINUE) {
|
||||
state.copy(
|
||||
gateRetryBudgets = state.gateRetryBudgets + (p.gate to 0),
|
||||
gateSalvageUsed = state.gateSalvageUsed + p.gate,
|
||||
)
|
||||
} else {
|
||||
state
|
||||
}
|
||||
|
||||
is RefinementIterationEvent -> state.copy(
|
||||
refinementIterations = state.refinementIterations + (p.cycleKey to p.iteration),
|
||||
)
|
||||
|
||||
+56
-13
@@ -17,6 +17,8 @@ import com.correx.core.events.events.ClarificationAnsweredEvent
|
||||
import com.correx.core.events.events.ArtifactValidatedEvent
|
||||
import com.correx.core.events.events.OrchestrationResumedEvent
|
||||
import com.correx.core.events.events.RefinementIterationEvent
|
||||
import com.correx.core.events.events.RetrySalvageDecidedEvent
|
||||
import com.correx.core.events.events.SalvageDecision
|
||||
import com.correx.core.events.events.StoredEvent
|
||||
import com.correx.core.events.events.TransitionExecutedEvent
|
||||
import com.correx.core.events.orchestration.OrchestrationState
|
||||
@@ -32,6 +34,7 @@ import com.correx.core.kernel.orchestration.subagent.InSessionSubagentRunner
|
||||
import com.correx.core.kernel.orchestration.subagent.SubagentRunRequest
|
||||
import com.correx.core.kernel.orchestration.subagent.SubagentRunner
|
||||
import com.correx.core.kernel.retry.RetryCoordinator
|
||||
import com.correx.core.kernel.retry.RetryDecision
|
||||
import com.correx.core.sessions.Session
|
||||
import com.correx.core.transitions.execution.StageExecutionResult
|
||||
import com.correx.core.transitions.graph.WorkflowGraph
|
||||
@@ -68,6 +71,11 @@ class DefaultSessionOrchestrator(
|
||||
override val cancellations: ConcurrentHashMap<SessionId, AtomicBoolean> =
|
||||
ConcurrentHashMap<SessionId, AtomicBoolean>()
|
||||
|
||||
// Hybrid-exhaustion salvage judge (T6/T7): consulted only when the "review" gate exhausts its
|
||||
// per-gate retry budget. Null = deterministic allow-one-reset-then-fail fallback (see
|
||||
// decideGateExhaustion) so the feature degrades safely without inference.
|
||||
private val salvageJudge: SalvageJudge? = engines.salvageJudge
|
||||
|
||||
override val subagentRunner: SubagentRunner = InSessionSubagentRunner(
|
||||
executeStage = { sid, stg, graph, session, cfg ->
|
||||
executeStage(sid, stg, graph, session, cfg, effectivesFor(cfg))
|
||||
@@ -442,29 +450,64 @@ class DefaultSessionOrchestrator(
|
||||
return StepResult.Continue(ctx.copy(stageCount = ctx.stageCount + 1))
|
||||
}
|
||||
val refreshedState = orchestrationRepository.getState(ctx.sessionId)
|
||||
val shouldRetry = retryCoordinator.shouldRetry(
|
||||
val gateDecision = retryCoordinator.decide(
|
||||
sessionId = ctx.sessionId,
|
||||
stageId = stageId,
|
||||
currentAttempt = refreshedState.retryCount,
|
||||
policy = ctx.config.retryPolicy,
|
||||
gate = result.gate,
|
||||
failureReason = result.reason,
|
||||
state = refreshedState,
|
||||
policy = ctx.config.retryPolicy,
|
||||
)
|
||||
if (!shouldRetry) {
|
||||
return StepResult.Terminal(
|
||||
failWorkflow(
|
||||
ctx.sessionId,
|
||||
stageId,
|
||||
result.reason,
|
||||
retryExhausted = true,
|
||||
),
|
||||
)
|
||||
when (gateDecision) {
|
||||
RetryDecision.Retry -> Unit // retry — loop and re-execute
|
||||
RetryDecision.Exhausted -> {
|
||||
val terminal = decideGateExhaustion(
|
||||
ctx.sessionId, stageId, result.gate, result.reason, refreshedState,
|
||||
)
|
||||
if (terminal != null) return StepResult.Terminal(terminal)
|
||||
// review-gate salvage said CONTINUE — budget was reset by the reducer;
|
||||
// loop and re-execute the stage with a fresh budget.
|
||||
}
|
||||
}
|
||||
// retry — loop and re-execute
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Hybrid exhaustion (design 2026-07-06-per-gate-retry-budgets.md §3/T6): a deterministic gate
|
||||
* exhausting its budget fails the stage terminally, unchanged from today's behaviour (just
|
||||
* per-gate now). The "review" gate instead gets one salvageability judgement — CONTINUE resets
|
||||
* its budget (via the reducer folding [RetrySalvageDecidedEvent]) and the caller loops to retry;
|
||||
* FAIL is terminal. A gate that already spent its one salvage reset (state.gateSalvageUsed) fails
|
||||
* immediately on a second exhaustion, regardless of what the judge would say.
|
||||
*
|
||||
* Returns the terminal [WorkflowResult] to fail with, or null to retry (loop and re-execute).
|
||||
*/
|
||||
private suspend fun decideGateExhaustion(
|
||||
sessionId: SessionId,
|
||||
stageId: StageId,
|
||||
gate: String,
|
||||
reason: String,
|
||||
state: OrchestrationState,
|
||||
): WorkflowResult? {
|
||||
if (gate != "review" || state.gateSalvageUsed.contains(gate)) {
|
||||
return failWorkflow(sessionId, stageId, reason, retryExhausted = true)
|
||||
}
|
||||
// No judge wired: degrade safely with a deterministic allow-one-reset-then-fail policy —
|
||||
// the gateSalvageUsed check above already ensures this fires at most once per gate.
|
||||
val judgment = salvageJudge?.judge(sessionId, stageId, reason)
|
||||
?: SalvageJudgment(
|
||||
SalvageDecision.CONTINUE,
|
||||
"no salvage judge wired — deterministic one-time reset",
|
||||
)
|
||||
emit(sessionId, RetrySalvageDecidedEvent(sessionId, stageId, gate, judgment.decision, judgment.rationale))
|
||||
return when (judgment.decision) {
|
||||
SalvageDecision.CONTINUE -> null
|
||||
SalvageDecision.FAIL -> failWorkflow(sessionId, stageId, reason, retryExhausted = true)
|
||||
}
|
||||
}
|
||||
|
||||
private fun ExecutionContext.enrich() = EnrichedExecutionContext(
|
||||
graph, sessionId, stageCount, currentStageId, config,
|
||||
session = repositories.sessionRepository.getSession(sessionId),
|
||||
|
||||
+4
@@ -43,4 +43,8 @@ data class OrchestratorEngines(
|
||||
// Gate 3). Null = no semantic review; the funnel then rests on the deterministic gates only. A
|
||||
// stage opts in via StageConfig.semanticReview.
|
||||
val semanticReviewer: SemanticReviewer? = null,
|
||||
// Hybrid-exhaustion salvage judge (design 2026-07-06-per-gate-retry-budgets.md §3/T7), consulted
|
||||
// only when the "review" gate exhausts its retry budget. Null = no LLM judge wired; the
|
||||
// orchestrator then falls back to a deterministic allow-one-reset-then-fail policy.
|
||||
val salvageJudge: SalvageJudge? = null,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package com.correx.core.kernel.orchestration
|
||||
|
||||
import com.correx.core.events.events.SalvageDecision
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.events.types.StageId
|
||||
|
||||
/** Result of a [SalvageJudge.judge] call. */
|
||||
data class SalvageJudgment(val decision: SalvageDecision, val rationale: String)
|
||||
|
||||
/**
|
||||
* Seam for the hybrid-exhaustion salvage judgement (design 2026-07-06-per-gate-retry-budgets.md
|
||||
* §3/T7): consulted when the "review" gate exhausts its per-gate retry budget. Like the other gate
|
||||
* seams ([SemanticReviewer], [ContractAssertionEvaluator]), the inference-touching implementation is
|
||||
* injected so the deterministic core never runs inference itself.
|
||||
*
|
||||
* The judgement is nondeterministic (LLM-backed), so invariant #9 requires the caller to record the
|
||||
* outcome as a [com.correx.core.events.events.RetrySalvageDecidedEvent] — replay reads that event
|
||||
* back and never re-invokes this judge. When no judge is wired (`null` in [OrchestratorEngines]),
|
||||
* the orchestrator falls back to a deterministic allow-one-reset-then-fail policy so the feature
|
||||
* degrades safely without inference (see `DefaultSessionOrchestrator.decideSalvage`).
|
||||
*/
|
||||
fun interface SalvageJudge {
|
||||
suspend fun judge(sessionId: SessionId, stageId: StageId, reason: String): SalvageJudgment
|
||||
}
|
||||
+199
-19
@@ -223,12 +223,16 @@ private const val MAX_CLARIFICATION_ROUNDS = 3
|
||||
// Static-analysis output caps: the tail retained in the recorded event vs. the (larger) slice fed
|
||||
// back to the model so it can fix the failure. Tails, because the error summary sits at the end.
|
||||
private const val CONTRACT_EVIDENCE_CAP = 400
|
||||
// Gate 3 semantic review: a FAIL blocks only on a correctness finding at/above this confidence, and
|
||||
// only until this many prior blocks have occurred for the stage — after which findings surface
|
||||
// without blocking, so a stuck reviewer can never trap a stage. Objective text fed to the reviewer
|
||||
// is capped to keep the review prompt bounded.
|
||||
// Gate 3 semantic review: a FAIL blocks only on a correctness finding at/above this confidence.
|
||||
// Per-gate retry budgeting + hybrid salvage (design 2026-07-06-per-gate-retry-budgets.md) is now the
|
||||
// authority on when review retries stop — the gate simply reports the block and the RetryCoordinator
|
||||
// charges/exhausts the "review" budget, then the salvage judge decides continue-vs-fail. This cap
|
||||
// remains only as an absolute backstop against a pathological run whose review findings *change*
|
||||
// every attempt (so the budget never charges, i.e. the run looks like perpetual "progress"): after
|
||||
// this many blocks, findings surface without blocking so a stuck reviewer can never trap a stage
|
||||
// forever. Set well above any legitimate budget+salvage sequence. Objective text is capped too.
|
||||
private const val REVIEW_BLOCK_MIN_CONFIDENCE = 0.7
|
||||
private const val REVIEW_BLOCK_RETRY_CAP = 2
|
||||
private const val REVIEW_BLOCK_RETRY_CAP = 20
|
||||
private const val REVIEW_OBJECTIVE_CAP = 4_000
|
||||
private const val STATIC_ANALYSIS_SUMMARY_CAP = 2_000
|
||||
private const val STATIC_ANALYSIS_FEEDBACK_CAP = 6_000
|
||||
@@ -1929,6 +1933,7 @@ abstract class SessionOrchestrator(
|
||||
return StageExecutionResult.Failure(
|
||||
"stage ${stageId.value} declared no artifacts and ran no tools",
|
||||
retryable = true,
|
||||
gate = "produces",
|
||||
)
|
||||
}
|
||||
val producedIds = stageConfig.produces.map { it.name }.toSet()
|
||||
@@ -1945,6 +1950,7 @@ abstract class SessionOrchestrator(
|
||||
return StageExecutionResult.Failure(
|
||||
"stage ${stageId.value} did not produce declared artifacts: $missingIds",
|
||||
retryable = true,
|
||||
gate = "produces",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -2015,6 +2021,7 @@ abstract class SessionOrchestrator(
|
||||
"stage ${stageId.value} references paths that do not exist in the workspace: " +
|
||||
"${missing.joinToString(", ")}. Reference only files you have read.",
|
||||
retryable = true,
|
||||
gate = "brief_grounding",
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -2077,6 +2084,7 @@ abstract class SessionOrchestrator(
|
||||
"files not in the brief: [${div.hallucinatedFiles.joinToString(", ")}]. " +
|
||||
"Restate the brief exactly; do not drop requirements or invent files.",
|
||||
retryable = true,
|
||||
gate = "brief_echo",
|
||||
)
|
||||
} else {
|
||||
StageExecutionResult.Success(emptyList())
|
||||
@@ -2174,6 +2182,7 @@ abstract class SessionOrchestrator(
|
||||
return StageExecutionResult.Failure(
|
||||
"stage ${stageId.value} did not satisfy its file contract. Fix these before review:\n$detail",
|
||||
retryable = true,
|
||||
gate = "contract",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -2209,6 +2218,7 @@ abstract class SessionOrchestrator(
|
||||
"stage ${stageId.value} produced an execution_plan that does not compile. " +
|
||||
"Fix the plan before it can be locked:\n$error",
|
||||
retryable = true,
|
||||
gate = "plan_compile",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -2283,6 +2293,7 @@ abstract class SessionOrchestrator(
|
||||
return StageExecutionResult.Failure(
|
||||
"stage ${stageId.value} did not pass static analysis. Fix these before review:\n\n$detail",
|
||||
retryable = true,
|
||||
gate = "static_analysis",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -2330,6 +2341,7 @@ abstract class SessionOrchestrator(
|
||||
"Fix these before proceeding:\n\n$ $command (exit ${run.exitCode})\n" +
|
||||
run.output.takeLast(STATIC_ANALYSIS_FEEDBACK_CAP),
|
||||
retryable = true,
|
||||
gate = "execution",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -2402,6 +2414,7 @@ abstract class SessionOrchestrator(
|
||||
return StageExecutionResult.Failure(
|
||||
"stage ${stageId.value} failed semantic review — fix these correctness issues:\n$detail",
|
||||
retryable = true,
|
||||
gate = "review",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -2560,6 +2573,7 @@ abstract class SessionOrchestrator(
|
||||
if (isCancelled(sessionId)) return InferenceResult.Cancelled
|
||||
|
||||
val responseArtifactId = artifactStore.put(response.text.toByteArray())
|
||||
val reasoningArtifactId = response.reasoning?.let { artifactStore.put(it.toByteArray()) }
|
||||
emit(
|
||||
sessionId,
|
||||
InferenceCompletedEvent(
|
||||
@@ -2570,6 +2584,7 @@ abstract class SessionOrchestrator(
|
||||
tokensUsed = response.tokensUsed,
|
||||
latencyMs = response.latencyMs,
|
||||
responseArtifactId = responseArtifactId,
|
||||
reasoningArtifactId = reasoningArtifactId,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -2687,8 +2702,35 @@ abstract class SessionOrchestrator(
|
||||
"[Orchestrator] FAILED session={} stage={} reason={} retryExhausted={}",
|
||||
sessionId.value, stageId.value, reason, retryExhausted,
|
||||
)
|
||||
correlateCritiqueOutcomes(sessionId)
|
||||
emit(sessionId, WorkflowFailedEvent(sessionId, stageId, reason, retryExhausted))
|
||||
// failWorkflow IS the clean-termination path: whatever goes wrong while recording it
|
||||
// (event-store hiccup, correlation lookup, or any other transient failure) must never be
|
||||
// allowed to escape as a raw exception — that would blow past this function's own
|
||||
// WorkflowFailedEvent (correct stageId/reason/retryExhausted) and land in the server's
|
||||
// top-level catch-all, which fabricates its own event with the wrong stage (graph.start)
|
||||
// and a garbage reason (the escaping throwable's message/toString). Record what we can and
|
||||
// still return a clean Failed result either way.
|
||||
// Critique correlation is best-effort bookkeeping; if it throws we log and press on, because
|
||||
// the terminal WorkflowFailedEvent below MUST still be recorded — skipping it would leave the
|
||||
// event log with no terminal marker for the session (breaks replay-completeness, invariants
|
||||
// #1/#8) and let the server catch-all fabricate a wrong one.
|
||||
runCatching { correlateCritiqueOutcomes(sessionId) }.onFailure { e ->
|
||||
log.error(
|
||||
"[Orchestrator] failWorkflow: critique correlation failed session={} stage={}",
|
||||
sessionId.value, stageId.value, e,
|
||||
)
|
||||
}
|
||||
// Last-ditch: the one unrecoverable case. If emitting the terminal event itself throws
|
||||
// (e.g. a serialization edge), we cannot record the failure at all — log it loudly with the
|
||||
// full throwable so it is never silent, then still return a clean Failed result.
|
||||
runCatching {
|
||||
emit(sessionId, WorkflowFailedEvent(sessionId, stageId, reason, retryExhausted))
|
||||
}.onFailure { e ->
|
||||
log.error(
|
||||
"[Orchestrator] failWorkflow: FAILED to record terminal WorkflowFailedEvent — event " +
|
||||
"log has no terminal marker for session={} stage={}",
|
||||
sessionId.value, stageId.value, e,
|
||||
)
|
||||
}
|
||||
cancellations.remove(sessionId)
|
||||
return WorkflowResult.Failed(sessionId, reason, retryExhausted)
|
||||
}
|
||||
@@ -2961,6 +3003,7 @@ abstract class SessionOrchestrator(
|
||||
private suspend fun computeToolPreview(toolName: String, parameters: Map<String, Any>): String? {
|
||||
if (toolName == "shell") return shellCommandPreview(parameters)
|
||||
if (toolName == "task_decompose") return renderDecomposePreview(parameters)
|
||||
if (toolName == "file_edit") return computeFileEditPreview(parameters)
|
||||
if (toolName != "file_write") return null
|
||||
val path = parameters["path"] as? String ?: return null
|
||||
// file_write no longer carries an `operation` param (delete was split into file_delete), so the
|
||||
@@ -2968,13 +3011,39 @@ private suspend fun computeToolPreview(toolName: String, parameters: Map<String,
|
||||
// silently bailed to the raw-JSON args fallback for every file_write.
|
||||
val proposedContent = parameters["content"] as? String ?: return null
|
||||
|
||||
val existingContent = withContext(Dispatchers.IO) {
|
||||
runCatching {
|
||||
val filePath = java.nio.file.Paths.get(path)
|
||||
if (java.nio.file.Files.exists(filePath)) {
|
||||
java.nio.file.Files.readString(filePath)
|
||||
} else null
|
||||
}.getOrNull()
|
||||
val existingContent = readFileIfExists(path)
|
||||
return buildDiffString(path, existingContent, proposedContent)
|
||||
}
|
||||
|
||||
private suspend fun readFileIfExists(path: String): String? = withContext(Dispatchers.IO) {
|
||||
runCatching {
|
||||
val filePath = java.nio.file.Paths.get(path)
|
||||
if (java.nio.file.Files.exists(filePath)) {
|
||||
java.nio.file.Files.readString(filePath)
|
||||
} else null
|
||||
}.getOrNull()
|
||||
}
|
||||
|
||||
/**
|
||||
* Preview for `file_edit`'s "replace" and "append" operations: reads the existing file and
|
||||
* simulates the edit locally to produce a diff, mirroring `file_write`'s preview. "patch" is left
|
||||
* to the raw-JSON-args fallback — simulating `patch -p1` application here isn't worth the risk of
|
||||
* the preview silently disagreeing with what the tool actually does.
|
||||
*/
|
||||
private suspend fun computeFileEditPreview(parameters: Map<String, Any>): String? {
|
||||
val path = parameters["path"] as? String ?: return null
|
||||
val operation = parameters["operation"] as? String ?: return null
|
||||
val existingContent = readFileIfExists(path) ?: return null
|
||||
|
||||
val proposedContent = when (operation) {
|
||||
"append" -> existingContent + (parameters["content"] as? String ?: return null)
|
||||
"replace" -> {
|
||||
val target = parameters["target"] as? String ?: return null
|
||||
val replacement = parameters["replacement"] as? String ?: return null
|
||||
if (!existingContent.contains(target)) return null
|
||||
existingContent.replaceFirst(target, replacement)
|
||||
}
|
||||
else -> return null
|
||||
}
|
||||
|
||||
return buildDiffString(path, existingContent, proposedContent)
|
||||
@@ -3003,25 +3072,136 @@ private fun shellQuoteToken(token: String): String {
|
||||
return if (needsQuote) "'" + token.replace("'", "'\\''") + "'" else token
|
||||
}
|
||||
|
||||
private enum class DiffOpKind { EQUAL, DELETE, INSERT }
|
||||
private data class DiffOp(val kind: DiffOpKind, val line: String)
|
||||
|
||||
/**
|
||||
* Build a unified-diff-style string for a full file replacement.
|
||||
* Build a unified-diff string for a full file replacement, using a line-level LCS diff
|
||||
* so a small edit to a large file shows as a tight hunk rather than a full-file replacement.
|
||||
* When the file doesn't exist yet (new file), only `+` lines are shown.
|
||||
* Uses a simple full-file diff: all old lines as `-`, all new lines as `+`.
|
||||
*/
|
||||
private fun buildDiffString(path: String, existingContent: String?, proposedContent: String): String {
|
||||
val existingLines = existingContent?.lines() ?: emptyList()
|
||||
val proposedLines = proposedContent.lines()
|
||||
if (existingLines == proposedLines) return " (no change)"
|
||||
|
||||
val ops = diffLines(existingLines, proposedLines)
|
||||
return buildString {
|
||||
appendLine("--- a/$path")
|
||||
appendLine("+++ b/$path")
|
||||
appendLine("@@ -1,${existingLines.size.coerceAtLeast(1)} +1,${proposedLines.size.coerceAtLeast(1)} @@")
|
||||
existingLines.forEach { appendLine("-$it") }
|
||||
proposedLines.forEach { appendLine("+$it") }
|
||||
append(renderHunks(ops))
|
||||
}.trimEnd()
|
||||
}
|
||||
|
||||
/** Line-level LCS diff producing an edit script of EQUAL/DELETE/INSERT ops. */
|
||||
private fun diffLines(oldLines: List<String>, newLines: List<String>): List<DiffOp> {
|
||||
val n = oldLines.size
|
||||
val m = newLines.size
|
||||
val lcs = Array(n + 1) { IntArray(m + 1) }
|
||||
for (i in n - 1 downTo 0) {
|
||||
for (j in m - 1 downTo 0) {
|
||||
lcs[i][j] = if (oldLines[i] == newLines[j]) {
|
||||
lcs[i + 1][j + 1] + 1
|
||||
} else {
|
||||
maxOf(lcs[i + 1][j], lcs[i][j + 1])
|
||||
}
|
||||
}
|
||||
}
|
||||
val ops = mutableListOf<DiffOp>()
|
||||
var i = 0
|
||||
var j = 0
|
||||
while (i < n && j < m) {
|
||||
when {
|
||||
oldLines[i] == newLines[j] -> {
|
||||
ops += DiffOp(DiffOpKind.EQUAL, oldLines[i])
|
||||
i++
|
||||
j++
|
||||
}
|
||||
lcs[i + 1][j] >= lcs[i][j + 1] -> {
|
||||
ops += DiffOp(DiffOpKind.DELETE, oldLines[i])
|
||||
i++
|
||||
}
|
||||
else -> {
|
||||
ops += DiffOp(DiffOpKind.INSERT, newLines[j])
|
||||
j++
|
||||
}
|
||||
}
|
||||
}
|
||||
while (i < n) {
|
||||
ops += DiffOp(DiffOpKind.DELETE, oldLines[i])
|
||||
i++
|
||||
}
|
||||
while (j < m) {
|
||||
ops += DiffOp(DiffOpKind.INSERT, newLines[j])
|
||||
j++
|
||||
}
|
||||
return ops
|
||||
}
|
||||
|
||||
/** Groups an edit script into unified-diff hunks with [context] lines of surrounding EQUAL context. */
|
||||
private fun renderHunks(ops: List<DiffOp>, context: Int = 3): String {
|
||||
val changedIndices = ops.indices.filter { ops[it].kind != DiffOpKind.EQUAL }
|
||||
if (changedIndices.isEmpty()) return ""
|
||||
|
||||
data class Hunk(val start: Int, val end: Int)
|
||||
val hunks = mutableListOf<Hunk>()
|
||||
var hunkStart = changedIndices.first()
|
||||
var hunkEnd = changedIndices.first()
|
||||
for (idx in changedIndices.drop(1)) {
|
||||
if (idx - hunkEnd <= context * 2) {
|
||||
hunkEnd = idx
|
||||
} else {
|
||||
hunks += Hunk(hunkStart, hunkEnd)
|
||||
hunkStart = idx
|
||||
hunkEnd = idx
|
||||
}
|
||||
}
|
||||
hunks += Hunk(hunkStart, hunkEnd)
|
||||
|
||||
var oldLine = 1
|
||||
var newLine = 1
|
||||
var opIdx = 0
|
||||
return buildString {
|
||||
for (hunk in hunks) {
|
||||
val from = maxOf(0, hunk.start - context)
|
||||
val to = minOf(ops.size - 1, hunk.end + context)
|
||||
while (opIdx < from) {
|
||||
if (ops[opIdx].kind != DiffOpKind.INSERT) oldLine++
|
||||
if (ops[opIdx].kind != DiffOpKind.DELETE) newLine++
|
||||
opIdx++
|
||||
}
|
||||
val oldStart = oldLine
|
||||
val newStart = newLine
|
||||
var oldCount = 0
|
||||
var newCount = 0
|
||||
val lines = StringBuilder()
|
||||
for (k in from..to) {
|
||||
val op = ops[k]
|
||||
when (op.kind) {
|
||||
DiffOpKind.EQUAL -> {
|
||||
lines.append(' ').append(op.line).append('\n')
|
||||
oldCount++
|
||||
newCount++
|
||||
}
|
||||
DiffOpKind.DELETE -> {
|
||||
lines.append('-').append(op.line).append('\n')
|
||||
oldCount++
|
||||
}
|
||||
DiffOpKind.INSERT -> {
|
||||
lines.append('+').append(op.line).append('\n')
|
||||
newCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
appendLine("@@ -$oldStart,${oldCount.coerceAtLeast(1)} +$newStart,${newCount.coerceAtLeast(1)} @@")
|
||||
append(lines)
|
||||
oldLine = oldStart + oldCount
|
||||
newLine = newStart + newCount
|
||||
opIdx = to + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The candidate in [paths] that shares [guessed]'s filename and the most trailing path segments,
|
||||
* or null when nothing shares the filename. Lets a wrong package/dir prefix
|
||||
|
||||
@@ -4,6 +4,7 @@ import com.correx.core.events.events.EventMetadata
|
||||
import com.correx.core.events.events.NewEvent
|
||||
import com.correx.core.events.events.RetryAttemptedEvent
|
||||
import com.correx.core.events.execution.RetryPolicy
|
||||
import com.correx.core.events.orchestration.OrchestrationState
|
||||
import com.correx.core.events.stores.EventStore
|
||||
import com.correx.core.events.types.EventId
|
||||
import com.correx.core.events.types.SessionId
|
||||
@@ -16,6 +17,53 @@ class DefaultRetryCoordinator(
|
||||
private val eventStore: EventStore,
|
||||
) : RetryCoordinator {
|
||||
|
||||
override suspend fun decide(
|
||||
sessionId: SessionId,
|
||||
stageId: StageId,
|
||||
gate: String,
|
||||
failureReason: String,
|
||||
state: OrchestrationState,
|
||||
policy: RetryPolicy,
|
||||
): RetryDecision {
|
||||
val fingerprint = FailureFingerprint.of(failureReason)
|
||||
val prevFingerprint = state.gateFailureFingerprints[gate]
|
||||
val progressed = prevFingerprint != fingerprint
|
||||
val charged = !progressed
|
||||
val currentCount = state.gateRetryBudgets[gate] ?: 0
|
||||
val newCount = if (charged) currentCount + 1 else currentCount
|
||||
val maxAttempts = policy.maxAttemptsFor(gate)
|
||||
if (charged && newCount > maxAttempts) {
|
||||
return RetryDecision.Exhausted
|
||||
}
|
||||
|
||||
eventStore.append(
|
||||
NewEvent(
|
||||
metadata = EventMetadata(
|
||||
eventId = EventId(UUID.randomUUID().toString()),
|
||||
sessionId = sessionId,
|
||||
timestamp = Clock.System.now(),
|
||||
schemaVersion = 1,
|
||||
causationId = null,
|
||||
correlationId = null,
|
||||
),
|
||||
payload = RetryAttemptedEvent(
|
||||
sessionId = sessionId,
|
||||
stageId = stageId,
|
||||
attemptNumber = newCount,
|
||||
maxAttempts = maxAttempts,
|
||||
failureReason = failureReason,
|
||||
gate = gate,
|
||||
fingerprint = fingerprint,
|
||||
charged = charged,
|
||||
),
|
||||
),
|
||||
)
|
||||
if (policy.backoffMs > 0L) {
|
||||
delay(policy.backoffMs)
|
||||
}
|
||||
return RetryDecision.Retry
|
||||
}
|
||||
|
||||
override suspend fun shouldRetry(
|
||||
sessionId: SessionId,
|
||||
stageId: StageId,
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
package com.correx.core.kernel.retry
|
||||
|
||||
// Collapses whitespace runs and digit runs so line numbers / byte counts / timestamps embedded in a
|
||||
// gate's failure reason don't make two otherwise-identical failures look like "progress". Gate
|
||||
// reasons already list the failing assertion/finding ids verbatim (see runContractGate,
|
||||
// runStaticAnalysis, runReviewGate in SessionOrchestrator), so this normalisation is sufficient to
|
||||
// treat "same root cause" retries as no-progress without needing per-gate parsing.
|
||||
private val VOLATILE_RUN = Regex("""[\s\d]+""")
|
||||
|
||||
/**
|
||||
* Stable fingerprint of a [StageExecutionResult.Failure] reason (design
|
||||
* 2026-07-06-per-gate-retry-budgets.md §5). Two failures with the same fingerprint are treated as
|
||||
* "no progress" for per-gate retry budgeting; a changed fingerprint means the run is moving and the
|
||||
* retry is free.
|
||||
*/
|
||||
object FailureFingerprint {
|
||||
fun of(reason: String): String {
|
||||
val normalized = reason.trim().lowercase().replace(VOLATILE_RUN, " ").trim()
|
||||
return Integer.toHexString(normalized.hashCode())
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,21 @@
|
||||
package com.correx.core.kernel.retry
|
||||
|
||||
import com.correx.core.events.execution.RetryPolicy
|
||||
import com.correx.core.events.orchestration.OrchestrationState
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.events.types.StageId
|
||||
|
||||
/** Outcome of [RetryCoordinator.decide] for a single gate's failure. */
|
||||
sealed interface RetryDecision {
|
||||
data object Retry : RetryDecision
|
||||
data object Exhausted : RetryDecision
|
||||
}
|
||||
|
||||
interface RetryCoordinator {
|
||||
/**
|
||||
* Legacy shared-budget entry point, kept for the non-gate transition-failure paths
|
||||
* (Stay/NoMatch in DefaultSessionOrchestrator.step) that predate per-gate budgeting.
|
||||
*/
|
||||
suspend fun shouldRetry(
|
||||
sessionId: SessionId,
|
||||
stageId: StageId,
|
||||
@@ -12,4 +23,23 @@ interface RetryCoordinator {
|
||||
policy: RetryPolicy,
|
||||
failureReason: String,
|
||||
): Boolean
|
||||
|
||||
/**
|
||||
* Per-gate, progress-aware retry decision (design 2026-07-06-per-gate-retry-budgets.md §5).
|
||||
* [failureReason] is fingerprinted internally (see [FailureFingerprint]) and charges [gate]'s
|
||||
* budget only when the fingerprint is unchanged from the last attempt recorded in [state] for
|
||||
* that gate (no progress); a changed fingerprint is free. On a charging decision that would
|
||||
* exceed the gate's budget, returns [RetryDecision.Exhausted] without emitting an event — the
|
||||
* caller (hybrid exhaustion, T6) decides what happens next. Otherwise emits
|
||||
* [com.correx.core.events.events.RetryAttemptedEvent] (with the fingerprint and the full
|
||||
* [failureReason] both recorded), applies backoff, and returns [RetryDecision.Retry].
|
||||
*/
|
||||
suspend fun decide(
|
||||
sessionId: SessionId,
|
||||
stageId: StageId,
|
||||
gate: String,
|
||||
failureReason: String,
|
||||
state: OrchestrationState,
|
||||
policy: RetryPolicy,
|
||||
): RetryDecision
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user