merge: integrate sonnet-vikunja (#299,#300,#297,#301) into codex handoff HEAD

# Conflicts: # core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/OrchestrationTuning.kt # examples/workflows/prompts/analyst_freestyle.md
2026-07-21 11:43:05 +04:00
parent 6e844ef1e1 8cc418a381
commit 28d369ebce
11 changed files with 714 additions and 6 deletions
@@ -66,6 +66,21 @@ data class OutsidePathAccessGrantedEvent(
    val path: String,
 ) : EventPayload

+/**
+ * Records that the operator approved widening the write scope/manifest to admit [path], after
+ * the same path was rejected `escalate_scope_after_n` times in a row (small models frequently
+ * fail to comply with the WRITE_SCOPE/PATH_OUTSIDE_MANIFEST remediation and thrash instead —
+ * see #301). Folded the same way [OutsidePathAccessGrantedEvent] widens out-of-workspace reads:
+ * subsequent writes to this path this session are admitted without re-prompting.
+ */
+@Serializable
+@SerialName("WriteScopeGranted")
+data class WriteScopeGrantedEvent(
+    val sessionId: SessionId,
+    val stageId: StageId,
+    val path: String,
+) : EventPayload
+
@Serializable
@SerialName("OrchestrationPaused")
 data class OrchestrationPausedEvent(
@@ -2,6 +2,7 @@ package com.correx.core.inference

 import com.correx.core.events.types.ProviderId
 import com.correx.core.events.types.StageId
+import kotlinx.coroutines.delay
 import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
 import kotlin.time.Duration
@@ -21,6 +22,12 @@ class DefaultInferenceRouter(
    private val strategy: RoutingStrategy,
    private val cacheTtl: Duration = 5.seconds,
    private val timeSource: TimeSource = TimeSource.Monotonic,
+    // A provider that briefly drops (crash + qa-stack restart) shouldn't collapse into a hard
+    // NoEligibleProvider abort — give it bounded time to come back before giving up. This only
+    // applies when the capability IS configured on some provider but that provider is currently
+    // unhealthy; a capability nobody ever declared fails immediately (see routeCapabilityCandidates).
+    private val unavailableRetryAttempts: Int = 3,
+    private val unavailableRetryDelay: Duration = 2.seconds,
 ) : InferenceRouter {

    private val cache = mutableMapOf<ProviderId, HealthEntry>()
@@ -45,12 +52,49 @@ class DefaultInferenceRouter(
        }
    }

+    private suspend fun refreshedHealth(provider: InferenceProvider): ProviderHealth =
+        lockFor(provider.id).withLock {
+            val fresh = provider.healthCheck()
+            mapMutex.withLock { cache[provider.id] = HealthEntry(fresh, timeSource.markNow()) }
+            fresh
+        }
+
+    // Bypasses healthCheck()/TTL entirely — writes Unavailable straight into the cache so the very
+    // next route() call gates on it, closing the ~18s reactive-poll lag (#300). A later cache-TTL
+    // expiry or the bounded-wait re-check in route() will naturally pick the provider back up once
+    // its own healthCheck() reports healthy again.
+    override suspend fun reportFailure(providerId: ProviderId, reason: String) {
+        lockFor(providerId).withLock {
+            mapMutex.withLock { cache[providerId] = HealthEntry(ProviderHealth.Unavailable(reason), timeSource.markNow()) }
+        }
+    }
+
    override suspend fun route(stageId: StageId, requiredCapabilities: Set<ModelCapability>): InferenceProvider {
        val candidates = requiredCapabilities
            .flatMap { registry.resolve(it) }
            .distinctBy { it.id }
            .ifEmpty { registry.listAll() }
-        val healthy = candidates.filter { cachedHealth(it) !is ProviderHealth.Unavailable }
+
+        // Nobody was ever configured with this capability set — no amount of waiting fixes that,
+        // fail fast instead of burning the bounded-wait budget below.
+        if (requiredCapabilities.isNotEmpty() &&
+            candidates.none { it.capabilities().map { c -> c.capability }.toSet().containsAll(requiredCapabilities) }
+        ) {
+            throw NoEligibleProviderException(stageId, requiredCapabilities)
+        }
+
+        var healthy = candidates.filter { cachedHealth(it) !is ProviderHealth.Unavailable }
+        var attempt = 0
+        while (healthy.isEmpty() && attempt < unavailableRetryAttempts) {
+            attempt++
+            log.warn(
+                "route: capability {} configured but all candidates unhealthy for stage={}" +
+                    " — waiting {} (attempt {}/{}) before declaring NoEligibleProvider",
+                requiredCapabilities, stageId.value, unavailableRetryDelay, attempt, unavailableRetryAttempts,
+            )
+            delay(unavailableRetryDelay)
+            healthy = candidates.filter { refreshedHealth(it) !is ProviderHealth.Unavailable }
+        }
        val selected = strategy.select(healthy, requiredCapabilities)
        // Post-selection re-check closes the TOCTOU window between initial filter and dispatch.
        when (val postHealth = selected.healthCheck()) {
@@ -39,6 +39,14 @@ interface InferenceRouter {
        requiredCapabilities: Set<ModelCapability>,
        modelId: String?,
    ): InferenceProvider = route(stageId, requiredCapabilities)
+
+    /**
+     * Event-driven health gate: called the moment a connection-level failure is observed on
+     * [providerId] (e.g. mid-request connection drop), so the NEXT route() call sees it as
+     * unavailable immediately instead of waiting for the next periodic health poll/cache TTL to
+     * catch up. Default no-op for routers that don't cache health.
+     */
+    suspend fun reportFailure(providerId: com.correx.core.events.types.ProviderId, reason: String) = Unit
 }

 class NoEligibleProviderException(
@@ -51,4 +51,12 @@ data class OrchestrationTuning(
    val stageFailureLoopLimit: Int = 6,
    /** Minimum confidence for a post-failure diagnostic proposal (#294) to be routed into recovery. */
    val diagnosisMinConfidence: Double = 0.5,
+    /**
+     * After this many consecutive WRITE_SCOPE/PATH_OUTSIDE_MANIFEST rejections of the SAME path in
+     * a session, stop hard-rejecting and escalate to user approval instead (#301) — small models
+     * frequently fail to comply with the remediation and thrash rather than widen scope themselves.
+     * 0 disables escalation (falls back to the hard-block-forever behavior). A headless session
+     * (no approver connected) auto-rejects the escalated prompt rather than hanging.
+     */
+    val escalateScopeAfterN: Int = 3,
 )
@@ -322,7 +322,15 @@ abstract class SessionOrchestrator(
        // ToolCalling score and routes the stage to the best tool-caller.
        val requiredCapabilities = stageConfig.requiredCapabilities +
            if (withTools && stageConfig.allowedTools.isNotEmpty()) setOf(ModelCapability.ToolCalling) else emptySet()
-        val provider = inferenceRouter.route(stageId, requiredCapabilities, stageConfig.modelId)
+        // Routing itself can fail transiently (provider mid-crash-recovery) — it must be retryable
+        // like any other inference failure, not escape and kill the whole session (see #299).
+        val provider = try {
+            inferenceRouter.route(stageId, requiredCapabilities, stageConfig.modelId)
+        } catch (e: CancellationException) {
+            throw e
+        } catch (e: Exception) {
+            return InferenceResult.Failed(e.message ?: "routing failed")
+        }
        log.debug(
            "[Orchestrator] inference session={} stage={} provider={} timeoutMs={}",
            sessionId.value, stageId.value, provider.id.value, timeoutMs,
@@ -423,6 +431,13 @@ abstract class SessionOrchestrator(
        } catch (e: CancellationException) {
            throw e // never swallow
        } catch (e: Exception) {
+            if (isConnectionLevelFailure(e)) {
+                // Mark it down NOW instead of waiting for the next periodic health poll (~18s lag,
+                // see #300) — the retry's route() call must see this provider as unavailable
+                // immediately so it gates/waits (#299) rather than instantly re-selecting the dead
+                // provider again.
+                inferenceRouter.reportFailure(provider.id, e.message ?: "connection failure")
+            }
            emit(
                sessionId,
                InferenceFailedEvent(
@@ -437,6 +452,19 @@ abstract class SessionOrchestrator(
        }
    }

+    // Connection-level failures (provider crashed/restarting mid-request) should gate routing
+    // immediately; other failures (bad request, model error, HTTP 4xx) should not mark the
+    // provider down since the provider itself is still reachable.
+    private fun isConnectionLevelFailure(e: Exception): Boolean {
+        val message = e.message.orEmpty()
+        return e is java.net.ConnectException ||
+            e is java.net.SocketException ||
+            e is java.io.IOException || // covers ktor/CIO's IOException, which extends java.io.IOException on the JVM
+            message.contains("prematurely closed", ignoreCase = true) ||
+            message.contains("connection refused", ignoreCase = true) ||
+            message.contains("connection reset", ignoreCase = true)
+    }
+
    // --- token estimation ---

    internal open suspend fun estimateTokens(content: String): Int {
@@ -31,6 +31,7 @@ import com.correx.core.events.events.ToolExecutionRejectedEvent
 import com.correx.core.events.events.ToolInvocationRequestedEvent
 import com.correx.core.events.events.ToolReceipt
 import com.correx.core.events.events.ToolRequest
+import com.correx.core.events.events.WriteScopeGrantedEvent
 import com.correx.core.events.risk.RiskAction
 import com.correx.core.events.risk.RiskSummary
 import com.correx.core.toolintent.ToolCallAssessmentInput
@@ -225,14 +226,36 @@ internal suspend fun SessionOrchestrator.dispatchToolCalls(
        // Per-task write scope: while a task is claimed, narrow the manifest to its affected
        // paths (recorded on the task) so the implementer can't write outside its unit of work.
        // Falls back to the stage's static manifest when nothing is claimed.
-        val effectiveManifest = taskClaimCoordinator?.activeScope(sessionId)?.takeIf { it.isNotEmpty() }
-            ?: stageConfig.writeManifest
+        val escalatedScope = escalatedWriteScopePaths(sessionId)
+        val effectiveManifest = (
+            taskClaimCoordinator?.activeScope(sessionId)?.takeIf { it.isNotEmpty() }
+                ?: stageConfig.writeManifest
+            ) + escalatedScope
        val plane2Risk: RiskSummary? = runPlane2Assessment(
            sessionId, stageId, invocationId, toolCall.function.name, request, tool, effectives,
            effectiveManifest,
        )?.let { assessment ->
            if (assessment.recommendedAction == RiskAction.BLOCK) {
                val rationale = assessment.rationale.joinToString("; ")
+                // #301: a write repeatedly rejected for being outside the claimed task's scope or
+                // the stage's manifest — never for anything else — escalates to user approval
+                // after N same-path rejections instead of hard-blocking forever. Small models
+                // routinely fail to comply with the "add its path via task_update" remediation and
+                // thrash the same call rather than widen scope themselves.
+                val escalationPath = (parameters["path"] as? String)
+                    ?.takeIf { isScopeBlockRationale(rationale) }
+                val escalateAfterN = tuning.escalateScopeAfterN
+                if (escalationPath != null && escalateAfterN > 0) {
+                    val priorRejections = priorScopeRejections(sessionId, escalationPath)
+                    if (priorRejections.size >= escalateAfterN) {
+                        val firstAttempt = priorRejections.first().first
+                        return@flatMap escalateWriteScopeBlock(
+                            sessionId, stageId, invocationId, toolCall, tier, escalationPath,
+                            firstAttempt, effectives, toolCallReasoning, approvalMode, assessment,
+                            fileWrittenSlots,
+                        )
+                    }
+                }
                // On a bad-path block, point the model at the closest real file so it fixes the
                // path instead of retrying the same wrong guess (deep package paths are easy to
                // misremember). Only fires when we can name a concrete match from the repo map.
@@ -516,6 +539,180 @@ internal suspend fun SessionOrchestrator.dispatchToolCalls(
    }
 }

+private fun parametersToArgumentsJson(parameters: Map<String, Any>): String =
+    kotlinx.serialization.json.buildJsonObject {
+        parameters.forEach { (k, v) ->
+            when (v) {
+                is List<*> -> put(
+                    k,
+                    kotlinx.serialization.json.buildJsonArray {
+                        v.forEach { add(kotlinx.serialization.json.JsonPrimitive(it.toString())) }
+                    },
+                )
+                else -> put(k, kotlinx.serialization.json.JsonPrimitive(v.toString()))
+            }
+        }
+    }.toString()
+
+/**
+ * #301: the Nth same-path WRITE_SCOPE/PATH_OUTSIDE_MANIFEST rejection routes into the existing
+ * approval/pause flow instead of hard-blocking again. [firstAttempt] is the FIRST invocation that
+ * was rejected for this path/reason (reached back into via [priorScopeRejections]) — its pristine
+ * arguments are what gets previewed and, on approval, executed; later attempts this session may
+ * have degraded (e.g. the model giving up and calling `task_update(action="block")` instead), so
+ * replaying those would not fulfil the original intent.
+ */
+internal suspend fun SessionOrchestrator.escalateWriteScopeBlock(
+    sessionId: SessionId,
+    stageId: StageId,
+    invocationId: ToolInvocationId,
+    toolCall: ToolCallRequest,
+    tier: Tier,
+    path: String,
+    firstAttempt: ToolInvocationRequestedEvent,
+    effectives: RunEffectives,
+    toolCallReasoning: String?,
+    approvalMode: ApprovalMode,
+    plane2Risk: RiskSummary?,
+    fileWrittenSlots: List<TypedArtifactSlot>,
+): List<ContextEntry> {
+    val sourceId = toolCall.id ?: invocationId.value
+    val assistantEntry = ContextEntry(
+        id = ContextEntryId(UUID.randomUUID().toString()),
+        layer = ContextLayer.L2,
+        sourceType = "assistantToolCall",
+        sourceId = sourceId,
+        content = Json.encodeToString(ToolCallRequest.serializer(), toolCall),
+        tokenEstimate = estimateTokens(toolCall.function.arguments),
+        role = EntryRole.ASSISTANT,
+        reasoning = toolCallReasoning,
+    )
+    suspend fun rejected(reason: String): List<ContextEntry> {
+        blockTaskOnScopeRejection(sessionId, toolCall.function.name, reason)
+        emit(
+            sessionId,
+            ToolExecutionRejectedEvent(
+                invocationId = invocationId,
+                sessionId = sessionId,
+                toolName = toolCall.function.name,
+                tier = tier,
+                reason = reason,
+            ),
+        )
+        return listOf(
+            assistantEntry,
+            ContextEntry(
+                id = ContextEntryId(UUID.randomUUID().toString()),
+                layer = ContextLayer.L2,
+                sourceType = "toolResult",
+                sourceId = sourceId,
+                content = "BLOCKED: $reason",
+                tokenEstimate = estimateTokens(reason),
+                role = EntryRole.TOOL,
+            ),
+        )
+    }
+
+    val projectId = effectives.policy?.workspaceRoot?.let { ProjectIdentity.of(it.toString()) }
+    val approvalCtx = ApprovalContext(
+        identity = ApprovalScopeIdentity(sessionId, stageId, projectId = projectId),
+        mode = approvalMode,
+    )
+    val requestId = ApprovalRequestId(UUID.randomUUID().toString())
+    val toolPreview = computeToolPreview(
+        firstAttempt.toolName, firstAttempt.request.parameters, effectives.policy?.workspaceRoot,
+    )
+    val previewArguments = parametersToArgumentsJson(firstAttempt.request.parameters)
+    val domainRequest = DomainApprovalRequest(
+        id = requestId,
+        tier = firstAttempt.tier,
+        validationReportId = ValidationReportId(UUID.randomUUID().toString()),
+        riskSummaryId = null,
+        timestamp = Clock.System.now(),
+        toolName = firstAttempt.toolName,
+        preview = toolPreview ?: previewArguments.take(200),
+    )
+    val sessionGrants = approvalRepository.getApprovalState(sessionId).grants.values
+    val ledgerGrants = approvalRepository.getApprovalState(GRANT_LEDGER_SESSION_ID).grants.values
+    val activeGrants = (sessionGrants + ledgerGrants).toList()
+    val engineDecision = approvalEngine.evaluate(domainRequest, approvalCtx, activeGrants, Clock.System.now())
+    val approved: Boolean
+    val denyReason: String?
+    if (engineDecision.state == ApprovalStatus.COMPLETED) {
+        // Headless/no-approver sessions resolve here (e.g. approvalMode DENY auto-completes to a
+        // denial) — the escalation never hangs waiting on a human who isn't connected.
+        emitDecisionResolved(sessionId, domainRequest, engineDecision)
+        approved = engineDecision.isApproved
+        denyReason = engineDecision.reason
+    } else {
+        val deferred = CompletableDeferred<ApprovalDecision>()
+        pendingApprovals[requestId] = deferred
+        emit(sessionId, OrchestrationPausedEvent(sessionId, stageId, "APPROVAL_PENDING"))
+        emit(
+            sessionId,
+            ApprovalRequestedEvent(
+                requestId = requestId,
+                tier = firstAttempt.tier,
+                validationReportId = domainRequest.validationReportId,
+                riskSummaryId = null,
+                riskSummary = plane2Risk,
+                sessionId = sessionId,
+                stageId = stageId,
+                projectId = null,
+                toolName = firstAttempt.toolName,
+                preview = toolPreview ?: previewArguments.take(200),
+            ),
+        )
+        val userDecision = try {
+            deferred.await()
+        } finally {
+            pendingApprovals.remove(requestId)
+        }
+        emitDecisionResolved(sessionId, domainRequest, userDecision)
+        emit(sessionId, OrchestrationResumedEvent(sessionId, stageId))
+        approved = userDecision.isApproved
+        denyReason = userDecision.reason
+    }
+    if (!approved) {
+        return rejected(denyReason ?: "scope-widening request denied")
+    }
+
+    // Approved: widen the write scope for this path for the rest of the session (future writes to
+    // it skip straight past the risk plane, mirroring OutsidePathAccessGrantedEvent), then execute
+    // the FIRST rejected attempt's pristine write rather than the model's current — possibly
+    // degraded — call.
+    emit(sessionId, WriteScopeGrantedEvent(sessionId, stageId, path))
+    val executor = effectives.executor ?: return rejected("no executor available to apply the approved write")
+    val tool = effectives.registry?.resolve(firstAttempt.toolName)
+    val result = executor.execute(firstAttempt.request)
+    val rendered = renderToolResult(firstAttempt.toolName, tool, result)
+    recordToolExecution(
+        sessionId,
+        stageId,
+        toolCall.copy(function = toolCall.function.copy(name = firstAttempt.toolName)),
+        invocationId,
+        tier,
+        result,
+        tool as? FileAffectingTool,
+        firstAttempt.request,
+        fileWrittenSlots,
+        rendered.fullOutputHash,
+    )
+    return listOf(
+        assistantEntry,
+        ContextEntry(
+            id = ContextEntryId(UUID.randomUUID().toString()),
+            layer = ContextLayer.L2,
+            sourceType = "toolResult",
+            sourceId = sourceId,
+            content = "APPROVED: the user widened write scope to admit '$path' after repeated rejection; " +
+                "the original write is applied. ${rendered.content}",
+            tokenEstimate = estimateTokens(rendered.content),
+            role = EntryRole.TOOL,
+        ),
+    )
+}
+
 /**
 * A rejected [SCOPE_PROPOSAL_TOOL] call means the operator denied widening the claimed task's
 * scope — block the task so the loop advances instead of the implementer retrying the same
@@ -4,7 +4,9 @@ import com.correx.core.events.events.ToolCallAssessedEvent
 import com.correx.core.events.events.RepoMapComputedEvent
 import com.correx.core.events.events.OutsidePathAccessGrantedEvent
 import com.correx.core.events.events.ToolExecutionCompletedEvent
+import com.correx.core.events.events.ToolExecutionRejectedEvent
 import com.correx.core.events.events.ToolInvocationRequestedEvent
+import com.correx.core.events.events.WriteScopeGrantedEvent
 import com.correx.core.events.risk.RiskAction
 import com.correx.core.toolintent.SessionContextProjection
 import com.correx.core.events.events.TransitionExecutedEvent
@@ -68,6 +70,55 @@ internal fun SessionOrchestrator.grantedOutsidePaths(sessionId: SessionId): Set<
        .map { it.path }
        .toSet()

+/**
+ * Write-scope/manifest paths the operator has approved widening into this session (folded from
+ * [WriteScopeGrantedEvent] — see #301). Replay-safe: reads the log, never re-prompts a path once
+ * granted.
+ */
+
+internal fun SessionOrchestrator.escalatedWriteScopePaths(sessionId: SessionId): Set<String> =
+    eventStore.read(sessionId)
+        .mapNotNull { it.payload as? WriteScopeGrantedEvent }
+        .filter { it.sessionId == sessionId }
+        .map { it.path }
+        .toSet()
+
+/** The rule codes whose BLOCK rationale (`"[CODE] message"`, see [toRiskSummary]) makes a
+ * rejection eligible for the #301 same-path escalation — a write rejected purely for being
+ * outside the claimed task's scope or the stage's manifest, not for any other reason (path
+ * traversal, privileged location, etc. are never escalated). */
+private val SCOPE_BLOCK_CODES = setOf("WRITE_SCOPE", "PATH_OUTSIDE_MANIFEST")
+
+internal fun SessionOrchestrator.isScopeBlockRationale(rationale: String): Boolean =
+    SCOPE_BLOCK_CODES.any { rationale.contains("[$it]") }
+
+/**
+ * Every (first-attempt-invocation, rejection) pair this session where [path] was rejected for a
+ * WRITE_SCOPE/PATH_OUTSIDE_MANIFEST reason, oldest first. Reached back into purely by folding the
+ * event log — no branching/replay-engine change needed (#301). The invocation carries the
+ * pristine [ToolRequest] parameters from that attempt, which may differ from the model's current
+ * (possibly degraded) call.
+ */
+
+internal fun SessionOrchestrator.priorScopeRejections(
+    sessionId: SessionId,
+    path: String,
+): List<Pair<ToolInvocationRequestedEvent, ToolExecutionRejectedEvent>> {
+    val events = eventStore.read(sessionId)
+    val invocationsById = events
+        .mapNotNull { it.payload as? ToolInvocationRequestedEvent }
+        .filter { it.sessionId == sessionId }
+        .associateBy { it.invocationId }
+    return events
+        .mapNotNull { it.payload as? ToolExecutionRejectedEvent }
+        .filter { it.sessionId == sessionId && isScopeBlockRationale(it.reason) }
+        .mapNotNull { rejected ->
+            val invocation = invocationsById[rejected.invocationId] ?: return@mapNotNull null
+            val invocationPath = invocation.request.parameters["path"] as? String
+            if (invocationPath == path) invocation to rejected else null
+        }
+}
+
 /**
 * Returns true when the agent must be offered only read-only tools on the next inference turn.
 * Active from the moment a READ_BEFORE_WRITE block lands until a FILE_READ completion follows it.