fix: unblock sessions stuck in PAUSED state after approval resolved
Sessions that had an approval resolved (or never needed one) but never received OrchestrationResumedEvent would show the last tool as RUNNING forever with no subsequent events. Three coordinated fixes: 1. SessionOrchestrator now emits OrchestrationResumedEvent in both the approved and rejected approval branches, so the paused flag clears correctly going forward. 2. SessionEventBridge.replaySnapshot() detects the stuck pattern (pendingApproval=true with no unresolved requests) and appends a synthetic OrchestrationResumedEvent so historical sessions self-heal on next TUI connect. 3. DomainEventMapper maps OrchestrationResumedEvent → SessionResumed; TUI SessionsReducer/SnapshotPhaseReducer handle the new message, clearing pendingApproval and setting status back to ACTIVE. Also adds server-restart recovery path in DefaultSessionOrchestrator (emit decision + resume events directly when no live deferred exists) and pre-registers pending approvals in ServerModule.start() to close the race between reconnect and ApprovalResponse routing.
This commit is contained in:
@@ -3,6 +3,8 @@ package com.correx.apps.server
|
||||
import com.correx.apps.server.approval.ApprovalCoordinator
|
||||
import com.correx.apps.server.registry.ProviderRegistry
|
||||
import com.correx.apps.server.registry.WorkflowRegistry
|
||||
import com.correx.core.approvals.ApprovalProjector
|
||||
import com.correx.core.approvals.DefaultApprovalReducer
|
||||
import com.correx.core.approvals.DefaultApprovalRepository
|
||||
import com.correx.core.artifactstore.ArtifactStore
|
||||
import com.correx.core.config.ApprovalConfig
|
||||
@@ -52,12 +54,33 @@ class ServerModule(
|
||||
/**
|
||||
* Subscribe to the event store and forward [ApprovalRequestedEvent] payloads to
|
||||
* the [ApprovalCoordinator]. Idempotent: calling twice is a no-op for the second call.
|
||||
*
|
||||
* Also pre-populates the coordinator's session lookup map from any approvals that were
|
||||
* pending when the server last stopped. This prevents a race where a reconnecting TUI
|
||||
* sends an ApprovalResponse before the per-connection replaySnapshot() has registered
|
||||
* the session mapping.
|
||||
*/
|
||||
fun start() {
|
||||
if (subscriptionJob != null) return
|
||||
preRegisterPendingApprovals()
|
||||
subscriptionJob = eventStore.subscribeAll()
|
||||
.filter { it.payload is ApprovalRequestedEvent }
|
||||
.onEach { approvalCoordinator.onApprovalRequested(it.payload as ApprovalRequestedEvent) }
|
||||
.launchIn(moduleScope)
|
||||
}
|
||||
|
||||
private fun preRegisterPendingApprovals() {
|
||||
val projector = ApprovalProjector(DefaultApprovalReducer())
|
||||
eventStore.allSessionIds().forEach { sessionId ->
|
||||
val orchState = orchestrationRepository.getState(sessionId)
|
||||
if (!orchState.pendingApproval) return@forEach
|
||||
val events = eventStore.read(sessionId)
|
||||
val approvalState = events.fold(projector.initial()) { state, event ->
|
||||
projector.apply(state, event)
|
||||
}
|
||||
approvalState.requests.values
|
||||
.filter { req -> approvalState.decisions.values.none { it.requestId == req.id } }
|
||||
.forEach { req -> approvalCoordinator.registerPendingRequest(req.id, sessionId) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,7 +85,7 @@ open class ApprovalCoordinator(
|
||||
requestSessions[requestId] = sessionId
|
||||
}
|
||||
|
||||
open fun handleResponse(msg: ClientMessage.ApprovalResponse, sessionId: SessionId): ServerMessage? {
|
||||
open suspend fun handleResponse(msg: ClientMessage.ApprovalResponse, sessionId: SessionId): ServerMessage? {
|
||||
if (resolved.putIfAbsent(msg.requestId, true) != null) {
|
||||
return ServerMessage.ProtocolError(
|
||||
message = "Approval request ${msg.requestId.value} already resolved",
|
||||
|
||||
@@ -10,6 +10,7 @@ import com.correx.core.events.events.WorkflowStartedEvent
|
||||
import com.correx.core.events.events.InferenceStartedEvent
|
||||
import com.correx.core.events.events.InferenceTimeoutEvent
|
||||
import com.correx.core.events.events.OrchestrationPausedEvent
|
||||
import com.correx.core.events.events.OrchestrationResumedEvent
|
||||
import com.correx.core.events.events.StageCompletedEvent
|
||||
import com.correx.core.events.events.StageFailedEvent
|
||||
import com.correx.core.events.events.StoredEvent
|
||||
@@ -90,6 +91,12 @@ suspend fun domainEventToServerMessage(
|
||||
)
|
||||
|
||||
is OrchestrationPausedEvent -> mapOrchestrationPaused(p, seq, sessionSequence)
|
||||
is OrchestrationResumedEvent -> ServerMessage.SessionResumed(
|
||||
sessionId = p.sessionId,
|
||||
stageId = p.stageId,
|
||||
sequence = seq,
|
||||
sessionSequence = sessionSequence,
|
||||
)
|
||||
is InferenceStartedEvent -> ServerMessage.InferenceStarted(
|
||||
sessionId = p.sessionId,
|
||||
stageId = p.stageId,
|
||||
|
||||
@@ -12,10 +12,13 @@ import com.correx.core.approvals.ApprovalProjector
|
||||
import com.correx.core.approvals.DefaultApprovalReducer
|
||||
import com.correx.core.approvals.model.ApprovalState
|
||||
import com.correx.core.artifactstore.ArtifactStore
|
||||
import com.correx.core.events.events.EventMetadata
|
||||
import com.correx.core.events.events.InferenceCompletedEvent
|
||||
import com.correx.core.events.events.InferenceStartedEvent
|
||||
import com.correx.core.events.events.InferenceTimeoutEvent
|
||||
import com.correx.core.events.events.NewEvent
|
||||
import com.correx.core.events.events.OrchestrationPausedEvent
|
||||
import com.correx.core.events.events.OrchestrationResumedEvent
|
||||
import com.correx.core.events.events.StageCompletedEvent
|
||||
import com.correx.core.events.events.StageFailedEvent
|
||||
import com.correx.core.events.events.StoredEvent
|
||||
@@ -27,6 +30,8 @@ import com.correx.core.events.events.TransitionExecutedEvent
|
||||
import com.correx.core.events.events.WorkflowCompletedEvent
|
||||
import com.correx.core.events.events.WorkflowFailedEvent
|
||||
import com.correx.core.events.orchestration.OrchestrationStatus
|
||||
import com.correx.core.events.types.EventId
|
||||
import kotlinx.datetime.Clock
|
||||
import com.correx.core.events.stores.EventStore
|
||||
import com.correx.core.events.types.ApprovalRequestId
|
||||
import com.correx.core.events.types.SessionId
|
||||
@@ -62,18 +67,46 @@ class SessionEventBridge(
|
||||
}
|
||||
} else null
|
||||
|
||||
val pendingApprovals = approvalState?.requests?.values
|
||||
?.filter { req -> approvalState.decisions.values.none { it.requestId == req.id } }
|
||||
?.sortedWith(compareBy({ it.timestamp }, { it.id.value }))
|
||||
?.map {
|
||||
ApprovalDto(
|
||||
requestId = it.id.value,
|
||||
tier = it.tier.name,
|
||||
toolName = it.toolName,
|
||||
preview = it.preview,
|
||||
val pendingApprovalRequests = approvalState?.let { state ->
|
||||
state.requests.values
|
||||
.filter { req -> state.decisions.values.none { it.requestId == req.id } }
|
||||
.sortedWith(compareBy({ it.timestamp }, { it.id.value }))
|
||||
} ?: emptyList()
|
||||
|
||||
// Fix stuck sessions: orchestration says PAUSED + pendingApproval but there are no
|
||||
// actual unresolved approval requests. This happens when approval was resolved but
|
||||
// OrchestrationResumedEvent was not emitted (pre-Feb-13 orchestrator bug). Append the
|
||||
// missing event permanently so the session state corrects on all future replays.
|
||||
if (orchState.pendingApproval && pendingApprovalRequests.isEmpty()) {
|
||||
val alreadyResumed = events.any { it.payload is OrchestrationResumedEvent }
|
||||
val stageId = orchState.currentStageId
|
||||
if (!alreadyResumed && stageId != null) {
|
||||
val resumeEvent = NewEvent(
|
||||
metadata = EventMetadata(
|
||||
eventId = EventId(java.util.UUID.randomUUID().toString()),
|
||||
sessionId = sessionId,
|
||||
timestamp = Clock.System.now(),
|
||||
schemaVersion = 1,
|
||||
causationId = null,
|
||||
correlationId = null,
|
||||
),
|
||||
payload = OrchestrationResumedEvent(
|
||||
sessionId = sessionId,
|
||||
stageId = stageId,
|
||||
),
|
||||
)
|
||||
eventStore.append(resumeEvent)
|
||||
}
|
||||
?: emptyList()
|
||||
}
|
||||
|
||||
val pendingApprovals = pendingApprovalRequests.map {
|
||||
ApprovalDto(
|
||||
requestId = it.id.value,
|
||||
tier = it.tier.name,
|
||||
toolName = it.toolName,
|
||||
preview = it.preview,
|
||||
)
|
||||
}
|
||||
|
||||
val recentEvents = events
|
||||
.mapNotNull { eventToEntry(it) }
|
||||
|
||||
@@ -53,6 +53,15 @@ sealed interface ServerMessage {
|
||||
override val sessionSequence: Long,
|
||||
) : ServerMessage, SessionMessage
|
||||
|
||||
@Serializable
|
||||
@SerialName("session.resumed")
|
||||
data class SessionResumed(
|
||||
val sessionId: SessionId,
|
||||
val stageId: StageId? = null,
|
||||
override val sequence: Long,
|
||||
override val sessionSequence: Long,
|
||||
) : ServerMessage, SessionMessage
|
||||
|
||||
@Serializable
|
||||
@SerialName("session.completed")
|
||||
data class SessionCompleted(
|
||||
|
||||
+4
-4
@@ -57,7 +57,7 @@ class ApprovalCoordinatorWiringTest {
|
||||
|
||||
private class RecordingGateway : ApprovalGateway {
|
||||
val submissions = CopyOnWriteArrayList<Pair<ApprovalRequestId, DomainApprovalDecision>>()
|
||||
override fun submitApprovalDecision(requestId: ApprovalRequestId, decision: DomainApprovalDecision) {
|
||||
override suspend fun submitApprovalDecision(requestId: ApprovalRequestId, decision: DomainApprovalDecision) {
|
||||
submissions.add(requestId to decision)
|
||||
}
|
||||
}
|
||||
@@ -90,7 +90,7 @@ class ApprovalCoordinatorWiringTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `handleResponse forwards to gateway and returns null on success`() {
|
||||
fun `handleResponse forwards to gateway and returns null on success`(): Unit = runBlocking {
|
||||
val gateway = RecordingGateway()
|
||||
val coord = ApprovalCoordinator(gateway, ApprovalConfig(timeoutMs = 300_000L), scope)
|
||||
val msg = ClientMessage.ApprovalResponse(
|
||||
@@ -107,7 +107,7 @@ class ApprovalCoordinatorWiringTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `duplicate ApprovalResponse for same requestId returns ProtocolError`() {
|
||||
fun `duplicate ApprovalResponse for same requestId returns ProtocolError`(): Unit = runBlocking {
|
||||
val gateway = RecordingGateway()
|
||||
val coord = ApprovalCoordinator(gateway, ApprovalConfig(timeoutMs = 300_000L), scope)
|
||||
val msg = ClientMessage.ApprovalResponse(
|
||||
@@ -177,7 +177,7 @@ class ApprovalCoordinatorWiringTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `lookupSession returns mapping registered by onApprovalRequested and cleared by handleResponse`() = runBlocking {
|
||||
fun `lookupSession returns mapping registered by onApprovalRequested and cleared by handleResponse`(): Unit = runBlocking {
|
||||
val gateway = RecordingGateway()
|
||||
val coord = ApprovalCoordinator(gateway, ApprovalConfig(timeoutMs = 300_000L), scope)
|
||||
val otherSessionId = SessionId("other-session")
|
||||
|
||||
@@ -165,6 +165,7 @@ object SessionsReducer {
|
||||
val (result, effects) = when (msg) {
|
||||
is ServerMessage.SessionStarted -> processSessionStartedMessage(msg, clock, sessions)
|
||||
is ServerMessage.SessionPaused -> processSessionPausedMessage(msg, sessions, clock)
|
||||
is ServerMessage.SessionResumed -> processSessionResumedMessage(msg, sessions, clock)
|
||||
is ServerMessage.SessionCompleted -> processSessionCompletedMessage(sessions, msg, clock)
|
||||
is ServerMessage.SessionFailed -> processSessionFailedMessage(sessions, msg, clock)
|
||||
is ServerMessage.StageStarted -> processStageStartedMessage(msg, sessions)
|
||||
@@ -189,6 +190,7 @@ object SessionsReducer {
|
||||
private fun sessionIdFromMessage(msg: ServerMessage): String? = when (msg) {
|
||||
is ServerMessage.SessionStarted -> msg.sessionId.value
|
||||
is ServerMessage.SessionPaused -> msg.sessionId.value
|
||||
is ServerMessage.SessionResumed -> msg.sessionId.value
|
||||
is ServerMessage.SessionCompleted -> msg.sessionId.value
|
||||
is ServerMessage.SessionFailed -> msg.sessionId.value
|
||||
is ServerMessage.StageStarted -> msg.sessionId.value
|
||||
@@ -507,6 +509,20 @@ object SessionsReducer {
|
||||
) to emptyList()
|
||||
}
|
||||
|
||||
private fun processSessionResumedMessage(
|
||||
msg: ServerMessage.SessionResumed,
|
||||
sessions: SessionsState,
|
||||
clock: () -> Long,
|
||||
): Pair<SessionsState, List<Effect>> = sessions.copy(
|
||||
sessions = sessions.sessions.map { s ->
|
||||
if (s.id == msg.sessionId.value) s.copy(
|
||||
status = "ACTIVE",
|
||||
pendingApproval = null,
|
||||
lastEventAt = clock(),
|
||||
) else s
|
||||
},
|
||||
) to emptyList()
|
||||
|
||||
private fun processSessionStartedMessage(
|
||||
msg: ServerMessage.SessionStarted,
|
||||
clock: () -> Long,
|
||||
|
||||
@@ -64,6 +64,7 @@ object SnapshotPhaseReducer {
|
||||
internal fun sessionIdOf(msg: ServerMessage): String? = when (msg) {
|
||||
is ServerMessage.SessionStarted -> msg.sessionId.value
|
||||
is ServerMessage.SessionPaused -> msg.sessionId.value
|
||||
is ServerMessage.SessionResumed -> msg.sessionId.value
|
||||
is ServerMessage.SessionCompleted -> msg.sessionId.value
|
||||
is ServerMessage.SessionFailed -> msg.sessionId.value
|
||||
is ServerMessage.StageStarted -> msg.sessionId.value
|
||||
|
||||
Reference in New Issue
Block a user