feat(validation): capability-gap reflection rung (#30 part 2)
Bounded LLM "are you sure?" pass over the capability gaps part 1 detects, run once at plan-lock before the operator is asked to approve — honest mistakes self-correct without escalation; only a genuine tool need reaches the human. - CapabilityGapReflector: fun-interface seam in core:kernel (plain-data in/out, no infrastructure:workflow dep), mirroring SemanticReviewer/SalvageJudge. - CapabilityGapReflectorImpl (apps/server): one InferenceRouter call for the whole gap batch, no per-gap loop, no retry; runCatching degrades to a safe RESOLVED-advisory default on any failure (unroutable/timeout/malformed JSON) — a broken pass can never manufacture a NEEDS_TOOL escalation or a grant. - CapabilityGapReflectedEvent(verdict RESOLVED|NEEDS_TOOL) recorded per gap and registered in eventModule; replay reads the event, never re-invokes inference (invariants #7/#8/#9). - FreestyleDriver.lockAndRun: RESOLVED is advisory (recorded, plan untouched — invariant #3); NEEDS_TOOL is appended to the existing requestPlanApproval preview so the operator decides — no auto-grant anywhere (#4/#5). - reflector nullable -> degrades to part-1 behavior when unwired. Verified: ./gradlew :core:events:test :core:kernel:test :infrastructure:workflow:test :apps:server:test green.
This commit is contained in:
@@ -518,6 +518,7 @@ fun main() {
|
|||||||
runPhase2 = { sid, graph, cfg -> orchestrator.run(sid, graph, cfg) },
|
runPhase2 = { sid, graph, cfg -> orchestrator.run(sid, graph, cfg) },
|
||||||
requestPlanApproval = { sid, planJson -> orchestrator.requestPlanApproval(sid, planJson) },
|
requestPlanApproval = { sid, planJson -> orchestrator.requestPlanApproval(sid, planJson) },
|
||||||
toolCapabilities = toolRegistry.all().associate { it.name to it.requiredCapabilities },
|
toolCapabilities = toolRegistry.all().associate { it.name to it.requiredCapabilities },
|
||||||
|
reflector = com.correx.apps.server.inference.CapabilityGapReflectorImpl(inferenceRouter),
|
||||||
)
|
)
|
||||||
// observability-spec §4: continuous health watch. Seed the monitor's last-status from the
|
// observability-spec §4: continuous health watch. Seed the monitor's last-status from the
|
||||||
// recorded system-session events so a restart doesn't re-emit a degraded already in the log.
|
// recorded system-session events so a restart doesn't re-emit a degraded already in the log.
|
||||||
|
|||||||
@@ -1,12 +1,15 @@
|
|||||||
package com.correx.apps.server.freestyle
|
package com.correx.apps.server.freestyle
|
||||||
|
|
||||||
import com.correx.core.events.events.CapabilityGapDetectedEvent
|
import com.correx.core.events.events.CapabilityGapDetectedEvent
|
||||||
|
import com.correx.core.events.events.CapabilityGapReflectedEvent
|
||||||
|
import com.correx.core.events.events.CapabilityGapVerdict
|
||||||
import com.correx.core.events.events.EventMetadata
|
import com.correx.core.events.events.EventMetadata
|
||||||
import com.correx.core.events.events.ExecutionPlanLockedEvent
|
import com.correx.core.events.events.ExecutionPlanLockedEvent
|
||||||
import com.correx.core.events.events.ExecutionPlanRejectedEvent
|
import com.correx.core.events.events.ExecutionPlanRejectedEvent
|
||||||
import com.correx.core.events.events.NewEvent
|
import com.correx.core.events.events.NewEvent
|
||||||
import com.correx.core.events.events.PlanLintCompletedEvent
|
import com.correx.core.events.events.PlanLintCompletedEvent
|
||||||
import com.correx.core.tools.contract.ToolCapability
|
import com.correx.core.tools.contract.ToolCapability
|
||||||
|
import com.correx.infrastructure.workflow.CapabilityGap
|
||||||
import com.correx.infrastructure.workflow.CapabilityGapDetector
|
import com.correx.infrastructure.workflow.CapabilityGapDetector
|
||||||
import com.correx.infrastructure.workflow.PlanLintResult
|
import com.correx.infrastructure.workflow.PlanLintResult
|
||||||
import com.correx.infrastructure.workflow.PlanLinter
|
import com.correx.infrastructure.workflow.PlanLinter
|
||||||
@@ -16,6 +19,8 @@ import com.correx.core.events.types.EventId
|
|||||||
import com.correx.core.events.types.SessionId
|
import com.correx.core.events.types.SessionId
|
||||||
import com.correx.core.events.types.StageId
|
import com.correx.core.events.types.StageId
|
||||||
import com.correx.core.kernel.execution.WorkflowResult
|
import com.correx.core.kernel.execution.WorkflowResult
|
||||||
|
import com.correx.core.kernel.orchestration.CapabilityGapInput
|
||||||
|
import com.correx.core.kernel.orchestration.CapabilityGapReflector
|
||||||
import com.correx.core.kernel.orchestration.OrchestrationConfig
|
import com.correx.core.kernel.orchestration.OrchestrationConfig
|
||||||
import com.correx.core.transitions.graph.WorkflowGraph
|
import com.correx.core.transitions.graph.WorkflowGraph
|
||||||
import com.correx.infrastructure.workflow.ExecutionPlanCompiler
|
import com.correx.infrastructure.workflow.ExecutionPlanCompiler
|
||||||
@@ -42,6 +47,9 @@ class FreestyleDriver(
|
|||||||
// capability-gap detector below. Empty = detector finds nothing (preserves callers/tests that
|
// capability-gap detector below. Empty = detector finds nothing (preserves callers/tests that
|
||||||
// don't care about it) rather than depending on a live ToolRegistry here.
|
// don't care about it) rather than depending on a live ToolRegistry here.
|
||||||
private val toolCapabilities: Map<String, Set<ToolCapability>> = emptyMap(),
|
private val toolCapabilities: Map<String, Set<ToolCapability>> = emptyMap(),
|
||||||
|
// Vikunja #30 part 2: bounded LLM "are you sure?" pass over capability gaps, consulted before
|
||||||
|
// requestPlanApproval. Null = feature degrades to part-1 behavior (gaps recorded, never reflected).
|
||||||
|
private val reflector: CapabilityGapReflector? = null,
|
||||||
) {
|
) {
|
||||||
suspend fun lockAndRun(sessionId: SessionId) {
|
suspend fun lockAndRun(sessionId: SessionId) {
|
||||||
val json = planContent(sessionId) ?: run {
|
val json = planContent(sessionId) ?: run {
|
||||||
@@ -69,16 +77,22 @@ class FreestyleDriver(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
// Capability-gap detector (Vikunja #30 part 1): advisory only — recorded, never blocking.
|
// Capability-gap detector (Vikunja #30 part 1): advisory only — recorded, never blocking.
|
||||||
// A gap does not fail the gate and does not grant the missing tool (invariants #3/#4/#5); a
|
// A gap does not fail the gate and does not grant the missing tool (invariants #3/#4/#5).
|
||||||
// future reflection rung (part 2) is what will act on these events.
|
val gaps = CapabilityGapDetector.detect(graph, toolCapabilities)
|
||||||
CapabilityGapDetector.detect(graph, toolCapabilities).forEach { gap ->
|
gaps.forEach { gap ->
|
||||||
log.info(
|
log.info(
|
||||||
"[FreestyleDriver] capability gap session={} stage={} implied={} evidence='{}'",
|
"[FreestyleDriver] capability gap session={} stage={} implied={} evidence='{}'",
|
||||||
sessionId.value, gap.stageId, gap.impliedCapability, gap.phrase,
|
sessionId.value, gap.stageId, gap.impliedCapability, gap.phrase,
|
||||||
)
|
)
|
||||||
emitCapabilityGap(sessionId, gap.stageId, gap.impliedCapability.name, gap.phrase)
|
emitCapabilityGap(sessionId, gap.stageId, gap.impliedCapability.name, gap.phrase)
|
||||||
}
|
}
|
||||||
val approved = requestPlanApproval(sessionId, json)
|
// Reflection rung (Vikunja #30 part 2): one bounded LLM pass over the whole batch of gaps
|
||||||
|
// before the human is bothered. RESOLVED is advisory-only (logged, never rewrites the locked
|
||||||
|
// plan — invariant #3 LLM-proposes/core-decides); NEEDS_TOOL is the only case surfaced into
|
||||||
|
// the plan-review approval preview below, since that is the existing human approval seam and
|
||||||
|
// a tool grant must never happen without it (invariants #4/#5).
|
||||||
|
val needsToolNote = reflectOnGaps(sessionId, gaps)
|
||||||
|
val approved = requestPlanApproval(sessionId, needsToolNote?.let { "$json\n\n$it" } ?: json)
|
||||||
if (!approved) {
|
if (!approved) {
|
||||||
log.info("freestyle: execution plan rejected by operator for session={}", sessionId.value)
|
log.info("freestyle: execution plan rejected by operator for session={}", sessionId.value)
|
||||||
emitRejected(sessionId, "rejected by operator at plan review", "operator")
|
emitRejected(sessionId, "rejected by operator at plan review", "operator")
|
||||||
@@ -167,6 +181,66 @@ class FreestyleDriver(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Runs the bounded reflection pass (if wired) over [gaps], emits one
|
||||||
|
* [CapabilityGapReflectedEvent] per gap, and returns an operator-facing note listing any
|
||||||
|
* NEEDS_TOOL verdicts — null when there are no gaps, no reflector, or nothing to escalate.
|
||||||
|
* A single call for the whole batch: no per-gap loop, no retry (bounded per invariant #9).
|
||||||
|
*/
|
||||||
|
private suspend fun reflectOnGaps(sessionId: SessionId, gaps: List<CapabilityGap>): String? {
|
||||||
|
val judge = reflector
|
||||||
|
if (gaps.isEmpty() || judge == null) return null
|
||||||
|
val inputs = gaps.map {
|
||||||
|
CapabilityGapInput(it.stageId, it.impliedCapability, it.phrase, it.grantedCapabilities)
|
||||||
|
}
|
||||||
|
val reflections = runCatching { judge.reflect(sessionId, inputs) }.getOrElse {
|
||||||
|
log.warn("[FreestyleDriver] capability gap reflection failed session={}: {}", sessionId.value, it.message)
|
||||||
|
emptyList()
|
||||||
|
}
|
||||||
|
reflections.forEach { r ->
|
||||||
|
log.info(
|
||||||
|
"[FreestyleDriver] capability gap reflected session={} stage={} implied={} verdict={}",
|
||||||
|
sessionId.value, r.stageId, r.impliedCapability, r.verdict,
|
||||||
|
)
|
||||||
|
emitCapabilityGapReflected(sessionId, r.stageId, r.impliedCapability.name, r.verdict, r.detail)
|
||||||
|
}
|
||||||
|
val escalations = reflections.filter { it.verdict == CapabilityGapVerdict.NEEDS_TOOL }
|
||||||
|
return escalations.takeIf { it.isNotEmpty() }?.let {
|
||||||
|
"CAPABILITY GAPS REQUESTED (reflection confirmed a genuine need — approve or reject the plan " +
|
||||||
|
"with this in mind; no tool is auto-granted):\n" +
|
||||||
|
it.joinToString("\n") { e -> "- stage ${e.stageId} needs ${e.impliedCapability}: ${e.detail}" }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private suspend fun emitCapabilityGapReflected(
|
||||||
|
sessionId: SessionId,
|
||||||
|
stageId: String,
|
||||||
|
impliedCapability: String,
|
||||||
|
verdict: CapabilityGapVerdict,
|
||||||
|
detail: String,
|
||||||
|
) {
|
||||||
|
eventStore.append(
|
||||||
|
NewEvent(
|
||||||
|
metadata = EventMetadata(
|
||||||
|
eventId = EventId(UUID.randomUUID().toString()),
|
||||||
|
sessionId = sessionId,
|
||||||
|
timestamp = Clock.System.now(),
|
||||||
|
schemaVersion = 1,
|
||||||
|
causationId = null,
|
||||||
|
correlationId = null,
|
||||||
|
),
|
||||||
|
payload = CapabilityGapReflectedEvent(
|
||||||
|
sessionId = sessionId,
|
||||||
|
stageId = StageId(stageId),
|
||||||
|
impliedCapability = impliedCapability,
|
||||||
|
verdict = verdict,
|
||||||
|
detail = detail,
|
||||||
|
timestampMs = Clock.System.now().toEpochMilliseconds(),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
private suspend fun emitRejected(sessionId: SessionId, reason: String, source: String) {
|
private suspend fun emitRejected(sessionId: SessionId, reason: String, source: String) {
|
||||||
eventStore.append(
|
eventStore.append(
|
||||||
NewEvent(
|
NewEvent(
|
||||||
|
|||||||
+173
@@ -0,0 +1,173 @@
|
|||||||
|
package com.correx.apps.server.inference
|
||||||
|
|
||||||
|
import com.correx.core.context.model.CompressionMetadata
|
||||||
|
import com.correx.core.context.model.ContextEntry
|
||||||
|
import com.correx.core.context.model.ContextLayer
|
||||||
|
import com.correx.core.context.model.ContextPack
|
||||||
|
import com.correx.core.context.model.EntryRole
|
||||||
|
import com.correx.core.events.events.CapabilityGapVerdict
|
||||||
|
import com.correx.core.events.types.ContextEntryId
|
||||||
|
import com.correx.core.events.types.ContextPackId
|
||||||
|
import com.correx.core.events.types.InferenceRequestId
|
||||||
|
import com.correx.core.events.types.SessionId
|
||||||
|
import com.correx.core.events.types.StageId
|
||||||
|
import com.correx.core.inference.GenerationConfig
|
||||||
|
import com.correx.core.inference.InferenceRequest
|
||||||
|
import com.correx.core.inference.InferenceRouter
|
||||||
|
import com.correx.core.inference.ModelCapability
|
||||||
|
import com.correx.core.inference.ResponseFormat
|
||||||
|
import com.correx.core.kernel.orchestration.CapabilityGapInput
|
||||||
|
import com.correx.core.kernel.orchestration.CapabilityGapReflection
|
||||||
|
import com.correx.core.kernel.orchestration.CapabilityGapReflector
|
||||||
|
import kotlinx.serialization.Serializable
|
||||||
|
import kotlinx.serialization.json.Json
|
||||||
|
import org.slf4j.LoggerFactory
|
||||||
|
import java.util.UUID
|
||||||
|
|
||||||
|
private const val REFLECTION_MAX_TOKENS = 4_096
|
||||||
|
private val REFLECTION_STAGE_ID = StageId("capability_gap_reflection")
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Vikunja #30 part 2: the "are you sure?" reflection judge. Given the plan-compile gate's
|
||||||
|
* capability-gap findings for a session, asks a single bounded question — revise onto an existing
|
||||||
|
* capability, or justify the need for a new one — and maps the answer to a [CapabilityGapReflection]
|
||||||
|
* per gap. Mirrors [SemanticReviewerImpl]: talks to the provider directly with a self-contained
|
||||||
|
* prompt, one inference call for the whole batch (no per-gap loop, no retry), and degrades to a
|
||||||
|
* safe RESOLVED/no-escalation default (see [defaultReflection]) on any failure so a broken
|
||||||
|
* reflection never wedges the plan-compile gate it sits behind.
|
||||||
|
*/
|
||||||
|
class CapabilityGapReflectorImpl(
|
||||||
|
private val inferenceRouter: InferenceRouter,
|
||||||
|
private val modelId: String? = null,
|
||||||
|
) : CapabilityGapReflector {
|
||||||
|
|
||||||
|
override suspend fun reflect(sessionId: SessionId, gaps: List<CapabilityGapInput>): List<CapabilityGapReflection> {
|
||||||
|
if (gaps.isEmpty()) return emptyList()
|
||||||
|
return runCatching {
|
||||||
|
val provider = inferenceRouter.route(REFLECTION_STAGE_ID, setOf(ModelCapability.General), modelId)
|
||||||
|
val prompt = buildPrompt(gaps)
|
||||||
|
val request = InferenceRequest(
|
||||||
|
requestId = InferenceRequestId(UUID.randomUUID().toString()),
|
||||||
|
sessionId = sessionId,
|
||||||
|
stageId = REFLECTION_STAGE_ID,
|
||||||
|
contextPack = pack(sessionId, prompt),
|
||||||
|
generationConfig = GenerationConfig(temperature = 0.2, topP = 0.9, maxTokens = REFLECTION_MAX_TOKENS),
|
||||||
|
responseFormat = ResponseFormat.Text,
|
||||||
|
)
|
||||||
|
parse(provider.infer(request).text, gaps)
|
||||||
|
}.getOrElse {
|
||||||
|
log.warn(
|
||||||
|
"[CapabilityGapReflector] reflection failed session={}: {}",
|
||||||
|
sessionId.value, it.message,
|
||||||
|
)
|
||||||
|
gaps.map(::defaultReflection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Malformed/missing verdict for a gap defaults to advisory RESOLVED-nothing — never NEEDS_TOOL
|
||||||
|
* by default, so a broken reflection pass cannot force an escalation on its own. */
|
||||||
|
private fun defaultReflection(gap: CapabilityGapInput) = CapabilityGapReflection(
|
||||||
|
stageId = gap.stageId,
|
||||||
|
impliedCapability = gap.impliedCapability,
|
||||||
|
verdict = CapabilityGapVerdict.RESOLVED,
|
||||||
|
detail = "reflection unavailable — treated as advisory, no escalation",
|
||||||
|
)
|
||||||
|
|
||||||
|
private fun buildPrompt(gaps: List<CapabilityGapInput>): String {
|
||||||
|
val findings = gaps.joinToString("\n\n") { gap ->
|
||||||
|
val granted = gap.grantedCapabilities.joinToString(", ").ifBlank { "(none)" }
|
||||||
|
"""
|
||||||
|
- stage: ${gap.stageId}
|
||||||
|
implied capability: ${gap.impliedCapability}
|
||||||
|
triggering phrase: "${gap.phrase}"
|
||||||
|
tools already granted to this stage give it: $granted
|
||||||
|
""".trimIndent()
|
||||||
|
}
|
||||||
|
return """
|
||||||
|
A deterministic scan of a compiled plan flagged stages whose brief implies a tool capability
|
||||||
|
they were not granted. Before this is escalated to a human operator, decide for EACH gap
|
||||||
|
whether it is a real gap or a false alarm.
|
||||||
|
|
||||||
|
IMPORTANT domain fact: the file_write tool auto-creates parent directories — there is no
|
||||||
|
separate mkdir/create-directory tool. A stage that says "create a directory" or "create the
|
||||||
|
file" and already has file_write does NOT need a new capability.
|
||||||
|
|
||||||
|
GAPS:
|
||||||
|
$findings
|
||||||
|
|
||||||
|
For EACH gap, respond with either:
|
||||||
|
- RESOLVED: the stage's existing tools are actually sufficient; state the revised intent.
|
||||||
|
- NEEDS_TOOL: the stage genuinely needs the missing capability; justify why no existing tool
|
||||||
|
covers it.
|
||||||
|
|
||||||
|
Respond with ONLY a JSON object, no prose, no markdown fence:
|
||||||
|
{"reflections":[
|
||||||
|
{"stage":"stage-id","verdict":"RESOLVED|NEEDS_TOOL","detail":"revised intent or justification"}
|
||||||
|
]}
|
||||||
|
""".trimIndent()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun pack(sessionId: SessionId, prompt: String) = ContextPack(
|
||||||
|
id = ContextPackId(UUID.randomUUID().toString()),
|
||||||
|
sessionId = sessionId,
|
||||||
|
stageId = REFLECTION_STAGE_ID,
|
||||||
|
layers = mapOf(
|
||||||
|
ContextLayer.L0 to listOf(
|
||||||
|
ContextEntry(
|
||||||
|
id = ContextEntryId(UUID.randomUUID().toString()),
|
||||||
|
layer = ContextLayer.L0,
|
||||||
|
content = prompt,
|
||||||
|
sourceType = "capability_gap_reflection",
|
||||||
|
sourceId = "reflector",
|
||||||
|
tokenEstimate = prompt.length / 4,
|
||||||
|
role = EntryRole.USER,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
budgetUsed = prompt.length / 4,
|
||||||
|
budgetLimit = prompt.length / 4,
|
||||||
|
compressionMetadata = CompressionMetadata(),
|
||||||
|
)
|
||||||
|
|
||||||
|
private fun parse(raw: String, gaps: List<CapabilityGapInput>): List<CapabilityGapReflection> {
|
||||||
|
val byStage = gaps.associateBy { it.stageId }
|
||||||
|
val dto = extractJson(raw)
|
||||||
|
?.let { json ->
|
||||||
|
runCatching { lenient.decodeFromString(ReflectionBatchDto.serializer(), json) }.getOrNull()
|
||||||
|
}
|
||||||
|
?: return gaps.map(::defaultReflection)
|
||||||
|
val byStageResult = dto.reflections.mapNotNull { r ->
|
||||||
|
val gap = byStage[r.stage] ?: return@mapNotNull null
|
||||||
|
val verdict = runCatching { CapabilityGapVerdict.valueOf(r.verdict.uppercase()) }
|
||||||
|
.getOrDefault(CapabilityGapVerdict.RESOLVED)
|
||||||
|
gap.stageId to CapabilityGapReflection(
|
||||||
|
stageId = gap.stageId,
|
||||||
|
impliedCapability = gap.impliedCapability,
|
||||||
|
verdict = verdict,
|
||||||
|
detail = r.detail.ifBlank { "no detail provided" },
|
||||||
|
)
|
||||||
|
}.toMap()
|
||||||
|
return gaps.map { gap -> byStageResult[gap.stageId] ?: defaultReflection(gap) }
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun extractJson(raw: String): String? {
|
||||||
|
val start = raw.indexOf('{')
|
||||||
|
val end = raw.lastIndexOf('}')
|
||||||
|
return if (start in 0 until end) raw.substring(start, end + 1) else null
|
||||||
|
}
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
private data class ReflectionBatchDto(val reflections: List<ReflectionDto> = emptyList())
|
||||||
|
|
||||||
|
@Serializable
|
||||||
|
private data class ReflectionDto(
|
||||||
|
val stage: String = "",
|
||||||
|
val verdict: String = "RESOLVED",
|
||||||
|
val detail: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
private val log = LoggerFactory.getLogger(CapabilityGapReflectorImpl::class.java)
|
||||||
|
private val lenient = Json { ignoreUnknownKeys = true; isLenient = true }
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,11 +3,16 @@ package com.correx.apps.server.freestyle
|
|||||||
import com.correx.core.artifacts.kind.ConfigArtifactKind
|
import com.correx.core.artifacts.kind.ConfigArtifactKind
|
||||||
import com.correx.core.artifacts.kind.DefaultArtifactKindRegistry
|
import com.correx.core.artifacts.kind.DefaultArtifactKindRegistry
|
||||||
import com.correx.core.artifacts.kind.JsonSchema
|
import com.correx.core.artifacts.kind.JsonSchema
|
||||||
|
import com.correx.core.events.events.CapabilityGapDetectedEvent
|
||||||
|
import com.correx.core.events.events.CapabilityGapReflectedEvent
|
||||||
|
import com.correx.core.events.events.CapabilityGapVerdict
|
||||||
import com.correx.core.events.events.ExecutionPlanLockedEvent
|
import com.correx.core.events.events.ExecutionPlanLockedEvent
|
||||||
import com.correx.core.events.events.ExecutionPlanRejectedEvent
|
import com.correx.core.events.events.ExecutionPlanRejectedEvent
|
||||||
import com.correx.core.events.events.PlanLintCompletedEvent
|
import com.correx.core.events.events.PlanLintCompletedEvent
|
||||||
import com.correx.core.events.types.SessionId
|
import com.correx.core.events.types.SessionId
|
||||||
import com.correx.core.kernel.execution.WorkflowResult
|
import com.correx.core.kernel.execution.WorkflowResult
|
||||||
|
import com.correx.core.kernel.orchestration.CapabilityGapReflection
|
||||||
|
import com.correx.core.kernel.orchestration.CapabilityGapReflector
|
||||||
import com.correx.core.kernel.orchestration.OrchestrationConfig
|
import com.correx.core.kernel.orchestration.OrchestrationConfig
|
||||||
import com.correx.core.transitions.graph.WorkflowGraph
|
import com.correx.core.transitions.graph.WorkflowGraph
|
||||||
import com.correx.infrastructure.persistence.InMemoryEventStore
|
import com.correx.infrastructure.persistence.InMemoryEventStore
|
||||||
@@ -316,4 +321,151 @@ class FreestyleDriverTest {
|
|||||||
assertEquals(1, runPhase2Invocations, "runPhase2 should be called exactly once after approval")
|
assertEquals(1, runPhase2Invocations, "runPhase2 should be called exactly once after approval")
|
||||||
assertTrue(approvalPreview != null, "requestPlanApproval should have been invoked")
|
assertTrue(approvalPreview != null, "requestPlanApproval should have been invoked")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A stage brief that trips the FILE_WRITE gap (verb "create") with zero granted tools, so
|
||||||
|
// CapabilityGapDetector.detect() always finds exactly one gap on stage "apply" for these tests.
|
||||||
|
private val gapPlanJson = """
|
||||||
|
{
|
||||||
|
"goal": "test plan with a capability gap",
|
||||||
|
"stages": [
|
||||||
|
{ "id": "analyse", "prompt": "Analyse the problem", "produces": "patch", "needs": [], "tools": [] },
|
||||||
|
{ "id": "apply", "prompt": "create the output file", "produces": "patch", "needs": ["patch"], "tools": [] }
|
||||||
|
],
|
||||||
|
"edges": [
|
||||||
|
{ "from": "analyse", "to": "apply", "condition": { "type": "always_true" } },
|
||||||
|
{ "from": "apply", "to": "done", "condition": { "type": "always_true" } }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
""".trimIndent()
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `a RESOLVED reflection is advisory only and does not touch the approval preview`(): Unit = runBlocking {
|
||||||
|
val sessionId = SessionId("driver-reflect-resolved-session")
|
||||||
|
val eventStore = InMemoryEventStore()
|
||||||
|
val compiler = ExecutionPlanCompiler(buildRegistry())
|
||||||
|
var approvalPreview: String? = null
|
||||||
|
|
||||||
|
val reflector = CapabilityGapReflector { _, gaps ->
|
||||||
|
gaps.map {
|
||||||
|
CapabilityGapReflection(
|
||||||
|
it.stageId, it.impliedCapability, CapabilityGapVerdict.RESOLVED, "revised: reuse file_write",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val driver = FreestyleDriver(
|
||||||
|
eventStore = eventStore,
|
||||||
|
compiler = compiler,
|
||||||
|
planContent = { gapPlanJson },
|
||||||
|
config = OrchestrationConfig(),
|
||||||
|
runPhase2 = { sid, graph, _ -> WorkflowResult.Completed(sid, graph.start) },
|
||||||
|
requestPlanApproval = { _, preview -> approvalPreview = preview; true },
|
||||||
|
reflector = reflector,
|
||||||
|
)
|
||||||
|
|
||||||
|
driver.lockAndRun(sessionId)
|
||||||
|
|
||||||
|
val payloads = eventStore.read(sessionId).map { it.payload }
|
||||||
|
assertEquals(1, payloads.filterIsInstance<CapabilityGapDetectedEvent>().size)
|
||||||
|
val reflected = payloads.filterIsInstance<CapabilityGapReflectedEvent>().single()
|
||||||
|
assertEquals(CapabilityGapVerdict.RESOLVED, reflected.verdict)
|
||||||
|
assertEquals(1, payloads.filterIsInstance<ExecutionPlanLockedEvent>().size, "RESOLVED must not block locking")
|
||||||
|
assertFalse(
|
||||||
|
approvalPreview.orEmpty().contains("CAPABILITY GAPS REQUESTED"),
|
||||||
|
"RESOLVED verdicts must not be surfaced as an escalation",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `a NEEDS_TOOL reflection is surfaced to the approval seam but never auto-grants`(): Unit = runBlocking {
|
||||||
|
val sessionId = SessionId("driver-reflect-needs-tool-session")
|
||||||
|
val eventStore = InMemoryEventStore()
|
||||||
|
val compiler = ExecutionPlanCompiler(buildRegistry())
|
||||||
|
var approvalPreview: String? = null
|
||||||
|
|
||||||
|
val reflector = CapabilityGapReflector { _, gaps ->
|
||||||
|
gaps.map {
|
||||||
|
CapabilityGapReflection(
|
||||||
|
it.stageId, it.impliedCapability, CapabilityGapVerdict.NEEDS_TOOL, "genuinely needs file_write",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val driver = FreestyleDriver(
|
||||||
|
eventStore = eventStore,
|
||||||
|
compiler = compiler,
|
||||||
|
planContent = { gapPlanJson },
|
||||||
|
config = OrchestrationConfig(),
|
||||||
|
runPhase2 = { sid, graph, _ -> WorkflowResult.Completed(sid, graph.start) },
|
||||||
|
requestPlanApproval = { _, preview -> approvalPreview = preview; true },
|
||||||
|
reflector = reflector,
|
||||||
|
)
|
||||||
|
|
||||||
|
driver.lockAndRun(sessionId)
|
||||||
|
|
||||||
|
val payloads = eventStore.read(sessionId).map { it.payload }
|
||||||
|
val reflected = payloads.filterIsInstance<CapabilityGapReflectedEvent>().single()
|
||||||
|
assertEquals(CapabilityGapVerdict.NEEDS_TOOL, reflected.verdict)
|
||||||
|
assertTrue(
|
||||||
|
approvalPreview.orEmpty().contains("CAPABILITY GAPS REQUESTED"),
|
||||||
|
"NEEDS_TOOL must be surfaced into the human approval preview",
|
||||||
|
)
|
||||||
|
assertTrue(approvalPreview.orEmpty().contains("apply"), "escalation note should name the stage")
|
||||||
|
// No tool grant mechanism exists on this seam at all — the plan still locks purely on the
|
||||||
|
// operator's approval decision, exactly as an ungapped plan would.
|
||||||
|
assertEquals(1, payloads.filterIsInstance<ExecutionPlanLockedEvent>().size)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `malformed reflection output degrades to advisory and never crashes lockAndRun`(): Unit = runBlocking {
|
||||||
|
val sessionId = SessionId("driver-reflect-malformed-session")
|
||||||
|
val eventStore = InMemoryEventStore()
|
||||||
|
val compiler = ExecutionPlanCompiler(buildRegistry())
|
||||||
|
|
||||||
|
val reflector = CapabilityGapReflector { _, _ -> throw IllegalStateException("boom") }
|
||||||
|
|
||||||
|
val driver = FreestyleDriver(
|
||||||
|
eventStore = eventStore,
|
||||||
|
compiler = compiler,
|
||||||
|
planContent = { gapPlanJson },
|
||||||
|
config = OrchestrationConfig(),
|
||||||
|
runPhase2 = { sid, graph, _ -> WorkflowResult.Completed(sid, graph.start) },
|
||||||
|
reflector = reflector,
|
||||||
|
)
|
||||||
|
|
||||||
|
driver.lockAndRun(sessionId)
|
||||||
|
|
||||||
|
val payloads = eventStore.read(sessionId).map { it.payload }
|
||||||
|
assertTrue(
|
||||||
|
payloads.filterIsInstance<CapabilityGapReflectedEvent>().isEmpty(),
|
||||||
|
"no verdict when the judge itself throws",
|
||||||
|
)
|
||||||
|
assertEquals(
|
||||||
|
1,
|
||||||
|
payloads.filterIsInstance<ExecutionPlanLockedEvent>().size,
|
||||||
|
"a broken reflector must not wedge the plan gate",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `no reflector wired preserves part-1 behavior`(): Unit = runBlocking {
|
||||||
|
val sessionId = SessionId("driver-no-reflector-session")
|
||||||
|
val eventStore = InMemoryEventStore()
|
||||||
|
val compiler = ExecutionPlanCompiler(buildRegistry())
|
||||||
|
|
||||||
|
val driver = FreestyleDriver(
|
||||||
|
eventStore = eventStore,
|
||||||
|
compiler = compiler,
|
||||||
|
planContent = { gapPlanJson },
|
||||||
|
config = OrchestrationConfig(),
|
||||||
|
runPhase2 = { sid, graph, _ -> WorkflowResult.Completed(sid, graph.start) },
|
||||||
|
)
|
||||||
|
|
||||||
|
driver.lockAndRun(sessionId)
|
||||||
|
|
||||||
|
val payloads = eventStore.read(sessionId).map { it.payload }
|
||||||
|
assertEquals(1, payloads.filterIsInstance<CapabilityGapDetectedEvent>().size)
|
||||||
|
assertTrue(payloads.filterIsInstance<CapabilityGapReflectedEvent>().isEmpty())
|
||||||
|
assertEquals(1, payloads.filterIsInstance<ExecutionPlanLockedEvent>().size)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+115
@@ -0,0 +1,115 @@
|
|||||||
|
package com.correx.apps.server.inference
|
||||||
|
|
||||||
|
import com.correx.core.events.events.CapabilityGapVerdict
|
||||||
|
import com.correx.core.events.types.ProviderId
|
||||||
|
import com.correx.core.events.types.SessionId
|
||||||
|
import com.correx.core.events.types.StageId
|
||||||
|
import com.correx.core.inference.CapabilityScore
|
||||||
|
import com.correx.core.inference.FinishReason
|
||||||
|
import com.correx.core.inference.InferenceProvider
|
||||||
|
import com.correx.core.inference.InferenceRequest
|
||||||
|
import com.correx.core.inference.InferenceResponse
|
||||||
|
import com.correx.core.inference.InferenceRouter
|
||||||
|
import com.correx.core.inference.ModelCapability
|
||||||
|
import com.correx.core.inference.ProviderHealth
|
||||||
|
import com.correx.core.inference.Token
|
||||||
|
import com.correx.core.inference.Tokenizer
|
||||||
|
import com.correx.core.inference.TokenUsage
|
||||||
|
import com.correx.core.kernel.orchestration.CapabilityGapInput
|
||||||
|
import com.correx.core.tools.contract.ToolCapability
|
||||||
|
import kotlinx.coroutines.runBlocking
|
||||||
|
import org.junit.jupiter.api.Assertions.assertEquals
|
||||||
|
import org.junit.jupiter.api.Assertions.assertTrue
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
|
||||||
|
private class StubTokenizer : Tokenizer {
|
||||||
|
override suspend fun tokenize(text: String): List<Token> = emptyList()
|
||||||
|
override suspend fun countTokens(text: String): Int = text.length / 4
|
||||||
|
}
|
||||||
|
|
||||||
|
private class StubProvider(private val response: String) : InferenceProvider {
|
||||||
|
override val id = ProviderId("stub")
|
||||||
|
override val name = "stub"
|
||||||
|
override val tokenizer = StubTokenizer()
|
||||||
|
override suspend fun infer(request: InferenceRequest): InferenceResponse = InferenceResponse(
|
||||||
|
requestId = request.requestId,
|
||||||
|
text = response,
|
||||||
|
finishReason = FinishReason.Stop,
|
||||||
|
tokensUsed = TokenUsage(0, 0),
|
||||||
|
latencyMs = 0,
|
||||||
|
)
|
||||||
|
override suspend fun healthCheck(): ProviderHealth = ProviderHealth.Healthy
|
||||||
|
override fun capabilities(): Set<CapabilityScore> = emptySet()
|
||||||
|
}
|
||||||
|
|
||||||
|
private class StubRouter(private val provider: InferenceProvider) : InferenceRouter {
|
||||||
|
override suspend fun route(
|
||||||
|
stageId: StageId,
|
||||||
|
requiredCapabilities: Set<ModelCapability>,
|
||||||
|
): InferenceProvider = provider
|
||||||
|
}
|
||||||
|
|
||||||
|
class CapabilityGapReflectorImplTest {
|
||||||
|
|
||||||
|
private val gap = CapabilityGapInput(
|
||||||
|
stageId = "apply",
|
||||||
|
impliedCapability = ToolCapability.FILE_WRITE,
|
||||||
|
phrase = "create",
|
||||||
|
grantedCapabilities = emptySet(),
|
||||||
|
)
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `a RESOLVED verdict from the model is parsed with its detail`(): Unit = runBlocking {
|
||||||
|
val response =
|
||||||
|
"""{"reflections":[{"stage":"apply","verdict":"RESOLVED","detail":"file_write already covers this"}]}"""
|
||||||
|
val reflector = CapabilityGapReflectorImpl(StubRouter(StubProvider(response)))
|
||||||
|
val result = reflector.reflect(SessionId("s1"), listOf(gap))
|
||||||
|
assertEquals(1, result.size)
|
||||||
|
assertEquals(CapabilityGapVerdict.RESOLVED, result.single().verdict)
|
||||||
|
assertEquals("file_write already covers this", result.single().detail)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `a NEEDS_TOOL verdict from the model is parsed with its justification`(): Unit = runBlocking {
|
||||||
|
val response =
|
||||||
|
"""{"reflections":[{"stage":"apply","verdict":"NEEDS_TOOL","detail":"no existing tool can do this"}]}"""
|
||||||
|
val reflector = CapabilityGapReflectorImpl(StubRouter(StubProvider(response)))
|
||||||
|
val result = reflector.reflect(SessionId("s1"), listOf(gap))
|
||||||
|
assertEquals(CapabilityGapVerdict.NEEDS_TOOL, result.single().verdict)
|
||||||
|
assertEquals("no existing tool can do this", result.single().detail)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `malformed model output degrades to a safe RESOLVED default without crashing`(): Unit = runBlocking {
|
||||||
|
val reflector = CapabilityGapReflectorImpl(StubRouter(StubProvider("not json at all")))
|
||||||
|
val result = reflector.reflect(SessionId("s1"), listOf(gap))
|
||||||
|
assertEquals(1, result.size)
|
||||||
|
assertEquals(CapabilityGapVerdict.RESOLVED, result.single().verdict)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `a provider that throws degrades to a safe RESOLVED default without crashing`(): Unit = runBlocking {
|
||||||
|
val throwingRouter = object : InferenceRouter {
|
||||||
|
override suspend fun route(
|
||||||
|
stageId: StageId,
|
||||||
|
requiredCapabilities: Set<ModelCapability>,
|
||||||
|
): InferenceProvider = error("no provider available")
|
||||||
|
}
|
||||||
|
val reflector = CapabilityGapReflectorImpl(throwingRouter)
|
||||||
|
val result = reflector.reflect(SessionId("s1"), listOf(gap))
|
||||||
|
assertEquals(1, result.size)
|
||||||
|
assertEquals(CapabilityGapVerdict.RESOLVED, result.single().verdict)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `empty gap list never invokes inference`(): Unit = runBlocking {
|
||||||
|
val router = object : InferenceRouter {
|
||||||
|
override suspend fun route(
|
||||||
|
stageId: StageId,
|
||||||
|
requiredCapabilities: Set<ModelCapability>,
|
||||||
|
): InferenceProvider = error("must not be called for an empty gap list")
|
||||||
|
}
|
||||||
|
val result = CapabilityGapReflectorImpl(router).reflect(SessionId("s1"), emptyList())
|
||||||
|
assertTrue(result.isEmpty())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -73,3 +73,28 @@ data class CapabilityGapDetectedEvent(
|
|||||||
val evidence: String,
|
val evidence: String,
|
||||||
val timestampMs: Long,
|
val timestampMs: Long,
|
||||||
) : EventPayload
|
) : EventPayload
|
||||||
|
|
||||||
|
/** Outcome of the bounded LLM "are you sure?" reflection pass over a [CapabilityGapDetectedEvent]. */
|
||||||
|
enum class CapabilityGapVerdict { RESOLVED, NEEDS_TOOL }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Vikunja #30 part 2: records the single-pass LLM reflection consulted on a capability gap before
|
||||||
|
* escalating to the operator. RESOLVED means the model self-corrected onto an already-granted
|
||||||
|
* capability ([detail] carries its revised intent, advisory only — never mutates the locked plan).
|
||||||
|
* NEEDS_TOOL means the model insists it genuinely needs [impliedCapability] ([detail] carries its
|
||||||
|
* justification) — this is the only case that should reach the human approval seam; the tool is
|
||||||
|
* never auto-granted (invariants #4/#5).
|
||||||
|
*
|
||||||
|
* The reflection itself is nondeterministic (LLM-backed, invariant #9): recorded here so replay
|
||||||
|
* reads this event back and never re-invokes inference (invariant #8).
|
||||||
|
*/
|
||||||
|
@Serializable
|
||||||
|
@SerialName("CapabilityGapReflected")
|
||||||
|
data class CapabilityGapReflectedEvent(
|
||||||
|
val sessionId: SessionId,
|
||||||
|
val stageId: StageId,
|
||||||
|
val impliedCapability: String,
|
||||||
|
val verdict: CapabilityGapVerdict,
|
||||||
|
val detail: String,
|
||||||
|
val timestampMs: Long,
|
||||||
|
) : EventPayload
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ import com.correx.core.events.events.EventPayload
|
|||||||
import com.correx.core.events.events.ExecutionPlanLockedEvent
|
import com.correx.core.events.events.ExecutionPlanLockedEvent
|
||||||
import com.correx.core.events.events.PlanCompileCheckedEvent
|
import com.correx.core.events.events.PlanCompileCheckedEvent
|
||||||
import com.correx.core.events.events.CapabilityGapDetectedEvent
|
import com.correx.core.events.events.CapabilityGapDetectedEvent
|
||||||
|
import com.correx.core.events.events.CapabilityGapReflectedEvent
|
||||||
import com.correx.core.events.events.ReviewFindingsRaisedEvent
|
import com.correx.core.events.events.ReviewFindingsRaisedEvent
|
||||||
import com.correx.core.events.events.ExecutionPlanRejectedEvent
|
import com.correx.core.events.events.ExecutionPlanRejectedEvent
|
||||||
import com.correx.core.events.events.HealthDegradedEvent
|
import com.correx.core.events.events.HealthDegradedEvent
|
||||||
@@ -173,6 +174,7 @@ val eventModule = SerializersModule {
|
|||||||
subclass(ExecutionPlanRejectedEvent::class)
|
subclass(ExecutionPlanRejectedEvent::class)
|
||||||
subclass(PlanCompileCheckedEvent::class)
|
subclass(PlanCompileCheckedEvent::class)
|
||||||
subclass(CapabilityGapDetectedEvent::class)
|
subclass(CapabilityGapDetectedEvent::class)
|
||||||
|
subclass(CapabilityGapReflectedEvent::class)
|
||||||
subclass(ReviewFindingsRaisedEvent::class)
|
subclass(ReviewFindingsRaisedEvent::class)
|
||||||
subclass(JournalCompactedEvent::class)
|
subclass(JournalCompactedEvent::class)
|
||||||
subclass(HealthDegradedEvent::class)
|
subclass(HealthDegradedEvent::class)
|
||||||
|
|||||||
+37
@@ -0,0 +1,37 @@
|
|||||||
|
package com.correx.core.kernel.orchestration
|
||||||
|
|
||||||
|
import com.correx.core.events.events.CapabilityGapVerdict
|
||||||
|
import com.correx.core.events.types.SessionId
|
||||||
|
import com.correx.core.tools.contract.ToolCapability
|
||||||
|
|
||||||
|
/** One capability gap finding from [com.correx.infrastructure.workflow.CapabilityGapDetector], as
|
||||||
|
* plain data so this module does not depend on infrastructure:workflow. */
|
||||||
|
data class CapabilityGapInput(
|
||||||
|
val stageId: String,
|
||||||
|
val impliedCapability: ToolCapability,
|
||||||
|
val phrase: String,
|
||||||
|
val grantedCapabilities: Set<ToolCapability>,
|
||||||
|
)
|
||||||
|
|
||||||
|
/** Result of reflecting on a single [CapabilityGapInput]. */
|
||||||
|
data class CapabilityGapReflection(
|
||||||
|
val stageId: String,
|
||||||
|
val impliedCapability: ToolCapability,
|
||||||
|
val verdict: CapabilityGapVerdict,
|
||||||
|
val detail: String,
|
||||||
|
)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Seam for the Vikunja #30 part 2 reflection rung: consulted once, bounded, on the capability gaps
|
||||||
|
* a plan-compile pass turned up, before any of them are surfaced to the operator. Like the other gate
|
||||||
|
* seams ([SemanticReviewer], [SalvageJudge]), the inference-touching implementation is injected so
|
||||||
|
* the deterministic core never runs inference itself and the caller stays unit-testable with a fake.
|
||||||
|
*
|
||||||
|
* A single call covers the whole batch of gaps for one plan — there is no per-gap loop and no retry;
|
||||||
|
* the caller records the result as a [com.correx.core.events.events.CapabilityGapReflectedEvent] per
|
||||||
|
* gap and never re-invokes this on replay (invariant #8 — the reflection is nondeterministic,
|
||||||
|
* invariant #9 requires recording the outcome, not re-observing it).
|
||||||
|
*/
|
||||||
|
fun interface CapabilityGapReflector {
|
||||||
|
suspend fun reflect(sessionId: SessionId, gaps: List<CapabilityGapInput>): List<CapabilityGapReflection>
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user