feat(validation): capability-gap reflection rung (#30 part 2)
Bounded LLM "are you sure?" pass over the capability gaps part 1 detects, run once at plan-lock before the operator is asked to approve — honest mistakes self-correct without escalation; only a genuine tool need reaches the human. - CapabilityGapReflector: fun-interface seam in core:kernel (plain-data in/out, no infrastructure:workflow dep), mirroring SemanticReviewer/SalvageJudge. - CapabilityGapReflectorImpl (apps/server): one InferenceRouter call for the whole gap batch, no per-gap loop, no retry; runCatching degrades to a safe RESOLVED-advisory default on any failure (unroutable/timeout/malformed JSON) — a broken pass can never manufacture a NEEDS_TOOL escalation or a grant. - CapabilityGapReflectedEvent(verdict RESOLVED|NEEDS_TOOL) recorded per gap and registered in eventModule; replay reads the event, never re-invokes inference (invariants #7/#8/#9). - FreestyleDriver.lockAndRun: RESOLVED is advisory (recorded, plan untouched — invariant #3); NEEDS_TOOL is appended to the existing requestPlanApproval preview so the operator decides — no auto-grant anywhere (#4/#5). - reflector nullable -> degrades to part-1 behavior when unwired. Verified: ./gradlew :core:events:test :core:kernel:test :infrastructure:workflow:test :apps:server:test green.
This commit is contained in:
@@ -518,6 +518,7 @@ fun main() {
|
||||
runPhase2 = { sid, graph, cfg -> orchestrator.run(sid, graph, cfg) },
|
||||
requestPlanApproval = { sid, planJson -> orchestrator.requestPlanApproval(sid, planJson) },
|
||||
toolCapabilities = toolRegistry.all().associate { it.name to it.requiredCapabilities },
|
||||
reflector = com.correx.apps.server.inference.CapabilityGapReflectorImpl(inferenceRouter),
|
||||
)
|
||||
// observability-spec §4: continuous health watch. Seed the monitor's last-status from the
|
||||
// recorded system-session events so a restart doesn't re-emit a degraded already in the log.
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
package com.correx.apps.server.freestyle
|
||||
|
||||
import com.correx.core.events.events.CapabilityGapDetectedEvent
|
||||
import com.correx.core.events.events.CapabilityGapReflectedEvent
|
||||
import com.correx.core.events.events.CapabilityGapVerdict
|
||||
import com.correx.core.events.events.EventMetadata
|
||||
import com.correx.core.events.events.ExecutionPlanLockedEvent
|
||||
import com.correx.core.events.events.ExecutionPlanRejectedEvent
|
||||
import com.correx.core.events.events.NewEvent
|
||||
import com.correx.core.events.events.PlanLintCompletedEvent
|
||||
import com.correx.core.tools.contract.ToolCapability
|
||||
import com.correx.infrastructure.workflow.CapabilityGap
|
||||
import com.correx.infrastructure.workflow.CapabilityGapDetector
|
||||
import com.correx.infrastructure.workflow.PlanLintResult
|
||||
import com.correx.infrastructure.workflow.PlanLinter
|
||||
@@ -16,6 +19,8 @@ import com.correx.core.events.types.EventId
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.events.types.StageId
|
||||
import com.correx.core.kernel.execution.WorkflowResult
|
||||
import com.correx.core.kernel.orchestration.CapabilityGapInput
|
||||
import com.correx.core.kernel.orchestration.CapabilityGapReflector
|
||||
import com.correx.core.kernel.orchestration.OrchestrationConfig
|
||||
import com.correx.core.transitions.graph.WorkflowGraph
|
||||
import com.correx.infrastructure.workflow.ExecutionPlanCompiler
|
||||
@@ -42,6 +47,9 @@ class FreestyleDriver(
|
||||
// capability-gap detector below. Empty = detector finds nothing (preserves callers/tests that
|
||||
// don't care about it) rather than depending on a live ToolRegistry here.
|
||||
private val toolCapabilities: Map<String, Set<ToolCapability>> = emptyMap(),
|
||||
// Vikunja #30 part 2: bounded LLM "are you sure?" pass over capability gaps, consulted before
|
||||
// requestPlanApproval. Null = feature degrades to part-1 behavior (gaps recorded, never reflected).
|
||||
private val reflector: CapabilityGapReflector? = null,
|
||||
) {
|
||||
suspend fun lockAndRun(sessionId: SessionId) {
|
||||
val json = planContent(sessionId) ?: run {
|
||||
@@ -69,16 +77,22 @@ class FreestyleDriver(
|
||||
return
|
||||
}
|
||||
// Capability-gap detector (Vikunja #30 part 1): advisory only — recorded, never blocking.
|
||||
// A gap does not fail the gate and does not grant the missing tool (invariants #3/#4/#5); a
|
||||
// future reflection rung (part 2) is what will act on these events.
|
||||
CapabilityGapDetector.detect(graph, toolCapabilities).forEach { gap ->
|
||||
// A gap does not fail the gate and does not grant the missing tool (invariants #3/#4/#5).
|
||||
val gaps = CapabilityGapDetector.detect(graph, toolCapabilities)
|
||||
gaps.forEach { gap ->
|
||||
log.info(
|
||||
"[FreestyleDriver] capability gap session={} stage={} implied={} evidence='{}'",
|
||||
sessionId.value, gap.stageId, gap.impliedCapability, gap.phrase,
|
||||
)
|
||||
emitCapabilityGap(sessionId, gap.stageId, gap.impliedCapability.name, gap.phrase)
|
||||
}
|
||||
val approved = requestPlanApproval(sessionId, json)
|
||||
// Reflection rung (Vikunja #30 part 2): one bounded LLM pass over the whole batch of gaps
|
||||
// before the human is bothered. RESOLVED is advisory-only (logged, never rewrites the locked
|
||||
// plan — invariant #3 LLM-proposes/core-decides); NEEDS_TOOL is the only case surfaced into
|
||||
// the plan-review approval preview below, since that is the existing human approval seam and
|
||||
// a tool grant must never happen without it (invariants #4/#5).
|
||||
val needsToolNote = reflectOnGaps(sessionId, gaps)
|
||||
val approved = requestPlanApproval(sessionId, needsToolNote?.let { "$json\n\n$it" } ?: json)
|
||||
if (!approved) {
|
||||
log.info("freestyle: execution plan rejected by operator for session={}", sessionId.value)
|
||||
emitRejected(sessionId, "rejected by operator at plan review", "operator")
|
||||
@@ -167,6 +181,66 @@ class FreestyleDriver(
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the bounded reflection pass (if wired) over [gaps], emits one
|
||||
* [CapabilityGapReflectedEvent] per gap, and returns an operator-facing note listing any
|
||||
* NEEDS_TOOL verdicts — null when there are no gaps, no reflector, or nothing to escalate.
|
||||
* A single call for the whole batch: no per-gap loop, no retry (bounded per invariant #9).
|
||||
*/
|
||||
private suspend fun reflectOnGaps(sessionId: SessionId, gaps: List<CapabilityGap>): String? {
|
||||
val judge = reflector
|
||||
if (gaps.isEmpty() || judge == null) return null
|
||||
val inputs = gaps.map {
|
||||
CapabilityGapInput(it.stageId, it.impliedCapability, it.phrase, it.grantedCapabilities)
|
||||
}
|
||||
val reflections = runCatching { judge.reflect(sessionId, inputs) }.getOrElse {
|
||||
log.warn("[FreestyleDriver] capability gap reflection failed session={}: {}", sessionId.value, it.message)
|
||||
emptyList()
|
||||
}
|
||||
reflections.forEach { r ->
|
||||
log.info(
|
||||
"[FreestyleDriver] capability gap reflected session={} stage={} implied={} verdict={}",
|
||||
sessionId.value, r.stageId, r.impliedCapability, r.verdict,
|
||||
)
|
||||
emitCapabilityGapReflected(sessionId, r.stageId, r.impliedCapability.name, r.verdict, r.detail)
|
||||
}
|
||||
val escalations = reflections.filter { it.verdict == CapabilityGapVerdict.NEEDS_TOOL }
|
||||
return escalations.takeIf { it.isNotEmpty() }?.let {
|
||||
"CAPABILITY GAPS REQUESTED (reflection confirmed a genuine need — approve or reject the plan " +
|
||||
"with this in mind; no tool is auto-granted):\n" +
|
||||
it.joinToString("\n") { e -> "- stage ${e.stageId} needs ${e.impliedCapability}: ${e.detail}" }
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun emitCapabilityGapReflected(
|
||||
sessionId: SessionId,
|
||||
stageId: String,
|
||||
impliedCapability: String,
|
||||
verdict: CapabilityGapVerdict,
|
||||
detail: String,
|
||||
) {
|
||||
eventStore.append(
|
||||
NewEvent(
|
||||
metadata = EventMetadata(
|
||||
eventId = EventId(UUID.randomUUID().toString()),
|
||||
sessionId = sessionId,
|
||||
timestamp = Clock.System.now(),
|
||||
schemaVersion = 1,
|
||||
causationId = null,
|
||||
correlationId = null,
|
||||
),
|
||||
payload = CapabilityGapReflectedEvent(
|
||||
sessionId = sessionId,
|
||||
stageId = StageId(stageId),
|
||||
impliedCapability = impliedCapability,
|
||||
verdict = verdict,
|
||||
detail = detail,
|
||||
timestampMs = Clock.System.now().toEpochMilliseconds(),
|
||||
),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
private suspend fun emitRejected(sessionId: SessionId, reason: String, source: String) {
|
||||
eventStore.append(
|
||||
NewEvent(
|
||||
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
package com.correx.apps.server.inference
|
||||
|
||||
import com.correx.core.context.model.CompressionMetadata
|
||||
import com.correx.core.context.model.ContextEntry
|
||||
import com.correx.core.context.model.ContextLayer
|
||||
import com.correx.core.context.model.ContextPack
|
||||
import com.correx.core.context.model.EntryRole
|
||||
import com.correx.core.events.events.CapabilityGapVerdict
|
||||
import com.correx.core.events.types.ContextEntryId
|
||||
import com.correx.core.events.types.ContextPackId
|
||||
import com.correx.core.events.types.InferenceRequestId
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.events.types.StageId
|
||||
import com.correx.core.inference.GenerationConfig
|
||||
import com.correx.core.inference.InferenceRequest
|
||||
import com.correx.core.inference.InferenceRouter
|
||||
import com.correx.core.inference.ModelCapability
|
||||
import com.correx.core.inference.ResponseFormat
|
||||
import com.correx.core.kernel.orchestration.CapabilityGapInput
|
||||
import com.correx.core.kernel.orchestration.CapabilityGapReflection
|
||||
import com.correx.core.kernel.orchestration.CapabilityGapReflector
|
||||
import kotlinx.serialization.Serializable
|
||||
import kotlinx.serialization.json.Json
|
||||
import org.slf4j.LoggerFactory
|
||||
import java.util.UUID
|
||||
|
||||
private const val REFLECTION_MAX_TOKENS = 4_096
|
||||
private val REFLECTION_STAGE_ID = StageId("capability_gap_reflection")
|
||||
|
||||
/**
|
||||
* Vikunja #30 part 2: the "are you sure?" reflection judge. Given the plan-compile gate's
|
||||
* capability-gap findings for a session, asks a single bounded question — revise onto an existing
|
||||
* capability, or justify the need for a new one — and maps the answer to a [CapabilityGapReflection]
|
||||
* per gap. Mirrors [SemanticReviewerImpl]: talks to the provider directly with a self-contained
|
||||
* prompt, one inference call for the whole batch (no per-gap loop, no retry), and degrades to a
|
||||
* safe RESOLVED/no-escalation default (see [defaultReflection]) on any failure so a broken
|
||||
* reflection never wedges the plan-compile gate it sits behind.
|
||||
*/
|
||||
class CapabilityGapReflectorImpl(
|
||||
private val inferenceRouter: InferenceRouter,
|
||||
private val modelId: String? = null,
|
||||
) : CapabilityGapReflector {
|
||||
|
||||
override suspend fun reflect(sessionId: SessionId, gaps: List<CapabilityGapInput>): List<CapabilityGapReflection> {
|
||||
if (gaps.isEmpty()) return emptyList()
|
||||
return runCatching {
|
||||
val provider = inferenceRouter.route(REFLECTION_STAGE_ID, setOf(ModelCapability.General), modelId)
|
||||
val prompt = buildPrompt(gaps)
|
||||
val request = InferenceRequest(
|
||||
requestId = InferenceRequestId(UUID.randomUUID().toString()),
|
||||
sessionId = sessionId,
|
||||
stageId = REFLECTION_STAGE_ID,
|
||||
contextPack = pack(sessionId, prompt),
|
||||
generationConfig = GenerationConfig(temperature = 0.2, topP = 0.9, maxTokens = REFLECTION_MAX_TOKENS),
|
||||
responseFormat = ResponseFormat.Text,
|
||||
)
|
||||
parse(provider.infer(request).text, gaps)
|
||||
}.getOrElse {
|
||||
log.warn(
|
||||
"[CapabilityGapReflector] reflection failed session={}: {}",
|
||||
sessionId.value, it.message,
|
||||
)
|
||||
gaps.map(::defaultReflection)
|
||||
}
|
||||
}
|
||||
|
||||
/** Malformed/missing verdict for a gap defaults to advisory RESOLVED-nothing — never NEEDS_TOOL
|
||||
* by default, so a broken reflection pass cannot force an escalation on its own. */
|
||||
private fun defaultReflection(gap: CapabilityGapInput) = CapabilityGapReflection(
|
||||
stageId = gap.stageId,
|
||||
impliedCapability = gap.impliedCapability,
|
||||
verdict = CapabilityGapVerdict.RESOLVED,
|
||||
detail = "reflection unavailable — treated as advisory, no escalation",
|
||||
)
|
||||
|
||||
private fun buildPrompt(gaps: List<CapabilityGapInput>): String {
|
||||
val findings = gaps.joinToString("\n\n") { gap ->
|
||||
val granted = gap.grantedCapabilities.joinToString(", ").ifBlank { "(none)" }
|
||||
"""
|
||||
- stage: ${gap.stageId}
|
||||
implied capability: ${gap.impliedCapability}
|
||||
triggering phrase: "${gap.phrase}"
|
||||
tools already granted to this stage give it: $granted
|
||||
""".trimIndent()
|
||||
}
|
||||
return """
|
||||
A deterministic scan of a compiled plan flagged stages whose brief implies a tool capability
|
||||
they were not granted. Before this is escalated to a human operator, decide for EACH gap
|
||||
whether it is a real gap or a false alarm.
|
||||
|
||||
IMPORTANT domain fact: the file_write tool auto-creates parent directories — there is no
|
||||
separate mkdir/create-directory tool. A stage that says "create a directory" or "create the
|
||||
file" and already has file_write does NOT need a new capability.
|
||||
|
||||
GAPS:
|
||||
$findings
|
||||
|
||||
For EACH gap, respond with either:
|
||||
- RESOLVED: the stage's existing tools are actually sufficient; state the revised intent.
|
||||
- NEEDS_TOOL: the stage genuinely needs the missing capability; justify why no existing tool
|
||||
covers it.
|
||||
|
||||
Respond with ONLY a JSON object, no prose, no markdown fence:
|
||||
{"reflections":[
|
||||
{"stage":"stage-id","verdict":"RESOLVED|NEEDS_TOOL","detail":"revised intent or justification"}
|
||||
]}
|
||||
""".trimIndent()
|
||||
}
|
||||
|
||||
private fun pack(sessionId: SessionId, prompt: String) = ContextPack(
|
||||
id = ContextPackId(UUID.randomUUID().toString()),
|
||||
sessionId = sessionId,
|
||||
stageId = REFLECTION_STAGE_ID,
|
||||
layers = mapOf(
|
||||
ContextLayer.L0 to listOf(
|
||||
ContextEntry(
|
||||
id = ContextEntryId(UUID.randomUUID().toString()),
|
||||
layer = ContextLayer.L0,
|
||||
content = prompt,
|
||||
sourceType = "capability_gap_reflection",
|
||||
sourceId = "reflector",
|
||||
tokenEstimate = prompt.length / 4,
|
||||
role = EntryRole.USER,
|
||||
),
|
||||
),
|
||||
),
|
||||
budgetUsed = prompt.length / 4,
|
||||
budgetLimit = prompt.length / 4,
|
||||
compressionMetadata = CompressionMetadata(),
|
||||
)
|
||||
|
||||
private fun parse(raw: String, gaps: List<CapabilityGapInput>): List<CapabilityGapReflection> {
|
||||
val byStage = gaps.associateBy { it.stageId }
|
||||
val dto = extractJson(raw)
|
||||
?.let { json ->
|
||||
runCatching { lenient.decodeFromString(ReflectionBatchDto.serializer(), json) }.getOrNull()
|
||||
}
|
||||
?: return gaps.map(::defaultReflection)
|
||||
val byStageResult = dto.reflections.mapNotNull { r ->
|
||||
val gap = byStage[r.stage] ?: return@mapNotNull null
|
||||
val verdict = runCatching { CapabilityGapVerdict.valueOf(r.verdict.uppercase()) }
|
||||
.getOrDefault(CapabilityGapVerdict.RESOLVED)
|
||||
gap.stageId to CapabilityGapReflection(
|
||||
stageId = gap.stageId,
|
||||
impliedCapability = gap.impliedCapability,
|
||||
verdict = verdict,
|
||||
detail = r.detail.ifBlank { "no detail provided" },
|
||||
)
|
||||
}.toMap()
|
||||
return gaps.map { gap -> byStageResult[gap.stageId] ?: defaultReflection(gap) }
|
||||
}
|
||||
|
||||
private fun extractJson(raw: String): String? {
|
||||
val start = raw.indexOf('{')
|
||||
val end = raw.lastIndexOf('}')
|
||||
return if (start in 0 until end) raw.substring(start, end + 1) else null
|
||||
}
|
||||
|
||||
@Serializable
|
||||
private data class ReflectionBatchDto(val reflections: List<ReflectionDto> = emptyList())
|
||||
|
||||
@Serializable
|
||||
private data class ReflectionDto(
|
||||
val stage: String = "",
|
||||
val verdict: String = "RESOLVED",
|
||||
val detail: String = "",
|
||||
)
|
||||
|
||||
companion object {
|
||||
private val log = LoggerFactory.getLogger(CapabilityGapReflectorImpl::class.java)
|
||||
private val lenient = Json { ignoreUnknownKeys = true; isLenient = true }
|
||||
}
|
||||
}
|
||||
@@ -3,11 +3,16 @@ package com.correx.apps.server.freestyle
|
||||
import com.correx.core.artifacts.kind.ConfigArtifactKind
|
||||
import com.correx.core.artifacts.kind.DefaultArtifactKindRegistry
|
||||
import com.correx.core.artifacts.kind.JsonSchema
|
||||
import com.correx.core.events.events.CapabilityGapDetectedEvent
|
||||
import com.correx.core.events.events.CapabilityGapReflectedEvent
|
||||
import com.correx.core.events.events.CapabilityGapVerdict
|
||||
import com.correx.core.events.events.ExecutionPlanLockedEvent
|
||||
import com.correx.core.events.events.ExecutionPlanRejectedEvent
|
||||
import com.correx.core.events.events.PlanLintCompletedEvent
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.kernel.execution.WorkflowResult
|
||||
import com.correx.core.kernel.orchestration.CapabilityGapReflection
|
||||
import com.correx.core.kernel.orchestration.CapabilityGapReflector
|
||||
import com.correx.core.kernel.orchestration.OrchestrationConfig
|
||||
import com.correx.core.transitions.graph.WorkflowGraph
|
||||
import com.correx.infrastructure.persistence.InMemoryEventStore
|
||||
@@ -316,4 +321,151 @@ class FreestyleDriverTest {
|
||||
assertEquals(1, runPhase2Invocations, "runPhase2 should be called exactly once after approval")
|
||||
assertTrue(approvalPreview != null, "requestPlanApproval should have been invoked")
|
||||
}
|
||||
|
||||
// A stage brief that trips the FILE_WRITE gap (verb "create") with zero granted tools, so
|
||||
// CapabilityGapDetector.detect() always finds exactly one gap on stage "apply" for these tests.
|
||||
private val gapPlanJson = """
|
||||
{
|
||||
"goal": "test plan with a capability gap",
|
||||
"stages": [
|
||||
{ "id": "analyse", "prompt": "Analyse the problem", "produces": "patch", "needs": [], "tools": [] },
|
||||
{ "id": "apply", "prompt": "create the output file", "produces": "patch", "needs": ["patch"], "tools": [] }
|
||||
],
|
||||
"edges": [
|
||||
{ "from": "analyse", "to": "apply", "condition": { "type": "always_true" } },
|
||||
{ "from": "apply", "to": "done", "condition": { "type": "always_true" } }
|
||||
]
|
||||
}
|
||||
""".trimIndent()
|
||||
|
||||
@Test
|
||||
fun `a RESOLVED reflection is advisory only and does not touch the approval preview`(): Unit = runBlocking {
|
||||
val sessionId = SessionId("driver-reflect-resolved-session")
|
||||
val eventStore = InMemoryEventStore()
|
||||
val compiler = ExecutionPlanCompiler(buildRegistry())
|
||||
var approvalPreview: String? = null
|
||||
|
||||
val reflector = CapabilityGapReflector { _, gaps ->
|
||||
gaps.map {
|
||||
CapabilityGapReflection(
|
||||
it.stageId, it.impliedCapability, CapabilityGapVerdict.RESOLVED, "revised: reuse file_write",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
val driver = FreestyleDriver(
|
||||
eventStore = eventStore,
|
||||
compiler = compiler,
|
||||
planContent = { gapPlanJson },
|
||||
config = OrchestrationConfig(),
|
||||
runPhase2 = { sid, graph, _ -> WorkflowResult.Completed(sid, graph.start) },
|
||||
requestPlanApproval = { _, preview -> approvalPreview = preview; true },
|
||||
reflector = reflector,
|
||||
)
|
||||
|
||||
driver.lockAndRun(sessionId)
|
||||
|
||||
val payloads = eventStore.read(sessionId).map { it.payload }
|
||||
assertEquals(1, payloads.filterIsInstance<CapabilityGapDetectedEvent>().size)
|
||||
val reflected = payloads.filterIsInstance<CapabilityGapReflectedEvent>().single()
|
||||
assertEquals(CapabilityGapVerdict.RESOLVED, reflected.verdict)
|
||||
assertEquals(1, payloads.filterIsInstance<ExecutionPlanLockedEvent>().size, "RESOLVED must not block locking")
|
||||
assertFalse(
|
||||
approvalPreview.orEmpty().contains("CAPABILITY GAPS REQUESTED"),
|
||||
"RESOLVED verdicts must not be surfaced as an escalation",
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a NEEDS_TOOL reflection is surfaced to the approval seam but never auto-grants`(): Unit = runBlocking {
|
||||
val sessionId = SessionId("driver-reflect-needs-tool-session")
|
||||
val eventStore = InMemoryEventStore()
|
||||
val compiler = ExecutionPlanCompiler(buildRegistry())
|
||||
var approvalPreview: String? = null
|
||||
|
||||
val reflector = CapabilityGapReflector { _, gaps ->
|
||||
gaps.map {
|
||||
CapabilityGapReflection(
|
||||
it.stageId, it.impliedCapability, CapabilityGapVerdict.NEEDS_TOOL, "genuinely needs file_write",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
val driver = FreestyleDriver(
|
||||
eventStore = eventStore,
|
||||
compiler = compiler,
|
||||
planContent = { gapPlanJson },
|
||||
config = OrchestrationConfig(),
|
||||
runPhase2 = { sid, graph, _ -> WorkflowResult.Completed(sid, graph.start) },
|
||||
requestPlanApproval = { _, preview -> approvalPreview = preview; true },
|
||||
reflector = reflector,
|
||||
)
|
||||
|
||||
driver.lockAndRun(sessionId)
|
||||
|
||||
val payloads = eventStore.read(sessionId).map { it.payload }
|
||||
val reflected = payloads.filterIsInstance<CapabilityGapReflectedEvent>().single()
|
||||
assertEquals(CapabilityGapVerdict.NEEDS_TOOL, reflected.verdict)
|
||||
assertTrue(
|
||||
approvalPreview.orEmpty().contains("CAPABILITY GAPS REQUESTED"),
|
||||
"NEEDS_TOOL must be surfaced into the human approval preview",
|
||||
)
|
||||
assertTrue(approvalPreview.orEmpty().contains("apply"), "escalation note should name the stage")
|
||||
// No tool grant mechanism exists on this seam at all — the plan still locks purely on the
|
||||
// operator's approval decision, exactly as an ungapped plan would.
|
||||
assertEquals(1, payloads.filterIsInstance<ExecutionPlanLockedEvent>().size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `malformed reflection output degrades to advisory and never crashes lockAndRun`(): Unit = runBlocking {
|
||||
val sessionId = SessionId("driver-reflect-malformed-session")
|
||||
val eventStore = InMemoryEventStore()
|
||||
val compiler = ExecutionPlanCompiler(buildRegistry())
|
||||
|
||||
val reflector = CapabilityGapReflector { _, _ -> throw IllegalStateException("boom") }
|
||||
|
||||
val driver = FreestyleDriver(
|
||||
eventStore = eventStore,
|
||||
compiler = compiler,
|
||||
planContent = { gapPlanJson },
|
||||
config = OrchestrationConfig(),
|
||||
runPhase2 = { sid, graph, _ -> WorkflowResult.Completed(sid, graph.start) },
|
||||
reflector = reflector,
|
||||
)
|
||||
|
||||
driver.lockAndRun(sessionId)
|
||||
|
||||
val payloads = eventStore.read(sessionId).map { it.payload }
|
||||
assertTrue(
|
||||
payloads.filterIsInstance<CapabilityGapReflectedEvent>().isEmpty(),
|
||||
"no verdict when the judge itself throws",
|
||||
)
|
||||
assertEquals(
|
||||
1,
|
||||
payloads.filterIsInstance<ExecutionPlanLockedEvent>().size,
|
||||
"a broken reflector must not wedge the plan gate",
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `no reflector wired preserves part-1 behavior`(): Unit = runBlocking {
|
||||
val sessionId = SessionId("driver-no-reflector-session")
|
||||
val eventStore = InMemoryEventStore()
|
||||
val compiler = ExecutionPlanCompiler(buildRegistry())
|
||||
|
||||
val driver = FreestyleDriver(
|
||||
eventStore = eventStore,
|
||||
compiler = compiler,
|
||||
planContent = { gapPlanJson },
|
||||
config = OrchestrationConfig(),
|
||||
runPhase2 = { sid, graph, _ -> WorkflowResult.Completed(sid, graph.start) },
|
||||
)
|
||||
|
||||
driver.lockAndRun(sessionId)
|
||||
|
||||
val payloads = eventStore.read(sessionId).map { it.payload }
|
||||
assertEquals(1, payloads.filterIsInstance<CapabilityGapDetectedEvent>().size)
|
||||
assertTrue(payloads.filterIsInstance<CapabilityGapReflectedEvent>().isEmpty())
|
||||
assertEquals(1, payloads.filterIsInstance<ExecutionPlanLockedEvent>().size)
|
||||
}
|
||||
}
|
||||
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
package com.correx.apps.server.inference
|
||||
|
||||
import com.correx.core.events.events.CapabilityGapVerdict
|
||||
import com.correx.core.events.types.ProviderId
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.events.types.StageId
|
||||
import com.correx.core.inference.CapabilityScore
|
||||
import com.correx.core.inference.FinishReason
|
||||
import com.correx.core.inference.InferenceProvider
|
||||
import com.correx.core.inference.InferenceRequest
|
||||
import com.correx.core.inference.InferenceResponse
|
||||
import com.correx.core.inference.InferenceRouter
|
||||
import com.correx.core.inference.ModelCapability
|
||||
import com.correx.core.inference.ProviderHealth
|
||||
import com.correx.core.inference.Token
|
||||
import com.correx.core.inference.Tokenizer
|
||||
import com.correx.core.inference.TokenUsage
|
||||
import com.correx.core.kernel.orchestration.CapabilityGapInput
|
||||
import com.correx.core.tools.contract.ToolCapability
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
private class StubTokenizer : Tokenizer {
|
||||
override suspend fun tokenize(text: String): List<Token> = emptyList()
|
||||
override suspend fun countTokens(text: String): Int = text.length / 4
|
||||
}
|
||||
|
||||
private class StubProvider(private val response: String) : InferenceProvider {
|
||||
override val id = ProviderId("stub")
|
||||
override val name = "stub"
|
||||
override val tokenizer = StubTokenizer()
|
||||
override suspend fun infer(request: InferenceRequest): InferenceResponse = InferenceResponse(
|
||||
requestId = request.requestId,
|
||||
text = response,
|
||||
finishReason = FinishReason.Stop,
|
||||
tokensUsed = TokenUsage(0, 0),
|
||||
latencyMs = 0,
|
||||
)
|
||||
override suspend fun healthCheck(): ProviderHealth = ProviderHealth.Healthy
|
||||
override fun capabilities(): Set<CapabilityScore> = emptySet()
|
||||
}
|
||||
|
||||
private class StubRouter(private val provider: InferenceProvider) : InferenceRouter {
|
||||
override suspend fun route(
|
||||
stageId: StageId,
|
||||
requiredCapabilities: Set<ModelCapability>,
|
||||
): InferenceProvider = provider
|
||||
}
|
||||
|
||||
class CapabilityGapReflectorImplTest {
|
||||
|
||||
private val gap = CapabilityGapInput(
|
||||
stageId = "apply",
|
||||
impliedCapability = ToolCapability.FILE_WRITE,
|
||||
phrase = "create",
|
||||
grantedCapabilities = emptySet(),
|
||||
)
|
||||
|
||||
@Test
|
||||
fun `a RESOLVED verdict from the model is parsed with its detail`(): Unit = runBlocking {
|
||||
val response =
|
||||
"""{"reflections":[{"stage":"apply","verdict":"RESOLVED","detail":"file_write already covers this"}]}"""
|
||||
val reflector = CapabilityGapReflectorImpl(StubRouter(StubProvider(response)))
|
||||
val result = reflector.reflect(SessionId("s1"), listOf(gap))
|
||||
assertEquals(1, result.size)
|
||||
assertEquals(CapabilityGapVerdict.RESOLVED, result.single().verdict)
|
||||
assertEquals("file_write already covers this", result.single().detail)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a NEEDS_TOOL verdict from the model is parsed with its justification`(): Unit = runBlocking {
|
||||
val response =
|
||||
"""{"reflections":[{"stage":"apply","verdict":"NEEDS_TOOL","detail":"no existing tool can do this"}]}"""
|
||||
val reflector = CapabilityGapReflectorImpl(StubRouter(StubProvider(response)))
|
||||
val result = reflector.reflect(SessionId("s1"), listOf(gap))
|
||||
assertEquals(CapabilityGapVerdict.NEEDS_TOOL, result.single().verdict)
|
||||
assertEquals("no existing tool can do this", result.single().detail)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `malformed model output degrades to a safe RESOLVED default without crashing`(): Unit = runBlocking {
|
||||
val reflector = CapabilityGapReflectorImpl(StubRouter(StubProvider("not json at all")))
|
||||
val result = reflector.reflect(SessionId("s1"), listOf(gap))
|
||||
assertEquals(1, result.size)
|
||||
assertEquals(CapabilityGapVerdict.RESOLVED, result.single().verdict)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a provider that throws degrades to a safe RESOLVED default without crashing`(): Unit = runBlocking {
|
||||
val throwingRouter = object : InferenceRouter {
|
||||
override suspend fun route(
|
||||
stageId: StageId,
|
||||
requiredCapabilities: Set<ModelCapability>,
|
||||
): InferenceProvider = error("no provider available")
|
||||
}
|
||||
val reflector = CapabilityGapReflectorImpl(throwingRouter)
|
||||
val result = reflector.reflect(SessionId("s1"), listOf(gap))
|
||||
assertEquals(1, result.size)
|
||||
assertEquals(CapabilityGapVerdict.RESOLVED, result.single().verdict)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `empty gap list never invokes inference`(): Unit = runBlocking {
|
||||
val router = object : InferenceRouter {
|
||||
override suspend fun route(
|
||||
stageId: StageId,
|
||||
requiredCapabilities: Set<ModelCapability>,
|
||||
): InferenceProvider = error("must not be called for an empty gap list")
|
||||
}
|
||||
val result = CapabilityGapReflectorImpl(router).reflect(SessionId("s1"), emptyList())
|
||||
assertTrue(result.isEmpty())
|
||||
}
|
||||
}
|
||||
@@ -73,3 +73,28 @@ data class CapabilityGapDetectedEvent(
|
||||
val evidence: String,
|
||||
val timestampMs: Long,
|
||||
) : EventPayload
|
||||
|
||||
/** Outcome of the bounded LLM "are you sure?" reflection pass over a [CapabilityGapDetectedEvent]. */
|
||||
enum class CapabilityGapVerdict { RESOLVED, NEEDS_TOOL }
|
||||
|
||||
/**
|
||||
* Vikunja #30 part 2: records the single-pass LLM reflection consulted on a capability gap before
|
||||
* escalating to the operator. RESOLVED means the model self-corrected onto an already-granted
|
||||
* capability ([detail] carries its revised intent, advisory only — never mutates the locked plan).
|
||||
* NEEDS_TOOL means the model insists it genuinely needs [impliedCapability] ([detail] carries its
|
||||
* justification) — this is the only case that should reach the human approval seam; the tool is
|
||||
* never auto-granted (invariants #4/#5).
|
||||
*
|
||||
* The reflection itself is nondeterministic (LLM-backed, invariant #9): recorded here so replay
|
||||
* reads this event back and never re-invokes inference (invariant #8).
|
||||
*/
|
||||
@Serializable
|
||||
@SerialName("CapabilityGapReflected")
|
||||
data class CapabilityGapReflectedEvent(
|
||||
val sessionId: SessionId,
|
||||
val stageId: StageId,
|
||||
val impliedCapability: String,
|
||||
val verdict: CapabilityGapVerdict,
|
||||
val detail: String,
|
||||
val timestampMs: Long,
|
||||
) : EventPayload
|
||||
|
||||
@@ -35,6 +35,7 @@ import com.correx.core.events.events.EventPayload
|
||||
import com.correx.core.events.events.ExecutionPlanLockedEvent
|
||||
import com.correx.core.events.events.PlanCompileCheckedEvent
|
||||
import com.correx.core.events.events.CapabilityGapDetectedEvent
|
||||
import com.correx.core.events.events.CapabilityGapReflectedEvent
|
||||
import com.correx.core.events.events.ReviewFindingsRaisedEvent
|
||||
import com.correx.core.events.events.ExecutionPlanRejectedEvent
|
||||
import com.correx.core.events.events.HealthDegradedEvent
|
||||
@@ -173,6 +174,7 @@ val eventModule = SerializersModule {
|
||||
subclass(ExecutionPlanRejectedEvent::class)
|
||||
subclass(PlanCompileCheckedEvent::class)
|
||||
subclass(CapabilityGapDetectedEvent::class)
|
||||
subclass(CapabilityGapReflectedEvent::class)
|
||||
subclass(ReviewFindingsRaisedEvent::class)
|
||||
subclass(JournalCompactedEvent::class)
|
||||
subclass(HealthDegradedEvent::class)
|
||||
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
package com.correx.core.kernel.orchestration
|
||||
|
||||
import com.correx.core.events.events.CapabilityGapVerdict
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.tools.contract.ToolCapability
|
||||
|
||||
/** One capability gap finding from [com.correx.infrastructure.workflow.CapabilityGapDetector], as
|
||||
* plain data so this module does not depend on infrastructure:workflow. */
|
||||
data class CapabilityGapInput(
|
||||
val stageId: String,
|
||||
val impliedCapability: ToolCapability,
|
||||
val phrase: String,
|
||||
val grantedCapabilities: Set<ToolCapability>,
|
||||
)
|
||||
|
||||
/** Result of reflecting on a single [CapabilityGapInput]. */
|
||||
data class CapabilityGapReflection(
|
||||
val stageId: String,
|
||||
val impliedCapability: ToolCapability,
|
||||
val verdict: CapabilityGapVerdict,
|
||||
val detail: String,
|
||||
)
|
||||
|
||||
/**
|
||||
* Seam for the Vikunja #30 part 2 reflection rung: consulted once, bounded, on the capability gaps
|
||||
* a plan-compile pass turned up, before any of them are surfaced to the operator. Like the other gate
|
||||
* seams ([SemanticReviewer], [SalvageJudge]), the inference-touching implementation is injected so
|
||||
* the deterministic core never runs inference itself and the caller stays unit-testable with a fake.
|
||||
*
|
||||
* A single call covers the whole batch of gaps for one plan — there is no per-gap loop and no retry;
|
||||
* the caller records the result as a [com.correx.core.events.events.CapabilityGapReflectedEvent] per
|
||||
* gap and never re-invokes this on replay (invariant #8 — the reflection is nondeterministic,
|
||||
* invariant #9 requires recording the outcome, not re-observing it).
|
||||
*/
|
||||
fun interface CapabilityGapReflector {
|
||||
suspend fun reflect(sessionId: SessionId, gaps: List<CapabilityGapInput>): List<CapabilityGapReflection>
|
||||
}
|
||||
Reference in New Issue
Block a user