diff --git a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/ScopeCoverage.kt b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/ScopeCoverage.kt new file mode 100644 index 00000000..1a585820 --- /dev/null +++ b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/ScopeCoverage.kt @@ -0,0 +1,127 @@ +package com.correx.core.kernel.orchestration + +import com.correx.core.events.types.SessionId +import com.correx.core.events.types.StageId +import com.correx.core.transitions.execution.StageExecutionResult +import com.correx.core.transitions.graph.StageConfig +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.intOrNull +import kotlinx.serialization.json.jsonPrimitive + +/** + * Pure deterministic check that a definition of done accounts for every in-scope item the + * discovery brief settled. + * + * Determinism / invariant #8: reads ONLY the two recorded artifact strings — no I/O, no external + * calls. Both are already in the event log (ArtifactCreatedEvent), so the result is recomputable + * on replay without emitting an observation event. + * + * The failure this closes: the analyst is the one-way funnel between the brief and every later + * stage. Session 954da1a9 turned an 8-item scope into four "Project Foundation" criteria, and + * because the architect, plan-compile gate and final reviewer all grade the *shrunken* DoD, a plan + * that delivered 5% of the request passed every gate. + * + * ponytail: index bookkeeping, not semantics — a criterion claiming `covers: [3]` without really + * proving scope[3] still passes. Forcing the analyst to name every index catches the silent + * collapse (a whole scope list dropped, unnoticed); judging whether a criterion is strong enough + * stays the reviewer's job. + */ +internal object ScopeCoverage { + + private val lenientJson = Json { ignoreUnknownKeys = true; isLenient = true } + + /** + * The discovery scope items no DoD criterion claims to cover, each rendered as + * `[index] item` so the retry feedback names the index the model must put in `covers`. + * + * Empty when the check does not apply: an unparseable discovery brief or an empty scope. An + * unparseable DoD, or one whose criteria declare no `covers` at all, reports the whole scope. + */ + fun uncoveredScope(discoveryJson: String, dodJson: String): List { + val scope = parse(discoveryJson) + ?.let { it["brief"] as? JsonObject } + ?.let { stringList(it, "scope") } + .orEmpty() + val covered = coveredIndexes(parse(dodJson)) + return scope.withIndex() + .filterNot { (index, _) -> index in covered } + .map { (index, item) -> "[$index] $item" } + } + + /** Every index listed in any criterion's `covers` array. */ + private fun coveredIndexes(dod: JsonObject?): Set = + (dod?.get("criteria") as? JsonArray) + ?.filterIsInstance() + ?.flatMap { criterion -> (criterion["covers"] as? JsonArray) ?: emptyList() } + ?.mapNotNull { runCatching { it.jsonPrimitive.intOrNull }.getOrNull() } + ?.toSet() + ?: emptySet() + + private fun parse(json: String): JsonObject? = + runCatching { lenientJson.parseToJsonElement(stripFence(json)) as? JsonObject }.getOrNull() + + private fun stringList(obj: JsonObject, key: String): List = + runCatching { + (obj[key] as? JsonArray)?.mapNotNull { it.jsonPrimitive.contentOrNull } ?: emptyList() + }.getOrElse { emptyList() } + + private fun stripFence(text: String): String { + val trimmed = text.trim() + if (!trimmed.startsWith("```") || !trimmed.endsWith("```")) return text + val withoutClose = trimmed.removeSuffix("```").trimEnd() + val firstNewline = withoutClose.indexOf('\n') + return if (firstNewline < 0) text else withoutClose.substring(firstNewline + 1) + } +} + +/** + * Scope-coverage gate: for a stage that produces a `dod` artifact and consumed the `discovery` + * brief, fail retryably when a settled scope item has no criterion claiming it. Both artifacts are + * already in the content cache (recorded via ArtifactCreatedEvent), so the verdict is a pure + * function of recorded data and needs no event of its own (invariants #8, #9). Unlike the + * plan-compile gate, nothing here leaves the process. + * + * Lives beside [ScopeCoverage] rather than in SessionOrchestratorGates.kt, which is already at its + * function budget. + */ +internal suspend fun SessionOrchestrator.runScopeCoverageGate( + sessionId: SessionId, + stageId: StageId, + stageConfig: StageConfig, +): StageExecutionResult { + val uncovered = uncoveredScopeItems(sessionId, stageConfig) + return if (uncovered.isEmpty()) { + StageExecutionResult.Success(emptyList()) + } else { + log.warn( + "[Orchestrator] scope-coverage gate failed session={} stage={} uncovered={}", + sessionId.value, stageId.value, uncovered.joinToString("; "), + ) + StageExecutionResult.Failure( + "stage ${stageId.value} produced a definition of done that drops settled in-scope " + + "work. These discovery scope items have no criterion:\n" + + uncovered.joinToString("\n") { "- $it" } + + "\nAdd a criterion for each and list its scope index in that criterion's `covers`.", + retryable = true, + gate = "scope_coverage", + ) + } +} + +/** Empty when the gate does not apply: no `dod` produced, or no `discovery` brief to compare. */ +private fun SessionOrchestrator.uncoveredScopeItems( + sessionId: SessionId, + stageConfig: StageConfig, +): List { + val dod = stageConfig.produces.firstOrNull { it.kind.id == "dod" } + ?.let { artifactContentCache["${sessionId.value}:${it.name.value}"] } + val discovery = artifactContentCache["${sessionId.value}:discovery"] + return if (dod == null || discovery == null) { + emptyList() + } else { + ScopeCoverage.uncoveredScope(discovery, dod) + } +} diff --git a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorGates.kt b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorGates.kt index ad3a0a86..9d0fb3d2 100644 --- a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorGates.kt +++ b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorGates.kt @@ -196,6 +196,7 @@ internal suspend fun SessionOrchestrator.runPostStageGates( val gates: List StageExecutionResult> = listOf( { groundBriefReferences(sessionId, stageId, stageConfig, effectives) }, { checkBriefEcho(sessionId, stageId, stageConfig) }, + { runScopeCoverageGate(sessionId, stageId, stageConfig) }, { runContractGate(sessionId, stageId, stageConfig, effectives) }, { runPlanCompileGate(sessionId, stageId, stageConfig) }, { runStaticAnalysis(sessionId, stageId, stageConfig, effectives) }, @@ -268,7 +269,9 @@ internal suspend fun SessionOrchestrator.evaluateStageContract( } /** The currently-failing assertions as (target, assertionId, evidence) triples for the checklist. */ -internal fun SessionOrchestrator.contractFailureItems(results: List): List> = +internal fun SessionOrchestrator.contractFailureItems( + results: List, +): List> = results.filterNot { it.passed }.map { Triple(it.target, it.assertionId, it.evidence) } internal suspend fun SessionOrchestrator.runContractGate( diff --git a/core/kernel/src/test/kotlin/com/correx/core/kernel/orchestration/ScopeCoverageTest.kt b/core/kernel/src/test/kotlin/com/correx/core/kernel/orchestration/ScopeCoverageTest.kt new file mode 100644 index 00000000..e9e73db0 --- /dev/null +++ b/core/kernel/src/test/kotlin/com/correx/core/kernel/orchestration/ScopeCoverageTest.kt @@ -0,0 +1,77 @@ +package com.correx.core.kernel.orchestration + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class ScopeCoverageTest { + + private fun discovery(vararg scope: String): String { + val items = scope.joinToString(",") { "\"$it\"" } + return """{"brief":{"what":"a ui","scope":[$items],"non_goals":[]},"ready":true,"questions":[]}""" + } + + private fun dod(vararg covers: List): String { + val criteria = covers.mapIndexed { i, c -> + """{"id":"c${i + 1}","statement":"s","part":"p","verified_by":"gate","covers":[${c.joinToString(",")}]}""" + }.joinToString(",") + return """{"summary":"s","criteria":[$criteria],"out_of_scope":[]}""" + } + + @Test + fun `every scope index covered leaves nothing uncovered`() { + val uncovered = ScopeCoverage.uncoveredScope( + discovery("sessions list", "events viewer"), + dod(listOf(0), listOf(1)), + ) + assertTrue(uncovered.isEmpty(), "expected full coverage, got $uncovered") + } + + @Test + fun `one criterion may cover several scope items and a run-level criterion covers none`() { + val uncovered = ScopeCoverage.uncoveredScope( + discovery("sessions list", "events viewer", "artifacts viewer"), + dod(listOf(0, 1, 2), emptyList()), + ) + assertTrue(uncovered.isEmpty(), "expected full coverage, got $uncovered") + } + + @Test + fun `dropped scope items are reported with their index`() { + val uncovered = ScopeCoverage.uncoveredScope( + discovery("session driver", "sessions list", "workflows", "events"), + dod(listOf(0), listOf(1)), + ) + assertEquals(listOf("[2] workflows", "[3] events"), uncovered) + } + + /** The regression this gate exists for: session 954da1a9's foundation-only DoD. */ + @Test + fun `a DoD with no covers at all reports the whole scope`() { + val uncovered = ScopeCoverage.uncoveredScope( + discovery("session driver", "sessions list"), + """{"summary":"init the stack","criteria":[ + {"id":"c1","statement":"Vite and React initialized","part":"Project Foundation","verified_by":"gate"} + ],"out_of_scope":[]}""", + ) + assertEquals(listOf("[0] session driver", "[1] sessions list"), uncovered) + } + + @Test + fun `an unparseable DoD reports the whole scope`() { + val uncovered = ScopeCoverage.uncoveredScope(discovery("sessions list"), "not json at all") + assertEquals(listOf("[0] sessions list"), uncovered) + } + + @Test + fun `a fenced DoD is read through the fence`() { + val fenced = "```json\n" + dod(listOf(0)) + "\n```" + assertTrue(ScopeCoverage.uncoveredScope(discovery("sessions list"), fenced).isEmpty()) + } + + @Test + fun `the check does not apply without a usable discovery scope`() { + assertTrue(ScopeCoverage.uncoveredScope("not json", dod(listOf(0))).isEmpty()) + assertTrue(ScopeCoverage.uncoveredScope(discovery(), dod(listOf(0))).isEmpty()) + } +} diff --git a/docs/schemas/dod.json b/docs/schemas/dod.json index 48fdf61d..ae7e8403 100644 --- a/docs/schemas/dod.json +++ b/docs/schemas/dod.json @@ -11,9 +11,14 @@ "id": { "type": "string" }, "statement": { "type": "string" }, "part": { "type": "string" }, - "verified_by": { "type": "string", "description": "one of: gate, reviewer" } + "verified_by": { "type": "string", "description": "one of: gate, reviewer" }, + "covers": { + "type": "array", + "items": { "type": "integer" }, + "description": "0-based indexes into the discovery brief's scope[] that this criterion proves. Every scope index must appear in at least one criterion or the scope-coverage gate fails the stage." + } }, - "required": ["id", "statement", "part", "verified_by"], + "required": ["id", "statement", "part", "verified_by", "covers"], "additionalProperties": false } }, diff --git a/examples/workflows/prompts/analyst_freestyle.md b/examples/workflows/prompts/analyst_freestyle.md index 4ca985e1..835d9a61 100644 --- a/examples/workflows/prompts/analyst_freestyle.md +++ b/examples/workflows/prompts/analyst_freestyle.md @@ -41,20 +41,27 @@ Emit the `dod` artifact once. Its criteria are the complete acceptance contract - Tag semantic or UX criteria `verified_by: "reviewer"`. - Copy discovery `brief.non_goals` into `out_of_scope`; this is a hard review boundary. - Cover the entire in-scope brief now. Later stages may not silently add criteria. +- Give every criterion a `covers` array: the 0-based indexes into discovery `brief.scope` it proves. + Walk `brief.scope` in order and account for every index. A scope-coverage gate fails this stage + and hands back the uncovered items verbatim. Two criteria proving one scope item repeat its index. + One criterion proving three items lists all three. A criterion that serves the run rather than a + scope item (the named task, a failure path) gets `[]`. - Include at least one criterion proving the named task is carried through to the implementation plan, and one criterion for each material failure or recovery path identified during discovery. Call `emit_artifact` with a JSON object matching this shape: `{"summary": string, "criteria": [{"id": string, "statement": string, "part": string, -"verified_by": "gate" | "reviewer"}], "out_of_scope": [string]}`. +"verified_by": "gate" | "reviewer", "covers": [integer]}], "out_of_scope": [string]}`. -Example: +Example, for a discovery brief whose `scope` is +`["Bounded validation gate", "Operator sees the diagnostic"]`: ```json { "summary": "Deliver the bounded validation gate for task gate-42.", "criteria": [ - {"id":"c1","statement":"The project typecheck passes before completion","part":"terminal gate","verified_by":"gate"}, - {"id":"c2","statement":"The operator sees the recorded diagnostic","part":"workflow UX","verified_by":"reviewer"} + {"id":"c1","statement":"The project typecheck passes before completion","part":"terminal gate","verified_by":"gate","covers":[0]}, + {"id":"c2","statement":"The operator sees the recorded diagnostic","part":"workflow UX","verified_by":"reviewer","covers":[1]}, + {"id":"c3","statement":"The implementation plan names task gate-42","part":"task threading","verified_by":"reviewer","covers":[]} ], "out_of_scope": ["Changing the workflow topology"] }