From 700f59ef0dc6b3b3819f078c60f0a4e18c3a10fb Mon Sep 17 00:00:00 2001 From: claude Date: Tue, 11 Aug 2026 19:56:43 +0400 Subject: [PATCH] feat(kernel): gate the DoD against discovery scope (#699) Session 954da1a9 asked for an eight-view web UI and shipped a Vite starter page. Discovery settled all eight items in brief.scope; the analyst emitted four criteria, all part="Project Foundation"; the architect planned against that DoD, so the run scaffolded Vite, Tailwind and TanStack Query and stopped. The plan-compile gate and the final reviewer both graded the shrunken DoD, so a plan delivering 5% of the request passed clean. Each DoD criterion now carries `covers`: the 0-based indexes into discovery brief.scope it proves. A post-stage scope_coverage gate fails the analyst retryably when an index has no criterion, handing back the dropped items verbatim. Pure function of two recorded artifacts, so replay recomputes it and no verdict event is needed. Ceiling is index bookkeeping, not semantics: a criterion claiming covers:[3] without really proving scope[3] still passes. It catches the silent collapse, not a weak criterion. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013dVqqci5H5b3s6xzv6Lojq --- .../kernel/orchestration/ScopeCoverage.kt | 127 ++++++++++++++++++ .../orchestration/SessionOrchestratorGates.kt | 5 +- .../kernel/orchestration/ScopeCoverageTest.kt | 77 +++++++++++ docs/schemas/dod.json | 9 +- .../workflows/prompts/analyst_freestyle.md | 15 ++- 5 files changed, 226 insertions(+), 7 deletions(-) create mode 100644 core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/ScopeCoverage.kt create mode 100644 core/kernel/src/test/kotlin/com/correx/core/kernel/orchestration/ScopeCoverageTest.kt diff --git a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/ScopeCoverage.kt b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/ScopeCoverage.kt new file mode 100644 index 00000000..1a585820 --- /dev/null +++ b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/ScopeCoverage.kt @@ -0,0 +1,127 @@ +package com.correx.core.kernel.orchestration + +import com.correx.core.events.types.SessionId +import com.correx.core.events.types.StageId +import com.correx.core.transitions.execution.StageExecutionResult +import com.correx.core.transitions.graph.StageConfig +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.intOrNull +import kotlinx.serialization.json.jsonPrimitive + +/** + * Pure deterministic check that a definition of done accounts for every in-scope item the + * discovery brief settled. + * + * Determinism / invariant #8: reads ONLY the two recorded artifact strings — no I/O, no external + * calls. Both are already in the event log (ArtifactCreatedEvent), so the result is recomputable + * on replay without emitting an observation event. + * + * The failure this closes: the analyst is the one-way funnel between the brief and every later + * stage. Session 954da1a9 turned an 8-item scope into four "Project Foundation" criteria, and + * because the architect, plan-compile gate and final reviewer all grade the *shrunken* DoD, a plan + * that delivered 5% of the request passed every gate. + * + * ponytail: index bookkeeping, not semantics — a criterion claiming `covers: [3]` without really + * proving scope[3] still passes. Forcing the analyst to name every index catches the silent + * collapse (a whole scope list dropped, unnoticed); judging whether a criterion is strong enough + * stays the reviewer's job. + */ +internal object ScopeCoverage { + + private val lenientJson = Json { ignoreUnknownKeys = true; isLenient = true } + + /** + * The discovery scope items no DoD criterion claims to cover, each rendered as + * `[index] item` so the retry feedback names the index the model must put in `covers`. + * + * Empty when the check does not apply: an unparseable discovery brief or an empty scope. An + * unparseable DoD, or one whose criteria declare no `covers` at all, reports the whole scope. + */ + fun uncoveredScope(discoveryJson: String, dodJson: String): List { + val scope = parse(discoveryJson) + ?.let { it["brief"] as? JsonObject } + ?.let { stringList(it, "scope") } + .orEmpty() + val covered = coveredIndexes(parse(dodJson)) + return scope.withIndex() + .filterNot { (index, _) -> index in covered } + .map { (index, item) -> "[$index] $item" } + } + + /** Every index listed in any criterion's `covers` array. */ + private fun coveredIndexes(dod: JsonObject?): Set = + (dod?.get("criteria") as? JsonArray) + ?.filterIsInstance() + ?.flatMap { criterion -> (criterion["covers"] as? JsonArray) ?: emptyList() } + ?.mapNotNull { runCatching { it.jsonPrimitive.intOrNull }.getOrNull() } + ?.toSet() + ?: emptySet() + + private fun parse(json: String): JsonObject? = + runCatching { lenientJson.parseToJsonElement(stripFence(json)) as? JsonObject }.getOrNull() + + private fun stringList(obj: JsonObject, key: String): List = + runCatching { + (obj[key] as? JsonArray)?.mapNotNull { it.jsonPrimitive.contentOrNull } ?: emptyList() + }.getOrElse { emptyList() } + + private fun stripFence(text: String): String { + val trimmed = text.trim() + if (!trimmed.startsWith("```") || !trimmed.endsWith("```")) return text + val withoutClose = trimmed.removeSuffix("```").trimEnd() + val firstNewline = withoutClose.indexOf('\n') + return if (firstNewline < 0) text else withoutClose.substring(firstNewline + 1) + } +} + +/** + * Scope-coverage gate: for a stage that produces a `dod` artifact and consumed the `discovery` + * brief, fail retryably when a settled scope item has no criterion claiming it. Both artifacts are + * already in the content cache (recorded via ArtifactCreatedEvent), so the verdict is a pure + * function of recorded data and needs no event of its own (invariants #8, #9). Unlike the + * plan-compile gate, nothing here leaves the process. + * + * Lives beside [ScopeCoverage] rather than in SessionOrchestratorGates.kt, which is already at its + * function budget. + */ +internal suspend fun SessionOrchestrator.runScopeCoverageGate( + sessionId: SessionId, + stageId: StageId, + stageConfig: StageConfig, +): StageExecutionResult { + val uncovered = uncoveredScopeItems(sessionId, stageConfig) + return if (uncovered.isEmpty()) { + StageExecutionResult.Success(emptyList()) + } else { + log.warn( + "[Orchestrator] scope-coverage gate failed session={} stage={} uncovered={}", + sessionId.value, stageId.value, uncovered.joinToString("; "), + ) + StageExecutionResult.Failure( + "stage ${stageId.value} produced a definition of done that drops settled in-scope " + + "work. These discovery scope items have no criterion:\n" + + uncovered.joinToString("\n") { "- $it" } + + "\nAdd a criterion for each and list its scope index in that criterion's `covers`.", + retryable = true, + gate = "scope_coverage", + ) + } +} + +/** Empty when the gate does not apply: no `dod` produced, or no `discovery` brief to compare. */ +private fun SessionOrchestrator.uncoveredScopeItems( + sessionId: SessionId, + stageConfig: StageConfig, +): List { + val dod = stageConfig.produces.firstOrNull { it.kind.id == "dod" } + ?.let { artifactContentCache["${sessionId.value}:${it.name.value}"] } + val discovery = artifactContentCache["${sessionId.value}:discovery"] + return if (dod == null || discovery == null) { + emptyList() + } else { + ScopeCoverage.uncoveredScope(discovery, dod) + } +} diff --git a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorGates.kt b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorGates.kt index ad3a0a86..9d0fb3d2 100644 --- a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorGates.kt +++ b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestratorGates.kt @@ -196,6 +196,7 @@ internal suspend fun SessionOrchestrator.runPostStageGates( val gates: List StageExecutionResult> = listOf( { groundBriefReferences(sessionId, stageId, stageConfig, effectives) }, { checkBriefEcho(sessionId, stageId, stageConfig) }, + { runScopeCoverageGate(sessionId, stageId, stageConfig) }, { runContractGate(sessionId, stageId, stageConfig, effectives) }, { runPlanCompileGate(sessionId, stageId, stageConfig) }, { runStaticAnalysis(sessionId, stageId, stageConfig, effectives) }, @@ -268,7 +269,9 @@ internal suspend fun SessionOrchestrator.evaluateStageContract( } /** The currently-failing assertions as (target, assertionId, evidence) triples for the checklist. */ -internal fun SessionOrchestrator.contractFailureItems(results: List): List> = +internal fun SessionOrchestrator.contractFailureItems( + results: List, +): List> = results.filterNot { it.passed }.map { Triple(it.target, it.assertionId, it.evidence) } internal suspend fun SessionOrchestrator.runContractGate( diff --git a/core/kernel/src/test/kotlin/com/correx/core/kernel/orchestration/ScopeCoverageTest.kt b/core/kernel/src/test/kotlin/com/correx/core/kernel/orchestration/ScopeCoverageTest.kt new file mode 100644 index 00000000..e9e73db0 --- /dev/null +++ b/core/kernel/src/test/kotlin/com/correx/core/kernel/orchestration/ScopeCoverageTest.kt @@ -0,0 +1,77 @@ +package com.correx.core.kernel.orchestration + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test + +class ScopeCoverageTest { + + private fun discovery(vararg scope: String): String { + val items = scope.joinToString(",") { "\"$it\"" } + return """{"brief":{"what":"a ui","scope":[$items],"non_goals":[]},"ready":true,"questions":[]}""" + } + + private fun dod(vararg covers: List): String { + val criteria = covers.mapIndexed { i, c -> + """{"id":"c${i + 1}","statement":"s","part":"p","verified_by":"gate","covers":[${c.joinToString(",")}]}""" + }.joinToString(",") + return """{"summary":"s","criteria":[$criteria],"out_of_scope":[]}""" + } + + @Test + fun `every scope index covered leaves nothing uncovered`() { + val uncovered = ScopeCoverage.uncoveredScope( + discovery("sessions list", "events viewer"), + dod(listOf(0), listOf(1)), + ) + assertTrue(uncovered.isEmpty(), "expected full coverage, got $uncovered") + } + + @Test + fun `one criterion may cover several scope items and a run-level criterion covers none`() { + val uncovered = ScopeCoverage.uncoveredScope( + discovery("sessions list", "events viewer", "artifacts viewer"), + dod(listOf(0, 1, 2), emptyList()), + ) + assertTrue(uncovered.isEmpty(), "expected full coverage, got $uncovered") + } + + @Test + fun `dropped scope items are reported with their index`() { + val uncovered = ScopeCoverage.uncoveredScope( + discovery("session driver", "sessions list", "workflows", "events"), + dod(listOf(0), listOf(1)), + ) + assertEquals(listOf("[2] workflows", "[3] events"), uncovered) + } + + /** The regression this gate exists for: session 954da1a9's foundation-only DoD. */ + @Test + fun `a DoD with no covers at all reports the whole scope`() { + val uncovered = ScopeCoverage.uncoveredScope( + discovery("session driver", "sessions list"), + """{"summary":"init the stack","criteria":[ + {"id":"c1","statement":"Vite and React initialized","part":"Project Foundation","verified_by":"gate"} + ],"out_of_scope":[]}""", + ) + assertEquals(listOf("[0] session driver", "[1] sessions list"), uncovered) + } + + @Test + fun `an unparseable DoD reports the whole scope`() { + val uncovered = ScopeCoverage.uncoveredScope(discovery("sessions list"), "not json at all") + assertEquals(listOf("[0] sessions list"), uncovered) + } + + @Test + fun `a fenced DoD is read through the fence`() { + val fenced = "```json\n" + dod(listOf(0)) + "\n```" + assertTrue(ScopeCoverage.uncoveredScope(discovery("sessions list"), fenced).isEmpty()) + } + + @Test + fun `the check does not apply without a usable discovery scope`() { + assertTrue(ScopeCoverage.uncoveredScope("not json", dod(listOf(0))).isEmpty()) + assertTrue(ScopeCoverage.uncoveredScope(discovery(), dod(listOf(0))).isEmpty()) + } +} diff --git a/docs/schemas/dod.json b/docs/schemas/dod.json index 48fdf61d..ae7e8403 100644 --- a/docs/schemas/dod.json +++ b/docs/schemas/dod.json @@ -11,9 +11,14 @@ "id": { "type": "string" }, "statement": { "type": "string" }, "part": { "type": "string" }, - "verified_by": { "type": "string", "description": "one of: gate, reviewer" } + "verified_by": { "type": "string", "description": "one of: gate, reviewer" }, + "covers": { + "type": "array", + "items": { "type": "integer" }, + "description": "0-based indexes into the discovery brief's scope[] that this criterion proves. Every scope index must appear in at least one criterion or the scope-coverage gate fails the stage." + } }, - "required": ["id", "statement", "part", "verified_by"], + "required": ["id", "statement", "part", "verified_by", "covers"], "additionalProperties": false } }, diff --git a/examples/workflows/prompts/analyst_freestyle.md b/examples/workflows/prompts/analyst_freestyle.md index 4ca985e1..835d9a61 100644 --- a/examples/workflows/prompts/analyst_freestyle.md +++ b/examples/workflows/prompts/analyst_freestyle.md @@ -41,20 +41,27 @@ Emit the `dod` artifact once. Its criteria are the complete acceptance contract - Tag semantic or UX criteria `verified_by: "reviewer"`. - Copy discovery `brief.non_goals` into `out_of_scope`; this is a hard review boundary. - Cover the entire in-scope brief now. Later stages may not silently add criteria. +- Give every criterion a `covers` array: the 0-based indexes into discovery `brief.scope` it proves. + Walk `brief.scope` in order and account for every index. A scope-coverage gate fails this stage + and hands back the uncovered items verbatim. Two criteria proving one scope item repeat its index. + One criterion proving three items lists all three. A criterion that serves the run rather than a + scope item (the named task, a failure path) gets `[]`. - Include at least one criterion proving the named task is carried through to the implementation plan, and one criterion for each material failure or recovery path identified during discovery. Call `emit_artifact` with a JSON object matching this shape: `{"summary": string, "criteria": [{"id": string, "statement": string, "part": string, -"verified_by": "gate" | "reviewer"}], "out_of_scope": [string]}`. +"verified_by": "gate" | "reviewer", "covers": [integer]}], "out_of_scope": [string]}`. -Example: +Example, for a discovery brief whose `scope` is +`["Bounded validation gate", "Operator sees the diagnostic"]`: ```json { "summary": "Deliver the bounded validation gate for task gate-42.", "criteria": [ - {"id":"c1","statement":"The project typecheck passes before completion","part":"terminal gate","verified_by":"gate"}, - {"id":"c2","statement":"The operator sees the recorded diagnostic","part":"workflow UX","verified_by":"reviewer"} + {"id":"c1","statement":"The project typecheck passes before completion","part":"terminal gate","verified_by":"gate","covers":[0]}, + {"id":"c2","statement":"The operator sees the recorded diagnostic","part":"workflow UX","verified_by":"reviewer","covers":[1]}, + {"id":"c3","statement":"The implementation plan names task gate-42","part":"task threading","verified_by":"reviewer","covers":[]} ], "out_of_scope": ["Changing the workflow topology"] }