feat(kernel): brief echo-back gate (plan-pipeline-addenda A1)
New brief_echo stage before the planner in role_pipeline: the model restates the analyst brief as a structured artifact, and a deterministic gate (checkBriefEcho, opt-in via brief_echo=true) diffs the restatement against the original brief. On divergence — dropped requirements (Jaccard coverage < 0.34) or hallucinated files — it emits BriefEchoMismatchEvent and fails the stage retryably, so a misread brief never reaches plan generation. The diff (BriefEchoDiff) is a pure function of two recorded artifacts, so it is replay-safe and records no observation; the event fires only on mismatch, for audit. Symbols are recorded but non-blocking in v1. Mirrors the groundBrief- References gate. role_pipeline only; freestyle_planning wiring is a follow-up. Runtime install (like research): copy brief_echo.json + brief_echo.md and the [[artifacts]] block into ~/.config/correx.
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
package com.correx.core.events.events
|
||||
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.events.types.StageId
|
||||
import kotlinx.serialization.SerialName
|
||||
import kotlinx.serialization.Serializable
|
||||
|
||||
/**
|
||||
* Emitted when the model's restatement of the analyst brief diverges beyond the acceptance
|
||||
* threshold: it dropped requirements or hallucinated file references. Recorded as a domain
|
||||
* signal for audit and narration. The gate then fails the stage (retryable) so the pipeline
|
||||
* cannot reach plan generation on a misread brief.
|
||||
*
|
||||
* Determinism note: this is NOT an environment observation (invariant #9). The diff is a pure
|
||||
* function of two already-recorded artifacts (the analysis brief and the brief_echo), so it is
|
||||
* deterministically recomputable on replay without re-emitting this event. The event is emitted
|
||||
* only on mismatch, as a domain signal.
|
||||
*/
|
||||
@Serializable
|
||||
@SerialName("BriefEchoMismatch")
|
||||
data class BriefEchoMismatchEvent(
|
||||
val sessionId: SessionId,
|
||||
val stageId: StageId,
|
||||
val uncoveredRequirements: List<String>,
|
||||
val hallucinatedFiles: List<String>,
|
||||
val hallucinatedSymbols: List<String>,
|
||||
) : EventPayload
|
||||
@@ -8,6 +8,7 @@ import com.correx.core.events.events.ArtifactContentStoredEvent
|
||||
import com.correx.core.events.events.ArtifactCreatedEvent
|
||||
import com.correx.core.events.events.ArtifactValidatedEvent
|
||||
import com.correx.core.events.events.ArtifactValidatingEvent
|
||||
import com.correx.core.events.events.BriefEchoMismatchEvent
|
||||
import com.correx.core.events.events.BriefGroundingCheckedEvent
|
||||
import com.correx.core.events.events.ChatSessionStartedEvent
|
||||
import com.correx.core.events.events.ClarificationAnsweredEvent
|
||||
@@ -108,6 +109,7 @@ val eventModule = SerializersModule {
|
||||
subclass(WorkspaceStateObservedEvent::class)
|
||||
subclass(RepoKnowledgeRetrievedEvent::class)
|
||||
subclass(BriefGroundingCheckedEvent::class)
|
||||
subclass(BriefEchoMismatchEvent::class)
|
||||
subclass(RiskAssessedEvent::class)
|
||||
subclass(ChatSessionStartedEvent::class)
|
||||
subclass(ChatTurnEvent::class)
|
||||
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
package com.correx.core.events.serialization
|
||||
|
||||
import com.correx.core.events.events.BriefEchoMismatchEvent
|
||||
import com.correx.core.events.events.EventPayload
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.events.types.StageId
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class BriefEchoMismatchEventSerializationTest {
|
||||
|
||||
private val sample = BriefEchoMismatchEvent(
|
||||
sessionId = SessionId("sess-1"),
|
||||
stageId = StageId("brief_echo"),
|
||||
uncoveredRequirements = listOf("must support retries", "must emit an event"),
|
||||
hallucinatedFiles = listOf("core/ghost/Phantom.kt"),
|
||||
hallucinatedSymbols = listOf("GhostService"),
|
||||
)
|
||||
|
||||
@Test
|
||||
fun `round-trips as polymorphic EventPayload`() {
|
||||
val encoded = eventJson.encodeToString(EventPayload.serializer(), sample)
|
||||
assertTrue(encoded.contains("\"type\":\"BriefEchoMismatch\""), "SerialName must be present: $encoded")
|
||||
val decoded = eventJson.decodeFromString(EventPayload.serializer(), encoded)
|
||||
assertEquals(sample, decoded)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `decodes hand-written BriefEchoMismatch JSON`() {
|
||||
val json = """
|
||||
{"type":"BriefEchoMismatch","sessionId":"s","stageId":"brief_echo",
|
||||
"uncoveredRequirements":["req-1"],"hallucinatedFiles":["a/b.kt"],"hallucinatedSymbols":[]}
|
||||
""".trimIndent()
|
||||
val decoded = eventJson.decodeFromString(EventPayload.serializer(), json)
|
||||
assertTrue(decoded is BriefEchoMismatchEvent)
|
||||
val event = decoded as BriefEchoMismatchEvent
|
||||
assertEquals(listOf("req-1"), event.uncoveredRequirements)
|
||||
assertEquals(listOf("a/b.kt"), event.hallucinatedFiles)
|
||||
assertTrue(event.hallucinatedSymbols.isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `empty lists round-trip correctly`() {
|
||||
val empty = sample.copy(
|
||||
uncoveredRequirements = emptyList(),
|
||||
hallucinatedFiles = emptyList(),
|
||||
hallucinatedSymbols = emptyList(),
|
||||
)
|
||||
val encoded = eventJson.encodeToString(EventPayload.serializer(), empty)
|
||||
val decoded = eventJson.decodeFromString(EventPayload.serializer(), encoded)
|
||||
assertEquals(empty, decoded)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package com.correx.core.kernel.orchestration
|
||||
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.contentOrNull
|
||||
import kotlinx.serialization.json.jsonPrimitive
|
||||
|
||||
/**
|
||||
* Pure deterministic diff between an analyst brief and a model's echo restatement.
|
||||
*
|
||||
* Determinism / invariant #8: reads ONLY the two recorded artifact strings — no I/O, no
|
||||
* external calls. Because both inputs are already in the event log (ArtifactCreatedEvent),
|
||||
* the result is recomputable on replay without re-emitting any observation event.
|
||||
*
|
||||
* isMismatch triggers gate failure when requirements are uncovered or files are hallucinated.
|
||||
* Hallucinated symbols are recorded for audit but do NOT block in v1 (display-only, mirroring
|
||||
* the role-reliability contradiction-check "display-only in v1" stance).
|
||||
*/
|
||||
data class BriefEchoDivergence(
|
||||
val uncoveredRequirements: List<String>,
|
||||
val hallucinatedFiles: List<String>,
|
||||
/** Recorded for audit/narration; does NOT contribute to isMismatch in v1. */
|
||||
val hallucinatedSymbols: List<String>,
|
||||
) {
|
||||
val isMismatch: Boolean
|
||||
get() = uncoveredRequirements.isNotEmpty() || hallucinatedFiles.isNotEmpty()
|
||||
}
|
||||
|
||||
internal object BriefEchoDiff {
|
||||
|
||||
/** Jaccard overlap below this threshold means a requirement is "uncovered" by the echo. */
|
||||
private const val COVERAGE_THRESHOLD = 0.34
|
||||
|
||||
/** Minimum token length for comparison — very short tokens add noise. */
|
||||
private const val MIN_TOKEN_LENGTH = 2
|
||||
|
||||
private val lenientJson = Json { ignoreUnknownKeys = true; isLenient = true }
|
||||
private val nonAlphanumeric = Regex("[^A-Za-z0-9]+")
|
||||
|
||||
/**
|
||||
* Compare the analyst brief JSON against the model's echo JSON, returning divergences.
|
||||
* On an unparseable echo, every brief requirement is reported as uncovered (total mismatch),
|
||||
* so a garbage echo always fails the gate.
|
||||
*/
|
||||
fun compare(briefJson: String, echoJson: String): BriefEchoDivergence {
|
||||
val brief = parseJson(briefJson)
|
||||
val echo = parseJson(echoJson)
|
||||
val briefFullTokens = tokenize(briefJson)
|
||||
val briefFiles = BriefReferenceExtractor.fileReferences(briefJson)
|
||||
return BriefEchoDivergence(
|
||||
uncoveredRequirements = uncoveredRequirements(brief, echo),
|
||||
hallucinatedFiles = hallucinatedFiles(echo, briefFullTokens, briefFiles),
|
||||
hallucinatedSymbols = hallucinatedSymbols(echo, briefFullTokens),
|
||||
)
|
||||
}
|
||||
|
||||
/** Each brief requirement whose best Jaccard overlap with any echo criterion is below threshold. */
|
||||
private fun uncoveredRequirements(brief: JsonObject?, echo: JsonObject?): List<String> {
|
||||
val requirements = brief?.let { stringList(it, "requirements") } ?: emptyList()
|
||||
val criteria = echo?.let { stringList(it, "acceptance_criteria") } ?: emptyList()
|
||||
return requirements.filter { req ->
|
||||
val reqTokens = tokenize(req)
|
||||
if (reqTokens.isEmpty()) return@filter false
|
||||
val bestOverlap = criteria.maxOfOrNull { jaccard(reqTokens, tokenize(it)) } ?: 0.0
|
||||
bestOverlap < COVERAGE_THRESHOLD
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Echo files not grounded in the brief. A file is hallucinated when no token of its basename
|
||||
* stem (extension dropped — an extension like "kt" matches any source file in the brief and is
|
||||
* pure noise) appears in the brief, AND it matches no extracted brief file path.
|
||||
*/
|
||||
private fun hallucinatedFiles(
|
||||
echo: JsonObject?,
|
||||
briefFullTokens: Set<String>,
|
||||
briefFiles: List<String>,
|
||||
): List<String> {
|
||||
val echoFiles = echo?.let { stringList(it, "referenced_files") } ?: emptyList()
|
||||
return echoFiles.filter { f ->
|
||||
val stemTokens = tokenize(f.substringAfterLast('/').substringBeforeLast('.'))
|
||||
if (stemTokens.isEmpty()) return@filter true
|
||||
stemTokens.none { it in briefFullTokens } &&
|
||||
briefFiles.none { it == f || it.endsWith(f) || f.endsWith(it) }
|
||||
}
|
||||
}
|
||||
|
||||
/** Echo symbols with no token present anywhere in the brief (recorded for audit, non-blocking). */
|
||||
private fun hallucinatedSymbols(echo: JsonObject?, briefFullTokens: Set<String>): List<String> {
|
||||
val echoSymbols = echo?.let { stringList(it, "referenced_symbols") } ?: emptyList()
|
||||
return echoSymbols.filter { sym -> tokenize(sym).none { it in briefFullTokens } }
|
||||
}
|
||||
|
||||
/** Lowercase, replace non-alphanumeric with spaces, split, drop tokens shorter than MIN_TOKEN_LENGTH. */
|
||||
internal fun tokenize(text: String): Set<String> =
|
||||
text.lowercase()
|
||||
.replace(nonAlphanumeric, " ")
|
||||
.trim()
|
||||
.split(" ")
|
||||
.filter { it.length >= MIN_TOKEN_LENGTH }
|
||||
.toSet()
|
||||
|
||||
private fun jaccard(a: Set<String>, b: Set<String>): Double {
|
||||
if (a.isEmpty() && b.isEmpty()) return 1.0
|
||||
val intersection = (a intersect b).size
|
||||
val union = (a union b).size
|
||||
return if (union == 0) 0.0 else intersection.toDouble() / union
|
||||
}
|
||||
|
||||
private fun parseJson(json: String): JsonObject? {
|
||||
val stripped = stripFence(json)
|
||||
return runCatching { lenientJson.parseToJsonElement(stripped) as? JsonObject }.getOrNull()
|
||||
}
|
||||
|
||||
private fun stringList(obj: JsonObject, key: String): List<String> =
|
||||
runCatching {
|
||||
(obj[key] as? JsonArray)?.mapNotNull { it.jsonPrimitive.contentOrNull } ?: emptyList()
|
||||
}.getOrElse { emptyList() }
|
||||
|
||||
private fun stripFence(text: String): String {
|
||||
val trimmed = text.trim()
|
||||
if (!trimmed.startsWith("```") || !trimmed.endsWith("```")) return text
|
||||
val withoutClose = trimmed.removeSuffix("```").trimEnd()
|
||||
val firstNewline = withoutClose.indexOf('\n')
|
||||
return if (firstNewline < 0) text else withoutClose.substring(firstNewline + 1)
|
||||
}
|
||||
}
|
||||
+70
-1
@@ -28,6 +28,7 @@ import com.correx.core.events.events.RefinementIterationEvent
|
||||
import com.correx.core.events.events.ApprovalRequestedEvent
|
||||
import com.correx.core.events.events.ArtifactContentStoredEvent
|
||||
import com.correx.core.events.events.ArtifactCreatedEvent
|
||||
import com.correx.core.events.events.BriefEchoMismatchEvent
|
||||
import com.correx.core.events.events.BriefGroundingCheckedEvent
|
||||
import com.correx.core.events.events.ContextTruncatedEvent
|
||||
import com.correx.core.events.events.ClarificationAnswer
|
||||
@@ -550,7 +551,11 @@ abstract class SessionOrchestrator(
|
||||
emitLlmArtifacts(sessionId, stageId, stageConfig)
|
||||
when (val produced = verifyProduces(sessionId, stageId, stageConfig)) {
|
||||
is StageExecutionResult.Success ->
|
||||
groundBriefReferences(sessionId, stageId, stageConfig, effectives)
|
||||
when (val grounded = groundBriefReferences(sessionId, stageId, stageConfig, effectives)) {
|
||||
is StageExecutionResult.Success ->
|
||||
checkBriefEcho(sessionId, stageId, stageConfig)
|
||||
is StageExecutionResult.Failure -> grounded
|
||||
}
|
||||
is StageExecutionResult.Failure -> produced
|
||||
}
|
||||
}
|
||||
@@ -1399,6 +1404,70 @@ abstract class SessionOrchestrator(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Brief echo gate (plan-pipeline-addenda §A1): for a stage that opted in (`brief_echo`),
|
||||
* diff the model's restatement artifact against the original brief. Both are already in the
|
||||
* artifact content cache (recorded via ArtifactCreatedEvent), so the diff is a pure function
|
||||
* of recorded data — deterministically recomputable on replay without re-probing any
|
||||
* environment (invariants #8, #9). Emits [BriefEchoMismatchEvent] on mismatch as a domain
|
||||
* signal, then fails the stage (retryable) so the pipeline cannot reach plan generation on a
|
||||
* misread brief.
|
||||
*
|
||||
* Brief-source selection among `needs`: prefers the artifact named "analysis"; if absent,
|
||||
* falls back to the single needs artifact present. When there are multiple needs and none is
|
||||
* named "analysis", the gate skips (ambiguous brief) and returns Success.
|
||||
*/
|
||||
private suspend fun checkBriefEcho(
|
||||
sessionId: SessionId,
|
||||
stageId: StageId,
|
||||
stageConfig: StageConfig,
|
||||
): StageExecutionResult {
|
||||
if (stageConfig.metadata["briefEcho"] != "true") return StageExecutionResult.Success(emptyList())
|
||||
|
||||
val echoSlot = stageConfig.produces.firstOrNull { it.kind.llmEmitted }
|
||||
?: return StageExecutionResult.Success(emptyList())
|
||||
val echo = artifactContentCache["${sessionId.value}:${echoSlot.name.value}"]
|
||||
?: return StageExecutionResult.Success(emptyList())
|
||||
|
||||
val brief = when {
|
||||
stageConfig.needs.any { it.value == "analysis" } ->
|
||||
artifactContentCache["${sessionId.value}:analysis"]
|
||||
stageConfig.needs.size == 1 ->
|
||||
artifactContentCache["${sessionId.value}:${stageConfig.needs.single().value}"]
|
||||
else -> null // ambiguous — skip
|
||||
} ?: return StageExecutionResult.Success(emptyList())
|
||||
|
||||
val div = BriefEchoDiff.compare(brief, echo)
|
||||
|
||||
return if (div.isMismatch) {
|
||||
emit(
|
||||
sessionId,
|
||||
BriefEchoMismatchEvent(
|
||||
sessionId = sessionId,
|
||||
stageId = stageId,
|
||||
uncoveredRequirements = div.uncoveredRequirements,
|
||||
hallucinatedFiles = div.hallucinatedFiles,
|
||||
hallucinatedSymbols = div.hallucinatedSymbols,
|
||||
),
|
||||
)
|
||||
log.warn(
|
||||
"[Orchestrator] brief echo mismatch session={} stage={} uncovered={} hallucinatedFiles={}",
|
||||
sessionId.value, stageId.value,
|
||||
div.uncoveredRequirements.joinToString(", "),
|
||||
div.hallucinatedFiles.joinToString(", "),
|
||||
)
|
||||
StageExecutionResult.Failure(
|
||||
"stage ${stageId.value} restated the brief unfaithfully — " +
|
||||
"dropped requirements: [${div.uncoveredRequirements.joinToString("; ")}]; " +
|
||||
"files not in the brief: [${div.hallucinatedFiles.joinToString(", ")}]. " +
|
||||
"Restate the brief exactly; do not drop requirements or invent files.",
|
||||
retryable = true,
|
||||
)
|
||||
} else {
|
||||
StageExecutionResult.Success(emptyList())
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun emitProcessResultEvents(
|
||||
sessionId: SessionId,
|
||||
stageId: StageId,
|
||||
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
package com.correx.core.kernel.orchestration
|
||||
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertFalse
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class BriefEchoDiffTest {
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
private fun brief(vararg requirements: String, files: List<String> = emptyList()): String {
|
||||
val reqs = requirements.joinToString(",") { "\"$it\"" }
|
||||
val fileArr = files.joinToString(",") { "\"$it\"" }
|
||||
return """{"summary":"do the thing","requirements":[$reqs],"affected_areas":[$fileArr]}"""
|
||||
}
|
||||
|
||||
private fun echo(
|
||||
vararg criteria: String,
|
||||
files: List<String> = emptyList(),
|
||||
symbols: List<String> = emptyList(),
|
||||
): String {
|
||||
val crit = criteria.joinToString(",") { "\"$it\"" }
|
||||
val fileArr = files.joinToString(",") { "\"$it\"" }
|
||||
val symArr = symbols.joinToString(",") { "\"$it\"" }
|
||||
return """{"goal":"do the thing","referenced_files":[$fileArr],""" +
|
||||
""""referenced_symbols":[$symArr],"acceptance_criteria":[$crit]}"""
|
||||
}
|
||||
|
||||
// --- faithful echo → no mismatch ---
|
||||
|
||||
@Test
|
||||
fun `faithful echo produces no mismatch`() {
|
||||
val b = brief("the system must retry on transient failures", "all retry attempts must be logged")
|
||||
val e = echo("the system must retry on transient failures", "all retry attempts must be logged")
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertFalse(div.isMismatch)
|
||||
assertTrue(div.uncoveredRequirements.isEmpty())
|
||||
assertTrue(div.hallucinatedFiles.isEmpty())
|
||||
}
|
||||
|
||||
// --- dropped requirement → uncovered ---
|
||||
|
||||
@Test
|
||||
fun `dropped requirement is reported as uncovered`() {
|
||||
val b = brief("must emit an event", "must support retries")
|
||||
val e = echo("must emit an event")
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertTrue(div.isMismatch)
|
||||
assertTrue(div.uncoveredRequirements.any { it.contains("retries") })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `all requirements dropped marks all as uncovered`() {
|
||||
val b = brief("req alpha", "req beta")
|
||||
val e = echo() // no criteria
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertTrue(div.isMismatch)
|
||||
assertEquals(2, div.uncoveredRequirements.size)
|
||||
}
|
||||
|
||||
// --- slightly paraphrased but high-overlap requirement → covered ---
|
||||
|
||||
@Test
|
||||
fun `near-verbatim paraphrase with high token overlap is covered`() {
|
||||
val b = brief("the gate must emit a BriefEchoMismatchEvent on divergence")
|
||||
val e = echo("gate must emit BriefEchoMismatchEvent on divergence")
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertFalse(div.isMismatch, "High-overlap paraphrase should be covered: ${div.uncoveredRequirements}")
|
||||
}
|
||||
|
||||
// --- hallucinated file ---
|
||||
|
||||
@Test
|
||||
fun `file not in brief is hallucinated`() {
|
||||
val b = brief("do something", files = listOf("core/kernel/Foo.kt"))
|
||||
val e = echo("do something", files = listOf("core/ghost/Phantom.kt"))
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertTrue(div.hallucinatedFiles.contains("core/ghost/Phantom.kt"))
|
||||
assertTrue(div.isMismatch)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `file present in brief is not hallucinated`() {
|
||||
val b = brief("do something", files = listOf("core/kernel/Foo.kt"))
|
||||
val e = echo("do something", files = listOf("core/kernel/Foo.kt"))
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertFalse(div.hallucinatedFiles.contains("core/kernel/Foo.kt"))
|
||||
assertFalse(div.isMismatch)
|
||||
}
|
||||
|
||||
// --- symbol divergence → recorded but isMismatch=false ---
|
||||
|
||||
@Test
|
||||
fun `hallucinated symbol is recorded but does not cause isMismatch`() {
|
||||
val b = brief("implement the interface")
|
||||
val e = echo("implement the interface", symbols = listOf("GhostService"))
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
// Symbol not in brief → hallucinated
|
||||
assertTrue(div.hallucinatedSymbols.contains("GhostService"))
|
||||
// But isMismatch should be false (symbols are display-only in v1)
|
||||
assertFalse(div.isMismatch, "Symbol-only divergence must not block in v1")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `symbol present in brief is not hallucinated`() {
|
||||
val b = brief("implement SessionOrchestrator interface")
|
||||
val e = echo("implement SessionOrchestrator interface", symbols = listOf("SessionOrchestrator"))
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertFalse(div.hallucinatedSymbols.contains("SessionOrchestrator"))
|
||||
}
|
||||
|
||||
// --- malformed / empty echo → total mismatch ---
|
||||
|
||||
@Test
|
||||
fun `malformed echo is treated as total mismatch`() {
|
||||
val b = brief("req A", "req B")
|
||||
val div = BriefEchoDiff.compare(b, "not valid json at all")
|
||||
assertTrue(div.isMismatch)
|
||||
assertEquals(2, div.uncoveredRequirements.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `empty echo JSON object is total mismatch`() {
|
||||
val b = brief("req A")
|
||||
val div = BriefEchoDiff.compare(b, "{}")
|
||||
assertTrue(div.isMismatch)
|
||||
assertTrue(div.uncoveredRequirements.isNotEmpty())
|
||||
}
|
||||
|
||||
// --- fenced JSON is tolerated ---
|
||||
|
||||
@Test
|
||||
fun `fenced echo JSON is tolerated`() {
|
||||
val b = brief("must support retries")
|
||||
val fenced = "```json\n${echo("must support retries")}\n```"
|
||||
val div = BriefEchoDiff.compare(b, fenced)
|
||||
assertFalse(div.isMismatch, "Fenced echo should parse and match: ${div.uncoveredRequirements}")
|
||||
}
|
||||
|
||||
// --- no requirements in brief → no mismatch ---
|
||||
|
||||
@Test
|
||||
fun `brief with no requirements yields no mismatch`() {
|
||||
val b = """{"summary":"something","requirements":[]}"""
|
||||
val e = echo()
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertFalse(div.isMismatch)
|
||||
assertTrue(div.uncoveredRequirements.isEmpty())
|
||||
}
|
||||
|
||||
// --- tokenize ---
|
||||
|
||||
@Test
|
||||
fun `tokenize lowercases and strips punctuation`() {
|
||||
val tokens = BriefEchoDiff.tokenize("Must EMIT a BriefEchoMismatchEvent!")
|
||||
assertTrue(tokens.contains("must"))
|
||||
assertTrue(tokens.contains("emit"))
|
||||
assertTrue(tokens.contains("briefechomismatchevent"))
|
||||
assertFalse(tokens.contains("a")) // length < 2
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"goal": {
|
||||
"type": "string",
|
||||
"description": "the analyst brief's goal restated in your own words"
|
||||
},
|
||||
"referenced_files": {
|
||||
"type": "array",
|
||||
"description": "workspace-relative file paths named in the brief, one per item",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"referenced_symbols": {
|
||||
"type": "array",
|
||||
"description": "class, interface, or function names named in the brief, one per item",
|
||||
"items": { "type": "string" }
|
||||
},
|
||||
"acceptance_criteria": {
|
||||
"type": "array",
|
||||
"description": "the brief's requirements / acceptance criteria restated faithfully, one per item",
|
||||
"minItems": 1,
|
||||
"items": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"required": ["goal", "acceptance_criteria"],
|
||||
"additionalProperties": true
|
||||
}
|
||||
@@ -29,6 +29,11 @@ id = "review_report"
|
||||
schema_path = "schemas/review_report.json"
|
||||
llm_emitted = true
|
||||
|
||||
[[artifacts]]
|
||||
id = "brief_echo"
|
||||
schema_path = "schemas/brief_echo.json"
|
||||
llm_emitted = true
|
||||
|
||||
[[artifacts]]
|
||||
id = "execution_plan"
|
||||
schema_path = "schemas/execution_plan.json"
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
You are about to plan. Before you do, RESTATE the analyst brief faithfully as JSON.
|
||||
|
||||
Rules:
|
||||
- Copy requirements exactly — do not merge, drop, or reword them.
|
||||
- Only reference files that are explicitly named in the brief.
|
||||
- Do not invent new scope, new files, or new symbols.
|
||||
|
||||
Emit your result as the `brief_echo` artifact (JSON, schema provided):
|
||||
- `goal`: the brief's goal in your own words (one sentence).
|
||||
- `referenced_files`: workspace-relative file paths named in the brief, verbatim.
|
||||
- `referenced_symbols`: class, interface, or function names named in the brief, verbatim.
|
||||
- `acceptance_criteria`: every requirement / acceptance criterion from the brief, one per item, verbatim.
|
||||
|
||||
If a field has no relevant content, emit an empty array. Do not add items that are not in the brief.
|
||||
@@ -1,4 +1,4 @@
|
||||
# Role pipeline: analyst → architect → planner → implementer ⇄ reviewer
|
||||
# Role pipeline: analyst → architect → brief_echo → planner → implementer ⇄ reviewer
|
||||
#
|
||||
# Each stage produces a typed artifact that the next stage `needs`, so work flows forward
|
||||
# without a human relaying notes. The decision journal (pinned into every stage's context)
|
||||
@@ -18,6 +18,8 @@
|
||||
# id = "impl_plan"; schema_path = "schemas/impl_plan.json"; llm_emitted = true
|
||||
# [[artifacts]]
|
||||
# id = "review_report"; schema_path = "schemas/review_report.json"; llm_emitted = true
|
||||
# [[artifacts]]
|
||||
# id = "brief_echo"; schema_path = "schemas/brief_echo.json"; llm_emitted = true
|
||||
#
|
||||
# Prompt files (prompts/*.md, relative to this workflow) must exist for a real run.
|
||||
|
||||
@@ -45,7 +47,20 @@ produces = [{ name = "design", kind = "design" }]
|
||||
token_budget = 16384
|
||||
max_retries = 2
|
||||
|
||||
# 3. Break the design into ordered, verifiable steps.
|
||||
# 3a. Before planning: echo the analyst brief back as a structured artifact.
|
||||
# The brief_echo gate diffs the restatement against the original analysis;
|
||||
# on divergence (dropped requirements or hallucinated files) it fails the
|
||||
# stage (retryable) so the pipeline cannot reach plan generation on a misread brief.
|
||||
[[stages]]
|
||||
id = "brief_echo"
|
||||
prompt = "prompts/brief_echo.md"
|
||||
needs = ["analysis"]
|
||||
produces = [{ name = "brief_echo", kind = "brief_echo" }]
|
||||
brief_echo = true
|
||||
token_budget = 8192
|
||||
max_retries = 2
|
||||
|
||||
# 3b. Break the design into ordered, verifiable steps.
|
||||
[[stages]]
|
||||
id = "planner"
|
||||
prompt = "prompts/planner.md"
|
||||
@@ -90,12 +105,19 @@ condition_type = "artifact_validated"
|
||||
condition_artifact_id = "analysis"
|
||||
|
||||
[[transitions]]
|
||||
id = "architect-to-planner"
|
||||
id = "architect-to-brief-echo"
|
||||
from = "architect"
|
||||
to = "planner"
|
||||
to = "brief_echo"
|
||||
condition_type = "artifact_validated"
|
||||
condition_artifact_id = "design"
|
||||
|
||||
[[transitions]]
|
||||
id = "brief-echo-to-planner"
|
||||
from = "brief_echo"
|
||||
to = "planner"
|
||||
condition_type = "artifact_validated"
|
||||
condition_artifact_id = "brief_echo"
|
||||
|
||||
[[transitions]]
|
||||
id = "planner-to-implementer"
|
||||
from = "planner"
|
||||
|
||||
+2
@@ -47,6 +47,7 @@ private data class StageSection(
|
||||
@param:JsonProperty("requires_approval") val requiresApproval: Boolean = false,
|
||||
@param:JsonProperty("inject_artifact_kinds") val injectArtifactKinds: Boolean = false,
|
||||
@param:JsonProperty("ground_references") val groundReferences: Boolean = false,
|
||||
@param:JsonProperty("brief_echo") val briefEcho: Boolean = false,
|
||||
)
|
||||
|
||||
// condition fields flattened into the transition row
|
||||
@@ -121,6 +122,7 @@ class TomlWorkflowLoader(
|
||||
if (s.requiresApproval) put("requiresApproval", "true")
|
||||
if (s.injectArtifactKinds) put("injectArtifactKinds", "true")
|
||||
if (s.groundReferences) put("groundReferences", "true")
|
||||
if (s.briefEcho) put("briefEcho", "true")
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
+3
-2
@@ -29,6 +29,7 @@ class RolePipelineWorkflowTest {
|
||||
private val registry = DefaultArtifactKindRegistry().apply {
|
||||
register(llmKind("analysis"))
|
||||
register(llmKind("design"))
|
||||
register(llmKind("brief_echo"))
|
||||
register(llmKind("impl_plan"))
|
||||
register(llmKind("review_report"))
|
||||
}
|
||||
@@ -53,10 +54,10 @@ class RolePipelineWorkflowTest {
|
||||
val graph = TomlWorkflowLoader(registry).load(path!!)
|
||||
|
||||
assertEquals(
|
||||
setOf("analyst", "architect", "planner", "implementer", "reviewer"),
|
||||
setOf("analyst", "architect", "brief_echo", "planner", "implementer", "reviewer"),
|
||||
graph.stages.keys.map { it.value }.toSet(),
|
||||
)
|
||||
assertEquals(6, graph.transitions.size)
|
||||
assertEquals(7, graph.transitions.size)
|
||||
|
||||
// Forward chaining via needs.
|
||||
assertTrue(graph.stages[StageId("reviewer")]!!.needs.map { it.value }.containsAll(listOf("patch", "impl_plan")))
|
||||
|
||||
Reference in New Issue
Block a user