feat(kernel): brief echo-back gate (plan-pipeline-addenda A1)
New brief_echo stage before the planner in role_pipeline: the model restates the analyst brief as a structured artifact, and a deterministic gate (checkBriefEcho, opt-in via brief_echo=true) diffs the restatement against the original brief. On divergence — dropped requirements (Jaccard coverage < 0.34) or hallucinated files — it emits BriefEchoMismatchEvent and fails the stage retryably, so a misread brief never reaches plan generation. The diff (BriefEchoDiff) is a pure function of two recorded artifacts, so it is replay-safe and records no observation; the event fires only on mismatch, for audit. Symbols are recorded but non-blocking in v1. Mirrors the groundBrief- References gate. role_pipeline only; freestyle_planning wiring is a follow-up. Runtime install (like research): copy brief_echo.json + brief_echo.md and the [[artifacts]] block into ~/.config/correx.
This commit is contained in:
@@ -0,0 +1,27 @@
|
|||||||
|
package com.correx.core.events.events
|
||||||
|
|
||||||
|
import com.correx.core.events.types.SessionId
|
||||||
|
import com.correx.core.events.types.StageId
|
||||||
|
import kotlinx.serialization.SerialName
|
||||||
|
import kotlinx.serialization.Serializable
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Emitted when the model's restatement of the analyst brief diverges beyond the acceptance
|
||||||
|
* threshold: it dropped requirements or hallucinated file references. Recorded as a domain
|
||||||
|
* signal for audit and narration. The gate then fails the stage (retryable) so the pipeline
|
||||||
|
* cannot reach plan generation on a misread brief.
|
||||||
|
*
|
||||||
|
* Determinism note: this is NOT an environment observation (invariant #9). The diff is a pure
|
||||||
|
* function of two already-recorded artifacts (the analysis brief and the brief_echo), so it is
|
||||||
|
* deterministically recomputable on replay without re-emitting this event. The event is emitted
|
||||||
|
* only on mismatch, as a domain signal.
|
||||||
|
*/
|
||||||
|
@Serializable
|
||||||
|
@SerialName("BriefEchoMismatch")
|
||||||
|
data class BriefEchoMismatchEvent(
|
||||||
|
val sessionId: SessionId,
|
||||||
|
val stageId: StageId,
|
||||||
|
val uncoveredRequirements: List<String>,
|
||||||
|
val hallucinatedFiles: List<String>,
|
||||||
|
val hallucinatedSymbols: List<String>,
|
||||||
|
) : EventPayload
|
||||||
@@ -8,6 +8,7 @@ import com.correx.core.events.events.ArtifactContentStoredEvent
|
|||||||
import com.correx.core.events.events.ArtifactCreatedEvent
|
import com.correx.core.events.events.ArtifactCreatedEvent
|
||||||
import com.correx.core.events.events.ArtifactValidatedEvent
|
import com.correx.core.events.events.ArtifactValidatedEvent
|
||||||
import com.correx.core.events.events.ArtifactValidatingEvent
|
import com.correx.core.events.events.ArtifactValidatingEvent
|
||||||
|
import com.correx.core.events.events.BriefEchoMismatchEvent
|
||||||
import com.correx.core.events.events.BriefGroundingCheckedEvent
|
import com.correx.core.events.events.BriefGroundingCheckedEvent
|
||||||
import com.correx.core.events.events.ChatSessionStartedEvent
|
import com.correx.core.events.events.ChatSessionStartedEvent
|
||||||
import com.correx.core.events.events.ClarificationAnsweredEvent
|
import com.correx.core.events.events.ClarificationAnsweredEvent
|
||||||
@@ -108,6 +109,7 @@ val eventModule = SerializersModule {
|
|||||||
subclass(WorkspaceStateObservedEvent::class)
|
subclass(WorkspaceStateObservedEvent::class)
|
||||||
subclass(RepoKnowledgeRetrievedEvent::class)
|
subclass(RepoKnowledgeRetrievedEvent::class)
|
||||||
subclass(BriefGroundingCheckedEvent::class)
|
subclass(BriefGroundingCheckedEvent::class)
|
||||||
|
subclass(BriefEchoMismatchEvent::class)
|
||||||
subclass(RiskAssessedEvent::class)
|
subclass(RiskAssessedEvent::class)
|
||||||
subclass(ChatSessionStartedEvent::class)
|
subclass(ChatSessionStartedEvent::class)
|
||||||
subclass(ChatTurnEvent::class)
|
subclass(ChatTurnEvent::class)
|
||||||
|
|||||||
+54
@@ -0,0 +1,54 @@
|
|||||||
|
package com.correx.core.events.serialization
|
||||||
|
|
||||||
|
import com.correx.core.events.events.BriefEchoMismatchEvent
|
||||||
|
import com.correx.core.events.events.EventPayload
|
||||||
|
import com.correx.core.events.types.SessionId
|
||||||
|
import com.correx.core.events.types.StageId
|
||||||
|
import kotlin.test.Test
|
||||||
|
import kotlin.test.assertEquals
|
||||||
|
import kotlin.test.assertTrue
|
||||||
|
|
||||||
|
class BriefEchoMismatchEventSerializationTest {
|
||||||
|
|
||||||
|
private val sample = BriefEchoMismatchEvent(
|
||||||
|
sessionId = SessionId("sess-1"),
|
||||||
|
stageId = StageId("brief_echo"),
|
||||||
|
uncoveredRequirements = listOf("must support retries", "must emit an event"),
|
||||||
|
hallucinatedFiles = listOf("core/ghost/Phantom.kt"),
|
||||||
|
hallucinatedSymbols = listOf("GhostService"),
|
||||||
|
)
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `round-trips as polymorphic EventPayload`() {
|
||||||
|
val encoded = eventJson.encodeToString(EventPayload.serializer(), sample)
|
||||||
|
assertTrue(encoded.contains("\"type\":\"BriefEchoMismatch\""), "SerialName must be present: $encoded")
|
||||||
|
val decoded = eventJson.decodeFromString(EventPayload.serializer(), encoded)
|
||||||
|
assertEquals(sample, decoded)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `decodes hand-written BriefEchoMismatch JSON`() {
|
||||||
|
val json = """
|
||||||
|
{"type":"BriefEchoMismatch","sessionId":"s","stageId":"brief_echo",
|
||||||
|
"uncoveredRequirements":["req-1"],"hallucinatedFiles":["a/b.kt"],"hallucinatedSymbols":[]}
|
||||||
|
""".trimIndent()
|
||||||
|
val decoded = eventJson.decodeFromString(EventPayload.serializer(), json)
|
||||||
|
assertTrue(decoded is BriefEchoMismatchEvent)
|
||||||
|
val event = decoded as BriefEchoMismatchEvent
|
||||||
|
assertEquals(listOf("req-1"), event.uncoveredRequirements)
|
||||||
|
assertEquals(listOf("a/b.kt"), event.hallucinatedFiles)
|
||||||
|
assertTrue(event.hallucinatedSymbols.isEmpty())
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `empty lists round-trip correctly`() {
|
||||||
|
val empty = sample.copy(
|
||||||
|
uncoveredRequirements = emptyList(),
|
||||||
|
hallucinatedFiles = emptyList(),
|
||||||
|
hallucinatedSymbols = emptyList(),
|
||||||
|
)
|
||||||
|
val encoded = eventJson.encodeToString(EventPayload.serializer(), empty)
|
||||||
|
val decoded = eventJson.decodeFromString(EventPayload.serializer(), encoded)
|
||||||
|
assertEquals(empty, decoded)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
package com.correx.core.kernel.orchestration
|
||||||
|
|
||||||
|
import kotlinx.serialization.json.Json
|
||||||
|
import kotlinx.serialization.json.JsonArray
|
||||||
|
import kotlinx.serialization.json.JsonObject
|
||||||
|
import kotlinx.serialization.json.contentOrNull
|
||||||
|
import kotlinx.serialization.json.jsonPrimitive
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pure deterministic diff between an analyst brief and a model's echo restatement.
|
||||||
|
*
|
||||||
|
* Determinism / invariant #8: reads ONLY the two recorded artifact strings — no I/O, no
|
||||||
|
* external calls. Because both inputs are already in the event log (ArtifactCreatedEvent),
|
||||||
|
* the result is recomputable on replay without re-emitting any observation event.
|
||||||
|
*
|
||||||
|
* isMismatch triggers gate failure when requirements are uncovered or files are hallucinated.
|
||||||
|
* Hallucinated symbols are recorded for audit but do NOT block in v1 (display-only, mirroring
|
||||||
|
* the role-reliability contradiction-check "display-only in v1" stance).
|
||||||
|
*/
|
||||||
|
data class BriefEchoDivergence(
|
||||||
|
val uncoveredRequirements: List<String>,
|
||||||
|
val hallucinatedFiles: List<String>,
|
||||||
|
/** Recorded for audit/narration; does NOT contribute to isMismatch in v1. */
|
||||||
|
val hallucinatedSymbols: List<String>,
|
||||||
|
) {
|
||||||
|
val isMismatch: Boolean
|
||||||
|
get() = uncoveredRequirements.isNotEmpty() || hallucinatedFiles.isNotEmpty()
|
||||||
|
}
|
||||||
|
|
||||||
|
internal object BriefEchoDiff {
|
||||||
|
|
||||||
|
/** Jaccard overlap below this threshold means a requirement is "uncovered" by the echo. */
|
||||||
|
private const val COVERAGE_THRESHOLD = 0.34
|
||||||
|
|
||||||
|
/** Minimum token length for comparison — very short tokens add noise. */
|
||||||
|
private const val MIN_TOKEN_LENGTH = 2
|
||||||
|
|
||||||
|
private val lenientJson = Json { ignoreUnknownKeys = true; isLenient = true }
|
||||||
|
private val nonAlphanumeric = Regex("[^A-Za-z0-9]+")
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compare the analyst brief JSON against the model's echo JSON, returning divergences.
|
||||||
|
* On an unparseable echo, every brief requirement is reported as uncovered (total mismatch),
|
||||||
|
* so a garbage echo always fails the gate.
|
||||||
|
*/
|
||||||
|
fun compare(briefJson: String, echoJson: String): BriefEchoDivergence {
|
||||||
|
val brief = parseJson(briefJson)
|
||||||
|
val echo = parseJson(echoJson)
|
||||||
|
val briefFullTokens = tokenize(briefJson)
|
||||||
|
val briefFiles = BriefReferenceExtractor.fileReferences(briefJson)
|
||||||
|
return BriefEchoDivergence(
|
||||||
|
uncoveredRequirements = uncoveredRequirements(brief, echo),
|
||||||
|
hallucinatedFiles = hallucinatedFiles(echo, briefFullTokens, briefFiles),
|
||||||
|
hallucinatedSymbols = hallucinatedSymbols(echo, briefFullTokens),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Each brief requirement whose best Jaccard overlap with any echo criterion is below threshold. */
|
||||||
|
private fun uncoveredRequirements(brief: JsonObject?, echo: JsonObject?): List<String> {
|
||||||
|
val requirements = brief?.let { stringList(it, "requirements") } ?: emptyList()
|
||||||
|
val criteria = echo?.let { stringList(it, "acceptance_criteria") } ?: emptyList()
|
||||||
|
return requirements.filter { req ->
|
||||||
|
val reqTokens = tokenize(req)
|
||||||
|
if (reqTokens.isEmpty()) return@filter false
|
||||||
|
val bestOverlap = criteria.maxOfOrNull { jaccard(reqTokens, tokenize(it)) } ?: 0.0
|
||||||
|
bestOverlap < COVERAGE_THRESHOLD
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Echo files not grounded in the brief. A file is hallucinated when no token of its basename
|
||||||
|
* stem (extension dropped — an extension like "kt" matches any source file in the brief and is
|
||||||
|
* pure noise) appears in the brief, AND it matches no extracted brief file path.
|
||||||
|
*/
|
||||||
|
private fun hallucinatedFiles(
|
||||||
|
echo: JsonObject?,
|
||||||
|
briefFullTokens: Set<String>,
|
||||||
|
briefFiles: List<String>,
|
||||||
|
): List<String> {
|
||||||
|
val echoFiles = echo?.let { stringList(it, "referenced_files") } ?: emptyList()
|
||||||
|
return echoFiles.filter { f ->
|
||||||
|
val stemTokens = tokenize(f.substringAfterLast('/').substringBeforeLast('.'))
|
||||||
|
if (stemTokens.isEmpty()) return@filter true
|
||||||
|
stemTokens.none { it in briefFullTokens } &&
|
||||||
|
briefFiles.none { it == f || it.endsWith(f) || f.endsWith(it) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Echo symbols with no token present anywhere in the brief (recorded for audit, non-blocking). */
|
||||||
|
private fun hallucinatedSymbols(echo: JsonObject?, briefFullTokens: Set<String>): List<String> {
|
||||||
|
val echoSymbols = echo?.let { stringList(it, "referenced_symbols") } ?: emptyList()
|
||||||
|
return echoSymbols.filter { sym -> tokenize(sym).none { it in briefFullTokens } }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Lowercase, replace non-alphanumeric with spaces, split, drop tokens shorter than MIN_TOKEN_LENGTH. */
|
||||||
|
internal fun tokenize(text: String): Set<String> =
|
||||||
|
text.lowercase()
|
||||||
|
.replace(nonAlphanumeric, " ")
|
||||||
|
.trim()
|
||||||
|
.split(" ")
|
||||||
|
.filter { it.length >= MIN_TOKEN_LENGTH }
|
||||||
|
.toSet()
|
||||||
|
|
||||||
|
private fun jaccard(a: Set<String>, b: Set<String>): Double {
|
||||||
|
if (a.isEmpty() && b.isEmpty()) return 1.0
|
||||||
|
val intersection = (a intersect b).size
|
||||||
|
val union = (a union b).size
|
||||||
|
return if (union == 0) 0.0 else intersection.toDouble() / union
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun parseJson(json: String): JsonObject? {
|
||||||
|
val stripped = stripFence(json)
|
||||||
|
return runCatching { lenientJson.parseToJsonElement(stripped) as? JsonObject }.getOrNull()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun stringList(obj: JsonObject, key: String): List<String> =
|
||||||
|
runCatching {
|
||||||
|
(obj[key] as? JsonArray)?.mapNotNull { it.jsonPrimitive.contentOrNull } ?: emptyList()
|
||||||
|
}.getOrElse { emptyList() }
|
||||||
|
|
||||||
|
private fun stripFence(text: String): String {
|
||||||
|
val trimmed = text.trim()
|
||||||
|
if (!trimmed.startsWith("```") || !trimmed.endsWith("```")) return text
|
||||||
|
val withoutClose = trimmed.removeSuffix("```").trimEnd()
|
||||||
|
val firstNewline = withoutClose.indexOf('\n')
|
||||||
|
return if (firstNewline < 0) text else withoutClose.substring(firstNewline + 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
+70
-1
@@ -28,6 +28,7 @@ import com.correx.core.events.events.RefinementIterationEvent
|
|||||||
import com.correx.core.events.events.ApprovalRequestedEvent
|
import com.correx.core.events.events.ApprovalRequestedEvent
|
||||||
import com.correx.core.events.events.ArtifactContentStoredEvent
|
import com.correx.core.events.events.ArtifactContentStoredEvent
|
||||||
import com.correx.core.events.events.ArtifactCreatedEvent
|
import com.correx.core.events.events.ArtifactCreatedEvent
|
||||||
|
import com.correx.core.events.events.BriefEchoMismatchEvent
|
||||||
import com.correx.core.events.events.BriefGroundingCheckedEvent
|
import com.correx.core.events.events.BriefGroundingCheckedEvent
|
||||||
import com.correx.core.events.events.ContextTruncatedEvent
|
import com.correx.core.events.events.ContextTruncatedEvent
|
||||||
import com.correx.core.events.events.ClarificationAnswer
|
import com.correx.core.events.events.ClarificationAnswer
|
||||||
@@ -550,7 +551,11 @@ abstract class SessionOrchestrator(
|
|||||||
emitLlmArtifacts(sessionId, stageId, stageConfig)
|
emitLlmArtifacts(sessionId, stageId, stageConfig)
|
||||||
when (val produced = verifyProduces(sessionId, stageId, stageConfig)) {
|
when (val produced = verifyProduces(sessionId, stageId, stageConfig)) {
|
||||||
is StageExecutionResult.Success ->
|
is StageExecutionResult.Success ->
|
||||||
groundBriefReferences(sessionId, stageId, stageConfig, effectives)
|
when (val grounded = groundBriefReferences(sessionId, stageId, stageConfig, effectives)) {
|
||||||
|
is StageExecutionResult.Success ->
|
||||||
|
checkBriefEcho(sessionId, stageId, stageConfig)
|
||||||
|
is StageExecutionResult.Failure -> grounded
|
||||||
|
}
|
||||||
is StageExecutionResult.Failure -> produced
|
is StageExecutionResult.Failure -> produced
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1399,6 +1404,70 @@ abstract class SessionOrchestrator(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Brief echo gate (plan-pipeline-addenda §A1): for a stage that opted in (`brief_echo`),
|
||||||
|
* diff the model's restatement artifact against the original brief. Both are already in the
|
||||||
|
* artifact content cache (recorded via ArtifactCreatedEvent), so the diff is a pure function
|
||||||
|
* of recorded data — deterministically recomputable on replay without re-probing any
|
||||||
|
* environment (invariants #8, #9). Emits [BriefEchoMismatchEvent] on mismatch as a domain
|
||||||
|
* signal, then fails the stage (retryable) so the pipeline cannot reach plan generation on a
|
||||||
|
* misread brief.
|
||||||
|
*
|
||||||
|
* Brief-source selection among `needs`: prefers the artifact named "analysis"; if absent,
|
||||||
|
* falls back to the single needs artifact present. When there are multiple needs and none is
|
||||||
|
* named "analysis", the gate skips (ambiguous brief) and returns Success.
|
||||||
|
*/
|
||||||
|
private suspend fun checkBriefEcho(
|
||||||
|
sessionId: SessionId,
|
||||||
|
stageId: StageId,
|
||||||
|
stageConfig: StageConfig,
|
||||||
|
): StageExecutionResult {
|
||||||
|
if (stageConfig.metadata["briefEcho"] != "true") return StageExecutionResult.Success(emptyList())
|
||||||
|
|
||||||
|
val echoSlot = stageConfig.produces.firstOrNull { it.kind.llmEmitted }
|
||||||
|
?: return StageExecutionResult.Success(emptyList())
|
||||||
|
val echo = artifactContentCache["${sessionId.value}:${echoSlot.name.value}"]
|
||||||
|
?: return StageExecutionResult.Success(emptyList())
|
||||||
|
|
||||||
|
val brief = when {
|
||||||
|
stageConfig.needs.any { it.value == "analysis" } ->
|
||||||
|
artifactContentCache["${sessionId.value}:analysis"]
|
||||||
|
stageConfig.needs.size == 1 ->
|
||||||
|
artifactContentCache["${sessionId.value}:${stageConfig.needs.single().value}"]
|
||||||
|
else -> null // ambiguous — skip
|
||||||
|
} ?: return StageExecutionResult.Success(emptyList())
|
||||||
|
|
||||||
|
val div = BriefEchoDiff.compare(brief, echo)
|
||||||
|
|
||||||
|
return if (div.isMismatch) {
|
||||||
|
emit(
|
||||||
|
sessionId,
|
||||||
|
BriefEchoMismatchEvent(
|
||||||
|
sessionId = sessionId,
|
||||||
|
stageId = stageId,
|
||||||
|
uncoveredRequirements = div.uncoveredRequirements,
|
||||||
|
hallucinatedFiles = div.hallucinatedFiles,
|
||||||
|
hallucinatedSymbols = div.hallucinatedSymbols,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
log.warn(
|
||||||
|
"[Orchestrator] brief echo mismatch session={} stage={} uncovered={} hallucinatedFiles={}",
|
||||||
|
sessionId.value, stageId.value,
|
||||||
|
div.uncoveredRequirements.joinToString(", "),
|
||||||
|
div.hallucinatedFiles.joinToString(", "),
|
||||||
|
)
|
||||||
|
StageExecutionResult.Failure(
|
||||||
|
"stage ${stageId.value} restated the brief unfaithfully — " +
|
||||||
|
"dropped requirements: [${div.uncoveredRequirements.joinToString("; ")}]; " +
|
||||||
|
"files not in the brief: [${div.hallucinatedFiles.joinToString(", ")}]. " +
|
||||||
|
"Restate the brief exactly; do not drop requirements or invent files.",
|
||||||
|
retryable = true,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
StageExecutionResult.Success(emptyList())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private suspend fun emitProcessResultEvents(
|
private suspend fun emitProcessResultEvents(
|
||||||
sessionId: SessionId,
|
sessionId: SessionId,
|
||||||
stageId: StageId,
|
stageId: StageId,
|
||||||
|
|||||||
+162
@@ -0,0 +1,162 @@
|
|||||||
|
package com.correx.core.kernel.orchestration
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions.assertEquals
|
||||||
|
import org.junit.jupiter.api.Assertions.assertFalse
|
||||||
|
import org.junit.jupiter.api.Assertions.assertTrue
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
|
||||||
|
class BriefEchoDiffTest {
|
||||||
|
|
||||||
|
// --- helpers ---
|
||||||
|
|
||||||
|
private fun brief(vararg requirements: String, files: List<String> = emptyList()): String {
|
||||||
|
val reqs = requirements.joinToString(",") { "\"$it\"" }
|
||||||
|
val fileArr = files.joinToString(",") { "\"$it\"" }
|
||||||
|
return """{"summary":"do the thing","requirements":[$reqs],"affected_areas":[$fileArr]}"""
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun echo(
|
||||||
|
vararg criteria: String,
|
||||||
|
files: List<String> = emptyList(),
|
||||||
|
symbols: List<String> = emptyList(),
|
||||||
|
): String {
|
||||||
|
val crit = criteria.joinToString(",") { "\"$it\"" }
|
||||||
|
val fileArr = files.joinToString(",") { "\"$it\"" }
|
||||||
|
val symArr = symbols.joinToString(",") { "\"$it\"" }
|
||||||
|
return """{"goal":"do the thing","referenced_files":[$fileArr],""" +
|
||||||
|
""""referenced_symbols":[$symArr],"acceptance_criteria":[$crit]}"""
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- faithful echo → no mismatch ---
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `faithful echo produces no mismatch`() {
|
||||||
|
val b = brief("the system must retry on transient failures", "all retry attempts must be logged")
|
||||||
|
val e = echo("the system must retry on transient failures", "all retry attempts must be logged")
|
||||||
|
val div = BriefEchoDiff.compare(b, e)
|
||||||
|
assertFalse(div.isMismatch)
|
||||||
|
assertTrue(div.uncoveredRequirements.isEmpty())
|
||||||
|
assertTrue(div.hallucinatedFiles.isEmpty())
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- dropped requirement → uncovered ---
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `dropped requirement is reported as uncovered`() {
|
||||||
|
val b = brief("must emit an event", "must support retries")
|
||||||
|
val e = echo("must emit an event")
|
||||||
|
val div = BriefEchoDiff.compare(b, e)
|
||||||
|
assertTrue(div.isMismatch)
|
||||||
|
assertTrue(div.uncoveredRequirements.any { it.contains("retries") })
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `all requirements dropped marks all as uncovered`() {
|
||||||
|
val b = brief("req alpha", "req beta")
|
||||||
|
val e = echo() // no criteria
|
||||||
|
val div = BriefEchoDiff.compare(b, e)
|
||||||
|
assertTrue(div.isMismatch)
|
||||||
|
assertEquals(2, div.uncoveredRequirements.size)
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- slightly paraphrased but high-overlap requirement → covered ---
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `near-verbatim paraphrase with high token overlap is covered`() {
|
||||||
|
val b = brief("the gate must emit a BriefEchoMismatchEvent on divergence")
|
||||||
|
val e = echo("gate must emit BriefEchoMismatchEvent on divergence")
|
||||||
|
val div = BriefEchoDiff.compare(b, e)
|
||||||
|
assertFalse(div.isMismatch, "High-overlap paraphrase should be covered: ${div.uncoveredRequirements}")
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- hallucinated file ---
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `file not in brief is hallucinated`() {
|
||||||
|
val b = brief("do something", files = listOf("core/kernel/Foo.kt"))
|
||||||
|
val e = echo("do something", files = listOf("core/ghost/Phantom.kt"))
|
||||||
|
val div = BriefEchoDiff.compare(b, e)
|
||||||
|
assertTrue(div.hallucinatedFiles.contains("core/ghost/Phantom.kt"))
|
||||||
|
assertTrue(div.isMismatch)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `file present in brief is not hallucinated`() {
|
||||||
|
val b = brief("do something", files = listOf("core/kernel/Foo.kt"))
|
||||||
|
val e = echo("do something", files = listOf("core/kernel/Foo.kt"))
|
||||||
|
val div = BriefEchoDiff.compare(b, e)
|
||||||
|
assertFalse(div.hallucinatedFiles.contains("core/kernel/Foo.kt"))
|
||||||
|
assertFalse(div.isMismatch)
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- symbol divergence → recorded but isMismatch=false ---
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `hallucinated symbol is recorded but does not cause isMismatch`() {
|
||||||
|
val b = brief("implement the interface")
|
||||||
|
val e = echo("implement the interface", symbols = listOf("GhostService"))
|
||||||
|
val div = BriefEchoDiff.compare(b, e)
|
||||||
|
// Symbol not in brief → hallucinated
|
||||||
|
assertTrue(div.hallucinatedSymbols.contains("GhostService"))
|
||||||
|
// But isMismatch should be false (symbols are display-only in v1)
|
||||||
|
assertFalse(div.isMismatch, "Symbol-only divergence must not block in v1")
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `symbol present in brief is not hallucinated`() {
|
||||||
|
val b = brief("implement SessionOrchestrator interface")
|
||||||
|
val e = echo("implement SessionOrchestrator interface", symbols = listOf("SessionOrchestrator"))
|
||||||
|
val div = BriefEchoDiff.compare(b, e)
|
||||||
|
assertFalse(div.hallucinatedSymbols.contains("SessionOrchestrator"))
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- malformed / empty echo → total mismatch ---
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `malformed echo is treated as total mismatch`() {
|
||||||
|
val b = brief("req A", "req B")
|
||||||
|
val div = BriefEchoDiff.compare(b, "not valid json at all")
|
||||||
|
assertTrue(div.isMismatch)
|
||||||
|
assertEquals(2, div.uncoveredRequirements.size)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `empty echo JSON object is total mismatch`() {
|
||||||
|
val b = brief("req A")
|
||||||
|
val div = BriefEchoDiff.compare(b, "{}")
|
||||||
|
assertTrue(div.isMismatch)
|
||||||
|
assertTrue(div.uncoveredRequirements.isNotEmpty())
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- fenced JSON is tolerated ---
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `fenced echo JSON is tolerated`() {
|
||||||
|
val b = brief("must support retries")
|
||||||
|
val fenced = "```json\n${echo("must support retries")}\n```"
|
||||||
|
val div = BriefEchoDiff.compare(b, fenced)
|
||||||
|
assertFalse(div.isMismatch, "Fenced echo should parse and match: ${div.uncoveredRequirements}")
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- no requirements in brief → no mismatch ---
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `brief with no requirements yields no mismatch`() {
|
||||||
|
val b = """{"summary":"something","requirements":[]}"""
|
||||||
|
val e = echo()
|
||||||
|
val div = BriefEchoDiff.compare(b, e)
|
||||||
|
assertFalse(div.isMismatch)
|
||||||
|
assertTrue(div.uncoveredRequirements.isEmpty())
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- tokenize ---
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `tokenize lowercases and strips punctuation`() {
|
||||||
|
val tokens = BriefEchoDiff.tokenize("Must EMIT a BriefEchoMismatchEvent!")
|
||||||
|
assertTrue(tokens.contains("must"))
|
||||||
|
assertTrue(tokens.contains("emit"))
|
||||||
|
assertTrue(tokens.contains("briefechomismatchevent"))
|
||||||
|
assertFalse(tokens.contains("a")) // length < 2
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"goal": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "the analyst brief's goal restated in your own words"
|
||||||
|
},
|
||||||
|
"referenced_files": {
|
||||||
|
"type": "array",
|
||||||
|
"description": "workspace-relative file paths named in the brief, one per item",
|
||||||
|
"items": { "type": "string" }
|
||||||
|
},
|
||||||
|
"referenced_symbols": {
|
||||||
|
"type": "array",
|
||||||
|
"description": "class, interface, or function names named in the brief, one per item",
|
||||||
|
"items": { "type": "string" }
|
||||||
|
},
|
||||||
|
"acceptance_criteria": {
|
||||||
|
"type": "array",
|
||||||
|
"description": "the brief's requirements / acceptance criteria restated faithfully, one per item",
|
||||||
|
"minItems": 1,
|
||||||
|
"items": { "type": "string" }
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["goal", "acceptance_criteria"],
|
||||||
|
"additionalProperties": true
|
||||||
|
}
|
||||||
@@ -29,6 +29,11 @@ id = "review_report"
|
|||||||
schema_path = "schemas/review_report.json"
|
schema_path = "schemas/review_report.json"
|
||||||
llm_emitted = true
|
llm_emitted = true
|
||||||
|
|
||||||
|
[[artifacts]]
|
||||||
|
id = "brief_echo"
|
||||||
|
schema_path = "schemas/brief_echo.json"
|
||||||
|
llm_emitted = true
|
||||||
|
|
||||||
[[artifacts]]
|
[[artifacts]]
|
||||||
id = "execution_plan"
|
id = "execution_plan"
|
||||||
schema_path = "schemas/execution_plan.json"
|
schema_path = "schemas/execution_plan.json"
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
You are about to plan. Before you do, RESTATE the analyst brief faithfully as JSON.
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
- Copy requirements exactly — do not merge, drop, or reword them.
|
||||||
|
- Only reference files that are explicitly named in the brief.
|
||||||
|
- Do not invent new scope, new files, or new symbols.
|
||||||
|
|
||||||
|
Emit your result as the `brief_echo` artifact (JSON, schema provided):
|
||||||
|
- `goal`: the brief's goal in your own words (one sentence).
|
||||||
|
- `referenced_files`: workspace-relative file paths named in the brief, verbatim.
|
||||||
|
- `referenced_symbols`: class, interface, or function names named in the brief, verbatim.
|
||||||
|
- `acceptance_criteria`: every requirement / acceptance criterion from the brief, one per item, verbatim.
|
||||||
|
|
||||||
|
If a field has no relevant content, emit an empty array. Do not add items that are not in the brief.
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
# Role pipeline: analyst → architect → planner → implementer ⇄ reviewer
|
# Role pipeline: analyst → architect → brief_echo → planner → implementer ⇄ reviewer
|
||||||
#
|
#
|
||||||
# Each stage produces a typed artifact that the next stage `needs`, so work flows forward
|
# Each stage produces a typed artifact that the next stage `needs`, so work flows forward
|
||||||
# without a human relaying notes. The decision journal (pinned into every stage's context)
|
# without a human relaying notes. The decision journal (pinned into every stage's context)
|
||||||
@@ -18,6 +18,8 @@
|
|||||||
# id = "impl_plan"; schema_path = "schemas/impl_plan.json"; llm_emitted = true
|
# id = "impl_plan"; schema_path = "schemas/impl_plan.json"; llm_emitted = true
|
||||||
# [[artifacts]]
|
# [[artifacts]]
|
||||||
# id = "review_report"; schema_path = "schemas/review_report.json"; llm_emitted = true
|
# id = "review_report"; schema_path = "schemas/review_report.json"; llm_emitted = true
|
||||||
|
# [[artifacts]]
|
||||||
|
# id = "brief_echo"; schema_path = "schemas/brief_echo.json"; llm_emitted = true
|
||||||
#
|
#
|
||||||
# Prompt files (prompts/*.md, relative to this workflow) must exist for a real run.
|
# Prompt files (prompts/*.md, relative to this workflow) must exist for a real run.
|
||||||
|
|
||||||
@@ -45,7 +47,20 @@ produces = [{ name = "design", kind = "design" }]
|
|||||||
token_budget = 16384
|
token_budget = 16384
|
||||||
max_retries = 2
|
max_retries = 2
|
||||||
|
|
||||||
# 3. Break the design into ordered, verifiable steps.
|
# 3a. Before planning: echo the analyst brief back as a structured artifact.
|
||||||
|
# The brief_echo gate diffs the restatement against the original analysis;
|
||||||
|
# on divergence (dropped requirements or hallucinated files) it fails the
|
||||||
|
# stage (retryable) so the pipeline cannot reach plan generation on a misread brief.
|
||||||
|
[[stages]]
|
||||||
|
id = "brief_echo"
|
||||||
|
prompt = "prompts/brief_echo.md"
|
||||||
|
needs = ["analysis"]
|
||||||
|
produces = [{ name = "brief_echo", kind = "brief_echo" }]
|
||||||
|
brief_echo = true
|
||||||
|
token_budget = 8192
|
||||||
|
max_retries = 2
|
||||||
|
|
||||||
|
# 3b. Break the design into ordered, verifiable steps.
|
||||||
[[stages]]
|
[[stages]]
|
||||||
id = "planner"
|
id = "planner"
|
||||||
prompt = "prompts/planner.md"
|
prompt = "prompts/planner.md"
|
||||||
@@ -90,12 +105,19 @@ condition_type = "artifact_validated"
|
|||||||
condition_artifact_id = "analysis"
|
condition_artifact_id = "analysis"
|
||||||
|
|
||||||
[[transitions]]
|
[[transitions]]
|
||||||
id = "architect-to-planner"
|
id = "architect-to-brief-echo"
|
||||||
from = "architect"
|
from = "architect"
|
||||||
to = "planner"
|
to = "brief_echo"
|
||||||
condition_type = "artifact_validated"
|
condition_type = "artifact_validated"
|
||||||
condition_artifact_id = "design"
|
condition_artifact_id = "design"
|
||||||
|
|
||||||
|
[[transitions]]
|
||||||
|
id = "brief-echo-to-planner"
|
||||||
|
from = "brief_echo"
|
||||||
|
to = "planner"
|
||||||
|
condition_type = "artifact_validated"
|
||||||
|
condition_artifact_id = "brief_echo"
|
||||||
|
|
||||||
[[transitions]]
|
[[transitions]]
|
||||||
id = "planner-to-implementer"
|
id = "planner-to-implementer"
|
||||||
from = "planner"
|
from = "planner"
|
||||||
|
|||||||
+2
@@ -47,6 +47,7 @@ private data class StageSection(
|
|||||||
@param:JsonProperty("requires_approval") val requiresApproval: Boolean = false,
|
@param:JsonProperty("requires_approval") val requiresApproval: Boolean = false,
|
||||||
@param:JsonProperty("inject_artifact_kinds") val injectArtifactKinds: Boolean = false,
|
@param:JsonProperty("inject_artifact_kinds") val injectArtifactKinds: Boolean = false,
|
||||||
@param:JsonProperty("ground_references") val groundReferences: Boolean = false,
|
@param:JsonProperty("ground_references") val groundReferences: Boolean = false,
|
||||||
|
@param:JsonProperty("brief_echo") val briefEcho: Boolean = false,
|
||||||
)
|
)
|
||||||
|
|
||||||
// condition fields flattened into the transition row
|
// condition fields flattened into the transition row
|
||||||
@@ -121,6 +122,7 @@ class TomlWorkflowLoader(
|
|||||||
if (s.requiresApproval) put("requiresApproval", "true")
|
if (s.requiresApproval) put("requiresApproval", "true")
|
||||||
if (s.injectArtifactKinds) put("injectArtifactKinds", "true")
|
if (s.injectArtifactKinds) put("injectArtifactKinds", "true")
|
||||||
if (s.groundReferences) put("groundReferences", "true")
|
if (s.groundReferences) put("groundReferences", "true")
|
||||||
|
if (s.briefEcho) put("briefEcho", "true")
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-2
@@ -29,6 +29,7 @@ class RolePipelineWorkflowTest {
|
|||||||
private val registry = DefaultArtifactKindRegistry().apply {
|
private val registry = DefaultArtifactKindRegistry().apply {
|
||||||
register(llmKind("analysis"))
|
register(llmKind("analysis"))
|
||||||
register(llmKind("design"))
|
register(llmKind("design"))
|
||||||
|
register(llmKind("brief_echo"))
|
||||||
register(llmKind("impl_plan"))
|
register(llmKind("impl_plan"))
|
||||||
register(llmKind("review_report"))
|
register(llmKind("review_report"))
|
||||||
}
|
}
|
||||||
@@ -53,10 +54,10 @@ class RolePipelineWorkflowTest {
|
|||||||
val graph = TomlWorkflowLoader(registry).load(path!!)
|
val graph = TomlWorkflowLoader(registry).load(path!!)
|
||||||
|
|
||||||
assertEquals(
|
assertEquals(
|
||||||
setOf("analyst", "architect", "planner", "implementer", "reviewer"),
|
setOf("analyst", "architect", "brief_echo", "planner", "implementer", "reviewer"),
|
||||||
graph.stages.keys.map { it.value }.toSet(),
|
graph.stages.keys.map { it.value }.toSet(),
|
||||||
)
|
)
|
||||||
assertEquals(6, graph.transitions.size)
|
assertEquals(7, graph.transitions.size)
|
||||||
|
|
||||||
// Forward chaining via needs.
|
// Forward chaining via needs.
|
||||||
assertTrue(graph.stages[StageId("reviewer")]!!.needs.map { it.value }.containsAll(listOf("patch", "impl_plan")))
|
assertTrue(graph.stages[StageId("reviewer")]!!.needs.map { it.value }.containsAll(listOf("patch", "impl_plan")))
|
||||||
|
|||||||
Reference in New Issue
Block a user