feat(kernel): brief echo-back gate (plan-pipeline-addenda A1)
New brief_echo stage before the planner in role_pipeline: the model restates the analyst brief as a structured artifact, and a deterministic gate (checkBriefEcho, opt-in via brief_echo=true) diffs the restatement against the original brief. On divergence — dropped requirements (Jaccard coverage < 0.34) or hallucinated files — it emits BriefEchoMismatchEvent and fails the stage retryably, so a misread brief never reaches plan generation. The diff (BriefEchoDiff) is a pure function of two recorded artifacts, so it is replay-safe and records no observation; the event fires only on mismatch, for audit. Symbols are recorded but non-blocking in v1. Mirrors the groundBrief- References gate. role_pipeline only; freestyle_planning wiring is a follow-up. Runtime install (like research): copy brief_echo.json + brief_echo.md and the [[artifacts]] block into ~/.config/correx.
This commit is contained in:
@@ -0,0 +1,128 @@
|
||||
package com.correx.core.kernel.orchestration
|
||||
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.contentOrNull
|
||||
import kotlinx.serialization.json.jsonPrimitive
|
||||
|
||||
/**
|
||||
* Pure deterministic diff between an analyst brief and a model's echo restatement.
|
||||
*
|
||||
* Determinism / invariant #8: reads ONLY the two recorded artifact strings — no I/O, no
|
||||
* external calls. Because both inputs are already in the event log (ArtifactCreatedEvent),
|
||||
* the result is recomputable on replay without re-emitting any observation event.
|
||||
*
|
||||
* isMismatch triggers gate failure when requirements are uncovered or files are hallucinated.
|
||||
* Hallucinated symbols are recorded for audit but do NOT block in v1 (display-only, mirroring
|
||||
* the role-reliability contradiction-check "display-only in v1" stance).
|
||||
*/
|
||||
data class BriefEchoDivergence(
|
||||
val uncoveredRequirements: List<String>,
|
||||
val hallucinatedFiles: List<String>,
|
||||
/** Recorded for audit/narration; does NOT contribute to isMismatch in v1. */
|
||||
val hallucinatedSymbols: List<String>,
|
||||
) {
|
||||
val isMismatch: Boolean
|
||||
get() = uncoveredRequirements.isNotEmpty() || hallucinatedFiles.isNotEmpty()
|
||||
}
|
||||
|
||||
internal object BriefEchoDiff {
|
||||
|
||||
/** Jaccard overlap below this threshold means a requirement is "uncovered" by the echo. */
|
||||
private const val COVERAGE_THRESHOLD = 0.34
|
||||
|
||||
/** Minimum token length for comparison — very short tokens add noise. */
|
||||
private const val MIN_TOKEN_LENGTH = 2
|
||||
|
||||
private val lenientJson = Json { ignoreUnknownKeys = true; isLenient = true }
|
||||
private val nonAlphanumeric = Regex("[^A-Za-z0-9]+")
|
||||
|
||||
/**
|
||||
* Compare the analyst brief JSON against the model's echo JSON, returning divergences.
|
||||
* On an unparseable echo, every brief requirement is reported as uncovered (total mismatch),
|
||||
* so a garbage echo always fails the gate.
|
||||
*/
|
||||
fun compare(briefJson: String, echoJson: String): BriefEchoDivergence {
|
||||
val brief = parseJson(briefJson)
|
||||
val echo = parseJson(echoJson)
|
||||
val briefFullTokens = tokenize(briefJson)
|
||||
val briefFiles = BriefReferenceExtractor.fileReferences(briefJson)
|
||||
return BriefEchoDivergence(
|
||||
uncoveredRequirements = uncoveredRequirements(brief, echo),
|
||||
hallucinatedFiles = hallucinatedFiles(echo, briefFullTokens, briefFiles),
|
||||
hallucinatedSymbols = hallucinatedSymbols(echo, briefFullTokens),
|
||||
)
|
||||
}
|
||||
|
||||
/** Each brief requirement whose best Jaccard overlap with any echo criterion is below threshold. */
|
||||
private fun uncoveredRequirements(brief: JsonObject?, echo: JsonObject?): List<String> {
|
||||
val requirements = brief?.let { stringList(it, "requirements") } ?: emptyList()
|
||||
val criteria = echo?.let { stringList(it, "acceptance_criteria") } ?: emptyList()
|
||||
return requirements.filter { req ->
|
||||
val reqTokens = tokenize(req)
|
||||
if (reqTokens.isEmpty()) return@filter false
|
||||
val bestOverlap = criteria.maxOfOrNull { jaccard(reqTokens, tokenize(it)) } ?: 0.0
|
||||
bestOverlap < COVERAGE_THRESHOLD
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Echo files not grounded in the brief. A file is hallucinated when no token of its basename
|
||||
* stem (extension dropped — an extension like "kt" matches any source file in the brief and is
|
||||
* pure noise) appears in the brief, AND it matches no extracted brief file path.
|
||||
*/
|
||||
private fun hallucinatedFiles(
|
||||
echo: JsonObject?,
|
||||
briefFullTokens: Set<String>,
|
||||
briefFiles: List<String>,
|
||||
): List<String> {
|
||||
val echoFiles = echo?.let { stringList(it, "referenced_files") } ?: emptyList()
|
||||
return echoFiles.filter { f ->
|
||||
val stemTokens = tokenize(f.substringAfterLast('/').substringBeforeLast('.'))
|
||||
if (stemTokens.isEmpty()) return@filter true
|
||||
stemTokens.none { it in briefFullTokens } &&
|
||||
briefFiles.none { it == f || it.endsWith(f) || f.endsWith(it) }
|
||||
}
|
||||
}
|
||||
|
||||
/** Echo symbols with no token present anywhere in the brief (recorded for audit, non-blocking). */
|
||||
private fun hallucinatedSymbols(echo: JsonObject?, briefFullTokens: Set<String>): List<String> {
|
||||
val echoSymbols = echo?.let { stringList(it, "referenced_symbols") } ?: emptyList()
|
||||
return echoSymbols.filter { sym -> tokenize(sym).none { it in briefFullTokens } }
|
||||
}
|
||||
|
||||
/** Lowercase, replace non-alphanumeric with spaces, split, drop tokens shorter than MIN_TOKEN_LENGTH. */
|
||||
internal fun tokenize(text: String): Set<String> =
|
||||
text.lowercase()
|
||||
.replace(nonAlphanumeric, " ")
|
||||
.trim()
|
||||
.split(" ")
|
||||
.filter { it.length >= MIN_TOKEN_LENGTH }
|
||||
.toSet()
|
||||
|
||||
private fun jaccard(a: Set<String>, b: Set<String>): Double {
|
||||
if (a.isEmpty() && b.isEmpty()) return 1.0
|
||||
val intersection = (a intersect b).size
|
||||
val union = (a union b).size
|
||||
return if (union == 0) 0.0 else intersection.toDouble() / union
|
||||
}
|
||||
|
||||
private fun parseJson(json: String): JsonObject? {
|
||||
val stripped = stripFence(json)
|
||||
return runCatching { lenientJson.parseToJsonElement(stripped) as? JsonObject }.getOrNull()
|
||||
}
|
||||
|
||||
private fun stringList(obj: JsonObject, key: String): List<String> =
|
||||
runCatching {
|
||||
(obj[key] as? JsonArray)?.mapNotNull { it.jsonPrimitive.contentOrNull } ?: emptyList()
|
||||
}.getOrElse { emptyList() }
|
||||
|
||||
private fun stripFence(text: String): String {
|
||||
val trimmed = text.trim()
|
||||
if (!trimmed.startsWith("```") || !trimmed.endsWith("```")) return text
|
||||
val withoutClose = trimmed.removeSuffix("```").trimEnd()
|
||||
val firstNewline = withoutClose.indexOf('\n')
|
||||
return if (firstNewline < 0) text else withoutClose.substring(firstNewline + 1)
|
||||
}
|
||||
}
|
||||
+70
-1
@@ -28,6 +28,7 @@ import com.correx.core.events.events.RefinementIterationEvent
|
||||
import com.correx.core.events.events.ApprovalRequestedEvent
|
||||
import com.correx.core.events.events.ArtifactContentStoredEvent
|
||||
import com.correx.core.events.events.ArtifactCreatedEvent
|
||||
import com.correx.core.events.events.BriefEchoMismatchEvent
|
||||
import com.correx.core.events.events.BriefGroundingCheckedEvent
|
||||
import com.correx.core.events.events.ContextTruncatedEvent
|
||||
import com.correx.core.events.events.ClarificationAnswer
|
||||
@@ -550,7 +551,11 @@ abstract class SessionOrchestrator(
|
||||
emitLlmArtifacts(sessionId, stageId, stageConfig)
|
||||
when (val produced = verifyProduces(sessionId, stageId, stageConfig)) {
|
||||
is StageExecutionResult.Success ->
|
||||
groundBriefReferences(sessionId, stageId, stageConfig, effectives)
|
||||
when (val grounded = groundBriefReferences(sessionId, stageId, stageConfig, effectives)) {
|
||||
is StageExecutionResult.Success ->
|
||||
checkBriefEcho(sessionId, stageId, stageConfig)
|
||||
is StageExecutionResult.Failure -> grounded
|
||||
}
|
||||
is StageExecutionResult.Failure -> produced
|
||||
}
|
||||
}
|
||||
@@ -1399,6 +1404,70 @@ abstract class SessionOrchestrator(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Brief echo gate (plan-pipeline-addenda §A1): for a stage that opted in (`brief_echo`),
|
||||
* diff the model's restatement artifact against the original brief. Both are already in the
|
||||
* artifact content cache (recorded via ArtifactCreatedEvent), so the diff is a pure function
|
||||
* of recorded data — deterministically recomputable on replay without re-probing any
|
||||
* environment (invariants #8, #9). Emits [BriefEchoMismatchEvent] on mismatch as a domain
|
||||
* signal, then fails the stage (retryable) so the pipeline cannot reach plan generation on a
|
||||
* misread brief.
|
||||
*
|
||||
* Brief-source selection among `needs`: prefers the artifact named "analysis"; if absent,
|
||||
* falls back to the single needs artifact present. When there are multiple needs and none is
|
||||
* named "analysis", the gate skips (ambiguous brief) and returns Success.
|
||||
*/
|
||||
private suspend fun checkBriefEcho(
|
||||
sessionId: SessionId,
|
||||
stageId: StageId,
|
||||
stageConfig: StageConfig,
|
||||
): StageExecutionResult {
|
||||
if (stageConfig.metadata["briefEcho"] != "true") return StageExecutionResult.Success(emptyList())
|
||||
|
||||
val echoSlot = stageConfig.produces.firstOrNull { it.kind.llmEmitted }
|
||||
?: return StageExecutionResult.Success(emptyList())
|
||||
val echo = artifactContentCache["${sessionId.value}:${echoSlot.name.value}"]
|
||||
?: return StageExecutionResult.Success(emptyList())
|
||||
|
||||
val brief = when {
|
||||
stageConfig.needs.any { it.value == "analysis" } ->
|
||||
artifactContentCache["${sessionId.value}:analysis"]
|
||||
stageConfig.needs.size == 1 ->
|
||||
artifactContentCache["${sessionId.value}:${stageConfig.needs.single().value}"]
|
||||
else -> null // ambiguous — skip
|
||||
} ?: return StageExecutionResult.Success(emptyList())
|
||||
|
||||
val div = BriefEchoDiff.compare(brief, echo)
|
||||
|
||||
return if (div.isMismatch) {
|
||||
emit(
|
||||
sessionId,
|
||||
BriefEchoMismatchEvent(
|
||||
sessionId = sessionId,
|
||||
stageId = stageId,
|
||||
uncoveredRequirements = div.uncoveredRequirements,
|
||||
hallucinatedFiles = div.hallucinatedFiles,
|
||||
hallucinatedSymbols = div.hallucinatedSymbols,
|
||||
),
|
||||
)
|
||||
log.warn(
|
||||
"[Orchestrator] brief echo mismatch session={} stage={} uncovered={} hallucinatedFiles={}",
|
||||
sessionId.value, stageId.value,
|
||||
div.uncoveredRequirements.joinToString(", "),
|
||||
div.hallucinatedFiles.joinToString(", "),
|
||||
)
|
||||
StageExecutionResult.Failure(
|
||||
"stage ${stageId.value} restated the brief unfaithfully — " +
|
||||
"dropped requirements: [${div.uncoveredRequirements.joinToString("; ")}]; " +
|
||||
"files not in the brief: [${div.hallucinatedFiles.joinToString(", ")}]. " +
|
||||
"Restate the brief exactly; do not drop requirements or invent files.",
|
||||
retryable = true,
|
||||
)
|
||||
} else {
|
||||
StageExecutionResult.Success(emptyList())
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun emitProcessResultEvents(
|
||||
sessionId: SessionId,
|
||||
stageId: StageId,
|
||||
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
package com.correx.core.kernel.orchestration
|
||||
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertFalse
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class BriefEchoDiffTest {
|
||||
|
||||
// --- helpers ---
|
||||
|
||||
private fun brief(vararg requirements: String, files: List<String> = emptyList()): String {
|
||||
val reqs = requirements.joinToString(",") { "\"$it\"" }
|
||||
val fileArr = files.joinToString(",") { "\"$it\"" }
|
||||
return """{"summary":"do the thing","requirements":[$reqs],"affected_areas":[$fileArr]}"""
|
||||
}
|
||||
|
||||
private fun echo(
|
||||
vararg criteria: String,
|
||||
files: List<String> = emptyList(),
|
||||
symbols: List<String> = emptyList(),
|
||||
): String {
|
||||
val crit = criteria.joinToString(",") { "\"$it\"" }
|
||||
val fileArr = files.joinToString(",") { "\"$it\"" }
|
||||
val symArr = symbols.joinToString(",") { "\"$it\"" }
|
||||
return """{"goal":"do the thing","referenced_files":[$fileArr],""" +
|
||||
""""referenced_symbols":[$symArr],"acceptance_criteria":[$crit]}"""
|
||||
}
|
||||
|
||||
// --- faithful echo → no mismatch ---
|
||||
|
||||
@Test
|
||||
fun `faithful echo produces no mismatch`() {
|
||||
val b = brief("the system must retry on transient failures", "all retry attempts must be logged")
|
||||
val e = echo("the system must retry on transient failures", "all retry attempts must be logged")
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertFalse(div.isMismatch)
|
||||
assertTrue(div.uncoveredRequirements.isEmpty())
|
||||
assertTrue(div.hallucinatedFiles.isEmpty())
|
||||
}
|
||||
|
||||
// --- dropped requirement → uncovered ---
|
||||
|
||||
@Test
|
||||
fun `dropped requirement is reported as uncovered`() {
|
||||
val b = brief("must emit an event", "must support retries")
|
||||
val e = echo("must emit an event")
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertTrue(div.isMismatch)
|
||||
assertTrue(div.uncoveredRequirements.any { it.contains("retries") })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `all requirements dropped marks all as uncovered`() {
|
||||
val b = brief("req alpha", "req beta")
|
||||
val e = echo() // no criteria
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertTrue(div.isMismatch)
|
||||
assertEquals(2, div.uncoveredRequirements.size)
|
||||
}
|
||||
|
||||
// --- slightly paraphrased but high-overlap requirement → covered ---
|
||||
|
||||
@Test
|
||||
fun `near-verbatim paraphrase with high token overlap is covered`() {
|
||||
val b = brief("the gate must emit a BriefEchoMismatchEvent on divergence")
|
||||
val e = echo("gate must emit BriefEchoMismatchEvent on divergence")
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertFalse(div.isMismatch, "High-overlap paraphrase should be covered: ${div.uncoveredRequirements}")
|
||||
}
|
||||
|
||||
// --- hallucinated file ---
|
||||
|
||||
@Test
|
||||
fun `file not in brief is hallucinated`() {
|
||||
val b = brief("do something", files = listOf("core/kernel/Foo.kt"))
|
||||
val e = echo("do something", files = listOf("core/ghost/Phantom.kt"))
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertTrue(div.hallucinatedFiles.contains("core/ghost/Phantom.kt"))
|
||||
assertTrue(div.isMismatch)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `file present in brief is not hallucinated`() {
|
||||
val b = brief("do something", files = listOf("core/kernel/Foo.kt"))
|
||||
val e = echo("do something", files = listOf("core/kernel/Foo.kt"))
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertFalse(div.hallucinatedFiles.contains("core/kernel/Foo.kt"))
|
||||
assertFalse(div.isMismatch)
|
||||
}
|
||||
|
||||
// --- symbol divergence → recorded but isMismatch=false ---
|
||||
|
||||
@Test
|
||||
fun `hallucinated symbol is recorded but does not cause isMismatch`() {
|
||||
val b = brief("implement the interface")
|
||||
val e = echo("implement the interface", symbols = listOf("GhostService"))
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
// Symbol not in brief → hallucinated
|
||||
assertTrue(div.hallucinatedSymbols.contains("GhostService"))
|
||||
// But isMismatch should be false (symbols are display-only in v1)
|
||||
assertFalse(div.isMismatch, "Symbol-only divergence must not block in v1")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `symbol present in brief is not hallucinated`() {
|
||||
val b = brief("implement SessionOrchestrator interface")
|
||||
val e = echo("implement SessionOrchestrator interface", symbols = listOf("SessionOrchestrator"))
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertFalse(div.hallucinatedSymbols.contains("SessionOrchestrator"))
|
||||
}
|
||||
|
||||
// --- malformed / empty echo → total mismatch ---
|
||||
|
||||
@Test
|
||||
fun `malformed echo is treated as total mismatch`() {
|
||||
val b = brief("req A", "req B")
|
||||
val div = BriefEchoDiff.compare(b, "not valid json at all")
|
||||
assertTrue(div.isMismatch)
|
||||
assertEquals(2, div.uncoveredRequirements.size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `empty echo JSON object is total mismatch`() {
|
||||
val b = brief("req A")
|
||||
val div = BriefEchoDiff.compare(b, "{}")
|
||||
assertTrue(div.isMismatch)
|
||||
assertTrue(div.uncoveredRequirements.isNotEmpty())
|
||||
}
|
||||
|
||||
// --- fenced JSON is tolerated ---
|
||||
|
||||
@Test
|
||||
fun `fenced echo JSON is tolerated`() {
|
||||
val b = brief("must support retries")
|
||||
val fenced = "```json\n${echo("must support retries")}\n```"
|
||||
val div = BriefEchoDiff.compare(b, fenced)
|
||||
assertFalse(div.isMismatch, "Fenced echo should parse and match: ${div.uncoveredRequirements}")
|
||||
}
|
||||
|
||||
// --- no requirements in brief → no mismatch ---
|
||||
|
||||
@Test
|
||||
fun `brief with no requirements yields no mismatch`() {
|
||||
val b = """{"summary":"something","requirements":[]}"""
|
||||
val e = echo()
|
||||
val div = BriefEchoDiff.compare(b, e)
|
||||
assertFalse(div.isMismatch)
|
||||
assertTrue(div.uncoveredRequirements.isEmpty())
|
||||
}
|
||||
|
||||
// --- tokenize ---
|
||||
|
||||
@Test
|
||||
fun `tokenize lowercases and strips punctuation`() {
|
||||
val tokens = BriefEchoDiff.tokenize("Must EMIT a BriefEchoMismatchEvent!")
|
||||
assertTrue(tokens.contains("must"))
|
||||
assertTrue(tokens.contains("emit"))
|
||||
assertTrue(tokens.contains("briefechomismatchevent"))
|
||||
assertFalse(tokens.contains("a")) // length < 2
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user