feat(kernel): analyst entity grounding (role-reliability §3)
Local analysts hallucinate file paths the same as planners. A stage can now
opt in with `ground_references = true`: after it produces its brief, every
workspace-relative file path the brief named is checked for existence, and a
path that doesn't exist fails the stage (retryable) with the misses fed back
("Reference only files you have read") so the model re-grounds.
Grounding probes the filesystem directly rather than the repo map — the map
is recency-ranked and depth/extension-limited, not a complete existence
oracle, so "absent from the map" can't be a hard failure. Existence is a
nondeterministic observation, so the result is recorded as a
BriefGroundingCheckedEvent (invariant #9); replay reads it back and never
re-probes.
- BriefGroundingCheckedEvent + GroundingReference, registered in eventModule
- BriefReferenceExtractor: pure, deterministic pull of relative file-path
references from a brief artifact JSON (needs a '/' and a dotted extension;
skips modules, bare names, URLs, absolute/parent paths — so coarse
"subsystem" mentions the brief is allowed to make aren't false-flagged)
- SessionOrchestrator.groundBriefReferences on the post-verifyProduces path,
gated by stage metadata; uses the existing worldProbe
- TomlWorkflowLoader `ground_references` → metadata; role_pipeline analyst
opts in
Scoped to file references with a '/' + extension (not bare basenames, which
a depth-limited probe can't resolve), so a flagged miss is high-confidence.
The symbol-grounding half of §3 is left for a real symbol index.
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
package com.correx.core.events.events
|
||||
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.events.types.StageId
|
||||
import kotlinx.serialization.SerialName
|
||||
import kotlinx.serialization.Serializable
|
||||
|
||||
/** One file-path reference from a brief and whether it grounds to an existing workspace entry. */
|
||||
@Serializable
|
||||
data class GroundingReference(
|
||||
val reference: String,
|
||||
val grounded: Boolean,
|
||||
)
|
||||
|
||||
/**
|
||||
* Environment observation (invariant #9): the workspace-relative file paths a stage's brief
|
||||
* referenced, each checked for existence at the moment the stage produced its artifact. Replay
|
||||
* reads these recorded facts and never re-probes the filesystem. A reference that does not
|
||||
* ground is a hallucinated path — the failure analysts share with planners — and fails the
|
||||
* stage so the model re-grounds (role-reliability §3 entity grounding).
|
||||
*/
|
||||
@Serializable
|
||||
@SerialName("BriefGroundingChecked")
|
||||
data class BriefGroundingCheckedEvent(
|
||||
val sessionId: SessionId,
|
||||
val stageId: StageId,
|
||||
val artifactId: String,
|
||||
val references: List<GroundingReference>,
|
||||
) : EventPayload
|
||||
@@ -8,6 +8,7 @@ import com.correx.core.events.events.ArtifactContentStoredEvent
|
||||
import com.correx.core.events.events.ArtifactCreatedEvent
|
||||
import com.correx.core.events.events.ArtifactValidatedEvent
|
||||
import com.correx.core.events.events.ArtifactValidatingEvent
|
||||
import com.correx.core.events.events.BriefGroundingCheckedEvent
|
||||
import com.correx.core.events.events.ChatSessionStartedEvent
|
||||
import com.correx.core.events.events.ChatTurnEvent
|
||||
import com.correx.core.events.events.RouterNarrationEvent
|
||||
@@ -99,6 +100,7 @@ val eventModule = SerializersModule {
|
||||
subclass(RepoMapComputedEvent::class)
|
||||
subclass(WorkspaceStateObservedEvent::class)
|
||||
subclass(RepoKnowledgeRetrievedEvent::class)
|
||||
subclass(BriefGroundingCheckedEvent::class)
|
||||
subclass(RiskAssessedEvent::class)
|
||||
subclass(ChatSessionStartedEvent::class)
|
||||
subclass(ChatTurnEvent::class)
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package com.correx.core.events.serialization
|
||||
|
||||
import com.correx.core.events.events.BriefGroundingCheckedEvent
|
||||
import com.correx.core.events.events.EventPayload
|
||||
import com.correx.core.events.events.GroundingReference
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.events.types.StageId
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class BriefGroundingCheckedEventSerializationTest {
|
||||
|
||||
private val sample = BriefGroundingCheckedEvent(
|
||||
sessionId = SessionId("sess-1"),
|
||||
stageId = StageId("analyst"),
|
||||
artifactId = "analyst",
|
||||
references = listOf(
|
||||
GroundingReference("core/validation/JsonSchemaValidator.kt", grounded = true),
|
||||
GroundingReference("core/ghost/Phantom.kt", grounded = false),
|
||||
),
|
||||
)
|
||||
|
||||
@Test
|
||||
fun `round-trips as polymorphic EventPayload`() {
|
||||
val encoded = eventJson.encodeToString(EventPayload.serializer(), sample)
|
||||
assertTrue(encoded.contains("\"type\":\"BriefGroundingChecked\""), "SerialName must be present: $encoded")
|
||||
val decoded = eventJson.decodeFromString(EventPayload.serializer(), encoded)
|
||||
assertEquals(sample, decoded)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `decodes hand-written BriefGroundingChecked JSON`() {
|
||||
val json = """
|
||||
{"type":"BriefGroundingChecked","sessionId":"s","stageId":"analyst","artifactId":"analyst",
|
||||
"references":[{"reference":"a/b.kt","grounded":false}]}
|
||||
""".trimIndent()
|
||||
val decoded = eventJson.decodeFromString(EventPayload.serializer(), json)
|
||||
assertTrue(decoded is BriefGroundingCheckedEvent)
|
||||
assertEquals(false, (decoded as BriefGroundingCheckedEvent).references.single().grounded)
|
||||
}
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
package com.correx.core.kernel.orchestration
|
||||
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
import kotlinx.serialization.json.JsonElement
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
|
||||
/**
|
||||
* Pulls the workspace-relative file-path references out of an LLM-emitted brief artifact (JSON),
|
||||
* for entity grounding (role-reliability §3). Deterministic: walks every string leaf, splits on
|
||||
* whitespace, strips surrounding prose punctuation, and keeps tokens that read as a relative
|
||||
* file path — one containing a '/' and ending in a short dotted extension.
|
||||
*
|
||||
* Module/subsystem names, bare words, URLs, and absolute or parent-relative paths are
|
||||
* intentionally skipped. Only concrete file references are grounded, so the check catches
|
||||
* hallucinated paths ("analysts hallucinate paths the same as planners") without false-flagging
|
||||
* the coarse "files, modules, or subsystems" the brief is allowed to name.
|
||||
*/
|
||||
internal object BriefReferenceExtractor {
|
||||
|
||||
// <segments…>/<name>.<ext>: at least one slash, a final dotted extension of 1–5 letters/digits.
|
||||
private val FILE_REF = Regex("""^[A-Za-z0-9_.\-/]+/[A-Za-z0-9_.\-]+\.[A-Za-z][A-Za-z0-9]{0,4}$""")
|
||||
|
||||
fun fileReferences(artifactJson: String): List<String> {
|
||||
val root = runCatching { Json.parseToJsonElement(artifactJson) }.getOrNull() ?: return emptyList()
|
||||
val out = LinkedHashSet<String>()
|
||||
collectStrings(root) { value ->
|
||||
for (token in value.split(WHITESPACE)) {
|
||||
val cleaned = token
|
||||
.trimStart('(', '[', '{', '`', '"', '\'')
|
||||
.trimEnd(')', ']', '}', ',', '.', ';', ':', '`', '"', '\'')
|
||||
if (cleaned.isFileReference()) out += cleaned
|
||||
}
|
||||
}
|
||||
return out.toList()
|
||||
}
|
||||
|
||||
private fun String.isFileReference(): Boolean =
|
||||
!startsWith("/") && !contains("..") && !contains("://") && FILE_REF.matches(this)
|
||||
|
||||
private fun collectStrings(element: JsonElement, sink: (String) -> Unit) {
|
||||
when (element) {
|
||||
is JsonPrimitive -> if (element.isString) sink(element.content)
|
||||
is JsonArray -> element.forEach { collectStrings(it, sink) }
|
||||
is JsonObject -> element.values.forEach { collectStrings(it, sink) }
|
||||
}
|
||||
}
|
||||
|
||||
private val WHITESPACE = Regex("""\s+""")
|
||||
}
|
||||
+51
-1
@@ -28,7 +28,9 @@ import com.correx.core.events.events.RefinementIterationEvent
|
||||
import com.correx.core.events.events.ApprovalRequestedEvent
|
||||
import com.correx.core.events.events.ArtifactContentStoredEvent
|
||||
import com.correx.core.events.events.ArtifactCreatedEvent
|
||||
import com.correx.core.events.events.BriefGroundingCheckedEvent
|
||||
import com.correx.core.events.events.ContextTruncatedEvent
|
||||
import com.correx.core.events.events.GroundingReference
|
||||
import com.correx.core.events.events.ToolCallAssessedEvent
|
||||
import com.correx.core.events.events.ArtifactValidatedEvent
|
||||
import com.correx.core.events.events.ArtifactValidatingEvent
|
||||
@@ -529,7 +531,11 @@ abstract class SessionOrchestrator(
|
||||
is StageExecutionResult.Success -> {
|
||||
emitToolArtifacts(sessionId, stageId, stageConfig)
|
||||
emitLlmArtifacts(sessionId, stageId, stageConfig)
|
||||
verifyProduces(sessionId, stageId, stageConfig)
|
||||
when (val produced = verifyProduces(sessionId, stageId, stageConfig)) {
|
||||
is StageExecutionResult.Success ->
|
||||
groundBriefReferences(sessionId, stageId, stageConfig, effectives)
|
||||
is StageExecutionResult.Failure -> produced
|
||||
}
|
||||
}
|
||||
|
||||
is StageExecutionResult.Failure -> outcome
|
||||
@@ -1265,6 +1271,50 @@ abstract class SessionOrchestrator(
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Entity grounding (role-reliability §3): for a stage that opted in (`ground_references`),
|
||||
* check every workspace-relative file path its brief named against the filesystem. Existence
|
||||
* is a nondeterministic observation, so the result is recorded as a [BriefGroundingCheckedEvent]
|
||||
* (invariant #9) — replay reads it back, never re-probes. A hallucinated path fails the stage
|
||||
* (retryable), feeding the missing paths back so the model re-grounds on the next attempt.
|
||||
*/
|
||||
private suspend fun groundBriefReferences(
|
||||
sessionId: SessionId,
|
||||
stageId: StageId,
|
||||
stageConfig: StageConfig,
|
||||
effectives: RunEffectives,
|
||||
): StageExecutionResult {
|
||||
if (stageConfig.metadata["groundReferences"] != "true") return StageExecutionResult.Success(emptyList())
|
||||
val workspaceRoot = effectives.policy?.workspaceRoot ?: return StageExecutionResult.Success(emptyList())
|
||||
val artifact = stageConfig.produces
|
||||
.filter { it.kind.llmEmitted }
|
||||
.firstNotNullOfOrNull { artifactContentCache["${sessionId.value}:${it.name.value}"] }
|
||||
?: return StageExecutionResult.Success(emptyList())
|
||||
|
||||
val references = BriefReferenceExtractor.fileReferences(artifact)
|
||||
if (references.isEmpty()) return StageExecutionResult.Success(emptyList())
|
||||
|
||||
val results = references.map { ref ->
|
||||
GroundingReference(ref, worldProbe.exists(workspaceRoot.resolve(ref)))
|
||||
}
|
||||
emit(sessionId, BriefGroundingCheckedEvent(sessionId, stageId, stageId.value, results))
|
||||
|
||||
val missing = results.filterNot { it.grounded }.map { it.reference }
|
||||
return if (missing.isEmpty()) {
|
||||
StageExecutionResult.Success(emptyList())
|
||||
} else {
|
||||
log.warn(
|
||||
"[Orchestrator] brief references nonexistent paths session={} stage={} missing={}",
|
||||
sessionId.value, stageId.value, missing.joinToString(", "),
|
||||
)
|
||||
StageExecutionResult.Failure(
|
||||
"stage ${stageId.value} references paths that do not exist in the workspace: " +
|
||||
"${missing.joinToString(", ")}. Reference only files you have read.",
|
||||
retryable = true,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private suspend fun emitProcessResultEvents(
|
||||
sessionId: SessionId,
|
||||
stageId: StageId,
|
||||
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.correx.core.kernel.orchestration
|
||||
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class BriefReferenceExtractorTest {
|
||||
|
||||
@Test
|
||||
fun `extracts relative file paths from string arrays`() {
|
||||
val json = """
|
||||
{
|
||||
"summary": "touch the validator",
|
||||
"affected_areas": [
|
||||
"core/validation/JsonSchemaValidator.kt",
|
||||
"the validation pipeline",
|
||||
"core:validation module"
|
||||
]
|
||||
}
|
||||
""".trimIndent()
|
||||
assertEquals(listOf("core/validation/JsonSchemaValidator.kt"), BriefReferenceExtractor.fileReferences(json))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `strips surrounding prose punctuation`() {
|
||||
val json = """{"requirements":["see (core/kernel/Foo.kt), then bar.","ends here: apps/cli/Main.kt."]}"""
|
||||
assertEquals(
|
||||
listOf("core/kernel/Foo.kt", "apps/cli/Main.kt"),
|
||||
BriefReferenceExtractor.fileReferences(json),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `skips urls, absolute and parent paths, and bare names`() {
|
||||
val json = """
|
||||
{
|
||||
"notes": [
|
||||
"https://example.com/x.html",
|
||||
"/etc/passwd",
|
||||
"../escape/secret.kt",
|
||||
"StageConfig.kt",
|
||||
"just prose with no paths"
|
||||
]
|
||||
}
|
||||
""".trimIndent()
|
||||
assertTrue(BriefReferenceExtractor.fileReferences(json).isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `dedupes and preserves first-seen order`() {
|
||||
val json = """{"a":["core/A.kt","docs/x.md"],"b":["core/A.kt"]}"""
|
||||
assertEquals(listOf("core/A.kt", "docs/x.md"), BriefReferenceExtractor.fileReferences(json))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `keeps nested dotfile-directory paths`() {
|
||||
val json = """{"areas":[".github/workflows/ci.yml"]}"""
|
||||
assertEquals(listOf(".github/workflows/ci.yml"), BriefReferenceExtractor.fileReferences(json))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `malformed json yields no references`() {
|
||||
assertTrue(BriefReferenceExtractor.fileReferences("not json at all").isEmpty())
|
||||
}
|
||||
}
|
||||
@@ -25,11 +25,14 @@ id = "role_pipeline"
|
||||
start = "analyst"
|
||||
|
||||
# 1. Understand the request and the relevant code. Read-only.
|
||||
# ground_references: every workspace-relative file path the analysis names is checked
|
||||
# for existence; a hallucinated path fails the stage and retries with the misses fed back.
|
||||
[[stages]]
|
||||
id = "analyst"
|
||||
prompt = "prompts/analyst.md"
|
||||
produces = [{ name = "analysis", kind = "analysis" }]
|
||||
allowed_tools = ["file_read", "ShellTool"]
|
||||
ground_references = true
|
||||
token_budget = 16384
|
||||
max_retries = 2
|
||||
|
||||
|
||||
+2
@@ -46,6 +46,7 @@ private data class StageSection(
|
||||
@param:JsonProperty("max_retries") val maxRetries: Int = 3,
|
||||
@param:JsonProperty("requires_approval") val requiresApproval: Boolean = false,
|
||||
@param:JsonProperty("inject_artifact_kinds") val injectArtifactKinds: Boolean = false,
|
||||
@param:JsonProperty("ground_references") val groundReferences: Boolean = false,
|
||||
)
|
||||
|
||||
// condition fields flattened into the transition row
|
||||
@@ -119,6 +120,7 @@ class TomlWorkflowLoader(
|
||||
s.systemPrompt?.let { put("systemPrompt", resolvePromptPath(it, workflowDir)) }
|
||||
if (s.requiresApproval) put("requiresApproval", "true")
|
||||
if (s.injectArtifactKinds) put("injectArtifactKinds", "true")
|
||||
if (s.groundReferences) put("groundReferences", "true")
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
+18
@@ -155,4 +155,22 @@ class TomlWorkflowLoaderTest {
|
||||
val collectStage = graph.stages[graph.start]!!
|
||||
assertEquals(emptyList(), collectStage.writeManifest)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `ground_references true maps to metadata key`() {
|
||||
val toml = validToml.replace(
|
||||
"prompt = \"prompts/collect.md\"",
|
||||
"prompt = \"prompts/collect.md\"\nground_references = true",
|
||||
)
|
||||
val path = Files.createTempFile("workflow", ".toml").also { it.writeText(toml) }
|
||||
val graph = loader.load(path)
|
||||
assertEquals("true", graph.stages[graph.start]!!.metadata["groundReferences"])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `ground_references absent means no metadata key`() {
|
||||
val path = Files.createTempFile("workflow", ".toml").also { it.writeText(validToml) }
|
||||
val graph = loader.load(path)
|
||||
assertEquals(null, graph.stages[graph.start]!!.metadata["groundReferences"])
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user