feat(kernel): analyst entity grounding (role-reliability §3)
Local analysts hallucinate file paths the same as planners. A stage can now
opt in with `ground_references = true`: after it produces its brief, every
workspace-relative file path the brief named is checked for existence, and a
path that doesn't exist fails the stage (retryable) with the misses fed back
("Reference only files you have read") so the model re-grounds.
Grounding probes the filesystem directly rather than the repo map — the map
is recency-ranked and depth/extension-limited, not a complete existence
oracle, so "absent from the map" can't be a hard failure. Existence is a
nondeterministic observation, so the result is recorded as a
BriefGroundingCheckedEvent (invariant #9); replay reads it back and never
re-probes.
- BriefGroundingCheckedEvent + GroundingReference, registered in eventModule
- BriefReferenceExtractor: pure, deterministic pull of relative file-path
references from a brief artifact JSON (needs a '/' and a dotted extension;
skips modules, bare names, URLs, absolute/parent paths — so coarse
"subsystem" mentions the brief is allowed to make aren't false-flagged)
- SessionOrchestrator.groundBriefReferences on the post-verifyProduces path,
gated by stage metadata; uses the existing worldProbe
- TomlWorkflowLoader `ground_references` → metadata; role_pipeline analyst
opts in
Scoped to file references with a '/' + extension (not bare basenames, which
a depth-limited probe can't resolve), so a flagged miss is high-confidence.
The symbol-grounding half of §3 is left for a real symbol index.
This commit is contained in:
@@ -0,0 +1,29 @@
|
|||||||
|
package com.correx.core.events.events
|
||||||
|
|
||||||
|
import com.correx.core.events.types.SessionId
|
||||||
|
import com.correx.core.events.types.StageId
|
||||||
|
import kotlinx.serialization.SerialName
|
||||||
|
import kotlinx.serialization.Serializable
|
||||||
|
|
||||||
|
/** One file-path reference from a brief and whether it grounds to an existing workspace entry. */
|
||||||
|
@Serializable
|
||||||
|
data class GroundingReference(
|
||||||
|
val reference: String,
|
||||||
|
val grounded: Boolean,
|
||||||
|
)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Environment observation (invariant #9): the workspace-relative file paths a stage's brief
|
||||||
|
* referenced, each checked for existence at the moment the stage produced its artifact. Replay
|
||||||
|
* reads these recorded facts and never re-probes the filesystem. A reference that does not
|
||||||
|
* ground is a hallucinated path — the failure analysts share with planners — and fails the
|
||||||
|
* stage so the model re-grounds (role-reliability §3 entity grounding).
|
||||||
|
*/
|
||||||
|
@Serializable
|
||||||
|
@SerialName("BriefGroundingChecked")
|
||||||
|
data class BriefGroundingCheckedEvent(
|
||||||
|
val sessionId: SessionId,
|
||||||
|
val stageId: StageId,
|
||||||
|
val artifactId: String,
|
||||||
|
val references: List<GroundingReference>,
|
||||||
|
) : EventPayload
|
||||||
@@ -8,6 +8,7 @@ import com.correx.core.events.events.ArtifactContentStoredEvent
|
|||||||
import com.correx.core.events.events.ArtifactCreatedEvent
|
import com.correx.core.events.events.ArtifactCreatedEvent
|
||||||
import com.correx.core.events.events.ArtifactValidatedEvent
|
import com.correx.core.events.events.ArtifactValidatedEvent
|
||||||
import com.correx.core.events.events.ArtifactValidatingEvent
|
import com.correx.core.events.events.ArtifactValidatingEvent
|
||||||
|
import com.correx.core.events.events.BriefGroundingCheckedEvent
|
||||||
import com.correx.core.events.events.ChatSessionStartedEvent
|
import com.correx.core.events.events.ChatSessionStartedEvent
|
||||||
import com.correx.core.events.events.ChatTurnEvent
|
import com.correx.core.events.events.ChatTurnEvent
|
||||||
import com.correx.core.events.events.RouterNarrationEvent
|
import com.correx.core.events.events.RouterNarrationEvent
|
||||||
@@ -99,6 +100,7 @@ val eventModule = SerializersModule {
|
|||||||
subclass(RepoMapComputedEvent::class)
|
subclass(RepoMapComputedEvent::class)
|
||||||
subclass(WorkspaceStateObservedEvent::class)
|
subclass(WorkspaceStateObservedEvent::class)
|
||||||
subclass(RepoKnowledgeRetrievedEvent::class)
|
subclass(RepoKnowledgeRetrievedEvent::class)
|
||||||
|
subclass(BriefGroundingCheckedEvent::class)
|
||||||
subclass(RiskAssessedEvent::class)
|
subclass(RiskAssessedEvent::class)
|
||||||
subclass(ChatSessionStartedEvent::class)
|
subclass(ChatSessionStartedEvent::class)
|
||||||
subclass(ChatTurnEvent::class)
|
subclass(ChatTurnEvent::class)
|
||||||
|
|||||||
+42
@@ -0,0 +1,42 @@
|
|||||||
|
package com.correx.core.events.serialization
|
||||||
|
|
||||||
|
import com.correx.core.events.events.BriefGroundingCheckedEvent
|
||||||
|
import com.correx.core.events.events.EventPayload
|
||||||
|
import com.correx.core.events.events.GroundingReference
|
||||||
|
import com.correx.core.events.types.SessionId
|
||||||
|
import com.correx.core.events.types.StageId
|
||||||
|
import kotlin.test.Test
|
||||||
|
import kotlin.test.assertEquals
|
||||||
|
import kotlin.test.assertTrue
|
||||||
|
|
||||||
|
class BriefGroundingCheckedEventSerializationTest {
|
||||||
|
|
||||||
|
private val sample = BriefGroundingCheckedEvent(
|
||||||
|
sessionId = SessionId("sess-1"),
|
||||||
|
stageId = StageId("analyst"),
|
||||||
|
artifactId = "analyst",
|
||||||
|
references = listOf(
|
||||||
|
GroundingReference("core/validation/JsonSchemaValidator.kt", grounded = true),
|
||||||
|
GroundingReference("core/ghost/Phantom.kt", grounded = false),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `round-trips as polymorphic EventPayload`() {
|
||||||
|
val encoded = eventJson.encodeToString(EventPayload.serializer(), sample)
|
||||||
|
assertTrue(encoded.contains("\"type\":\"BriefGroundingChecked\""), "SerialName must be present: $encoded")
|
||||||
|
val decoded = eventJson.decodeFromString(EventPayload.serializer(), encoded)
|
||||||
|
assertEquals(sample, decoded)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `decodes hand-written BriefGroundingChecked JSON`() {
|
||||||
|
val json = """
|
||||||
|
{"type":"BriefGroundingChecked","sessionId":"s","stageId":"analyst","artifactId":"analyst",
|
||||||
|
"references":[{"reference":"a/b.kt","grounded":false}]}
|
||||||
|
""".trimIndent()
|
||||||
|
val decoded = eventJson.decodeFromString(EventPayload.serializer(), json)
|
||||||
|
assertTrue(decoded is BriefGroundingCheckedEvent)
|
||||||
|
assertEquals(false, (decoded as BriefGroundingCheckedEvent).references.single().grounded)
|
||||||
|
}
|
||||||
|
}
|
||||||
+51
@@ -0,0 +1,51 @@
|
|||||||
|
package com.correx.core.kernel.orchestration
|
||||||
|
|
||||||
|
import kotlinx.serialization.json.Json
|
||||||
|
import kotlinx.serialization.json.JsonArray
|
||||||
|
import kotlinx.serialization.json.JsonElement
|
||||||
|
import kotlinx.serialization.json.JsonObject
|
||||||
|
import kotlinx.serialization.json.JsonPrimitive
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pulls the workspace-relative file-path references out of an LLM-emitted brief artifact (JSON),
|
||||||
|
* for entity grounding (role-reliability §3). Deterministic: walks every string leaf, splits on
|
||||||
|
* whitespace, strips surrounding prose punctuation, and keeps tokens that read as a relative
|
||||||
|
* file path — one containing a '/' and ending in a short dotted extension.
|
||||||
|
*
|
||||||
|
* Module/subsystem names, bare words, URLs, and absolute or parent-relative paths are
|
||||||
|
* intentionally skipped. Only concrete file references are grounded, so the check catches
|
||||||
|
* hallucinated paths ("analysts hallucinate paths the same as planners") without false-flagging
|
||||||
|
* the coarse "files, modules, or subsystems" the brief is allowed to name.
|
||||||
|
*/
|
||||||
|
internal object BriefReferenceExtractor {
|
||||||
|
|
||||||
|
// <segments…>/<name>.<ext>: at least one slash, a final dotted extension of 1–5 letters/digits.
|
||||||
|
private val FILE_REF = Regex("""^[A-Za-z0-9_.\-/]+/[A-Za-z0-9_.\-]+\.[A-Za-z][A-Za-z0-9]{0,4}$""")
|
||||||
|
|
||||||
|
fun fileReferences(artifactJson: String): List<String> {
|
||||||
|
val root = runCatching { Json.parseToJsonElement(artifactJson) }.getOrNull() ?: return emptyList()
|
||||||
|
val out = LinkedHashSet<String>()
|
||||||
|
collectStrings(root) { value ->
|
||||||
|
for (token in value.split(WHITESPACE)) {
|
||||||
|
val cleaned = token
|
||||||
|
.trimStart('(', '[', '{', '`', '"', '\'')
|
||||||
|
.trimEnd(')', ']', '}', ',', '.', ';', ':', '`', '"', '\'')
|
||||||
|
if (cleaned.isFileReference()) out += cleaned
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out.toList()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun String.isFileReference(): Boolean =
|
||||||
|
!startsWith("/") && !contains("..") && !contains("://") && FILE_REF.matches(this)
|
||||||
|
|
||||||
|
private fun collectStrings(element: JsonElement, sink: (String) -> Unit) {
|
||||||
|
when (element) {
|
||||||
|
is JsonPrimitive -> if (element.isString) sink(element.content)
|
||||||
|
is JsonArray -> element.forEach { collectStrings(it, sink) }
|
||||||
|
is JsonObject -> element.values.forEach { collectStrings(it, sink) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private val WHITESPACE = Regex("""\s+""")
|
||||||
|
}
|
||||||
+51
-1
@@ -28,7 +28,9 @@ import com.correx.core.events.events.RefinementIterationEvent
|
|||||||
import com.correx.core.events.events.ApprovalRequestedEvent
|
import com.correx.core.events.events.ApprovalRequestedEvent
|
||||||
import com.correx.core.events.events.ArtifactContentStoredEvent
|
import com.correx.core.events.events.ArtifactContentStoredEvent
|
||||||
import com.correx.core.events.events.ArtifactCreatedEvent
|
import com.correx.core.events.events.ArtifactCreatedEvent
|
||||||
|
import com.correx.core.events.events.BriefGroundingCheckedEvent
|
||||||
import com.correx.core.events.events.ContextTruncatedEvent
|
import com.correx.core.events.events.ContextTruncatedEvent
|
||||||
|
import com.correx.core.events.events.GroundingReference
|
||||||
import com.correx.core.events.events.ToolCallAssessedEvent
|
import com.correx.core.events.events.ToolCallAssessedEvent
|
||||||
import com.correx.core.events.events.ArtifactValidatedEvent
|
import com.correx.core.events.events.ArtifactValidatedEvent
|
||||||
import com.correx.core.events.events.ArtifactValidatingEvent
|
import com.correx.core.events.events.ArtifactValidatingEvent
|
||||||
@@ -529,7 +531,11 @@ abstract class SessionOrchestrator(
|
|||||||
is StageExecutionResult.Success -> {
|
is StageExecutionResult.Success -> {
|
||||||
emitToolArtifacts(sessionId, stageId, stageConfig)
|
emitToolArtifacts(sessionId, stageId, stageConfig)
|
||||||
emitLlmArtifacts(sessionId, stageId, stageConfig)
|
emitLlmArtifacts(sessionId, stageId, stageConfig)
|
||||||
verifyProduces(sessionId, stageId, stageConfig)
|
when (val produced = verifyProduces(sessionId, stageId, stageConfig)) {
|
||||||
|
is StageExecutionResult.Success ->
|
||||||
|
groundBriefReferences(sessionId, stageId, stageConfig, effectives)
|
||||||
|
is StageExecutionResult.Failure -> produced
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
is StageExecutionResult.Failure -> outcome
|
is StageExecutionResult.Failure -> outcome
|
||||||
@@ -1265,6 +1271,50 @@ abstract class SessionOrchestrator(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Entity grounding (role-reliability §3): for a stage that opted in (`ground_references`),
|
||||||
|
* check every workspace-relative file path its brief named against the filesystem. Existence
|
||||||
|
* is a nondeterministic observation, so the result is recorded as a [BriefGroundingCheckedEvent]
|
||||||
|
* (invariant #9) — replay reads it back, never re-probes. A hallucinated path fails the stage
|
||||||
|
* (retryable), feeding the missing paths back so the model re-grounds on the next attempt.
|
||||||
|
*/
|
||||||
|
private suspend fun groundBriefReferences(
|
||||||
|
sessionId: SessionId,
|
||||||
|
stageId: StageId,
|
||||||
|
stageConfig: StageConfig,
|
||||||
|
effectives: RunEffectives,
|
||||||
|
): StageExecutionResult {
|
||||||
|
if (stageConfig.metadata["groundReferences"] != "true") return StageExecutionResult.Success(emptyList())
|
||||||
|
val workspaceRoot = effectives.policy?.workspaceRoot ?: return StageExecutionResult.Success(emptyList())
|
||||||
|
val artifact = stageConfig.produces
|
||||||
|
.filter { it.kind.llmEmitted }
|
||||||
|
.firstNotNullOfOrNull { artifactContentCache["${sessionId.value}:${it.name.value}"] }
|
||||||
|
?: return StageExecutionResult.Success(emptyList())
|
||||||
|
|
||||||
|
val references = BriefReferenceExtractor.fileReferences(artifact)
|
||||||
|
if (references.isEmpty()) return StageExecutionResult.Success(emptyList())
|
||||||
|
|
||||||
|
val results = references.map { ref ->
|
||||||
|
GroundingReference(ref, worldProbe.exists(workspaceRoot.resolve(ref)))
|
||||||
|
}
|
||||||
|
emit(sessionId, BriefGroundingCheckedEvent(sessionId, stageId, stageId.value, results))
|
||||||
|
|
||||||
|
val missing = results.filterNot { it.grounded }.map { it.reference }
|
||||||
|
return if (missing.isEmpty()) {
|
||||||
|
StageExecutionResult.Success(emptyList())
|
||||||
|
} else {
|
||||||
|
log.warn(
|
||||||
|
"[Orchestrator] brief references nonexistent paths session={} stage={} missing={}",
|
||||||
|
sessionId.value, stageId.value, missing.joinToString(", "),
|
||||||
|
)
|
||||||
|
StageExecutionResult.Failure(
|
||||||
|
"stage ${stageId.value} references paths that do not exist in the workspace: " +
|
||||||
|
"${missing.joinToString(", ")}. Reference only files you have read.",
|
||||||
|
retryable = true,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private suspend fun emitProcessResultEvents(
|
private suspend fun emitProcessResultEvents(
|
||||||
sessionId: SessionId,
|
sessionId: SessionId,
|
||||||
stageId: StageId,
|
stageId: StageId,
|
||||||
|
|||||||
+65
@@ -0,0 +1,65 @@
|
|||||||
|
package com.correx.core.kernel.orchestration
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions.assertEquals
|
||||||
|
import org.junit.jupiter.api.Assertions.assertTrue
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
|
||||||
|
class BriefReferenceExtractorTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `extracts relative file paths from string arrays`() {
|
||||||
|
val json = """
|
||||||
|
{
|
||||||
|
"summary": "touch the validator",
|
||||||
|
"affected_areas": [
|
||||||
|
"core/validation/JsonSchemaValidator.kt",
|
||||||
|
"the validation pipeline",
|
||||||
|
"core:validation module"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
""".trimIndent()
|
||||||
|
assertEquals(listOf("core/validation/JsonSchemaValidator.kt"), BriefReferenceExtractor.fileReferences(json))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `strips surrounding prose punctuation`() {
|
||||||
|
val json = """{"requirements":["see (core/kernel/Foo.kt), then bar.","ends here: apps/cli/Main.kt."]}"""
|
||||||
|
assertEquals(
|
||||||
|
listOf("core/kernel/Foo.kt", "apps/cli/Main.kt"),
|
||||||
|
BriefReferenceExtractor.fileReferences(json),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `skips urls, absolute and parent paths, and bare names`() {
|
||||||
|
val json = """
|
||||||
|
{
|
||||||
|
"notes": [
|
||||||
|
"https://example.com/x.html",
|
||||||
|
"/etc/passwd",
|
||||||
|
"../escape/secret.kt",
|
||||||
|
"StageConfig.kt",
|
||||||
|
"just prose with no paths"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
""".trimIndent()
|
||||||
|
assertTrue(BriefReferenceExtractor.fileReferences(json).isEmpty())
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `dedupes and preserves first-seen order`() {
|
||||||
|
val json = """{"a":["core/A.kt","docs/x.md"],"b":["core/A.kt"]}"""
|
||||||
|
assertEquals(listOf("core/A.kt", "docs/x.md"), BriefReferenceExtractor.fileReferences(json))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `keeps nested dotfile-directory paths`() {
|
||||||
|
val json = """{"areas":[".github/workflows/ci.yml"]}"""
|
||||||
|
assertEquals(listOf(".github/workflows/ci.yml"), BriefReferenceExtractor.fileReferences(json))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `malformed json yields no references`() {
|
||||||
|
assertTrue(BriefReferenceExtractor.fileReferences("not json at all").isEmpty())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,11 +25,14 @@ id = "role_pipeline"
|
|||||||
start = "analyst"
|
start = "analyst"
|
||||||
|
|
||||||
# 1. Understand the request and the relevant code. Read-only.
|
# 1. Understand the request and the relevant code. Read-only.
|
||||||
|
# ground_references: every workspace-relative file path the analysis names is checked
|
||||||
|
# for existence; a hallucinated path fails the stage and retries with the misses fed back.
|
||||||
[[stages]]
|
[[stages]]
|
||||||
id = "analyst"
|
id = "analyst"
|
||||||
prompt = "prompts/analyst.md"
|
prompt = "prompts/analyst.md"
|
||||||
produces = [{ name = "analysis", kind = "analysis" }]
|
produces = [{ name = "analysis", kind = "analysis" }]
|
||||||
allowed_tools = ["file_read", "ShellTool"]
|
allowed_tools = ["file_read", "ShellTool"]
|
||||||
|
ground_references = true
|
||||||
token_budget = 16384
|
token_budget = 16384
|
||||||
max_retries = 2
|
max_retries = 2
|
||||||
|
|
||||||
|
|||||||
+2
@@ -46,6 +46,7 @@ private data class StageSection(
|
|||||||
@param:JsonProperty("max_retries") val maxRetries: Int = 3,
|
@param:JsonProperty("max_retries") val maxRetries: Int = 3,
|
||||||
@param:JsonProperty("requires_approval") val requiresApproval: Boolean = false,
|
@param:JsonProperty("requires_approval") val requiresApproval: Boolean = false,
|
||||||
@param:JsonProperty("inject_artifact_kinds") val injectArtifactKinds: Boolean = false,
|
@param:JsonProperty("inject_artifact_kinds") val injectArtifactKinds: Boolean = false,
|
||||||
|
@param:JsonProperty("ground_references") val groundReferences: Boolean = false,
|
||||||
)
|
)
|
||||||
|
|
||||||
// condition fields flattened into the transition row
|
// condition fields flattened into the transition row
|
||||||
@@ -119,6 +120,7 @@ class TomlWorkflowLoader(
|
|||||||
s.systemPrompt?.let { put("systemPrompt", resolvePromptPath(it, workflowDir)) }
|
s.systemPrompt?.let { put("systemPrompt", resolvePromptPath(it, workflowDir)) }
|
||||||
if (s.requiresApproval) put("requiresApproval", "true")
|
if (s.requiresApproval) put("requiresApproval", "true")
|
||||||
if (s.injectArtifactKinds) put("injectArtifactKinds", "true")
|
if (s.injectArtifactKinds) put("injectArtifactKinds", "true")
|
||||||
|
if (s.groundReferences) put("groundReferences", "true")
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
+18
@@ -155,4 +155,22 @@ class TomlWorkflowLoaderTest {
|
|||||||
val collectStage = graph.stages[graph.start]!!
|
val collectStage = graph.stages[graph.start]!!
|
||||||
assertEquals(emptyList(), collectStage.writeManifest)
|
assertEquals(emptyList(), collectStage.writeManifest)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `ground_references true maps to metadata key`() {
|
||||||
|
val toml = validToml.replace(
|
||||||
|
"prompt = \"prompts/collect.md\"",
|
||||||
|
"prompt = \"prompts/collect.md\"\nground_references = true",
|
||||||
|
)
|
||||||
|
val path = Files.createTempFile("workflow", ".toml").also { it.writeText(toml) }
|
||||||
|
val graph = loader.load(path)
|
||||||
|
assertEquals("true", graph.stages[graph.start]!!.metadata["groundReferences"])
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `ground_references absent means no metadata key`() {
|
||||||
|
val path = Files.createTempFile("workflow", ".toml").also { it.writeText(validToml) }
|
||||||
|
val graph = loader.load(path)
|
||||||
|
assertEquals(null, graph.stages[graph.start]!!.metadata["groundReferences"])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user