feat(context): reasoning-thread continuity, L3 doc filtering, and gate hardening

Threads reasoning_content across inference calls and journal renders, filters markdown noise out of L3 repo-knowledge retrieval, adds PlanLinter H3 checks, and tightens filesystem tool output/dir-listing behavior surfaced by prior live QA (see project_readloop_campaign memory).
2026-07-10 11:13:43 +04:00
parent f51a8dada4
commit 3d5e05c1fb
32 changed files with 742 additions and 85 deletions
@@ -142,9 +142,15 @@ class LlamaCppInferenceProvider(

        val baseMessages = PromptRenderer.render(request.contextPack)
        val messages = baseMessages.map { msg ->
+            val toolCalls = if (msg.role == "assistant") {
+                parseAssistantToolCalls(msg.content)
+            } else emptyList()
            LlamaCppChatMessage(
                role = msg.role,
-                content = msg.content,
+                content = if (toolCalls.isNotEmpty()) null else msg.content,
+                reasoningContent = msg.reasoning,
+                toolCalls = toolCalls,
+                toolCallId = msg.toolCallId,
            )
        }

@@ -230,5 +236,40 @@ class LlamaCppInferenceProvider(
        ProviderHealth.Unavailable("Health check failed: ${e.message}")
    }

+    /**
+     * Parses an assistant message's content back into [ToolCallRequest] objects.
+     *
+     * The context builder's FORMAT_COMPRESS stage converts raw ToolCallRequest JSON into a compact
+     * dotted-line format (id = ..., function.name = ...). If compression is disabled, the content
+     * is still valid JSON. This handles both forms so the Gemma4 Jinja template sees the native
+     * `tool_calls` array — which it requires to render `reasoning_content` as a `<|channel>thought`
+     * block (without `tool_calls` the reasoning field is silently dropped and the model re-enters
+     * thought mode from scratch, causing the degenerate `<|channel>thought`~16384-token loop).
+     */
+    private fun parseAssistantToolCalls(content: String): List<ToolCallRequest> {
+        // Try JSON parse first (FORMAT_COMPRESS disabled — raw ToolCallRequest JSON).
+        if (content.startsWith("{\"id\"")) {
+            runCatching {
+                return listOf(json.decodeFromString<ToolCallRequest>(content))
+            }
+        }
+        // Try dotted-line format (FORMAT_COMPRESS enabled).
+        if (!content.contains("function.name")) return emptyList()
+        val lines = content.split('\n')
+        var id: String? = null
+        var name: String? = null
+        var arguments: String? = null
+        for (line in lines) {
+            when {
+                line.startsWith("id = ") -> id = line.removePrefix("id = ").trim()
+                line.startsWith("function.name = ") -> name = line.removePrefix("function.name = ").trim()
+                line.startsWith("function.arguments = ") -> arguments = line.removePrefix("function.arguments = ").trim()
+            }
+        }
+        return if (name != null && arguments != null) {
+            listOf(ToolCallRequest(id = id, function = ToolCallFunction(name = name, arguments = arguments)))
+        } else emptyList()
+    }
+
    override fun capabilities(): Set<CapabilityScore> = descriptor.capabilities
 }
@@ -88,7 +88,7 @@ class OpenAiCompatInferenceProvider(
            // tool_calls turn (we don't carry tool_call_ids through the context pack). Fold any
            // non-standard role into a user turn so the model still sees the content.
            if (msg.role in STANDARD_ROLES) {
-                OpenAiChatMessage(role = msg.role, content = msg.content)
+                OpenAiChatMessage(role = msg.role, content = msg.content, reasoningContent = msg.reasoning)
            } else {
                OpenAiChatMessage(role = "user", content = "[${msg.role}] ${msg.content}")
            }
@@ -135,8 +135,18 @@ class FileEditTool(
            operation == "patch" && !request.parameters.containsKey("patch") ->
                ValidationResult.Invalid("Missing 'patch' parameter for 'patch' operation.")

+            // Don't point the model at 'file_read'/'list_dir' here — this rejection is commonly hit
+            // in exactly the round those tools have been deliberately withheld (the read-loop
+            // breaker forces a write-only round after repeated reads with no write), so telling it
+            // to go use tools that aren't in its tool list this round would just compound the stall.
+            operation == "read" ->
+                ValidationResult.Invalid(
+                    "'edit_file' has no 'read' operation. It only supports 'append', 'replace', or 'patch' — " +
+                        "proceed with one of those using what you already know about the file.",
+                )
+
            operation !in listOf("append", "replace", "patch") ->
-                ValidationResult.Invalid("Unknown operation: $operation")
+                ValidationResult.Invalid("Unknown operation: $operation. Expected 'append', 'replace', or 'patch'.")

            else -> ValidationResult.Valid
        }
@@ -154,6 +154,7 @@ class FileWriteTool(
        val originalContent = runCatching { Files.readString(path) }.getOrDefault("")

        val result = runCatching {
+            Files.createDirectories(path.parent)
            AtomicFileWriter.write(path, content.toByteArray(Charsets.UTF_8))
            ToolResult.Success(
                invocationId = request.invocationId,
@@ -97,8 +97,17 @@ class ListDirTool(
        val recursive = request.parameters["recursive"]?.toString()?.toBoolean() ?: false
        val root = resolvePath(pathString)
        when {
-            !Files.exists(root) ->
-                ToolResult.Failure(request.invocationId, "Directory not found: $pathString", recoverable = true)
+            !Files.exists(root) -> {
+                val suggestions = similarEntries(root)
+                val msg = buildString {
+                    append("Directory not found: $pathString")
+                    if (suggestions.isNotEmpty()) {
+                        append(". Did you mean ")
+                        append(suggestions.joinToString(", ", postfix = "?"))
+                    }
+                }
+                ToolResult.Failure(request.invocationId, msg, recoverable = true)
+            }
            !Files.isDirectory(root) ->
                ToolResult.Failure(request.invocationId, "Not a directory: $pathString", recoverable = true)
            else -> runCatching { walk(root, recursive, request) }.getOrElse {
@@ -109,8 +118,13 @@ class ListDirTool(

    private fun walk(root: Path, recursive: Boolean, request: ToolRequest): ToolResult {
        val matcher = GitignoreMatcher(workingDir ?: root, root)
+        // The caller explicitly targeted `root` — if a rule already covers it (e.g. an entire
+        // subtree gitignored as scaffold output), don't let that same rule keep pruning everything
+        // found underneath; other, unrelated rules (node_modules/ inside it, etc.) still apply.
+        matcher.exemptRulesMatching(root)
        val out = ArrayList<String>()
        var truncated = false
+        var ignoredCount = 0
        // Emit one entry; TERMINATE the walk once the cap is hit so a huge tree can't run away.
        fun emit(path: Path, isDir: Boolean): FileVisitResult {
            val slash = if (isDir) "/" else ""
@@ -121,8 +135,11 @@ class ListDirTool(
        val maxDepth = if (recursive) Int.MAX_VALUE else 1
        Files.walkFileTree(root, emptySet(), maxDepth, object : SimpleFileVisitor<Path>() {
            override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult {
-                if (dir != root && (dir.name == ".git" || matcher.ignored(dir, isDir = true)))
+                if (dir != root && dir.name == ".git") return FileVisitResult.SKIP_SUBTREE
+                if (dir != root && matcher.ignored(dir, isDir = true)) {
+                    ignoredCount++
                    return FileVisitResult.SKIP_SUBTREE
+                }
                matcher.loadFrom(dir)
                return if (dir == root) FileVisitResult.CONTINUE else emit(dir, isDir = true)
            }
@@ -130,15 +147,83 @@ class ListDirTool(
                // A directory sitting exactly at maxDepth is delivered here (walkFileTree won't
                // descend it), so honour its dir-ness for both the ignore check and the trailing '/'.
                val isDir = attrs.isDirectory
-                return if (matcher.ignored(file, isDir)) FileVisitResult.CONTINUE else emit(file, isDir)
+                if (matcher.ignored(file, isDir)) {
+                    ignoredCount++
+                    return FileVisitResult.CONTINUE
+                }
+                return emit(file, isDir)
            }
            override fun visitFileFailed(file: Path, exc: java.io.IOException) = FileVisitResult.CONTINUE
        })
        out.sort()
-        val body = out.joinToString("\n").ifEmpty { "(empty)" }
-        val note =
-            if (truncated) "\n… truncated at $MAX_ENTRIES entries; narrow the path or list a sub-directory." else ""
-        return ToolResult.Success(request.invocationId, output = body + note)
+        return ToolResult.Success(request.invocationId, output = render(root, out, truncated, ignoredCount))
+    }
+
+    /**
+     * `<path>/<type>/<entries>` wrapper (matching the shape other coding-agent tools use) instead of
+     * a bare newline list — the count and explicit `directory` tag let a small model tell "empty
+     * listing" from "malformed output" and know how many entries to expect. Notes on truncation/
+     * gitignore-pruning are surfaced here too, so a shorter-than-expected listing isn't mistaken for
+     * the real tree.
+     */
+    private fun render(root: Path, entries: List<String>, truncated: Boolean, ignoredCount: Int): String {
+        val notes = buildList {
+            if (truncated) add("truncated at $MAX_ENTRIES entries; narrow the path or list a sub-directory")
+            if (ignoredCount > 0) add("$ignoredCount ${if (ignoredCount == 1) "entry" else "entries"} hidden by .gitignore")
+        }
+        return buildString {
+            appendLine("<path>$root</path>")
+            appendLine("<type>directory</type>")
+            appendLine("<entries>")
+            appendLine(entries.joinToString("\n").ifEmpty { "(empty)" })
+            if (notes.isNotEmpty()) appendLine(notes.joinToString("; "))
+            append("(${entries.size} ${if (entries.size == 1) "entry" else "entries"})")
+            appendLine()
+            append("</entries>")
+        }
+    }
+
+    private fun similarEntries(target: Path): List<String> {
+        val parent = target.parent ?: return emptyList()
+        if (!Files.isDirectory(parent)) return emptyList()
+        val targetName = target.fileName?.toString() ?: return emptyList()
+        return try {
+            Files.list(parent).use { stream ->
+                stream.map { it.fileName.toString() }
+                    .filter { it != targetName && isSimilarName(it, targetName) }
+                    .toList()
+                    .sortedWith(compareByDescending<String> { commonPrefixLen(it, targetName) }.thenBy { it })
+                    .take(3)
+            }
+        } catch (_: Exception) { emptyList() }
+    }
+
+    private fun isSimilarName(a: String, b: String): Boolean {
+        val prefix = commonPrefixLen(a, b)
+        return prefix >= 2 || levenshtein(a, b) <= 3
+    }
+
+    private fun commonPrefixLen(a: String, b: String): Int {
+        var i = 0
+        val minLen = minOf(a.length, b.length)
+        while (i < minLen && a[i].lowercaseChar() == b[i].lowercaseChar()) i++
+        return i
+    }
+
+    private fun levenshtein(a: String, b: String): Int {
+        val dp = Array(a.length + 1) { IntArray(b.length + 1) }
+        for (i in 0..a.length) dp[i][0] = i
+        for (j in 0..b.length) dp[0][j] = j
+        for (i in 1..a.length) {
+            for (j in 1..b.length) {
+                dp[i][j] = minOf(
+                    dp[i - 1][j] + 1,
+                    dp[i][j - 1] + 1,
+                    dp[i - 1][j - 1] + if (a[i - 1].lowercaseChar() == b[i - 1].lowercaseChar()) 0 else 1
+                )
+            }
+        }
+        return dp[a.length][b.length]
    }

    private companion object {
@@ -176,10 +261,17 @@ internal class GitignoreMatcher(anchor: Path, target: Path) {
        }
    }

-    fun ignored(path: Path, isDir: Boolean): Boolean = rules.any { r ->
+    fun ignored(path: Path, isDir: Boolean): Boolean = rules.any { matches(it, path, isDir) }
+
+    /** Drop whichever rules currently cover [path] — used so an explicitly-targeted (already
+     * gitignored) listing root doesn't have that same rule keep pruning its own children. */
+    fun exemptRulesMatching(path: Path) {
+        rules.removeAll { matches(it, path, isDir = true) }
+    }
+
+    private fun matches(r: Rule, path: Path, isDir: Boolean): Boolean =
        (!r.dirOnly || isDir) && path.startsWith(r.base) &&
            r.regex.matches(r.base.relativize(path).toString().replace('\\', '/'))
-    }

    private fun compile(base: Path, raw: String): Rule {
        var pat = raw
@@ -41,7 +41,7 @@ object PlanLinter {

    fun lint(graph: WorkflowGraph, seeds: Set<String> = seedArtifacts): PlanLintResult =
        PlanLintResult(
-            hardFailures = unproducedNeeds(graph, seeds) + trapStates(graph),
+            hardFailures = unproducedNeeds(graph, seeds) + trapStates(graph) + unreferencedPromptArtifacts(graph, seeds),
            softFindings = stageCount(graph) + fanOut(graph) + emptyBriefs(graph) + duplicateBriefs(graph),
        )

@@ -89,6 +89,35 @@ object PlanLinter {
        }
    }

+    /**
+     * H3: a stage's inline prompt mentions an artifact ID that is produced by the plan or
+     * provided by a seed, but the stage does not declare it in its `needs`. The stage will
+     * not have access to that artifact at runtime, so any reference in the prompt is a dangling
+     * assumption that will confuse or stall the model — force the architect to either add the
+     * artifact to `needs` or reword the prompt.
+     */
+    private fun unreferencedPromptArtifacts(graph: WorkflowGraph, seeds: Set<String>): List<PlanLintFinding> {
+        val produced = graph.stages.values.flatMap { it.produces }.map { it.name.value }.toSet() + seeds
+        return graph.stages.entries.flatMap { (id, stage) ->
+            val prompt = briefOf(stage)
+            val needIds = stage.needs.map { it.value }.toSet()
+            val ownProduces = stage.produces.map { it.name.value }.toSet()
+            produced.filter { prodId ->
+                prompt.isNotBlank() &&
+                    prodId.length >= 2 && // skip single-char IDs (false-positive risk with substring matching)
+                    Regex("\\b${Regex.escape(prodId)}\\b").containsMatchIn(prompt) &&
+                    prodId !in needIds &&
+                    prodId !in ownProduces
+            }.map { prodId ->
+                PlanLintFinding(
+                    code = "unreferenced_prompt_artifact",
+                    stageId = id.value,
+                    detail = "prompt mentions '$prodId' but it is not declared in needs",
+                )
+            }
+        }
+    }
+
    /** S1: more stages than the ceiling. */
    private fun stageCount(graph: WorkflowGraph): List<PlanLintFinding> =
        if (graph.stages.size > STAGE_CEILING) {
@@ -92,6 +92,37 @@ class PlanLinterTest {
        assertTrue(result.passed, "analysis is a planning-phase seed, not an unproduced need")
    }

+    @Test
+    fun `a stage mentioning its own produced artifact in the prompt is not a failure`() {
+        val g = graph(
+            stages = mapOf(
+                "a" to stage(produces = listOf("plan"), prompt = "write the plan artifact"),
+                "b" to stage(needs = listOf("plan"), prompt = "build"),
+            ),
+            edges = listOf("a" to "b", "b" to "done"),
+            start = "a",
+        )
+        val result = PlanLinter.lint(g)
+        assertTrue(result.passed, "expected pass, hard=${result.hardFailures}")
+    }
+
+    @Test
+    fun `a prompt mentioning another stage's artifact not in needs is a hard failure`() {
+        val g = graph(
+            stages = mapOf(
+                "a" to stage(produces = listOf("plan"), prompt = "write the plan artifact"),
+                "b" to stage(produces = listOf("report"), prompt = "use plan to build"),
+            ),
+            edges = listOf("a" to "b", "b" to "done"),
+            start = "a",
+        )
+        val result = PlanLinter.lint(g)
+        assertFalse(result.passed)
+        val h = result.hardFailures.single { it.code == "unreferenced_prompt_artifact" }
+        assertEquals("b", h.stageId)
+        assertTrue(h.detail.contains("plan"))
+    }
+
    @Test
    fun `a cycle with no exit is a trap-state hard failure`() {
        val g = graph(