fix(context,tools): context hygiene + tool ergonomics from freestyle QA
Found while live-QAing freestyle_planning on a 12B local model: - list_dir tool: recursive, .gitignore-aware listing so weak models stop flooding context with `ls -R` over node_modules/build/dist. Wired into the fileRead toggle + advertised to the planner (architect_freestyle). - ContextClassifier: assistantToolCall turns are STRUCTURED, so the token pruner never shreds the model's own tool-call history — that was causing amnesia loops (re-issuing calls it had already made). - Retire instruction-doc LLMLingua pruning (DOC_SOURCE_TYPES emptied): it fused load-bearing procedural text into unparseable soup. The static block stays small by dropping CLAUDE.md at the loader instead. - AgentInstructionsLoader: load only AGENTS.md, not CLAUDE.md — the latter targets the outer assistant and polluted the agent's stage context. - DefaultSessionReducer: WorkflowFailed flips session status to FAILED (was stuck ACTIVE forever, so clients/approval loops never saw a terminal). - ShellTool: run shell command lines (cd/&&/pipes) via `sh -c`; unrunnable program is recoverable instead of an uncaught IOException killing the stage; malformed argv (non-string/collapsed-array) rejected with guidance. - llmlingua sidecar: cap force_tokens to max_force_token (big docs blew the assert and 500'd, so doc pruning silently failed open). Tests added/updated across all of the above.
This commit is contained in:
+201
@@ -0,0 +1,201 @@
|
||||
package com.correx.infrastructure.tools.filesystem
|
||||
|
||||
import com.correx.core.approvals.Tier
|
||||
import com.correx.core.events.events.ToolRequest
|
||||
import com.correx.core.tools.contract.ParamRole
|
||||
import com.correx.core.tools.contract.Tool
|
||||
import com.correx.core.tools.contract.ToolCapability
|
||||
import com.correx.core.tools.contract.ToolExecutor
|
||||
import com.correx.core.tools.contract.ToolResult
|
||||
import com.correx.core.tools.contract.ValidationResult
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.buildJsonArray
|
||||
import kotlinx.serialization.json.buildJsonObject
|
||||
import kotlinx.serialization.json.put
|
||||
import kotlinx.serialization.json.putJsonObject
|
||||
import java.nio.file.FileVisitResult
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.InvalidPathException
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
import java.nio.file.SimpleFileVisitor
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
import kotlin.io.path.name
|
||||
import kotlin.io.path.readText
|
||||
|
||||
/**
|
||||
* Recursive, `.gitignore`-aware directory listing — the affordance weak local models should reach
|
||||
* for instead of shell `ls -R`, which dumps `node_modules`/`build`/`dist` and drowns a small model
|
||||
* in noise. Read-only (Tier T1, FILE_READ): it shares [FileReadTool]'s path jail and workspace
|
||||
* anchor. `.gitignore` is honoured for *enumeration only* — the mutation tools deliberately do NOT
|
||||
* respect it (you legitimately write to ignored paths like `.env`/`dist`).
|
||||
*/
|
||||
class ListDirTool(
|
||||
private val allowedPaths: Set<Path> = emptySet(),
|
||||
private val workingDir: Path? = null,
|
||||
) : Tool, ToolExecutor {
|
||||
|
||||
private fun resolvePath(pathString: String): Path {
|
||||
val raw = Paths.get(pathString)
|
||||
return when {
|
||||
raw.isAbsolute -> raw.normalize()
|
||||
workingDir != null -> workingDir.resolve(raw).normalize()
|
||||
else -> raw.toAbsolutePath().normalize()
|
||||
}
|
||||
}
|
||||
|
||||
override val name: String = "list_dir"
|
||||
override val description: String =
|
||||
"List a directory tree, skipping anything matched by .gitignore (node_modules, build output, " +
|
||||
"logs, etc). Prefer this over shell 'ls -R' or 'find' for exploring project structure."
|
||||
override val parametersSchema: JsonObject = buildJsonObject {
|
||||
put("type", "object")
|
||||
putJsonObject("properties") {
|
||||
putJsonObject("path") {
|
||||
put("type", "string")
|
||||
put("description", "Relative directory to list. Omit or '.' for the workspace root.")
|
||||
}
|
||||
putJsonObject("recursive") {
|
||||
put("type", "boolean")
|
||||
put("description", "Descend into sub-directories (gitignored subtrees are pruned). Default true.")
|
||||
}
|
||||
}
|
||||
put("required", buildJsonArray {})
|
||||
}
|
||||
override val tier: Tier = Tier.T1
|
||||
override val requiredCapabilities: Set<ToolCapability> = setOf(ToolCapability.FILE_READ)
|
||||
override val paramRoles: Map<String, ParamRole> = mapOf("path" to ParamRole.PATH)
|
||||
|
||||
override fun validateRequest(request: ToolRequest): ValidationResult {
|
||||
val pathString = (request.parameters["path"] as? String) ?: "."
|
||||
return runCatching {
|
||||
val path = resolvePath(pathString)
|
||||
if (!PathJail.isContained(path, allowedPaths))
|
||||
ValidationResult.Invalid("Path '$pathString' is not in the allowed list")
|
||||
else ValidationResult.Valid
|
||||
}.getOrElse {
|
||||
when (it) {
|
||||
is InvalidPathException -> ValidationResult.Invalid("Invalid path format: ${it.message}")
|
||||
else -> ValidationResult.Invalid(it.message ?: "Unknown error occurred")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override suspend fun execute(request: ToolRequest): ToolResult = withContext(Dispatchers.IO) {
|
||||
(validateRequest(request) as? ValidationResult.Invalid)?.let {
|
||||
return@withContext ToolResult.Failure(request.invocationId, it.reason, recoverable = false)
|
||||
}
|
||||
val pathString = (request.parameters["path"] as? String) ?: "."
|
||||
val recursive = request.parameters["recursive"]?.toString()?.toBoolean() ?: true
|
||||
val root = resolvePath(pathString)
|
||||
when {
|
||||
!Files.exists(root) ->
|
||||
ToolResult.Failure(request.invocationId, "Directory not found: $pathString", recoverable = true)
|
||||
!Files.isDirectory(root) ->
|
||||
ToolResult.Failure(request.invocationId, "Not a directory: $pathString", recoverable = true)
|
||||
else -> runCatching { walk(root, recursive, request) }.getOrElse {
|
||||
ToolResult.Failure(request.invocationId, "Failed to list dir: ${it.message}", recoverable = false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun walk(root: Path, recursive: Boolean, request: ToolRequest): ToolResult {
|
||||
val matcher = GitignoreMatcher(workingDir ?: root, root)
|
||||
val out = ArrayList<String>()
|
||||
var truncated = false
|
||||
// Emit one entry; TERMINATE the walk once the cap is hit so a huge tree can't run away.
|
||||
fun emit(path: Path, isDir: Boolean): FileVisitResult {
|
||||
val slash = if (isDir) "/" else ""
|
||||
out += root.relativize(path).toString().replace('\\', '/') + slash
|
||||
if (out.size >= MAX_ENTRIES) truncated = true
|
||||
return if (truncated) FileVisitResult.TERMINATE else FileVisitResult.CONTINUE
|
||||
}
|
||||
val maxDepth = if (recursive) Int.MAX_VALUE else 1
|
||||
Files.walkFileTree(root, emptySet(), maxDepth, object : SimpleFileVisitor<Path>() {
|
||||
override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult {
|
||||
if (dir != root && (dir.name == ".git" || matcher.ignored(dir, isDir = true)))
|
||||
return FileVisitResult.SKIP_SUBTREE
|
||||
matcher.loadFrom(dir)
|
||||
return if (dir == root) FileVisitResult.CONTINUE else emit(dir, isDir = true)
|
||||
}
|
||||
override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
|
||||
// A directory sitting exactly at maxDepth is delivered here (walkFileTree won't
|
||||
// descend it), so honour its dir-ness for both the ignore check and the trailing '/'.
|
||||
val isDir = attrs.isDirectory
|
||||
return if (matcher.ignored(file, isDir)) FileVisitResult.CONTINUE else emit(file, isDir)
|
||||
}
|
||||
override fun visitFileFailed(file: Path, exc: java.io.IOException) = FileVisitResult.CONTINUE
|
||||
})
|
||||
out.sort()
|
||||
val body = out.joinToString("\n").ifEmpty { "(empty)" }
|
||||
val note =
|
||||
if (truncated) "\n… truncated at $MAX_ENTRIES entries; narrow the path or list a sub-directory." else ""
|
||||
return ToolResult.Success(request.invocationId, output = body + note)
|
||||
}
|
||||
|
||||
private companion object {
|
||||
const val MAX_ENTRIES = 400
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal `.gitignore` matcher: enough to prune the common noise (node_modules, dist, build, logs,
|
||||
* editor dirs). Loads a `.gitignore` per directory as the walk descends, plus the target's ancestors
|
||||
* once at construction, so nested ignore files scope to their own subtree.
|
||||
*
|
||||
* ponytail: no negation (`!`) and `**` is treated as a loose `.*`. If a real repo needs full
|
||||
* gitignore semantics, back this with `git check-ignore` / a spec-complete matcher.
|
||||
*/
|
||||
internal class GitignoreMatcher(anchor: Path, target: Path) {
|
||||
private data class Rule(val base: Path, val regex: Regex, val dirOnly: Boolean)
|
||||
|
||||
private val rules = ArrayList<Rule>()
|
||||
|
||||
init {
|
||||
// Ancestor .gitignore files (target up to and including anchor) apply to the whole listing;
|
||||
// load them outermost-first so the target's own rules are added last.
|
||||
generateSequence(target) { if (it == anchor) null else it.parent }
|
||||
.toList().asReversed().forEach { loadFrom(it) }
|
||||
}
|
||||
|
||||
fun loadFrom(dir: Path) {
|
||||
val gi = dir.resolve(".gitignore")
|
||||
if (!Files.isRegularFile(gi)) return
|
||||
runCatching { gi.readText() }.getOrNull()?.lineSequence()?.forEach { line ->
|
||||
val pat = line.trim()
|
||||
if (pat.isEmpty() || pat.startsWith("#") || pat.startsWith("!")) return@forEach
|
||||
rules += compile(dir, pat)
|
||||
}
|
||||
}
|
||||
|
||||
fun ignored(path: Path, isDir: Boolean): Boolean = rules.any { r ->
|
||||
(!r.dirOnly || isDir) && path.startsWith(r.base) &&
|
||||
r.regex.matches(r.base.relativize(path).toString().replace('\\', '/'))
|
||||
}
|
||||
|
||||
private fun compile(base: Path, raw: String): Rule {
|
||||
var pat = raw
|
||||
val dirOnly = pat.endsWith("/").also { if (it) pat = pat.dropLast(1) }
|
||||
val anchored = pat.startsWith("/").also { if (it) pat = pat.drop(1) }
|
||||
val body = StringBuilder()
|
||||
var i = 0
|
||||
while (i < pat.length) {
|
||||
val c = pat[i]
|
||||
when (c) {
|
||||
'*' -> if (i + 1 < pat.length && pat[i + 1] == '*') { body.append(".*"); i++ } else body.append("[^/]*")
|
||||
'?' -> body.append("[^/]")
|
||||
'.', '(', ')', '+', '|', '^', '$', '{', '}', '[', ']', '\\' -> body.append('\\').append(c)
|
||||
else -> body.append(c)
|
||||
}
|
||||
i++
|
||||
}
|
||||
// A slash-less, unanchored pattern matches the entry at any depth (git semantics); otherwise
|
||||
// it is anchored to the .gitignore's directory. Trailing "(/.*)?" so a matched dir also
|
||||
// covers everything under it.
|
||||
val prefix = if (anchored || pat.contains('/')) "" else "(.*/)?"
|
||||
return Rule(base, Regex("$prefix$body(/.*)?"), dirOnly)
|
||||
}
|
||||
}
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
package com.correx.infrastructure.tools.filesystem
|
||||
|
||||
import com.correx.core.events.events.ToolRequest
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.events.types.StageId
|
||||
import com.correx.core.events.types.ToolInvocationId
|
||||
import com.correx.core.tools.contract.ToolResult
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.junit.jupiter.api.Assertions.assertFalse
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.nio.file.Files
|
||||
import java.util.*
|
||||
|
||||
class ListDirToolTest {
|
||||
|
||||
private fun request(path: String? = null, recursive: Boolean? = null) = ToolRequest(
|
||||
invocationId = ToolInvocationId(UUID.randomUUID().toString()),
|
||||
sessionId = SessionId(UUID.randomUUID().toString()),
|
||||
stageId = StageId(UUID.randomUUID().toString()),
|
||||
toolName = "list_dir",
|
||||
parameters = buildMap {
|
||||
path?.let { put("path", it) }
|
||||
recursive?.let { put("recursive", it) }
|
||||
},
|
||||
)
|
||||
|
||||
@Test
|
||||
fun `recursive listing prunes gitignored dirs and files`(): Unit = runBlocking {
|
||||
val root = Files.createTempDirectory("listdir")
|
||||
Files.writeString(root.resolve(".gitignore"), "node_modules\ndist\n*.log\n")
|
||||
Files.createDirectories(root.resolve("node_modules/pkg")).also { Files.writeString(it.resolve("x.js"), "x") }
|
||||
Files.createDirectories(root.resolve("dist")).also { Files.writeString(it.resolve("bundle.js"), "b") }
|
||||
Files.createDirectories(root.resolve("src")).also { Files.writeString(it.resolve("main.ts"), "m") }
|
||||
Files.writeString(root.resolve("debug.log"), "noise")
|
||||
Files.writeString(root.resolve("package.json"), "{}")
|
||||
|
||||
val tool = ListDirTool(allowedPaths = setOf(root), workingDir = root)
|
||||
val out = (tool.execute(request()) as ToolResult.Success).output
|
||||
|
||||
assertTrue(out.contains("src/main.ts"), out)
|
||||
assertTrue(out.contains("package.json"), out)
|
||||
assertFalse(out.contains("node_modules"), out) // whole subtree pruned, never descended
|
||||
assertFalse(out.contains("dist"), out)
|
||||
assertFalse(out.contains("debug.log"), out)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `non-recursive lists only immediate children`(): Unit = runBlocking {
|
||||
val root = Files.createTempDirectory("listdir")
|
||||
Files.createDirectories(root.resolve("src/deep")).also { Files.writeString(it.resolve("f.ts"), "f") }
|
||||
val tool = ListDirTool(allowedPaths = setOf(root), workingDir = root)
|
||||
val out = (tool.execute(request(recursive = false)) as ToolResult.Success).output
|
||||
assertTrue(out.contains("src/"), out)
|
||||
assertFalse(out.contains("deep"), out)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user