feat(tools): glob + grep read-only workspace search tools
Weak local models reached for shell find/grep -r (unjailed, dumps .gitignore'd trees, floods context) to answer 'does frontend/ exist?' / 'where is symbol X?'. Add two jailed, gitignore-aware read-only tools: - GlobTool (name 'glob', T1, FILE_READ+DIRECTORY_LIST): find files by glob pattern; survey-exempt from anti-hallucination read gates like list_dir. - GrepTool (name 'grep', T1, FILE_READ): regex content search, returns path:line: text; skips gitignored, binary (NUL), and >2MB files. Both reuse GitignoreMatcher + PathJail from the filesystem package, share FileReadTool's jail/anchor/toggle, cap at 200 results. Registered in ToolConfig.buildTools (fileRead block) and added to StageConfig.ALWAYS_AVAILABLE_READ_TOOLS so every tool-granting stage can call them. 6 tests green. Vikunja #37.
This commit is contained in:
@@ -66,6 +66,6 @@ data class StageConfig(
|
||||
|
||||
companion object {
|
||||
/** Read-only tools every tool-granting stage may call regardless of its declared set. */
|
||||
val ALWAYS_AVAILABLE_READ_TOOLS: Set<String> = setOf("file_read", "list_dir")
|
||||
val ALWAYS_AVAILABLE_READ_TOOLS: Set<String> = setOf("file_read", "list_dir", "glob", "grep")
|
||||
}
|
||||
}
|
||||
|
||||
+246
@@ -0,0 +1,246 @@
|
||||
package com.correx.infrastructure.tools.filesystem
|
||||
|
||||
import com.correx.core.approvals.Tier
|
||||
import com.correx.core.events.events.ToolRequest
|
||||
import com.correx.core.tools.contract.ParamRole
|
||||
import com.correx.core.tools.contract.Tool
|
||||
import com.correx.core.tools.contract.ToolCapability
|
||||
import com.correx.core.tools.contract.ToolExecutor
|
||||
import com.correx.core.tools.contract.ToolResult
|
||||
import com.correx.core.tools.contract.ValidationResult
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.buildJsonArray
|
||||
import kotlinx.serialization.json.buildJsonObject
|
||||
import kotlinx.serialization.json.put
|
||||
import kotlinx.serialization.json.putJsonObject
|
||||
import java.nio.file.FileSystems
|
||||
import java.nio.file.FileVisitResult
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.InvalidPathException
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.Paths
|
||||
import java.nio.file.SimpleFileVisitor
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
import kotlin.io.path.name
|
||||
import kotlin.io.path.readText
|
||||
|
||||
// Shared read-only search cap. The list_dir incident (a 400-entry dump burying the answer) is the
|
||||
// cautionary tale: a search that floods the context is worse than useless to a small model, so cap
|
||||
// aggressively and tell it to narrow rather than returning everything.
|
||||
private const val MAX_RESULTS = 200
|
||||
|
||||
// Skip files larger than this when grepping — a multi-MB minified bundle or lockfile is never what a
|
||||
// symbol search wants, and reading it line-by-line stalls the walk. ponytail: fixed byte cap; make it
|
||||
// a param if a real use case needs to grep large generated files.
|
||||
private const val GREP_MAX_FILE_BYTES = 2_000_000L
|
||||
|
||||
/**
|
||||
* `.gitignore`-aware path search by glob pattern — the affordance a model should reach for to answer
|
||||
* "does `frontend/` exist?" / "where are the `.tsx` files?" instead of `shell find`, which is unjailed
|
||||
* and dumps ignored trees. Read-only (Tier T1, FILE_READ + DIRECTORY_LIST — like [ListDirTool], a glob
|
||||
* is a survey, so anti-hallucination read gates exempt it: globbing a not-yet-created path truthfully
|
||||
* reports "no matches"). Shares [FileReadTool]'s path jail + workspace anchor. Results are relative
|
||||
* paths, sorted, capped at [MAX_RESULTS].
|
||||
*/
|
||||
class GlobTool(
|
||||
private val allowedPaths: Set<Path> = emptySet(),
|
||||
private val workingDir: Path? = null,
|
||||
) : Tool, ToolExecutor {
|
||||
|
||||
override val name: String = "glob"
|
||||
override val description: String =
|
||||
"Find files by glob pattern (e.g. '**/*.tsx', 'src/**/use*.ts'), skipping .gitignore'd paths. " +
|
||||
"Prefer this over shell 'find' to locate files. Returns matching relative paths."
|
||||
override val parametersSchema: JsonObject = buildJsonObject {
|
||||
put("type", "object")
|
||||
putJsonObject("properties") {
|
||||
putJsonObject("pattern") {
|
||||
put("type", "string")
|
||||
put("description", "Glob pattern, e.g. '**/*.kt' or 'src/**/use*.ts'. Matched against paths relative to 'path'.")
|
||||
}
|
||||
putJsonObject("path") {
|
||||
put("type", "string")
|
||||
put("description", "Relative directory to search under. Omit or '.' for the workspace root.")
|
||||
}
|
||||
}
|
||||
put("required", buildJsonArray { add(kotlinx.serialization.json.JsonPrimitive("pattern")) })
|
||||
}
|
||||
override val tier: Tier = Tier.T1
|
||||
override val requiredCapabilities: Set<ToolCapability> =
|
||||
setOf(ToolCapability.FILE_READ, ToolCapability.DIRECTORY_LIST)
|
||||
override val paramRoles: Map<String, ParamRole> = mapOf("path" to ParamRole.PATH)
|
||||
|
||||
override fun validateRequest(request: ToolRequest): ValidationResult =
|
||||
validateSearchPath(request, allowedPaths, workingDir)
|
||||
|
||||
override suspend fun execute(request: ToolRequest): ToolResult = withContext(Dispatchers.IO) {
|
||||
(validateRequest(request) as? ValidationResult.Invalid)?.let {
|
||||
return@withContext ToolResult.Failure(request.invocationId, it.reason, recoverable = false)
|
||||
}
|
||||
val pattern = (request.parameters["pattern"] as? String)?.takeIf { it.isNotBlank() }
|
||||
?: return@withContext ToolResult.Failure(request.invocationId, "glob requires a non-empty 'pattern'", recoverable = true)
|
||||
val base = resolveSearchPath(request, workingDir)
|
||||
if (!Files.isDirectory(base)) {
|
||||
return@withContext ToolResult.Failure(request.invocationId, "Not a directory: ${request.parameters["path"] ?: "."}", recoverable = true)
|
||||
}
|
||||
val matcher = runCatching { FileSystems.getDefault().getPathMatcher("glob:$pattern") }
|
||||
.getOrElse { return@withContext ToolResult.Failure(request.invocationId, "Invalid glob pattern: ${it.message}", recoverable = true) }
|
||||
|
||||
val hits = ArrayList<String>()
|
||||
var truncated = false
|
||||
walkGitignored(base) { rel, _ ->
|
||||
if (matcher.matches(Paths.get(rel))) {
|
||||
hits += rel
|
||||
if (hits.size >= MAX_RESULTS) truncated = true
|
||||
}
|
||||
!truncated
|
||||
}
|
||||
hits.sort()
|
||||
ToolResult.Success(request.invocationId, output = renderResults("glob '$pattern'", hits, truncated))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* `.gitignore`-aware content search — find lines matching a regex across the workspace, so a model can
|
||||
* locate a symbol/string ("where's `web-ui-design-spec`?") in one call instead of `shell grep -r`.
|
||||
* Read-only (Tier T1, FILE_READ). Shares the reader's jail + anchor. Returns `path:line: text`, capped
|
||||
* at [MAX_RESULTS] total matches; an optional `glob` narrows which files are scanned.
|
||||
*/
|
||||
class GrepTool(
|
||||
private val allowedPaths: Set<Path> = emptySet(),
|
||||
private val workingDir: Path? = null,
|
||||
) : Tool, ToolExecutor {
|
||||
|
||||
override val name: String = "grep"
|
||||
override val description: String =
|
||||
"Search file contents for a regex across the workspace (skips .gitignore'd paths). Prefer this " +
|
||||
"over shell 'grep -r' to find a symbol or string. Returns 'path:line: matched text'."
|
||||
override val parametersSchema: JsonObject = buildJsonObject {
|
||||
put("type", "object")
|
||||
putJsonObject("properties") {
|
||||
putJsonObject("pattern") {
|
||||
put("type", "string")
|
||||
put("description", "Regular expression to search for in file contents.")
|
||||
}
|
||||
putJsonObject("path") {
|
||||
put("type", "string")
|
||||
put("description", "Relative directory to search under. Omit or '.' for the workspace root.")
|
||||
}
|
||||
putJsonObject("glob") {
|
||||
put("type", "string")
|
||||
put("description", "Optional glob to limit which files are scanned, e.g. '**/*.kt'.")
|
||||
}
|
||||
}
|
||||
put("required", buildJsonArray { add(kotlinx.serialization.json.JsonPrimitive("pattern")) })
|
||||
}
|
||||
override val tier: Tier = Tier.T1
|
||||
override val requiredCapabilities: Set<ToolCapability> = setOf(ToolCapability.FILE_READ)
|
||||
override val paramRoles: Map<String, ParamRole> = mapOf("path" to ParamRole.PATH)
|
||||
|
||||
override fun validateRequest(request: ToolRequest): ValidationResult =
|
||||
validateSearchPath(request, allowedPaths, workingDir)
|
||||
|
||||
override suspend fun execute(request: ToolRequest): ToolResult = withContext(Dispatchers.IO) {
|
||||
(validateRequest(request) as? ValidationResult.Invalid)?.let {
|
||||
return@withContext ToolResult.Failure(request.invocationId, it.reason, recoverable = false)
|
||||
}
|
||||
val patternStr = (request.parameters["pattern"] as? String)?.takeIf { it.isNotBlank() }
|
||||
?: return@withContext ToolResult.Failure(request.invocationId, "grep requires a non-empty 'pattern'", recoverable = true)
|
||||
val regex = runCatching { Regex(patternStr) }
|
||||
.getOrElse { return@withContext ToolResult.Failure(request.invocationId, "Invalid regex: ${it.message}", recoverable = true) }
|
||||
val base = resolveSearchPath(request, workingDir)
|
||||
if (!Files.isDirectory(base)) {
|
||||
return@withContext ToolResult.Failure(request.invocationId, "Not a directory: ${request.parameters["path"] ?: "."}", recoverable = true)
|
||||
}
|
||||
val fileFilter = (request.parameters["glob"] as? String)?.takeIf { it.isNotBlank() }?.let {
|
||||
runCatching { FileSystems.getDefault().getPathMatcher("glob:$it") }
|
||||
.getOrElse { e -> return@withContext ToolResult.Failure(request.invocationId, "Invalid glob: ${e.message}", recoverable = true) }
|
||||
}
|
||||
|
||||
val hits = ArrayList<String>()
|
||||
var truncated = false
|
||||
walkGitignored(base) { rel, file ->
|
||||
if (fileFilter == null || fileFilter.matches(Paths.get(rel))) {
|
||||
if (Files.size(file) <= GREP_MAX_FILE_BYTES) {
|
||||
val text = runCatching { file.readText() }.getOrNull()
|
||||
// Skip apparent binaries (NUL byte) — a symbol search never wants them.
|
||||
if (text != null && !text.contains('\u0000')) {
|
||||
text.lineSequence().forEachIndexed { idx, line ->
|
||||
if (!truncated && regex.containsMatchIn(line)) {
|
||||
hits += "$rel:${idx + 1}: ${line.trim().take(200)}"
|
||||
if (hits.size >= MAX_RESULTS) truncated = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
!truncated
|
||||
}
|
||||
ToolResult.Success(request.invocationId, output = renderResults("grep '$patternStr'", hits, truncated))
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- shared helpers ----------
|
||||
|
||||
private fun resolveSearchPath(request: ToolRequest, workingDir: Path?): Path {
|
||||
val pathString = (request.parameters["path"] as? String)?.takeIf { it.isNotBlank() } ?: "."
|
||||
val raw = Paths.get(pathString)
|
||||
return when {
|
||||
raw.isAbsolute -> raw.normalize()
|
||||
workingDir != null -> workingDir.resolve(raw).normalize()
|
||||
else -> raw.toAbsolutePath().normalize()
|
||||
}
|
||||
}
|
||||
|
||||
private fun validateSearchPath(request: ToolRequest, allowedPaths: Set<Path>, workingDir: Path?): ValidationResult =
|
||||
runCatching {
|
||||
val path = resolveSearchPath(request, workingDir)
|
||||
val effectiveRoots = allowedPaths + request.grantedPaths.map { Paths.get(it) }
|
||||
if (!PathJail.isContained(path, effectiveRoots)) {
|
||||
ValidationResult.Invalid("Path '${request.parameters["path"] ?: "."}' is not in the allowed list")
|
||||
} else {
|
||||
ValidationResult.Valid
|
||||
}
|
||||
}.getOrElse {
|
||||
when (it) {
|
||||
is InvalidPathException -> ValidationResult.Invalid("Invalid path format: ${it.message}")
|
||||
else -> ValidationResult.Invalid(it.message ?: "Unknown error occurred")
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk [base] gitignore-pruned (reusing [GitignoreMatcher] — same rules as list_dir), invoking [onFile]
|
||||
* for every non-ignored regular file with its base-relative path and absolute [Path]. [onFile] returns
|
||||
* false to stop the walk early (used to terminate once a result cap is hit). `.git` is always skipped.
|
||||
*/
|
||||
private fun walkGitignored(base: Path, onFile: (rel: String, file: Path) -> Boolean) {
|
||||
val matcher = GitignoreMatcher(base, base)
|
||||
Files.walkFileTree(base, object : SimpleFileVisitor<Path>() {
|
||||
override fun preVisitDirectory(dir: Path, attrs: BasicFileAttributes): FileVisitResult {
|
||||
if (dir != base && dir.name == ".git") return FileVisitResult.SKIP_SUBTREE
|
||||
if (dir != base && matcher.ignored(dir, isDir = true)) return FileVisitResult.SKIP_SUBTREE
|
||||
matcher.loadFrom(dir)
|
||||
return FileVisitResult.CONTINUE
|
||||
}
|
||||
|
||||
override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
|
||||
if (attrs.isDirectory || matcher.ignored(file, isDir = false)) return FileVisitResult.CONTINUE
|
||||
val rel = base.relativize(file).toString().replace('\\', '/')
|
||||
return if (onFile(rel, file)) FileVisitResult.CONTINUE else FileVisitResult.TERMINATE
|
||||
}
|
||||
|
||||
override fun visitFileFailed(file: Path, exc: java.io.IOException) = FileVisitResult.CONTINUE
|
||||
})
|
||||
}
|
||||
|
||||
private fun renderResults(query: String, hits: List<String>, truncated: Boolean): String = buildString {
|
||||
appendLine("<search>$query</search>")
|
||||
appendLine("<results>")
|
||||
appendLine(hits.joinToString("\n").ifEmpty { "(no matches)" })
|
||||
if (truncated) appendLine("truncated at $MAX_RESULTS matches; narrow the pattern or scope with 'path'/'glob'")
|
||||
append("(${hits.size} ${if (hits.size == 1) "match" else "matches"})")
|
||||
appendLine()
|
||||
append("</results>")
|
||||
}
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
package com.correx.infrastructure.tools.filesystem
|
||||
|
||||
import com.correx.core.events.events.ToolRequest
|
||||
import com.correx.core.events.types.SessionId
|
||||
import com.correx.core.events.types.StageId
|
||||
import com.correx.core.events.types.ToolInvocationId
|
||||
import com.correx.core.tools.contract.ToolResult
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.junit.jupiter.api.Assertions.assertFalse
|
||||
import org.junit.jupiter.api.Assertions.assertTrue
|
||||
import org.junit.jupiter.api.Test
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.util.*
|
||||
|
||||
class WorkspaceSearchToolsTest {
|
||||
|
||||
private fun request(toolName: String, params: Map<String, Any>) = ToolRequest(
|
||||
invocationId = ToolInvocationId(UUID.randomUUID().toString()),
|
||||
sessionId = SessionId(UUID.randomUUID().toString()),
|
||||
stageId = StageId(UUID.randomUUID().toString()),
|
||||
toolName = toolName,
|
||||
parameters = params,
|
||||
)
|
||||
|
||||
private fun fixture(): Path {
|
||||
val root = Files.createTempDirectory("search")
|
||||
Files.writeString(root.resolve(".gitignore"), "node_modules\ndist\n")
|
||||
Files.createDirectories(root.resolve("src/hooks"))
|
||||
Files.writeString(root.resolve("src/hooks/useAuth.ts"), "export const useAuth = () => {}\n")
|
||||
Files.writeString(root.resolve("src/main.ts"), "import { useAuth } from './hooks/useAuth'\n")
|
||||
Files.writeString(root.resolve("README.md"), "# web-ui-design-spec lives here\n")
|
||||
Files.createDirectories(root.resolve("node_modules/pkg"))
|
||||
Files.writeString(root.resolve("node_modules/pkg/useAuth.ts"), "junk useAuth\n")
|
||||
Files.createDirectories(root.resolve("dist"))
|
||||
Files.writeString(root.resolve("dist/bundle.js"), "useAuth\n")
|
||||
return root
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `glob finds matching paths and prunes gitignored trees`(): Unit = runBlocking {
|
||||
val root = fixture()
|
||||
val tool = GlobTool(allowedPaths = setOf(root), workingDir = root)
|
||||
val out = (tool.execute(request("glob", mapOf("pattern" to "**/*.ts"))) as ToolResult.Success).output
|
||||
|
||||
assertTrue(out.contains("src/hooks/useAuth.ts"), out)
|
||||
assertTrue(out.contains("src/main.ts"), out)
|
||||
assertFalse(out.contains("node_modules"), out) // gitignored subtree never matched
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `glob with a leading path segment scopes the search`(): Unit = runBlocking {
|
||||
val root = fixture()
|
||||
val tool = GlobTool(allowedPaths = setOf(root), workingDir = root)
|
||||
val out = (tool.execute(request("glob", mapOf("pattern" to "src/**/use*.ts"))) as ToolResult.Success).output
|
||||
assertTrue(out.contains("src/hooks/useAuth.ts"), out)
|
||||
assertFalse(out.contains("main.ts"), out)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `grep finds a literal across files, skipping gitignored ones`(): Unit = runBlocking {
|
||||
val root = fixture()
|
||||
val tool = GrepTool(allowedPaths = setOf(root), workingDir = root)
|
||||
val out = (tool.execute(request("grep", mapOf("pattern" to "web-ui-design-spec"))) as ToolResult.Success).output
|
||||
assertTrue(out.contains("README.md:1:"), out)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `grep respects an optional file glob and prunes gitignore`(): Unit = runBlocking {
|
||||
val root = fixture()
|
||||
val tool = GrepTool(allowedPaths = setOf(root), workingDir = root)
|
||||
val out = (tool.execute(request("grep", mapOf("pattern" to "useAuth", "glob" to "**/*.ts"))) as ToolResult.Success).output
|
||||
assertTrue(out.contains("src/hooks/useAuth.ts:"), out)
|
||||
assertTrue(out.contains("src/main.ts:"), out)
|
||||
assertFalse(out.contains("node_modules"), out) // gitignored
|
||||
assertFalse(out.contains("bundle.js"), out) // not matched by *.ts glob (and gitignored)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `grep rejects an invalid regex recoverably`(): Unit = runBlocking {
|
||||
val root = fixture()
|
||||
val tool = GrepTool(allowedPaths = setOf(root), workingDir = root)
|
||||
val result = tool.execute(request("grep", mapOf("pattern" to "[unterminated")))
|
||||
assertTrue(result is ToolResult.Failure && result.recoverable, result.toString())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `search outside the jail is rejected`(): Unit = runBlocking {
|
||||
val root = fixture()
|
||||
val tool = GlobTool(allowedPaths = setOf(root), workingDir = root)
|
||||
val result = tool.execute(request("glob", mapOf("pattern" to "*", "path" to "/etc")))
|
||||
assertTrue(result is ToolResult.Failure, result.toString())
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import com.correx.infrastructure.tools.filesystem.FileDeleteTool
|
||||
import com.correx.infrastructure.tools.filesystem.FileEditTool
|
||||
import com.correx.infrastructure.tools.filesystem.FileReadTool
|
||||
import com.correx.infrastructure.tools.filesystem.FileWriteTool
|
||||
import com.correx.infrastructure.tools.filesystem.GlobTool
|
||||
import com.correx.infrastructure.tools.filesystem.GrepTool
|
||||
import com.correx.infrastructure.tools.filesystem.ListDirTool
|
||||
import com.correx.infrastructure.tools.shell.ShellTool
|
||||
import com.correx.infrastructure.tools.web.WebFetchTool
|
||||
@@ -82,6 +84,10 @@ fun ToolConfig.buildTools(): List<Tool> = buildList {
|
||||
workingDir = fileRead.workingDir,
|
||||
),
|
||||
)
|
||||
// glob (find by pattern) + grep (search contents) — read-only search, same jail/anchor/toggle;
|
||||
// the affordance weak models need instead of shell find/grep -r.
|
||||
add(GlobTool(allowedPaths = fileRead.allowedPaths, workingDir = fileRead.workingDir))
|
||||
add(GrepTool(allowedPaths = fileRead.allowedPaths, workingDir = fileRead.workingDir))
|
||||
}
|
||||
if (fileWrite.enabled) {
|
||||
add(
|
||||
|
||||
Reference in New Issue
Block a user