feat(context): doc catalog — frontmatter/H1 descriptor as read-on-demand index

Docs were indexed as up to 40 h1-h3 headings per .md (a TOC flood) and
force-fed or excluded wholesale by a binary docs gate. Replace with a
compact, always-on catalog the agent reads on demand.

- RepoMapIndexer: .md now yields ONE descriptor (frontmatter
  description/summary/title -> first # H1 -> first prose line), not a
  heading dump. Replay-safe (still List<String> symbols).
- SessionOrchestrator: always-on '## Docs available (file_read to open)'
  catalog — top-N docs by recency as 'path — descriptor', hard-capped at
  DOCS_CATALOG_MAX. One line each, so it stays present without the
  poisoning the docs gate suppressed. Source layout still excludes doc
  content for non-doc prompts.
This commit is contained in:
2026-07-07 14:52:49 +04:00
parent a8c496e909
commit 82c55ec9c7
3 changed files with 119 additions and 18 deletions
@@ -59,20 +59,58 @@ class RepoMapIndexer(
}
private fun extractSymbols(path: Path): List<String> {
val regex = SYMBOL_PATTERNS[path.extension] ?: return emptyList()
val text = runCatching { path.readText() }.getOrElse { return emptyList() }
return regex.findAll(text)
.mapNotNull { it.groups["name"]?.value }
.distinct()
.take(MAX_SYMBOLS_PER_FILE)
.toList()
// Docs are indexed as a single "what is this" descriptor, not a table-of-contents dump: a
// heading flood (40 h1h3 per doc across a large docs/ tree) buries real source and force-feeds
// content the stage may never need. One descriptor lets the agent decide whether to file_read
// the doc (2026-07-07 doc-injection rework). Non-doc source keeps top-level symbol extraction.
if (path.extension.equals("md", ignoreCase = true)) return listOfNotNull(docDescriptor(path))
val regex = SYMBOL_PATTERNS[path.extension]
val text = regex?.let { runCatching { path.readText() }.getOrNull() }
return text?.let {
regex.findAll(it)
.mapNotNull { m -> m.groups["name"]?.value }
.distinct()
.take(MAX_SYMBOLS_PER_FILE)
.toList()
} ?: emptyList()
}
/**
* One-line "what is this doc about" descriptor, preferred in order: YAML frontmatter
* `description`/`summary`/`title`, then the first `# H1` heading, then the first prose line.
* This is the read-on-demand index the doc catalog surfaces — never the doc body.
*/
private fun docDescriptor(path: Path): String? {
val lines = runCatching { path.readText() }.getOrNull()?.lines() ?: return null
val raw = frontmatterDescriptor(lines)
?: lines.firstOrNull { it.trimStart().startsWith("#") }
?.trimStart()?.trimStart('#')?.trim()?.takeIf { it.isNotBlank() }
?: lines.firstOrNull { it.isNotBlank() && !it.startsWith("---") }?.trim()
return raw?.truncateDescriptor()
}
private fun frontmatterDescriptor(lines: List<String>): String? {
val hasOpen = lines.firstOrNull()?.trim() == "---"
val end = if (hasOpen) lines.drop(1).indexOfFirst { it.trim() == "---" } else -1
if (!hasOpen || end < 0) return null
return FRONTMATTER_KEYS.firstNotNullOfOrNull { key ->
lines.subList(1, end + 1)
.firstOrNull { it.trimStart().startsWith("$key:", ignoreCase = true) }
?.substringAfter(':')?.trim()?.trim('"', '\'')
?.takeIf { it.isNotBlank() }
}
}
private fun String.truncateDescriptor(): String =
if (length <= MAX_DESCRIPTOR_CHARS) this else take(MAX_DESCRIPTOR_CHARS).trimEnd() + ""
companion object {
/** File extensions considered for indexing — reused by [WorkspaceStateProbe] for fingerprinting. */
val INDEXED_EXTENSIONS: Set<String> get() = SYMBOL_PATTERNS.keys
private const val MAX_SYMBOLS_PER_FILE = 40
private const val MAX_DESCRIPTOR_CHARS = 120
private val FRONTMATTER_KEYS = listOf("description", "summary", "title")
// Top-level declarations only — best-effort per language. Bodies/locals are never matched.
val SYMBOL_PATTERNS: Map<String, Regex> = mapOf(
@@ -112,27 +112,45 @@ class RepoMapIndexerTest {
}
@Test
fun `extracts markdown headings as symbols`(@TempDir root: Path) {
fun `markdown is indexed as a single H1 descriptor, not a heading dump`(@TempDir root: Path) {
root.resolve("design.md").writeText(
"""
# Game Design Document
intro text with an inline # hash
## Combat System
body
### Damage Formula
#### Too Deep To Matter
""".trimIndent(),
)
val entry = RepoMapIndexer().index(root).single()
assertEquals("design.md", entry.path)
assertTrue(
entry.symbols.containsAll(listOf("Game Design Document", "Combat System", "Damage Formula")),
"missing headings in: ${entry.symbols}",
assertEquals(listOf("Game Design Document"), entry.symbols)
}
@Test
fun `markdown descriptor prefers frontmatter description over the H1`(@TempDir root: Path) {
root.resolve("spec.md").writeText(
"""
---
title: Ignored Title
description: The authoritative sessions module spec
---
# Sessions Module
body
""".trimIndent(),
)
assertFalse(entry.symbols.any { it.contains("Too Deep") }, "h4+ headings must not be matched")
val entry = RepoMapIndexer().index(root).single()
assertEquals(listOf("The authoritative sessions module spec"), entry.symbols)
}
@Test
fun `markdown descriptor falls back to first prose line when there is no heading`(@TempDir root: Path) {
root.resolve("notes.md").writeText("just a plain note with no heading at all")
val entry = RepoMapIndexer().index(root).single()
assertEquals(listOf("just a plain note with no heading at all"), entry.symbols)
}
@Test