feat(context): doc catalog — frontmatter/H1 descriptor as read-on-demand index

Docs were indexed as up to 40 h1-h3 headings per .md (a TOC flood) and
force-fed or excluded wholesale by a binary docs gate. Replace with a
compact, always-on catalog the agent reads on demand.

- RepoMapIndexer: .md now yields ONE descriptor (frontmatter
  description/summary/title -> first # H1 -> first prose line), not a
  heading dump. Replay-safe (still List<String> symbols).
- SessionOrchestrator: always-on '## Docs available (file_read to open)'
  catalog — top-N docs by recency as 'path — descriptor', hard-capped at
  DOCS_CATALOG_MAX. One line each, so it stays present without the
  poisoning the docs gate suppressed. Source layout still excludes doc
  content for non-doc prompts.
This commit is contained in:
2026-07-07 14:52:49 +04:00
parent a8c496e909
commit 82c55ec9c7
3 changed files with 119 additions and 18 deletions
@@ -59,20 +59,58 @@ class RepoMapIndexer(
} }
private fun extractSymbols(path: Path): List<String> { private fun extractSymbols(path: Path): List<String> {
val regex = SYMBOL_PATTERNS[path.extension] ?: return emptyList() // Docs are indexed as a single "what is this" descriptor, not a table-of-contents dump: a
val text = runCatching { path.readText() }.getOrElse { return emptyList() } // heading flood (40 h1h3 per doc across a large docs/ tree) buries real source and force-feeds
return regex.findAll(text) // content the stage may never need. One descriptor lets the agent decide whether to file_read
.mapNotNull { it.groups["name"]?.value } // the doc (2026-07-07 doc-injection rework). Non-doc source keeps top-level symbol extraction.
if (path.extension.equals("md", ignoreCase = true)) return listOfNotNull(docDescriptor(path))
val regex = SYMBOL_PATTERNS[path.extension]
val text = regex?.let { runCatching { path.readText() }.getOrNull() }
return text?.let {
regex.findAll(it)
.mapNotNull { m -> m.groups["name"]?.value }
.distinct() .distinct()
.take(MAX_SYMBOLS_PER_FILE) .take(MAX_SYMBOLS_PER_FILE)
.toList() .toList()
} ?: emptyList()
} }
/**
* One-line "what is this doc about" descriptor, preferred in order: YAML frontmatter
* `description`/`summary`/`title`, then the first `# H1` heading, then the first prose line.
* This is the read-on-demand index the doc catalog surfaces — never the doc body.
*/
private fun docDescriptor(path: Path): String? {
val lines = runCatching { path.readText() }.getOrNull()?.lines() ?: return null
val raw = frontmatterDescriptor(lines)
?: lines.firstOrNull { it.trimStart().startsWith("#") }
?.trimStart()?.trimStart('#')?.trim()?.takeIf { it.isNotBlank() }
?: lines.firstOrNull { it.isNotBlank() && !it.startsWith("---") }?.trim()
return raw?.truncateDescriptor()
}
private fun frontmatterDescriptor(lines: List<String>): String? {
val hasOpen = lines.firstOrNull()?.trim() == "---"
val end = if (hasOpen) lines.drop(1).indexOfFirst { it.trim() == "---" } else -1
if (!hasOpen || end < 0) return null
return FRONTMATTER_KEYS.firstNotNullOfOrNull { key ->
lines.subList(1, end + 1)
.firstOrNull { it.trimStart().startsWith("$key:", ignoreCase = true) }
?.substringAfter(':')?.trim()?.trim('"', '\'')
?.takeIf { it.isNotBlank() }
}
}
private fun String.truncateDescriptor(): String =
if (length <= MAX_DESCRIPTOR_CHARS) this else take(MAX_DESCRIPTOR_CHARS).trimEnd() + ""
companion object { companion object {
/** File extensions considered for indexing — reused by [WorkspaceStateProbe] for fingerprinting. */ /** File extensions considered for indexing — reused by [WorkspaceStateProbe] for fingerprinting. */
val INDEXED_EXTENSIONS: Set<String> get() = SYMBOL_PATTERNS.keys val INDEXED_EXTENSIONS: Set<String> get() = SYMBOL_PATTERNS.keys
private const val MAX_SYMBOLS_PER_FILE = 40 private const val MAX_SYMBOLS_PER_FILE = 40
private const val MAX_DESCRIPTOR_CHARS = 120
private val FRONTMATTER_KEYS = listOf("description", "summary", "title")
// Top-level declarations only — best-effort per language. Bodies/locals are never matched. // Top-level declarations only — best-effort per language. Bodies/locals are never matched.
val SYMBOL_PATTERNS: Map<String, Regex> = mapOf( val SYMBOL_PATTERNS: Map<String, Regex> = mapOf(
@@ -112,27 +112,45 @@ class RepoMapIndexerTest {
} }
@Test @Test
fun `extracts markdown headings as symbols`(@TempDir root: Path) { fun `markdown is indexed as a single H1 descriptor, not a heading dump`(@TempDir root: Path) {
root.resolve("design.md").writeText( root.resolve("design.md").writeText(
""" """
# Game Design Document # Game Design Document
intro text with an inline # hash intro text with an inline # hash
## Combat System ## Combat System
body
### Damage Formula ### Damage Formula
#### Too Deep To Matter
""".trimIndent(), """.trimIndent(),
) )
val entry = RepoMapIndexer().index(root).single() val entry = RepoMapIndexer().index(root).single()
assertEquals("design.md", entry.path) assertEquals("design.md", entry.path)
assertTrue( assertEquals(listOf("Game Design Document"), entry.symbols)
entry.symbols.containsAll(listOf("Game Design Document", "Combat System", "Damage Formula")), }
"missing headings in: ${entry.symbols}",
@Test
fun `markdown descriptor prefers frontmatter description over the H1`(@TempDir root: Path) {
root.resolve("spec.md").writeText(
"""
---
title: Ignored Title
description: The authoritative sessions module spec
---
# Sessions Module
body
""".trimIndent(),
) )
assertFalse(entry.symbols.any { it.contains("Too Deep") }, "h4+ headings must not be matched")
val entry = RepoMapIndexer().index(root).single()
assertEquals(listOf("The authoritative sessions module spec"), entry.symbols)
}
@Test
fun `markdown descriptor falls back to first prose line when there is no heading`(@TempDir root: Path) {
root.resolve("notes.md").writeText("just a plain note with no heading at all")
val entry = RepoMapIndexer().index(root).single()
assertEquals(listOf("just a plain note with no heading at all"), entry.symbols)
} }
@Test @Test
@@ -197,6 +197,11 @@ private const val REFERENCE_EXISTS_CODE = "REFERENCE_EXISTS"
private const val OUTPUT_SUMMARY_LIMIT = 500 private const val OUTPUT_SUMMARY_LIMIT = 500
private const val REPO_MAP_INJECT_TOP_K = 30 private const val REPO_MAP_INJECT_TOP_K = 30
private const val REPO_MAP_FILES_PER_DIR = 8 private const val REPO_MAP_FILES_PER_DIR = 8
// Read-on-demand doc catalog: the top-N docs (by repo-map recency score) surfaced as
// `path — descriptor` so a stage learns which docs exist and can file_read one when relevant,
// instead of docs being force-fed (or excluded outright). One line each, hard-capped, so it can
// stay always-on without re-poisoning context (2026-07-07 doc-injection rework).
private const val DOCS_CATALOG_MAX = 20
// A stage prompt must explicitly ask about documentation for .md/docs paths to enter the repo // A stage prompt must explicitly ask about documentation for .md/docs paths to enter the repo
// layout listing; otherwise docs only reach context through a real retrieval hit. // layout listing; otherwise docs only reach context through a real retrieval hit.
@@ -1698,9 +1703,12 @@ abstract class SessionOrchestrator(
val recorded = eventStore.read(sessionId) val recorded = eventStore.read(sessionId)
.mapNotNull { it.payload as? RepoKnowledgeRetrievedEvent } .mapNotNull { it.payload as? RepoKnowledgeRetrievedEvent }
.lastOrNull { it.stageId == stageId } .lastOrNull { it.stageId == stageId }
if (recorded != null) return repoEntriesOrMapFloor(sessionId, recorded.hits, stagePrompt) if (recorded != null) {
return repoEntriesOrMapFloor(sessionId, recorded.hits, stagePrompt) + buildDocsCatalogEntry(sessionId)
}
val retriever = repoKnowledgeRetriever ?: return buildRepoMapEntries(sessionId, stagePrompt) val retriever = repoKnowledgeRetriever
?: return buildRepoMapEntries(sessionId, stagePrompt) + buildDocsCatalogEntry(sessionId)
val hits = runCatching { retriever.retrieve(sessionId, stagePrompt, REPO_MAP_INJECT_TOP_K) } val hits = runCatching { retriever.retrieve(sessionId, stagePrompt, REPO_MAP_INJECT_TOP_K) }
.getOrElse { e -> .getOrElse { e ->
if (e is CancellationException) throw e if (e is CancellationException) throw e
@@ -1710,7 +1718,44 @@ abstract class SessionOrchestrator(
// Record the raw retrieval for audit (a degenerate all-zero result is itself the signal that // Record the raw retrieval for audit (a degenerate all-zero result is itself the signal that
// the embedder is noop / the index is empty), but only useful hits ever reach the agent. // the embedder is noop / the index is empty), but only useful hits ever reach the agent.
if (hits.isNotEmpty()) emit(sessionId, RepoKnowledgeRetrievedEvent(sessionId, stageId, stagePrompt, hits)) if (hits.isNotEmpty()) emit(sessionId, RepoKnowledgeRetrievedEvent(sessionId, stageId, stagePrompt, hits))
return repoEntriesOrMapFloor(sessionId, hits, stagePrompt) return repoEntriesOrMapFloor(sessionId, hits, stagePrompt) + buildDocsCatalogEntry(sessionId)
}
/**
* Always-on, capped catalog of the docs that exist, as `path — descriptor` lines (the descriptor
* is the doc's frontmatter/H1/first-line summary recorded on [RepoMapEntry.symbols]). This is the
* read-on-demand doc index: one line per doc lets a stage decide whether to file_read it, so docs
* stay discoverable without the heading-flood the docs gate was added to suppress. Distinct from
* [buildRepoMapEntries]/[repoEntriesOrMapFloor], which still exclude doc *content* from the source
* layout for non-doc prompts. Reads the recorded map (replay-safe — never re-scans the FS).
*/
private suspend fun buildDocsCatalogEntry(sessionId: SessionId): List<ContextEntry> {
val map = eventStore.read(sessionId)
.mapNotNull { it.payload as? RepoMapComputedEvent }
.lastOrNull() ?: return emptyList()
val docs = map.entries
.filter { isDocPath(it.path) }
.sortedByDescending { it.score }
.take(DOCS_CATALOG_MAX)
if (docs.isEmpty()) return emptyList()
val content = buildString {
appendLine("## Docs available (file_read to open — do not assume contents)")
docs.forEach { entry ->
val descriptor = entry.symbols.firstOrNull()?.takeIf { it.isNotBlank() }
appendLine(if (descriptor != null) "- ${entry.path}$descriptor" else "- ${entry.path}")
}
}.trimEnd()
return listOf(
ContextEntry(
id = ContextEntryId(UUID.randomUUID().toString()),
layer = ContextLayer.L3,
content = content,
sourceType = "docsCatalog",
sourceId = "docs-catalog",
tokenEstimate = estimateTokens(content),
role = EntryRole.SYSTEM,
),
)
} }
/** /**