feat(context): doc catalog — frontmatter/H1 descriptor as read-on-demand index
Docs were indexed as up to 40 h1-h3 headings per .md (a TOC flood) and force-fed or excluded wholesale by a binary docs gate. Replace with a compact, always-on catalog the agent reads on demand. - RepoMapIndexer: .md now yields ONE descriptor (frontmatter description/summary/title -> first # H1 -> first prose line), not a heading dump. Replay-safe (still List<String> symbols). - SessionOrchestrator: always-on '## Docs available (file_read to open)' catalog — top-N docs by recency as 'path — descriptor', hard-capped at DOCS_CATALOG_MAX. One line each, so it stays present without the poisoning the docs gate suppressed. Source layout still excludes doc content for non-doc prompts.
This commit is contained in:
@@ -59,20 +59,58 @@ class RepoMapIndexer(
|
|||||||
}
|
}
|
||||||
|
|
||||||
private fun extractSymbols(path: Path): List<String> {
|
private fun extractSymbols(path: Path): List<String> {
|
||||||
val regex = SYMBOL_PATTERNS[path.extension] ?: return emptyList()
|
// Docs are indexed as a single "what is this" descriptor, not a table-of-contents dump: a
|
||||||
val text = runCatching { path.readText() }.getOrElse { return emptyList() }
|
// heading flood (40 h1–h3 per doc across a large docs/ tree) buries real source and force-feeds
|
||||||
return regex.findAll(text)
|
// content the stage may never need. One descriptor lets the agent decide whether to file_read
|
||||||
.mapNotNull { it.groups["name"]?.value }
|
// the doc (2026-07-07 doc-injection rework). Non-doc source keeps top-level symbol extraction.
|
||||||
|
if (path.extension.equals("md", ignoreCase = true)) return listOfNotNull(docDescriptor(path))
|
||||||
|
val regex = SYMBOL_PATTERNS[path.extension]
|
||||||
|
val text = regex?.let { runCatching { path.readText() }.getOrNull() }
|
||||||
|
return text?.let {
|
||||||
|
regex.findAll(it)
|
||||||
|
.mapNotNull { m -> m.groups["name"]?.value }
|
||||||
.distinct()
|
.distinct()
|
||||||
.take(MAX_SYMBOLS_PER_FILE)
|
.take(MAX_SYMBOLS_PER_FILE)
|
||||||
.toList()
|
.toList()
|
||||||
|
} ?: emptyList()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One-line "what is this doc about" descriptor, preferred in order: YAML frontmatter
|
||||||
|
* `description`/`summary`/`title`, then the first `# H1` heading, then the first prose line.
|
||||||
|
* This is the read-on-demand index the doc catalog surfaces — never the doc body.
|
||||||
|
*/
|
||||||
|
private fun docDescriptor(path: Path): String? {
|
||||||
|
val lines = runCatching { path.readText() }.getOrNull()?.lines() ?: return null
|
||||||
|
val raw = frontmatterDescriptor(lines)
|
||||||
|
?: lines.firstOrNull { it.trimStart().startsWith("#") }
|
||||||
|
?.trimStart()?.trimStart('#')?.trim()?.takeIf { it.isNotBlank() }
|
||||||
|
?: lines.firstOrNull { it.isNotBlank() && !it.startsWith("---") }?.trim()
|
||||||
|
return raw?.truncateDescriptor()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun frontmatterDescriptor(lines: List<String>): String? {
|
||||||
|
val hasOpen = lines.firstOrNull()?.trim() == "---"
|
||||||
|
val end = if (hasOpen) lines.drop(1).indexOfFirst { it.trim() == "---" } else -1
|
||||||
|
if (!hasOpen || end < 0) return null
|
||||||
|
return FRONTMATTER_KEYS.firstNotNullOfOrNull { key ->
|
||||||
|
lines.subList(1, end + 1)
|
||||||
|
.firstOrNull { it.trimStart().startsWith("$key:", ignoreCase = true) }
|
||||||
|
?.substringAfter(':')?.trim()?.trim('"', '\'')
|
||||||
|
?.takeIf { it.isNotBlank() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun String.truncateDescriptor(): String =
|
||||||
|
if (length <= MAX_DESCRIPTOR_CHARS) this else take(MAX_DESCRIPTOR_CHARS).trimEnd() + "…"
|
||||||
|
|
||||||
companion object {
|
companion object {
|
||||||
/** File extensions considered for indexing — reused by [WorkspaceStateProbe] for fingerprinting. */
|
/** File extensions considered for indexing — reused by [WorkspaceStateProbe] for fingerprinting. */
|
||||||
val INDEXED_EXTENSIONS: Set<String> get() = SYMBOL_PATTERNS.keys
|
val INDEXED_EXTENSIONS: Set<String> get() = SYMBOL_PATTERNS.keys
|
||||||
|
|
||||||
private const val MAX_SYMBOLS_PER_FILE = 40
|
private const val MAX_SYMBOLS_PER_FILE = 40
|
||||||
|
private const val MAX_DESCRIPTOR_CHARS = 120
|
||||||
|
private val FRONTMATTER_KEYS = listOf("description", "summary", "title")
|
||||||
|
|
||||||
// Top-level declarations only — best-effort per language. Bodies/locals are never matched.
|
// Top-level declarations only — best-effort per language. Bodies/locals are never matched.
|
||||||
val SYMBOL_PATTERNS: Map<String, Regex> = mapOf(
|
val SYMBOL_PATTERNS: Map<String, Regex> = mapOf(
|
||||||
|
|||||||
@@ -112,27 +112,45 @@ class RepoMapIndexerTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun `extracts markdown headings as symbols`(@TempDir root: Path) {
|
fun `markdown is indexed as a single H1 descriptor, not a heading dump`(@TempDir root: Path) {
|
||||||
root.resolve("design.md").writeText(
|
root.resolve("design.md").writeText(
|
||||||
"""
|
"""
|
||||||
# Game Design Document
|
# Game Design Document
|
||||||
intro text with an inline # hash
|
intro text with an inline # hash
|
||||||
|
|
||||||
## Combat System
|
## Combat System
|
||||||
body
|
|
||||||
|
|
||||||
### Damage Formula
|
### Damage Formula
|
||||||
#### Too Deep To Matter
|
|
||||||
""".trimIndent(),
|
""".trimIndent(),
|
||||||
)
|
)
|
||||||
|
|
||||||
val entry = RepoMapIndexer().index(root).single()
|
val entry = RepoMapIndexer().index(root).single()
|
||||||
assertEquals("design.md", entry.path)
|
assertEquals("design.md", entry.path)
|
||||||
assertTrue(
|
assertEquals(listOf("Game Design Document"), entry.symbols)
|
||||||
entry.symbols.containsAll(listOf("Game Design Document", "Combat System", "Damage Formula")),
|
}
|
||||||
"missing headings in: ${entry.symbols}",
|
|
||||||
|
@Test
|
||||||
|
fun `markdown descriptor prefers frontmatter description over the H1`(@TempDir root: Path) {
|
||||||
|
root.resolve("spec.md").writeText(
|
||||||
|
"""
|
||||||
|
---
|
||||||
|
title: Ignored Title
|
||||||
|
description: The authoritative sessions module spec
|
||||||
|
---
|
||||||
|
# Sessions Module
|
||||||
|
body
|
||||||
|
""".trimIndent(),
|
||||||
)
|
)
|
||||||
assertFalse(entry.symbols.any { it.contains("Too Deep") }, "h4+ headings must not be matched")
|
|
||||||
|
val entry = RepoMapIndexer().index(root).single()
|
||||||
|
assertEquals(listOf("The authoritative sessions module spec"), entry.symbols)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `markdown descriptor falls back to first prose line when there is no heading`(@TempDir root: Path) {
|
||||||
|
root.resolve("notes.md").writeText("just a plain note with no heading at all")
|
||||||
|
|
||||||
|
val entry = RepoMapIndexer().index(root).single()
|
||||||
|
assertEquals(listOf("just a plain note with no heading at all"), entry.symbols)
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
+48
-3
@@ -197,6 +197,11 @@ private const val REFERENCE_EXISTS_CODE = "REFERENCE_EXISTS"
|
|||||||
private const val OUTPUT_SUMMARY_LIMIT = 500
|
private const val OUTPUT_SUMMARY_LIMIT = 500
|
||||||
private const val REPO_MAP_INJECT_TOP_K = 30
|
private const val REPO_MAP_INJECT_TOP_K = 30
|
||||||
private const val REPO_MAP_FILES_PER_DIR = 8
|
private const val REPO_MAP_FILES_PER_DIR = 8
|
||||||
|
// Read-on-demand doc catalog: the top-N docs (by repo-map recency score) surfaced as
|
||||||
|
// `path — descriptor` so a stage learns which docs exist and can file_read one when relevant,
|
||||||
|
// instead of docs being force-fed (or excluded outright). One line each, hard-capped, so it can
|
||||||
|
// stay always-on without re-poisoning context (2026-07-07 doc-injection rework).
|
||||||
|
private const val DOCS_CATALOG_MAX = 20
|
||||||
|
|
||||||
// A stage prompt must explicitly ask about documentation for .md/docs paths to enter the repo
|
// A stage prompt must explicitly ask about documentation for .md/docs paths to enter the repo
|
||||||
// layout listing; otherwise docs only reach context through a real retrieval hit.
|
// layout listing; otherwise docs only reach context through a real retrieval hit.
|
||||||
@@ -1698,9 +1703,12 @@ abstract class SessionOrchestrator(
|
|||||||
val recorded = eventStore.read(sessionId)
|
val recorded = eventStore.read(sessionId)
|
||||||
.mapNotNull { it.payload as? RepoKnowledgeRetrievedEvent }
|
.mapNotNull { it.payload as? RepoKnowledgeRetrievedEvent }
|
||||||
.lastOrNull { it.stageId == stageId }
|
.lastOrNull { it.stageId == stageId }
|
||||||
if (recorded != null) return repoEntriesOrMapFloor(sessionId, recorded.hits, stagePrompt)
|
if (recorded != null) {
|
||||||
|
return repoEntriesOrMapFloor(sessionId, recorded.hits, stagePrompt) + buildDocsCatalogEntry(sessionId)
|
||||||
|
}
|
||||||
|
|
||||||
val retriever = repoKnowledgeRetriever ?: return buildRepoMapEntries(sessionId, stagePrompt)
|
val retriever = repoKnowledgeRetriever
|
||||||
|
?: return buildRepoMapEntries(sessionId, stagePrompt) + buildDocsCatalogEntry(sessionId)
|
||||||
val hits = runCatching { retriever.retrieve(sessionId, stagePrompt, REPO_MAP_INJECT_TOP_K) }
|
val hits = runCatching { retriever.retrieve(sessionId, stagePrompt, REPO_MAP_INJECT_TOP_K) }
|
||||||
.getOrElse { e ->
|
.getOrElse { e ->
|
||||||
if (e is CancellationException) throw e
|
if (e is CancellationException) throw e
|
||||||
@@ -1710,7 +1718,44 @@ abstract class SessionOrchestrator(
|
|||||||
// Record the raw retrieval for audit (a degenerate all-zero result is itself the signal that
|
// Record the raw retrieval for audit (a degenerate all-zero result is itself the signal that
|
||||||
// the embedder is noop / the index is empty), but only useful hits ever reach the agent.
|
// the embedder is noop / the index is empty), but only useful hits ever reach the agent.
|
||||||
if (hits.isNotEmpty()) emit(sessionId, RepoKnowledgeRetrievedEvent(sessionId, stageId, stagePrompt, hits))
|
if (hits.isNotEmpty()) emit(sessionId, RepoKnowledgeRetrievedEvent(sessionId, stageId, stagePrompt, hits))
|
||||||
return repoEntriesOrMapFloor(sessionId, hits, stagePrompt)
|
return repoEntriesOrMapFloor(sessionId, hits, stagePrompt) + buildDocsCatalogEntry(sessionId)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Always-on, capped catalog of the docs that exist, as `path — descriptor` lines (the descriptor
|
||||||
|
* is the doc's frontmatter/H1/first-line summary recorded on [RepoMapEntry.symbols]). This is the
|
||||||
|
* read-on-demand doc index: one line per doc lets a stage decide whether to file_read it, so docs
|
||||||
|
* stay discoverable without the heading-flood the docs gate was added to suppress. Distinct from
|
||||||
|
* [buildRepoMapEntries]/[repoEntriesOrMapFloor], which still exclude doc *content* from the source
|
||||||
|
* layout for non-doc prompts. Reads the recorded map (replay-safe — never re-scans the FS).
|
||||||
|
*/
|
||||||
|
private suspend fun buildDocsCatalogEntry(sessionId: SessionId): List<ContextEntry> {
|
||||||
|
val map = eventStore.read(sessionId)
|
||||||
|
.mapNotNull { it.payload as? RepoMapComputedEvent }
|
||||||
|
.lastOrNull() ?: return emptyList()
|
||||||
|
val docs = map.entries
|
||||||
|
.filter { isDocPath(it.path) }
|
||||||
|
.sortedByDescending { it.score }
|
||||||
|
.take(DOCS_CATALOG_MAX)
|
||||||
|
if (docs.isEmpty()) return emptyList()
|
||||||
|
val content = buildString {
|
||||||
|
appendLine("## Docs available (file_read to open — do not assume contents)")
|
||||||
|
docs.forEach { entry ->
|
||||||
|
val descriptor = entry.symbols.firstOrNull()?.takeIf { it.isNotBlank() }
|
||||||
|
appendLine(if (descriptor != null) "- ${entry.path} — $descriptor" else "- ${entry.path}")
|
||||||
|
}
|
||||||
|
}.trimEnd()
|
||||||
|
return listOf(
|
||||||
|
ContextEntry(
|
||||||
|
id = ContextEntryId(UUID.randomUUID().toString()),
|
||||||
|
layer = ContextLayer.L3,
|
||||||
|
content = content,
|
||||||
|
sourceType = "docsCatalog",
|
||||||
|
sourceId = "docs-catalog",
|
||||||
|
tokenEstimate = estimateTokens(content),
|
||||||
|
role = EntryRole.SYSTEM,
|
||||||
|
),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user