From 82c55ec9c76f7d890b284179e0f15dcebf296e1a Mon Sep 17 00:00:00 2001 From: kami Date: Tue, 7 Jul 2026 14:52:49 +0400 Subject: [PATCH] =?UTF-8?q?feat(context):=20doc=20catalog=20=E2=80=94=20fr?= =?UTF-8?q?ontmatter/H1=20descriptor=20as=20read-on-demand=20index?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docs were indexed as up to 40 h1-h3 headings per .md (a TOC flood) and force-fed or excluded wholesale by a binary docs gate. Replace with a compact, always-on catalog the agent reads on demand. - RepoMapIndexer: .md now yields ONE descriptor (frontmatter description/summary/title -> first # H1 -> first prose line), not a heading dump. Replay-safe (still List symbols). - SessionOrchestrator: always-on '## Docs available (file_read to open)' catalog — top-N docs by recency as 'path — descriptor', hard-capped at DOCS_CATALOG_MAX. One line each, so it stays present without the poisoning the docs gate suppressed. Source layout still excludes doc content for non-doc prompts. --- .../apps/server/memory/RepoMapIndexer.kt | 52 ++++++++++++++++--- .../apps/server/memory/RepoMapIndexerTest.kt | 34 +++++++++--- .../orchestration/SessionOrchestrator.kt | 51 ++++++++++++++++-- 3 files changed, 119 insertions(+), 18 deletions(-) diff --git a/apps/server/src/main/kotlin/com/correx/apps/server/memory/RepoMapIndexer.kt b/apps/server/src/main/kotlin/com/correx/apps/server/memory/RepoMapIndexer.kt index a85a9f99..3734bcc1 100644 --- a/apps/server/src/main/kotlin/com/correx/apps/server/memory/RepoMapIndexer.kt +++ b/apps/server/src/main/kotlin/com/correx/apps/server/memory/RepoMapIndexer.kt @@ -59,20 +59,58 @@ class RepoMapIndexer( } private fun extractSymbols(path: Path): List { - val regex = SYMBOL_PATTERNS[path.extension] ?: return emptyList() - val text = runCatching { path.readText() }.getOrElse { return emptyList() } - return regex.findAll(text) - .mapNotNull { it.groups["name"]?.value } - .distinct() - .take(MAX_SYMBOLS_PER_FILE) - .toList() + // Docs are indexed as a single "what is this" descriptor, not a table-of-contents dump: a + // heading flood (40 h1–h3 per doc across a large docs/ tree) buries real source and force-feeds + // content the stage may never need. One descriptor lets the agent decide whether to file_read + // the doc (2026-07-07 doc-injection rework). Non-doc source keeps top-level symbol extraction. + if (path.extension.equals("md", ignoreCase = true)) return listOfNotNull(docDescriptor(path)) + val regex = SYMBOL_PATTERNS[path.extension] + val text = regex?.let { runCatching { path.readText() }.getOrNull() } + return text?.let { + regex.findAll(it) + .mapNotNull { m -> m.groups["name"]?.value } + .distinct() + .take(MAX_SYMBOLS_PER_FILE) + .toList() + } ?: emptyList() } + /** + * One-line "what is this doc about" descriptor, preferred in order: YAML frontmatter + * `description`/`summary`/`title`, then the first `# H1` heading, then the first prose line. + * This is the read-on-demand index the doc catalog surfaces — never the doc body. + */ + private fun docDescriptor(path: Path): String? { + val lines = runCatching { path.readText() }.getOrNull()?.lines() ?: return null + val raw = frontmatterDescriptor(lines) + ?: lines.firstOrNull { it.trimStart().startsWith("#") } + ?.trimStart()?.trimStart('#')?.trim()?.takeIf { it.isNotBlank() } + ?: lines.firstOrNull { it.isNotBlank() && !it.startsWith("---") }?.trim() + return raw?.truncateDescriptor() + } + + private fun frontmatterDescriptor(lines: List): String? { + val hasOpen = lines.firstOrNull()?.trim() == "---" + val end = if (hasOpen) lines.drop(1).indexOfFirst { it.trim() == "---" } else -1 + if (!hasOpen || end < 0) return null + return FRONTMATTER_KEYS.firstNotNullOfOrNull { key -> + lines.subList(1, end + 1) + .firstOrNull { it.trimStart().startsWith("$key:", ignoreCase = true) } + ?.substringAfter(':')?.trim()?.trim('"', '\'') + ?.takeIf { it.isNotBlank() } + } + } + + private fun String.truncateDescriptor(): String = + if (length <= MAX_DESCRIPTOR_CHARS) this else take(MAX_DESCRIPTOR_CHARS).trimEnd() + "…" + companion object { /** File extensions considered for indexing — reused by [WorkspaceStateProbe] for fingerprinting. */ val INDEXED_EXTENSIONS: Set get() = SYMBOL_PATTERNS.keys private const val MAX_SYMBOLS_PER_FILE = 40 + private const val MAX_DESCRIPTOR_CHARS = 120 + private val FRONTMATTER_KEYS = listOf("description", "summary", "title") // Top-level declarations only — best-effort per language. Bodies/locals are never matched. val SYMBOL_PATTERNS: Map = mapOf( diff --git a/apps/server/src/test/kotlin/com/correx/apps/server/memory/RepoMapIndexerTest.kt b/apps/server/src/test/kotlin/com/correx/apps/server/memory/RepoMapIndexerTest.kt index 9d3fd7a8..d471ebe5 100644 --- a/apps/server/src/test/kotlin/com/correx/apps/server/memory/RepoMapIndexerTest.kt +++ b/apps/server/src/test/kotlin/com/correx/apps/server/memory/RepoMapIndexerTest.kt @@ -112,27 +112,45 @@ class RepoMapIndexerTest { } @Test - fun `extracts markdown headings as symbols`(@TempDir root: Path) { + fun `markdown is indexed as a single H1 descriptor, not a heading dump`(@TempDir root: Path) { root.resolve("design.md").writeText( """ # Game Design Document intro text with an inline # hash ## Combat System - body - ### Damage Formula - #### Too Deep To Matter """.trimIndent(), ) val entry = RepoMapIndexer().index(root).single() assertEquals("design.md", entry.path) - assertTrue( - entry.symbols.containsAll(listOf("Game Design Document", "Combat System", "Damage Formula")), - "missing headings in: ${entry.symbols}", + assertEquals(listOf("Game Design Document"), entry.symbols) + } + + @Test + fun `markdown descriptor prefers frontmatter description over the H1`(@TempDir root: Path) { + root.resolve("spec.md").writeText( + """ + --- + title: Ignored Title + description: The authoritative sessions module spec + --- + # Sessions Module + body + """.trimIndent(), ) - assertFalse(entry.symbols.any { it.contains("Too Deep") }, "h4+ headings must not be matched") + + val entry = RepoMapIndexer().index(root).single() + assertEquals(listOf("The authoritative sessions module spec"), entry.symbols) + } + + @Test + fun `markdown descriptor falls back to first prose line when there is no heading`(@TempDir root: Path) { + root.resolve("notes.md").writeText("just a plain note with no heading at all") + + val entry = RepoMapIndexer().index(root).single() + assertEquals(listOf("just a plain note with no heading at all"), entry.symbols) } @Test diff --git a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestrator.kt b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestrator.kt index 4710c31c..d159f443 100644 --- a/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestrator.kt +++ b/core/kernel/src/main/kotlin/com/correx/core/kernel/orchestration/SessionOrchestrator.kt @@ -197,6 +197,11 @@ private const val REFERENCE_EXISTS_CODE = "REFERENCE_EXISTS" private const val OUTPUT_SUMMARY_LIMIT = 500 private const val REPO_MAP_INJECT_TOP_K = 30 private const val REPO_MAP_FILES_PER_DIR = 8 +// Read-on-demand doc catalog: the top-N docs (by repo-map recency score) surfaced as +// `path — descriptor` so a stage learns which docs exist and can file_read one when relevant, +// instead of docs being force-fed (or excluded outright). One line each, hard-capped, so it can +// stay always-on without re-poisoning context (2026-07-07 doc-injection rework). +private const val DOCS_CATALOG_MAX = 20 // A stage prompt must explicitly ask about documentation for .md/docs paths to enter the repo // layout listing; otherwise docs only reach context through a real retrieval hit. @@ -1698,9 +1703,12 @@ abstract class SessionOrchestrator( val recorded = eventStore.read(sessionId) .mapNotNull { it.payload as? RepoKnowledgeRetrievedEvent } .lastOrNull { it.stageId == stageId } - if (recorded != null) return repoEntriesOrMapFloor(sessionId, recorded.hits, stagePrompt) + if (recorded != null) { + return repoEntriesOrMapFloor(sessionId, recorded.hits, stagePrompt) + buildDocsCatalogEntry(sessionId) + } - val retriever = repoKnowledgeRetriever ?: return buildRepoMapEntries(sessionId, stagePrompt) + val retriever = repoKnowledgeRetriever + ?: return buildRepoMapEntries(sessionId, stagePrompt) + buildDocsCatalogEntry(sessionId) val hits = runCatching { retriever.retrieve(sessionId, stagePrompt, REPO_MAP_INJECT_TOP_K) } .getOrElse { e -> if (e is CancellationException) throw e @@ -1710,7 +1718,44 @@ abstract class SessionOrchestrator( // Record the raw retrieval for audit (a degenerate all-zero result is itself the signal that // the embedder is noop / the index is empty), but only useful hits ever reach the agent. if (hits.isNotEmpty()) emit(sessionId, RepoKnowledgeRetrievedEvent(sessionId, stageId, stagePrompt, hits)) - return repoEntriesOrMapFloor(sessionId, hits, stagePrompt) + return repoEntriesOrMapFloor(sessionId, hits, stagePrompt) + buildDocsCatalogEntry(sessionId) + } + + /** + * Always-on, capped catalog of the docs that exist, as `path — descriptor` lines (the descriptor + * is the doc's frontmatter/H1/first-line summary recorded on [RepoMapEntry.symbols]). This is the + * read-on-demand doc index: one line per doc lets a stage decide whether to file_read it, so docs + * stay discoverable without the heading-flood the docs gate was added to suppress. Distinct from + * [buildRepoMapEntries]/[repoEntriesOrMapFloor], which still exclude doc *content* from the source + * layout for non-doc prompts. Reads the recorded map (replay-safe — never re-scans the FS). + */ + private suspend fun buildDocsCatalogEntry(sessionId: SessionId): List { + val map = eventStore.read(sessionId) + .mapNotNull { it.payload as? RepoMapComputedEvent } + .lastOrNull() ?: return emptyList() + val docs = map.entries + .filter { isDocPath(it.path) } + .sortedByDescending { it.score } + .take(DOCS_CATALOG_MAX) + if (docs.isEmpty()) return emptyList() + val content = buildString { + appendLine("## Docs available (file_read to open — do not assume contents)") + docs.forEach { entry -> + val descriptor = entry.symbols.firstOrNull()?.takeIf { it.isNotBlank() } + appendLine(if (descriptor != null) "- ${entry.path} — $descriptor" else "- ${entry.path}") + } + }.trimEnd() + return listOf( + ContextEntry( + id = ContextEntryId(UUID.randomUUID().toString()), + layer = ContextLayer.L3, + content = content, + sourceType = "docsCatalog", + sourceId = "docs-catalog", + tokenEstimate = estimateTokens(content), + role = EntryRole.SYSTEM, + ), + ) } /**