feat(server,kernel): repo-map indexer + droppable L3 context injection
This commit is contained in:
@@ -298,6 +298,10 @@ fun main() {
|
|||||||
l3MemoryStore = l3MemoryStore,
|
l3MemoryStore = l3MemoryStore,
|
||||||
journalRepository = decisionJournalRepository,
|
journalRepository = decisionJournalRepository,
|
||||||
eventStore = eventStore,
|
eventStore = eventStore,
|
||||||
|
indexer = com.correx.apps.server.memory.RepoMapIndexer(
|
||||||
|
maxDepth = correxConfig.project.maxDepth,
|
||||||
|
ignoreGlobs = correxConfig.project.ignoreGlobs,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
null
|
null
|
||||||
|
|||||||
@@ -129,8 +129,14 @@ class ServerModule(
|
|||||||
) {
|
) {
|
||||||
activeSessionJobs.computeIfAbsent(sessionId) {
|
activeSessionJobs.computeIfAbsent(sessionId) {
|
||||||
moduleScope.launch {
|
moduleScope.launch {
|
||||||
// Seed prior-session project memory before the run so stages see it in context.
|
// Record the repo map + seed prior-session memory before the run so stages
|
||||||
projectMemory?.let { pm -> runCatching { pm.retrieveAndSeed(sessionId, pm.repoRoot()) } }
|
// see both in context.
|
||||||
|
projectMemory?.let { pm ->
|
||||||
|
runCatching {
|
||||||
|
pm.indexAndRecord(sessionId, pm.repoRoot())
|
||||||
|
pm.retrieveAndSeed(sessionId, pm.repoRoot())
|
||||||
|
}
|
||||||
|
}
|
||||||
runCatching {
|
runCatching {
|
||||||
orchestrator.run(sessionId, graph, sessionConfig)
|
orchestrator.run(sessionId, graph, sessionConfig)
|
||||||
// Distil this run's decisions into durable project memory on completion.
|
// Distil this run's decisions into durable project memory on completion.
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package com.correx.apps.server.memory
|
|||||||
import com.correx.core.config.ProjectConfig
|
import com.correx.core.config.ProjectConfig
|
||||||
import com.correx.core.events.events.EventMetadata
|
import com.correx.core.events.events.EventMetadata
|
||||||
import com.correx.core.events.events.NewEvent
|
import com.correx.core.events.events.NewEvent
|
||||||
|
import com.correx.core.events.events.RepoMapComputedEvent
|
||||||
import com.correx.core.events.events.SteeringNoteAddedEvent
|
import com.correx.core.events.events.SteeringNoteAddedEvent
|
||||||
import com.correx.core.events.stores.EventStore
|
import com.correx.core.events.stores.EventStore
|
||||||
import com.correx.core.events.types.EventId
|
import com.correx.core.events.types.EventId
|
||||||
@@ -39,12 +40,47 @@ class ProjectMemoryService(
|
|||||||
private val journalRepository: DefaultDecisionJournalRepository,
|
private val journalRepository: DefaultDecisionJournalRepository,
|
||||||
private val eventStore: EventStore,
|
private val eventStore: EventStore,
|
||||||
private val distiller: ProjectMemoryDistiller = ProjectMemoryDistiller(),
|
private val distiller: ProjectMemoryDistiller = ProjectMemoryDistiller(),
|
||||||
|
private val indexer: RepoMapIndexer = RepoMapIndexer(),
|
||||||
) {
|
) {
|
||||||
private fun tag(repoRoot: String) = "project:$repoRoot"
|
private fun tag(repoRoot: String) = "project:$repoRoot"
|
||||||
|
|
||||||
/** Resolved repo-root key: configured [ProjectConfig.root], else the working dir. */
|
/** Resolved repo-root key: configured [ProjectConfig.root], else the working dir. */
|
||||||
fun repoRoot(): String = config.root.ifBlank { System.getProperty("user.dir") ?: "." }
|
fun repoRoot(): String = config.root.ifBlank { System.getProperty("user.dir") ?: "." }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Walk [repoRoot] and record the ranked file/symbol map as a [RepoMapComputedEvent] once
|
||||||
|
* per session. The full map is recorded (the log); only a top-K slice is injected into
|
||||||
|
* context downstream. Replay reads the recorded event, never re-scanning (invariant #9).
|
||||||
|
*/
|
||||||
|
suspend fun indexAndRecord(sessionId: SessionId, repoRoot: String) {
|
||||||
|
if (!config.enabled) return
|
||||||
|
runCatching {
|
||||||
|
val entries = indexer.index(java.nio.file.Path.of(repoRoot))
|
||||||
|
if (entries.isEmpty()) return
|
||||||
|
eventStore.append(
|
||||||
|
NewEvent(
|
||||||
|
metadata = EventMetadata(
|
||||||
|
eventId = EventId(UUID.randomUUID().toString()),
|
||||||
|
sessionId = sessionId,
|
||||||
|
timestamp = Clock.System.now(),
|
||||||
|
schemaVersion = 1,
|
||||||
|
causationId = null,
|
||||||
|
correlationId = null,
|
||||||
|
),
|
||||||
|
payload = RepoMapComputedEvent(
|
||||||
|
sessionId = sessionId,
|
||||||
|
repoRoot = repoRoot,
|
||||||
|
entries = entries,
|
||||||
|
computedAt = Clock.System.now(),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}.exceptionOrNull()?.let { e ->
|
||||||
|
if (e is CancellationException) throw e
|
||||||
|
log.warn("repo-map indexing failed for {}: {}", repoRoot, e.message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Distil the session's decision journal into durable, repo-tagged L3 entries. */
|
/** Distil the session's decision journal into durable, repo-tagged L3 entries. */
|
||||||
suspend fun persist(sessionId: SessionId, repoRoot: String) {
|
suspend fun persist(sessionId: SessionId, repoRoot: String) {
|
||||||
if (!config.enabled) return
|
if (!config.enabled) return
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
package com.correx.apps.server.memory
|
||||||
|
|
||||||
|
import com.correx.core.events.events.RepoMapEntry
|
||||||
|
import java.nio.file.Files
|
||||||
|
import java.nio.file.Path
|
||||||
|
import kotlin.io.path.extension
|
||||||
|
import kotlin.io.path.getLastModifiedTime
|
||||||
|
import kotlin.io.path.isDirectory
|
||||||
|
import kotlin.io.path.isRegularFile
|
||||||
|
import kotlin.io.path.name
|
||||||
|
import kotlin.io.path.readText
|
||||||
|
import kotlin.io.path.relativeTo
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Walks a repo and produces a ranked file/symbol index. Reads the filesystem (a
|
||||||
|
* nondeterministic environment observation) — the caller records the result as a
|
||||||
|
* [com.correx.core.events.events.RepoMapComputedEvent] so replay reads recorded facts and
|
||||||
|
* never re-scans (invariant #9). Paths + top-level symbol names only, never file bodies.
|
||||||
|
*
|
||||||
|
* Scoring is recency-based: the most-recently-modified source file scores 1.0, the oldest
|
||||||
|
* ~0.0, linearly between. Recency is a cheap centrality proxy — files under active work rank
|
||||||
|
* highest, which is what a fresh session most wants to see.
|
||||||
|
*/
|
||||||
|
class RepoMapIndexer(
|
||||||
|
private val maxDepth: Int = 4,
|
||||||
|
private val ignoreGlobs: List<String> = emptyList(),
|
||||||
|
) {
|
||||||
|
fun index(repoRoot: Path): List<RepoMapEntry> {
|
||||||
|
if (!repoRoot.isDirectory()) return emptyList()
|
||||||
|
val ignore = ignoreGlobs.toSet()
|
||||||
|
|
||||||
|
val files = Files.walk(repoRoot, maxDepth).use { stream ->
|
||||||
|
stream.filter { it.isRegularFile() }
|
||||||
|
.filter { it.extension in SYMBOL_PATTERNS.keys }
|
||||||
|
.filter { path -> path.relativeTo(repoRoot).none { it.name in ignore } }
|
||||||
|
.toList()
|
||||||
|
}
|
||||||
|
if (files.isEmpty()) return emptyList()
|
||||||
|
|
||||||
|
val mtimes = files.associateWith { runCatching { it.getLastModifiedTime().toMillis() }.getOrDefault(0L) }
|
||||||
|
val min = mtimes.values.min()
|
||||||
|
val max = mtimes.values.max()
|
||||||
|
val span = (max - min).coerceAtLeast(1L)
|
||||||
|
|
||||||
|
return files
|
||||||
|
.map { path ->
|
||||||
|
RepoMapEntry(
|
||||||
|
path = path.relativeTo(repoRoot).toString(),
|
||||||
|
score = (mtimes.getValue(path) - min).toDouble() / span,
|
||||||
|
symbols = extractSymbols(path),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
.sortedByDescending { it.score }
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun extractSymbols(path: Path): List<String> {
|
||||||
|
val regex = SYMBOL_PATTERNS[path.extension] ?: return emptyList()
|
||||||
|
val text = runCatching { path.readText() }.getOrElse { return emptyList() }
|
||||||
|
return regex.findAll(text)
|
||||||
|
.mapNotNull { it.groups["name"]?.value }
|
||||||
|
.distinct()
|
||||||
|
.take(MAX_SYMBOLS_PER_FILE)
|
||||||
|
.toList()
|
||||||
|
}
|
||||||
|
|
||||||
|
private companion object {
|
||||||
|
const val MAX_SYMBOLS_PER_FILE = 40
|
||||||
|
|
||||||
|
// Top-level declarations only — best-effort per language. Bodies/locals are never matched.
|
||||||
|
val SYMBOL_PATTERNS: Map<String, Regex> = mapOf(
|
||||||
|
"kt" to Regex("""(?m)^\s*(?:public |internal |open |abstract |sealed |data )*(?:class|interface|object|fun)\s+(?<name>[A-Za-z_][A-Za-z0-9_]*)"""),
|
||||||
|
"java" to Regex("""(?m)^\s*(?:public |private |protected |final |abstract |static )*(?:class|interface|enum|record)\s+(?<name>[A-Za-z_][A-Za-z0-9_]*)"""),
|
||||||
|
"py" to Regex("""(?m)^(?:def|class)\s+(?<name>[A-Za-z_][A-Za-z0-9_]*)"""),
|
||||||
|
"go" to Regex("""(?m)^\s*(?:func\s+(?:\([^)]*\)\s*)?|type\s+)(?<name>[A-Za-z_][A-Za-z0-9_]*)"""),
|
||||||
|
"js" to Regex("""(?m)^\s*(?:export\s+)?(?:default\s+)?(?:async\s+)?(?:function|class)\s+(?<name>[A-Za-z_$][A-Za-z0-9_$]*)"""),
|
||||||
|
"ts" to Regex("""(?m)^\s*(?:export\s+)?(?:default\s+)?(?:async\s+)?(?:function|class|interface|type|enum)\s+(?<name>[A-Za-z_$][A-Za-z0-9_$]*)"""),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
package com.correx.apps.server.memory
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions.assertEquals
|
||||||
|
import org.junit.jupiter.api.Assertions.assertFalse
|
||||||
|
import org.junit.jupiter.api.Assertions.assertTrue
|
||||||
|
import org.junit.jupiter.api.io.TempDir
|
||||||
|
import java.nio.file.Files
|
||||||
|
import java.nio.file.Path
|
||||||
|
import kotlin.io.path.createDirectories
|
||||||
|
import kotlin.io.path.writeText
|
||||||
|
import org.junit.jupiter.api.Test
|
||||||
|
|
||||||
|
class RepoMapIndexerTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `indexes source files with top-level symbols, ranks recent highest, prunes ignores`(@TempDir root: Path) {
|
||||||
|
root.resolve("Old.kt").writeText("class Old\nfun helper() {}\n")
|
||||||
|
Thread.sleep(10)
|
||||||
|
root.resolve("New.kt").writeText("interface New\nobject Thing\n")
|
||||||
|
|
||||||
|
// Pruned directory must not appear.
|
||||||
|
root.resolve("build").createDirectories()
|
||||||
|
root.resolve("build/Generated.kt").writeText("class Generated\n")
|
||||||
|
|
||||||
|
// Non-source file must be ignored.
|
||||||
|
root.resolve("README.md").writeText("# hi\n")
|
||||||
|
|
||||||
|
val entries = RepoMapIndexer(maxDepth = 4, ignoreGlobs = listOf("build")).index(root)
|
||||||
|
|
||||||
|
val paths = entries.map { it.path }
|
||||||
|
assertTrue(paths.contains("New.kt"))
|
||||||
|
assertTrue(paths.contains("Old.kt"))
|
||||||
|
assertFalse(paths.any { it.contains("build") }, "ignored dir leaked: $paths")
|
||||||
|
assertFalse(paths.contains("README.md"), "non-source file leaked")
|
||||||
|
|
||||||
|
// Most-recently-modified ranks first.
|
||||||
|
assertEquals("New.kt", entries.first().path)
|
||||||
|
|
||||||
|
val newEntry = entries.first { it.path == "New.kt" }
|
||||||
|
assertTrue(newEntry.symbols.containsAll(listOf("New", "Thing")))
|
||||||
|
assertTrue(entries.first { it.path == "Old.kt" }.symbols.contains("Old"))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `empty or missing root yields empty map`(@TempDir root: Path) {
|
||||||
|
assertTrue(RepoMapIndexer().index(root).isEmpty())
|
||||||
|
assertTrue(RepoMapIndexer().index(root.resolve("does-not-exist")).isEmpty())
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `maxDepth bounds recursion`(@TempDir root: Path) {
|
||||||
|
root.resolve("a/b/c").createDirectories()
|
||||||
|
root.resolve("Top.kt").writeText("class Top\n")
|
||||||
|
root.resolve("a/b/c/Deep.kt").writeText("class Deep\n")
|
||||||
|
|
||||||
|
val shallow = RepoMapIndexer(maxDepth = 1).index(root).map { it.path }
|
||||||
|
assertTrue(shallow.contains("Top.kt"))
|
||||||
|
assertFalse(shallow.any { it.contains("Deep") }, "depth cap breached: $shallow")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -418,6 +418,9 @@ object ConfigLoader {
|
|||||||
enabled = asBoolean(projectSection["enabled"], false),
|
enabled = asBoolean(projectSection["enabled"], false),
|
||||||
root = asString(projectSection["root"], ""),
|
root = asString(projectSection["root"], ""),
|
||||||
memoryK = asInt(projectSection["memory_k"], 5),
|
memoryK = asInt(projectSection["memory_k"], 5),
|
||||||
|
maxDepth = asInt(projectSection["max_depth"], 4),
|
||||||
|
ignoreGlobs = asStringList(projectSection["ignore_globs"]).ifEmpty { ProjectConfig.DEFAULT_IGNORES },
|
||||||
|
injectTopK = asInt(projectSection["inject_top_k"], 30),
|
||||||
)
|
)
|
||||||
|
|
||||||
val modelsSettings = ModelsSettings(
|
val modelsSettings = ModelsSettings(
|
||||||
|
|||||||
@@ -26,7 +26,16 @@ data class ProjectConfig(
|
|||||||
val enabled: Boolean = false,
|
val enabled: Boolean = false,
|
||||||
val root: String = "",
|
val root: String = "",
|
||||||
val memoryK: Int = 5,
|
val memoryK: Int = 5,
|
||||||
)
|
val maxDepth: Int = 4,
|
||||||
|
val ignoreGlobs: List<String> = DEFAULT_IGNORES,
|
||||||
|
val injectTopK: Int = 30,
|
||||||
|
) {
|
||||||
|
companion object {
|
||||||
|
val DEFAULT_IGNORES: List<String> = listOf(
|
||||||
|
".git", "node_modules", "build", "target", ".gradle", "dist", ".idea", ".opencode",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A user-declared artifact kind. [schemaPath] points to a JSON file holding the
|
* A user-declared artifact kind. [schemaPath] points to a JSON file holding the
|
||||||
|
|||||||
+37
-1
@@ -30,6 +30,7 @@ import com.correx.core.events.events.ArtifactValidatingEvent
|
|||||||
import com.correx.core.events.events.EventMetadata
|
import com.correx.core.events.events.EventMetadata
|
||||||
import com.correx.core.events.events.EventPayload
|
import com.correx.core.events.events.EventPayload
|
||||||
import com.correx.core.events.events.InferenceCompletedEvent
|
import com.correx.core.events.events.InferenceCompletedEvent
|
||||||
|
import com.correx.core.events.events.RepoMapComputedEvent
|
||||||
import com.correx.core.events.events.InferenceFailedEvent
|
import com.correx.core.events.events.InferenceFailedEvent
|
||||||
import com.correx.core.events.events.InferenceStartedEvent
|
import com.correx.core.events.events.InferenceStartedEvent
|
||||||
import com.correx.core.events.events.InferenceTimeoutEvent
|
import com.correx.core.events.events.InferenceTimeoutEvent
|
||||||
@@ -117,6 +118,7 @@ import kotlin.coroutines.cancellation.CancellationException
|
|||||||
|
|
||||||
private const val MAX_TOOL_ROUNDS = 10
|
private const val MAX_TOOL_ROUNDS = 10
|
||||||
private const val STAGE_COMPLETE_TOOL = "stage_complete"
|
private const val STAGE_COMPLETE_TOOL = "stage_complete"
|
||||||
|
private const val REPO_MAP_INJECT_TOP_K = 30
|
||||||
|
|
||||||
@SuppressWarnings(
|
@SuppressWarnings(
|
||||||
"ForbiddenComment",
|
"ForbiddenComment",
|
||||||
@@ -303,7 +305,9 @@ abstract class SessionOrchestrator(
|
|||||||
role = EntryRole.SYSTEM,
|
role = EntryRole.SYSTEM,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
var accumulatedEntries = systemPrompt + journalEntries + schemaEntries + promptEntries + steeringEntries
|
val repoMapEntries = buildRepoMapEntries(sessionId)
|
||||||
|
var accumulatedEntries =
|
||||||
|
systemPrompt + journalEntries + repoMapEntries + schemaEntries + promptEntries + steeringEntries
|
||||||
val contextPack = contextPackBuilder.build(
|
val contextPack = contextPackBuilder.build(
|
||||||
id = ContextPackId(UUID.randomUUID().toString()),
|
id = ContextPackId(UUID.randomUUID().toString()),
|
||||||
sessionId = sessionId,
|
sessionId = sessionId,
|
||||||
@@ -807,6 +811,38 @@ abstract class SessionOrchestrator(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Renders a top-K slice of the recorded repo map as a droppable L3 context entry. Reads
|
||||||
|
* the latest [RepoMapComputedEvent] from the log (replay-safe — never re-scans the FS).
|
||||||
|
* Not pinned: under budget pressure this yields before the working set (paths + symbols
|
||||||
|
* only, so the model can `file_read` for detail).
|
||||||
|
*/
|
||||||
|
private suspend fun buildRepoMapEntries(sessionId: SessionId): List<ContextEntry> {
|
||||||
|
val map = eventStore.read(sessionId)
|
||||||
|
.mapNotNull { it.payload as? RepoMapComputedEvent }
|
||||||
|
.lastOrNull() ?: return emptyList()
|
||||||
|
val top = map.entries.sortedByDescending { it.score }.take(REPO_MAP_INJECT_TOP_K)
|
||||||
|
if (top.isEmpty()) return emptyList()
|
||||||
|
val content = buildString {
|
||||||
|
appendLine("## Repo map (top files by recency — read files for detail)")
|
||||||
|
top.forEach { entry ->
|
||||||
|
val symbols = if (entry.symbols.isEmpty()) "" else ": ${entry.symbols.joinToString(", ")}"
|
||||||
|
appendLine("- ${entry.path}$symbols")
|
||||||
|
}
|
||||||
|
}.trimEnd()
|
||||||
|
return listOf(
|
||||||
|
ContextEntry(
|
||||||
|
id = ContextEntryId(UUID.randomUUID().toString()),
|
||||||
|
layer = ContextLayer.L3,
|
||||||
|
content = content,
|
||||||
|
sourceType = "repoMap",
|
||||||
|
sourceId = "repo-map",
|
||||||
|
tokenEstimate = estimateTokens(content),
|
||||||
|
role = EntryRole.SYSTEM,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
private suspend fun emitToolArtifacts(
|
private suspend fun emitToolArtifacts(
|
||||||
sessionId: SessionId,
|
sessionId: SessionId,
|
||||||
stageId: StageId,
|
stageId: StageId,
|
||||||
|
|||||||
@@ -80,6 +80,11 @@ capabilities = { General = 1.0, Coding = 0.7, Reasoning = 0.6, Summarization = 0
|
|||||||
enabled = false
|
enabled = false
|
||||||
root = "" # repo root key; empty = current working dir
|
root = "" # repo root key; empty = current working dir
|
||||||
memory_k = 5 # distilled decisions retrieved per session start
|
memory_k = 5 # distilled decisions retrieved per session start
|
||||||
|
# Repo-map indexer (emits a ranked file/symbol index as a recorded observation at session
|
||||||
|
# start; a top-K slice rides in context as droppable L3 memory — paths + symbol names only).
|
||||||
|
max_depth = 4 # directory recursion cap
|
||||||
|
inject_top_k = 30 # entries injected into context (ranked by score)
|
||||||
|
# ignore_globs = [".git", "node_modules", "build", "target", ".gradle", "dist", ".idea"]
|
||||||
|
|
||||||
# Router configuration (optional, defaults shown below)
|
# Router configuration (optional, defaults shown below)
|
||||||
[router]
|
[router]
|
||||||
|
|||||||
Reference in New Issue
Block a user