feat(concept): heuristic concept compiler (ACR fold-in)

Promotes recurring validated failure->fix patterns into L3 as retrieval-on-demand
concepts. Deterministic core: ConceptCompilerProjection clusters
RetryAttempted->StageCompleted pairs by fingerprint (gate-agnostic), promotes at
N=3 cross-session validated fixes, never-contradicted. ConceptPromotedEvent is the
sole authoritative write (idempotent under replay); ConceptCompilerService appends
it + best-effort injects to L3 (non-authoritative, inv #6). Wired live in
ServerModule.start() on StageCompleted.

Design: docs/plans/2026-07-12-acr-concept-compiler.md

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DhFXmKe4WisSSPf9LrmmTg
This commit is contained in:
2026-07-15 13:35:17 +04:00
parent d69cb12ce9
commit 3a48ecd24f
9 changed files with 525 additions and 0 deletions
@@ -535,6 +535,11 @@ fun main() {
} else {
null
}
// Heuristic concept compiler (design 2026-07-12-acr-concept-compiler.md): promotes recurring
// validated failure→fix patterns into L3 as retrieval-on-demand concepts. Fires only on ≥N
// cross-session validated fixes, so it's harmless to run unconditionally.
val conceptCompilerService =
com.correx.apps.server.concept.ConceptCompilerService(eventStore, embedder, l3MemoryStore)
// Built from a config snapshot and reused by ConfigService's rebuild hook so toggling
// project.enabled / personalization.* applies live to the next session.
fun buildProjectMemory(cfg: CorrexConfig): com.correx.apps.server.memory.ProjectMemoryService? =
@@ -638,6 +643,7 @@ fun main() {
narrationMaxPerRun = correxConfig.talkie.narration.maxPerRun,
projectMemory = projectMemory,
architectContradictionChecker = architectContradictionChecker,
conceptCompilerService = conceptCompilerService,
configHolder = configHolder,
freestyleDriver = freestyleDriver,
operatorProfile = operatorProfile,
@@ -20,6 +20,7 @@ import com.correx.core.config.ProjectProfileLoader
import com.correx.apps.server.memory.ArchitectContradictionChecker
import com.correx.core.events.events.AgentInstructionsBoundEvent
import com.correx.core.events.events.ApprovalRequestedEvent
import com.correx.core.events.events.StageCompletedEvent
import com.correx.core.events.events.ArtifactContentStoredEvent
import com.correx.core.events.events.ArtifactCreatedEvent
import com.correx.core.events.events.EventMetadata
@@ -103,6 +104,9 @@ class ServerModule(
// (tests / project.enabled=false). Unlike projectMemory there is no live config-rebuild hook:
// a server restart re-reads project.enabled, which is enough for this informational flag.
private val architectContradictionChecker: ArchitectContradictionChecker? = null,
// Heuristic concept compiler write-side (design 2026-07-12-acr-concept-compiler.md). Null disables
// the promotion hook (tests). Live-only like the subscriptions below — never runs under replay.
private val conceptCompilerService: com.correx.apps.server.concept.ConceptCompilerService? = null,
// Live, swappable config. Null only in tests that don't exercise config editing; defaults to a
// holder seeded from defaults so callers always have a value to read.
val configHolder: com.correx.core.config.ConfigHolder =
@@ -240,6 +244,21 @@ class ServerModule(
}
.launchIn(moduleScope)
}
// Heuristic concept compiler (design 2026-07-12-acr-concept-compiler.md). Live-only like the
// hooks above: subscribeAll() replays nothing and ServerModule is never built under replay, so
// promotion never re-fires on restart/replay (invariant #8). A StageCompleted may resolve a
// validated failure→fix; re-fold the log and promote any newly-eligible fingerprint. Failures
// are logged and swallowed — promotion is best-effort enrichment, never on the stage's path.
conceptCompilerService?.let { compiler ->
eventStore.subscribeAll()
.filter { it.payload is StageCompletedEvent }
.onEach {
runCatching { compiler.runOnce() }
.onFailure { e -> log.warn("concept compiler run failed: {}", e.message) }
}
.launchIn(moduleScope)
}
}
/**
@@ -0,0 +1,105 @@
package com.correx.apps.server.concept
import com.correx.core.events.events.ConceptPromotedEvent
import com.correx.core.events.events.EventMetadata
import com.correx.core.events.events.NewEvent
import com.correx.core.events.stores.EventStore
import com.correx.core.events.types.EventId
import com.correx.core.events.types.SessionId
import com.correx.core.inference.Embedder
import com.correx.core.kernel.concept.ConceptCluster
import com.correx.core.kernel.concept.ConceptCompilerProjection
import com.correx.core.kernel.concept.DEFAULT_PROMOTION_THRESHOLD
import com.correx.core.talkie.l3.L3MemoryEntry
import com.correx.core.talkie.l3.L3MemoryStore
import kotlinx.coroutines.CancellationException
import kotlinx.datetime.Clock
import org.slf4j.LoggerFactory
import java.util.UUID
private val log = LoggerFactory.getLogger(ConceptCompilerService::class.java)
/**
* The write-side of the heuristic concept compiler (design 2026-07-12-acr-concept-compiler.md).
* [ConceptCompilerProjection] does the deterministic clustering; this service is the single "write":
* it folds the whole cross-session log, and for every fingerprint that has just become promotable it
* appends a [ConceptPromotedEvent] (the authoritative record, idempotent under replay) and best-effort
* injects the concept into L3 so it is retrieved on demand into future stage context — the same
* pull/top-k path RepoKnowledge already uses, no broadcast. The L3 write is non-authoritative
* (invariant #6); the event is truth, so an embed/store failure is logged and swallowed.
*
* Deterministic rule firing → no assessor gate needed (invariant #3/#7 hold trivially).
*
* ponytail: [runOnce] re-folds `allEvents()` on each call — O(n) in log size. Fine at local-first
* scale and called on stage completions, not per-event. Switch to an incremental in-memory fold over
* `subscribeAll()` if the log outgrows a full rescan.
*/
class ConceptCompilerService(
private val eventStore: EventStore,
private val embedder: Embedder,
private val l3MemoryStore: L3MemoryStore,
private val threshold: Int = DEFAULT_PROMOTION_THRESHOLD,
) {
private val projection = ConceptCompilerProjection()
/** Fold the full log, promote every newly-promotable fingerprint. Safe to call repeatedly. */
suspend fun runOnce() {
val state = eventStore.allEvents().fold(projection.initial(), projection::apply)
state.promotable(threshold).forEach { promote(it) }
}
private suspend fun promote(cluster: ConceptCluster) {
val text = conceptText(cluster)
eventStore.append(
NewEvent(
metadata = metadata(),
payload = ConceptPromotedEvent(
sessionId = SYSTEM_SESSION,
fingerprint = cluster.fingerprint,
gate = cluster.gate,
conceptText = text,
occurrences = cluster.validatedFixes,
),
),
)
injectToL3(cluster.fingerprint, text)
log.info("promoted concept {} ({}x, gate={})", cluster.fingerprint, cluster.validatedFixes, cluster.gate)
}
private suspend fun injectToL3(fingerprint: String, text: String) {
runCatching {
val now = Clock.System.now().toEpochMilliseconds()
l3MemoryStore.store(
L3MemoryEntry(
id = "concept:$fingerprint",
sessionId = SYSTEM_SESSION,
turnId = "concept:$fingerprint",
text = text,
vector = embedder.embed(text),
timestampMs = now,
),
)
}.exceptionOrNull()?.let { e ->
if (e is CancellationException) throw e
log.warn("L3 inject failed for concept {}: {}", fingerprint, e.message)
}
}
private fun conceptText(c: ConceptCluster) =
"Recurring ${c.gate} failure (validated-fixed ${c.validatedFixes}× across sessions): " +
"${c.signature}. This failure class has a known resolution — check prior fixes before re-deriving."
private fun metadata() = EventMetadata(
eventId = EventId(UUID.randomUUID().toString()),
sessionId = SYSTEM_SESSION,
timestamp = Clock.System.now(),
schemaVersion = 1,
causationId = null,
correlationId = null,
)
companion object {
/** Cross-session concepts live on their own stream, out of any user session's replay. */
val SYSTEM_SESSION = SessionId("__concept_compiler__")
}
}
@@ -0,0 +1,109 @@
package com.correx.apps.server.concept
import com.correx.core.events.events.ConceptPromotedEvent
import com.correx.core.events.events.EventMetadata
import com.correx.core.events.events.EventPayload
import com.correx.core.events.events.NewEvent
import com.correx.core.events.events.RetryAttemptedEvent
import com.correx.core.events.events.StageCompletedEvent
import com.correx.core.events.stores.EventStore
import com.correx.core.events.types.EventId
import com.correx.core.events.types.SessionId
import com.correx.core.events.types.StageId
import com.correx.core.events.types.TransitionId
import com.correx.core.inference.Embedder
import com.correx.core.talkie.l3.InMemoryL3MemoryStore
import com.correx.core.talkie.l3.L3Query
import com.correx.infrastructure.persistence.InMemoryEventStore
import kotlinx.coroutines.flow.toList
import kotlinx.coroutines.runBlocking
import kotlinx.datetime.Clock
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
import java.util.UUID
private class ConstantEmbedder(override val dimension: Int = 8) : Embedder {
override suspend fun embed(text: String): FloatArray = FloatArray(dimension) { 1f }
}
class ConceptCompilerServiceTest {
private val store = InMemoryEventStore()
private val l3 = InMemoryL3MemoryStore()
private val service = ConceptCompilerService(store, ConstantEmbedder(), l3, threshold = 3)
private suspend fun append(payload: EventPayload, session: String) {
store.append(
NewEvent(
metadata = EventMetadata(
eventId = EventId(UUID.randomUUID().toString()),
sessionId = SessionId(session),
timestamp = Clock.System.now(),
schemaVersion = 1,
causationId = null,
correlationId = null,
),
payload = payload,
),
)
}
private suspend fun validatedFix(fp: String, session: String) {
append(
RetryAttemptedEvent(
sessionId = SessionId(session),
stageId = StageId("impl"),
attemptNumber = 1,
maxAttempts = 3,
failureReason = "detekt: MagicNumber\ntrace",
gate = "lint",
fingerprint = fp,
),
session,
)
append(StageCompletedEvent(SessionId(session), StageId("impl"), TransitionId("t")), session)
}
private suspend fun promotedEvents(): List<ConceptPromotedEvent> =
store.allEvents().mapNotNull { it.payload as? ConceptPromotedEvent }.toList()
@Test
fun `promotes a fingerprint that crossed the threshold and injects it into L3`(): Unit = runBlocking {
validatedFix("fp1", "sa")
validatedFix("fp1", "sb")
validatedFix("fp1", "sc")
service.runOnce()
val promoted = promotedEvents()
assertEquals(1, promoted.size)
assertEquals("fp1", promoted.single().fingerprint)
assertEquals(3, promoted.single().occurrences)
val hits = l3.query(L3Query(vector = FloatArray(8) { 1f }, k = 5))
assertTrue(hits.any { it.entry.turnId == "concept:fp1" }) { "concept not injected into L3" }
}
@Test
fun `does not promote below the threshold`(): Unit = runBlocking {
validatedFix("fp1", "sa")
validatedFix("fp1", "sb")
service.runOnce()
assertTrue(promotedEvents().isEmpty())
}
@Test
fun `repeated runs do not re-promote the same fingerprint`(): Unit = runBlocking {
validatedFix("fp1", "sa")
validatedFix("fp1", "sb")
validatedFix("fp1", "sc")
service.runOnce()
service.runOnce()
assertEquals(1, promotedEvents().size)
}
}
@@ -0,0 +1,33 @@
package com.correx.core.events.events
import com.correx.core.events.types.SessionId
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
/**
* A validated failure→fix pattern the heuristic concept compiler has seen resolved reliably enough to
* promote (design 2026-07-12-acr-concept-compiler.md). The only "write" the compiler makes: a
* deterministic rule firing — a [fingerprint] whose validated-fix count crossed the promotion
* threshold, never contradicted — so it needs no assessor gate (invariant #3/#7 hold trivially:
* nothing model-proposed reaches state here).
*
* Read-only over the log otherwise: the promotion is derivable from the RetryAttempted→StageCompleted
* stream, and this event makes the promotion idempotent under replay (the compiler folds already-emitted
* fingerprints into its promoted set and never re-promotes). Best-effort injected into L3 as a
* retrieval-on-demand concept; the L3 write is non-authoritative (invariant #6) — this event is truth.
*/
@Serializable
@SerialName("ConceptPromoted")
data class ConceptPromotedEvent(
// A dedicated system session (ConceptCompilerService.SYSTEM_SESSION) — concepts are cross-session,
// so they live on their own stream rather than polluting any user session's replay.
val sessionId: SessionId,
// Gate-agnostic fingerprint of the recurring failure (see FailureFingerprint).
val fingerprint: String,
// The gate the failure recurred on ("contract" | "review" | "plan-compile" | "lint" | "static" | …).
val gate: String,
// Templated, retrieval-friendly concept text injected into L3.
val conceptText: String,
// Count of validated fixes observed at promotion time (≥ threshold).
val occurrences: Int,
) : EventPayload
@@ -14,6 +14,7 @@ import com.correx.core.events.events.ArtifactValidatedEvent
import com.correx.core.events.events.ArtifactValidatingEvent
import com.correx.core.events.events.BriefEchoMismatchEvent
import com.correx.core.events.events.BriefGroundingCheckedEvent
import com.correx.core.events.events.ConceptPromotedEvent
import com.correx.core.events.events.StaticAnalysisCompletedEvent
import com.correx.core.events.events.ContractGateEvaluatedEvent
import com.correx.core.events.events.PlanLintCompletedEvent
@@ -128,6 +129,7 @@ val eventModule = SerializersModule {
subclass(SessionNamedEvent::class)
subclass(StageFailedEvent::class)
subclass(StageCompletedEvent::class)
subclass(ConceptPromotedEvent::class)
subclass(TransitionExecutedEvent::class)
subclass(ApprovalRequestedEvent::class)
subclass(ApprovalDecisionResolvedEvent::class)
@@ -0,0 +1,27 @@
package com.correx.core.events.serialization
import com.correx.core.events.events.ConceptPromotedEvent
import com.correx.core.events.events.EventPayload
import com.correx.core.events.types.SessionId
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class ConceptPromotedEventSerializationTest {
private val sample = ConceptPromotedEvent(
sessionId = SessionId("__concept_compiler__"),
fingerprint = "1a2b3c",
gate = "lint",
conceptText = "Recurring lint failure (validated-fixed 3x across sessions): detekt: MagicNumber.",
occurrences = 3,
)
@Test
fun `round-trips as polymorphic EventPayload`() {
val encoded = eventJson.encodeToString(EventPayload.serializer(), sample)
assertTrue(encoded.contains("\"type\":\"ConceptPromoted\""), "SerialName must be present: $encoded")
val decoded = eventJson.decodeFromString(EventPayload.serializer(), encoded)
assertEquals(sample, decoded)
}
}
@@ -0,0 +1,100 @@
package com.correx.core.kernel.concept
import com.correx.core.events.events.ConceptPromotedEvent
import com.correx.core.events.events.RetryAttemptedEvent
import com.correx.core.events.events.StageCompletedEvent
import com.correx.core.events.events.StageFailedEvent
import com.correx.core.events.events.StoredEvent
import com.correx.core.events.events.WorkflowFailedEvent
import com.correx.core.sessions.projections.Projection
/** Default validated-fix count a fingerprint must reach before promotion (design §"Promotion rule"). */
const val DEFAULT_PROMOTION_THRESHOLD = 3
/** A recurring failure signature and its resolution tally, keyed by [RetryAttemptedEvent.fingerprint]. */
data class ConceptCluster(
val fingerprint: String,
val gate: String,
// First line of the recurring failure reason — the retrieval-friendly signature.
val signature: String,
// Count of times a stage retrying this fingerprint went on to complete = validated failure→fix.
val validatedFixes: Int = 0,
// Any StageFailed / WorkflowFailed observed while this fingerprint was open: the fix did NOT hold.
val contradicted: Boolean = false,
)
/** A fingerprint currently being fought by a stage — resolved by the next StageCompleted/Failed. */
data class OpenFailure(val fingerprint: String, val gate: String, val signature: String)
data class ConceptCompilerState(
val clusters: Map<String, ConceptCluster> = emptyMap(),
// fingerprints already emitted as ConceptPromotedEvent — folded back so replay never re-promotes.
val promoted: Set<String> = emptySet(),
// Per-stage open failure the stage is retrying; keyed by "sessionId/stageId" so concurrent sessions
// in one cross-session fold don't collide.
val open: Map<String, OpenFailure> = emptyMap(),
) {
/**
* Clusters that have earned promotion but haven't been emitted yet: validated ≥ [threshold], never
* contradicted, not already promoted. Pure — the caller (a service) appends the ConceptPromotedEvent.
*/
fun promotable(threshold: Int = DEFAULT_PROMOTION_THRESHOLD): List<ConceptCluster> =
clusters.values.filter {
it.fingerprint !in promoted && !it.contradicted && it.validatedFixes >= threshold
}
}
/**
* Read-only projection over the event log that clusters validated failure→fix pairs by
* [RetryAttemptedEvent.fingerprint] (gate-agnostic — contract/review/plan-compile/lint/static all
* flow through the same retry event) and tracks which fingerprints are promotable. Fed the full
* cross-session stream by its driver; the fold is deterministic and replayable (invariant #8).
*
* Signal: a stage emits RetryAttemptedEvent(fingerprint=fp) then StageCompleted ⇒ fp was validated-fixed.
* A StageFailed/WorkflowFailed while fp is still open ⇒ the fix didn't hold (contradiction).
*/
class ConceptCompilerProjection : Projection<ConceptCompilerState> {
override fun initial() = ConceptCompilerState()
override fun apply(state: ConceptCompilerState, event: StoredEvent): ConceptCompilerState =
when (val p = event.payload) {
is RetryAttemptedEvent -> {
val key = "${p.sessionId.value}/${p.stageId.value}"
val sig = p.failureReason.lineSequence().firstOrNull()?.take(SIGNATURE_MAX)?.trim().orEmpty()
state.copy(open = state.open + (key to OpenFailure(p.fingerprint, p.gate, sig)))
}
is StageCompletedEvent -> resolve(state, "${p.sessionId.value}/${p.stageId.value}", fixed = true)
is StageFailedEvent -> resolve(state, "${p.sessionId.value}/${p.stageId.value}", fixed = false)
is WorkflowFailedEvent -> failAllInSession(state, p.sessionId.value)
is ConceptPromotedEvent -> state.copy(promoted = state.promoted + p.fingerprint)
else -> state
}
private fun resolve(state: ConceptCompilerState, key: String, fixed: Boolean): ConceptCompilerState {
val failure = state.open[key] ?: return state
val existing = state.clusters[failure.fingerprint]
?: ConceptCluster(failure.fingerprint, failure.gate, failure.signature)
val updated = if (fixed) {
existing.copy(validatedFixes = existing.validatedFixes + 1)
} else {
existing.copy(contradicted = true)
}
return state.copy(
clusters = state.clusters + (failure.fingerprint to updated),
open = state.open - key,
)
}
// A workflow that failed terminally contradicts every fingerprint still open in that session.
private fun failAllInSession(state: ConceptCompilerState, sessionId: String): ConceptCompilerState {
val prefix = "$sessionId/"
var acc = state
state.open.keys.filter { it.startsWith(prefix) }.forEach { acc = resolve(acc, it, fixed = false) }
return acc
}
private companion object {
const val SIGNATURE_MAX = 200
}
}
@@ -0,0 +1,124 @@
package com.correx.core.kernel.concept
import com.correx.core.events.events.ConceptPromotedEvent
import com.correx.core.events.events.EventMetadata
import com.correx.core.events.events.EventPayload
import com.correx.core.events.events.RetryAttemptedEvent
import com.correx.core.events.events.StageCompletedEvent
import com.correx.core.events.events.StageFailedEvent
import com.correx.core.events.events.StoredEvent
import com.correx.core.events.events.WorkflowFailedEvent
import com.correx.core.events.types.EventId
import com.correx.core.events.types.SessionId
import com.correx.core.events.types.StageId
import com.correx.core.events.types.TransitionId
import kotlinx.datetime.Instant
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
class ConceptCompilerProjectionTest {
private val projection = ConceptCompilerProjection()
private var seq = 0L
private fun stored(payload: EventPayload, session: String = "s1") = StoredEvent(
metadata = EventMetadata(
eventId = EventId("e${seq++}"),
sessionId = SessionId(session),
timestamp = Instant.parse("2026-01-01T00:00:00Z"),
schemaVersion = 1,
causationId = null,
correlationId = null,
),
sequence = seq,
sessionSequence = seq,
payload = payload,
)
private fun retry(fp: String, stage: String = "impl", session: String = "s1", reason: String = "boom\ndetail") =
stored(
RetryAttemptedEvent(
sessionId = SessionId(session),
stageId = StageId(stage),
attemptNumber = 1,
maxAttempts = 3,
failureReason = reason,
gate = "lint",
fingerprint = fp,
),
session,
)
private fun completed(stage: String = "impl", session: String = "s1") =
stored(StageCompletedEvent(SessionId(session), StageId(stage), TransitionId("t")), session)
private fun failed(stage: String = "impl", session: String = "s1") =
stored(StageFailedEvent(SessionId(session), StageId(stage), TransitionId("t"), "x"), session)
private fun fold(events: List<StoredEvent>) =
events.fold(projection.initial(), projection::apply)
@Test
fun `retry then StageCompleted counts one validated fix`() {
val state = fold(listOf(retry("fp1"), completed()))
assertEquals(1, state.clusters["fp1"]?.validatedFixes)
assertTrue(state.promotable().isEmpty())
}
@Test
fun `three cross-session validated fixes cross the threshold`() {
val state = fold(
listOf(
retry("fp1", session = "sa"), completed(session = "sa"),
retry("fp1", session = "sb"), completed(session = "sb"),
retry("fp1", session = "sc"), completed(session = "sc"),
),
)
assertEquals(3, state.clusters["fp1"]?.validatedFixes)
assertEquals(listOf("fp1"), state.promotable().map { it.fingerprint })
}
@Test
fun `StageFailed while fingerprint open contradicts the cluster`() {
val state = fold(
listOf(
retry("fp1", session = "sa"), completed(session = "sa"),
retry("fp1", session = "sb"), completed(session = "sb"),
retry("fp1", session = "sc"), failed(session = "sc"),
),
)
assertTrue(state.clusters["fp1"]!!.contradicted)
assertTrue(state.promotable().isEmpty())
}
@Test
fun `WorkflowFailed contradicts every open fingerprint in that session`() {
val state = fold(
listOf(
retry("fp1", stage = "a", session = "sa"),
retry("fp2", stage = "b", session = "sa"),
stored(WorkflowFailedEvent(SessionId("sa"), StageId("a"), "dead", true), "sa"),
),
)
assertTrue(state.clusters["fp1"]!!.contradicted)
assertTrue(state.clusters["fp2"]!!.contradicted)
}
@Test
fun `promotion is idempotent - a promoted fingerprint is not promotable again`() {
val base = listOf(
retry("fp1", session = "sa"), completed(session = "sa"),
retry("fp1", session = "sb"), completed(session = "sb"),
retry("fp1", session = "sc"), completed(session = "sc"),
)
val promoted = base + stored(ConceptPromotedEvent(SessionId("sc"), "fp1", "lint", "concept", 3))
assertTrue(fold(promoted).promotable().isEmpty())
}
@Test
fun `signature is the first line of the failure reason`() {
val state = fold(listOf(retry("fp1", reason = "detekt: MagicNumber\ntrace..."), completed()))
assertEquals("detekt: MagicNumber", state.clusters["fp1"]?.signature)
}
}