kami/correx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(critique): structured CritiqueFinding + per-model calibration projection

Browse Source 
- CritiqueFinding/CritiqueSeverity/CritiqueRole shared type (BACKLOG B6)
- CritiqueOutcomeCorrelatedEvent + serialization registration (BACKLOG C-A3)
- core/critique module: CriticCalibration state/reducer/projector, keyed by (modelHash, role)
- schema+projection only; live reviewer-loop producer wiring deferred

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
kami
2026-06-20 07:45:23 +00:00
parent b1c0f88dee
commit f783eed4be
11 changed files with 346 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/critique/build.gradle
+11
View File
@@ -0,0 +1,11 @@
plugins {
id 'java-library'
id 'org.jetbrains.kotlin.jvm'
id 'org.jetbrains.kotlin.plugin.serialization'
}
dependencies {
implementation(project(":core:events"))
testImplementation "org.jetbrains.kotlin:kotlin-test"
}
tasks.named("koverVerify").configure { enabled = false }
core/critique/src/main/kotlin/com/correx/core/critique/CriticCalibrationProjector.kt
+13
View File
@@ -0,0 +1,13 @@
package com.correx.core.critique
import com.correx.core.critique.model.CriticCalibrationState
import com.correx.core.events.events.StoredEvent
import com.correx.core.sessions.projections.Projection
class CriticCalibrationProjector(
private val reducer: CriticCalibrationReducer,
) : Projection<CriticCalibrationState> {
override fun initial(): CriticCalibrationState = CriticCalibrationState()
override fun apply(state: CriticCalibrationState, event: StoredEvent): CriticCalibrationState =
reducer.reduce(state, event)
}
core/critique/src/main/kotlin/com/correx/core/critique/CriticCalibrationReducer.kt
+8
View File
@@ -0,0 +1,8 @@
package com.correx.core.critique
import com.correx.core.critique.model.CriticCalibrationState
import com.correx.core.events.events.StoredEvent
interface CriticCalibrationReducer {
fun reduce(state: CriticCalibrationState, event: StoredEvent): CriticCalibrationState
}
core/critique/src/main/kotlin/com/correx/core/critique/DefaultCriticCalibrationReducer.kt
+22
View File
@@ -0,0 +1,22 @@
package com.correx.core.critique
import com.correx.core.critique.model.CalibrationKey
import com.correx.core.critique.model.CalibrationStats
import com.correx.core.critique.model.CriticCalibrationState
import com.correx.core.events.events.CritiqueOutcome
import com.correx.core.events.events.CritiqueOutcomeCorrelatedEvent
import com.correx.core.events.events.StoredEvent
class DefaultCriticCalibrationReducer : CriticCalibrationReducer {
override fun reduce(state: CriticCalibrationState, event: StoredEvent): CriticCalibrationState {
val payload = event.payload as? CritiqueOutcomeCorrelatedEvent ?: return state
val key = CalibrationKey(payload.modelHash, payload.role)
val current = state.byKey[key] ?: CalibrationStats()
val updated = when (payload.outcome) {
CritiqueOutcome.UPHELD -> current.copy(upheld = current.upheld + 1)
CritiqueOutcome.DISMISSED -> current.copy(dismissed = current.dismissed + 1)
CritiqueOutcome.INCONCLUSIVE -> current.copy(inconclusive = current.inconclusive + 1)
}
return state.copy(byKey = state.byKey + (key to updated))
}
}
core/critique/src/main/kotlin/com/correx/core/critique/model/CriticCalibrationState.kt
+33
View File
@@ -0,0 +1,33 @@
package com.correx.core.critique.model
import com.correx.core.events.events.CritiqueRole
import kotlinx.serialization.Serializable
/** Composite key: critic precision is tracked per model and per role. */
@Serializable
data class CalibrationKey(val modelHash: String, val role: CritiqueRole)
/**
* Outcome tallies for one [CalibrationKey]. [precision] = upheld / (upheld + dismissed) — the
* share of *decisive* findings that held up — or null when no decisive findings exist yet.
* INCONCLUSIVE outcomes count toward [total] but never toward precision.
*/
@Serializable
data class CalibrationStats(
val upheld: Int = 0,
val dismissed: Int = 0,
val inconclusive: Int = 0,
) {
val total: Int get() = upheld + dismissed + inconclusive
val decisive: Int get() = upheld + dismissed
val precision: Double? get() = if (decisive == 0) null else upheld.toDouble() / decisive
}
/**
* Replay-built view of how each (model, role) critic has performed. Rebuilt from
* [com.correx.core.events.events.CritiqueOutcomeCorrelatedEvent]s; never persisted directly.
*/
@Serializable
data class CriticCalibrationState(
val byKey: Map<CalibrationKey, CalibrationStats> = emptyMap(),
)
core/critique/src/test/kotlin/com/correx/core/critique/DefaultCriticCalibrationReducerTest.kt
+156
View File
@@ -0,0 +1,156 @@
package com.correx.core.critique
import com.correx.core.critique.model.CalibrationKey
import com.correx.core.critique.model.CriticCalibrationState
import com.correx.core.events.events.CritiqueOutcome
import com.correx.core.events.events.CritiqueOutcomeCorrelatedEvent
import com.correx.core.events.events.CritiqueRole
import com.correx.core.events.events.CritiqueSeverity
import com.correx.core.events.events.EventMetadata
import com.correx.core.events.events.EventPayload
import com.correx.core.events.events.StageCompletedEvent
import com.correx.core.events.events.StoredEvent
import com.correx.core.events.types.EventId
import com.correx.core.events.types.SessionId
import com.correx.core.events.types.StageId
import com.correx.core.events.types.TransitionId
import kotlinx.datetime.Instant
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull
import kotlin.test.assertTrue
class DefaultCriticCalibrationReducerTest {
private val reducer = DefaultCriticCalibrationReducer()
private val projector = CriticCalibrationProjector(reducer)
private var seq = 0L
private fun stored(payload: EventPayload): StoredEvent {
seq += 1
return StoredEvent(
metadata = EventMetadata(
eventId = EventId("e-$seq"),
sessionId = SessionId("s1"),
timestamp = Instant.parse("2026-01-01T00:00:00Z"),
schemaVersion = 1,
causationId = null,
correlationId = null,
),
sequence = seq,
sessionSequence = seq,
payload = payload,
)
}
private fun outcome(
outcome: CritiqueOutcome,
modelHash: String = "m1",
role: CritiqueRole = CritiqueRole.CODE_REVIEWER,
severity: CritiqueSeverity = CritiqueSeverity.MAJOR,
) = stored(
CritiqueOutcomeCorrelatedEvent(
sessionId = SessionId("s1"),
findingId = "f-$seq",
role = role,
modelHash = modelHash,
severity = severity,
outcome = outcome,
),
)
private fun fold(events: List<StoredEvent>): CriticCalibrationState =
events.fold(projector.initial(), projector::apply)
@Test
fun `initial state is empty`() {
assertTrue(projector.initial().byKey.isEmpty())
}
@Test
fun `single UPHELD increments upheld and yields precision 1`() {
val state = fold(listOf(outcome(CritiqueOutcome.UPHELD)))
val stats = state.byKey.getValue(CalibrationKey("m1", CritiqueRole.CODE_REVIEWER))
assertEquals(1, stats.upheld)
assertEquals(0, stats.dismissed)
assertEquals(1, stats.total)
assertEquals(1.0, stats.precision)
}
@Test
fun `single DISMISSED increments dismissed and yields precision 0`() {
val state = fold(listOf(outcome(CritiqueOutcome.DISMISSED)))
val stats = state.byKey.getValue(CalibrationKey("m1", CritiqueRole.CODE_REVIEWER))
assertEquals(1, stats.dismissed)
assertEquals(0.0, stats.precision)
}
@Test
fun `INCONCLUSIVE counts toward total but not precision`() {
val state = fold(listOf(outcome(CritiqueOutcome.INCONCLUSIVE)))
val stats = state.byKey.getValue(CalibrationKey("m1", CritiqueRole.CODE_REVIEWER))
assertEquals(1, stats.inconclusive)
assertEquals(1, stats.total)
assertEquals(0, stats.decisive)
assertNull(stats.precision, "precision is undefined with no decisive findings")
}
@Test
fun `distinct model-role keys are tracked independently`() {
val state = fold(
listOf(
outcome(CritiqueOutcome.UPHELD, modelHash = "m1", role = CritiqueRole.CODE_REVIEWER),
outcome(CritiqueOutcome.DISMISSED, modelHash = "m2", role = CritiqueRole.CODE_REVIEWER),
outcome(CritiqueOutcome.UPHELD, modelHash = "m1", role = CritiqueRole.PLAN_CRITIC),
),
)
assertEquals(3, state.byKey.size)
assertEquals(1.0, state.byKey.getValue(CalibrationKey("m1", CritiqueRole.CODE_REVIEWER)).precision)
assertEquals(0.0, state.byKey.getValue(CalibrationKey("m2", CritiqueRole.CODE_REVIEWER)).precision)
assertEquals(1.0, state.byKey.getValue(CalibrationKey("m1", CritiqueRole.PLAN_CRITIC)).precision)
}
@Test
fun `precision is computed over a mix of outcomes for one key`() {
val state = fold(
listOf(
outcome(CritiqueOutcome.UPHELD),
outcome(CritiqueOutcome.UPHELD),
outcome(CritiqueOutcome.UPHELD),
outcome(CritiqueOutcome.DISMISSED),
outcome(CritiqueOutcome.INCONCLUSIVE),
),
)
val stats = state.byKey.getValue(CalibrationKey("m1", CritiqueRole.CODE_REVIEWER))
assertEquals(3, stats.upheld)
assertEquals(1, stats.dismissed)
assertEquals(1, stats.inconclusive)
assertEquals(5, stats.total)
assertEquals(0.75, stats.precision) // 3 / (3 + 1)
}
@Test
fun `unrelated events pass through unchanged`() {
val state = fold(
listOf(
stored(
StageCompletedEvent(
sessionId = SessionId("s1"),
stageId = StageId("plan"),
transitionId = TransitionId("t1"),
),
),
outcome(CritiqueOutcome.UPHELD),
stored(
StageCompletedEvent(
sessionId = SessionId("s1"),
stageId = StageId("review"),
transitionId = TransitionId("t2"),
),
),
),
)
assertEquals(1, state.byKey.size)
assertEquals(1, state.byKey.getValue(CalibrationKey("m1", CritiqueRole.CODE_REVIEWER)).upheld)
}
}