feat(critique): structured CritiqueFinding + per-model calibration projection

- CritiqueFinding/CritiqueSeverity/CritiqueRole shared type (BACKLOG B6)
- CritiqueOutcomeCorrelatedEvent + serialization registration (BACKLOG C-A3)
- core/critique module: CriticCalibration state/reducer/projector, keyed by (modelHash, role)
- schema+projection only; live reviewer-loop producer wiring deferred

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-06-20 07:45:23 +00:00
parent b1c0f88dee
commit f783eed4be
11 changed files with 346 additions and 0 deletions
+11
View File
@@ -0,0 +1,11 @@
plugins {
id 'java-library'
id 'org.jetbrains.kotlin.jvm'
id 'org.jetbrains.kotlin.plugin.serialization'
}
dependencies {
implementation(project(":core:events"))
testImplementation "org.jetbrains.kotlin:kotlin-test"
}
tasks.named("koverVerify").configure { enabled = false }
@@ -0,0 +1,13 @@
package com.correx.core.critique
import com.correx.core.critique.model.CriticCalibrationState
import com.correx.core.events.events.StoredEvent
import com.correx.core.sessions.projections.Projection
class CriticCalibrationProjector(
private val reducer: CriticCalibrationReducer,
) : Projection<CriticCalibrationState> {
override fun initial(): CriticCalibrationState = CriticCalibrationState()
override fun apply(state: CriticCalibrationState, event: StoredEvent): CriticCalibrationState =
reducer.reduce(state, event)
}
@@ -0,0 +1,8 @@
package com.correx.core.critique
import com.correx.core.critique.model.CriticCalibrationState
import com.correx.core.events.events.StoredEvent
interface CriticCalibrationReducer {
fun reduce(state: CriticCalibrationState, event: StoredEvent): CriticCalibrationState
}
@@ -0,0 +1,22 @@
package com.correx.core.critique
import com.correx.core.critique.model.CalibrationKey
import com.correx.core.critique.model.CalibrationStats
import com.correx.core.critique.model.CriticCalibrationState
import com.correx.core.events.events.CritiqueOutcome
import com.correx.core.events.events.CritiqueOutcomeCorrelatedEvent
import com.correx.core.events.events.StoredEvent
class DefaultCriticCalibrationReducer : CriticCalibrationReducer {
override fun reduce(state: CriticCalibrationState, event: StoredEvent): CriticCalibrationState {
val payload = event.payload as? CritiqueOutcomeCorrelatedEvent ?: return state
val key = CalibrationKey(payload.modelHash, payload.role)
val current = state.byKey[key] ?: CalibrationStats()
val updated = when (payload.outcome) {
CritiqueOutcome.UPHELD -> current.copy(upheld = current.upheld + 1)
CritiqueOutcome.DISMISSED -> current.copy(dismissed = current.dismissed + 1)
CritiqueOutcome.INCONCLUSIVE -> current.copy(inconclusive = current.inconclusive + 1)
}
return state.copy(byKey = state.byKey + (key to updated))
}
}
@@ -0,0 +1,33 @@
package com.correx.core.critique.model
import com.correx.core.events.events.CritiqueRole
import kotlinx.serialization.Serializable
/** Composite key: critic precision is tracked per model and per role. */
@Serializable
data class CalibrationKey(val modelHash: String, val role: CritiqueRole)
/**
* Outcome tallies for one [CalibrationKey]. [precision] = upheld / (upheld + dismissed) — the
* share of *decisive* findings that held up — or null when no decisive findings exist yet.
* INCONCLUSIVE outcomes count toward [total] but never toward precision.
*/
@Serializable
data class CalibrationStats(
val upheld: Int = 0,
val dismissed: Int = 0,
val inconclusive: Int = 0,
) {
val total: Int get() = upheld + dismissed + inconclusive
val decisive: Int get() = upheld + dismissed
val precision: Double? get() = if (decisive == 0) null else upheld.toDouble() / decisive
}
/**
* Replay-built view of how each (model, role) critic has performed. Rebuilt from
* [com.correx.core.events.events.CritiqueOutcomeCorrelatedEvent]s; never persisted directly.
*/
@Serializable
data class CriticCalibrationState(
val byKey: Map<CalibrationKey, CalibrationStats> = emptyMap(),
)
@@ -0,0 +1,156 @@
package com.correx.core.critique
import com.correx.core.critique.model.CalibrationKey
import com.correx.core.critique.model.CriticCalibrationState
import com.correx.core.events.events.CritiqueOutcome
import com.correx.core.events.events.CritiqueOutcomeCorrelatedEvent
import com.correx.core.events.events.CritiqueRole
import com.correx.core.events.events.CritiqueSeverity
import com.correx.core.events.events.EventMetadata
import com.correx.core.events.events.EventPayload
import com.correx.core.events.events.StageCompletedEvent
import com.correx.core.events.events.StoredEvent
import com.correx.core.events.types.EventId
import com.correx.core.events.types.SessionId
import com.correx.core.events.types.StageId
import com.correx.core.events.types.TransitionId
import kotlinx.datetime.Instant
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull
import kotlin.test.assertTrue
class DefaultCriticCalibrationReducerTest {
private val reducer = DefaultCriticCalibrationReducer()
private val projector = CriticCalibrationProjector(reducer)
private var seq = 0L
private fun stored(payload: EventPayload): StoredEvent {
seq += 1
return StoredEvent(
metadata = EventMetadata(
eventId = EventId("e-$seq"),
sessionId = SessionId("s1"),
timestamp = Instant.parse("2026-01-01T00:00:00Z"),
schemaVersion = 1,
causationId = null,
correlationId = null,
),
sequence = seq,
sessionSequence = seq,
payload = payload,
)
}
private fun outcome(
outcome: CritiqueOutcome,
modelHash: String = "m1",
role: CritiqueRole = CritiqueRole.CODE_REVIEWER,
severity: CritiqueSeverity = CritiqueSeverity.MAJOR,
) = stored(
CritiqueOutcomeCorrelatedEvent(
sessionId = SessionId("s1"),
findingId = "f-$seq",
role = role,
modelHash = modelHash,
severity = severity,
outcome = outcome,
),
)
private fun fold(events: List<StoredEvent>): CriticCalibrationState =
events.fold(projector.initial(), projector::apply)
@Test
fun `initial state is empty`() {
assertTrue(projector.initial().byKey.isEmpty())
}
@Test
fun `single UPHELD increments upheld and yields precision 1`() {
val state = fold(listOf(outcome(CritiqueOutcome.UPHELD)))
val stats = state.byKey.getValue(CalibrationKey("m1", CritiqueRole.CODE_REVIEWER))
assertEquals(1, stats.upheld)
assertEquals(0, stats.dismissed)
assertEquals(1, stats.total)
assertEquals(1.0, stats.precision)
}
@Test
fun `single DISMISSED increments dismissed and yields precision 0`() {
val state = fold(listOf(outcome(CritiqueOutcome.DISMISSED)))
val stats = state.byKey.getValue(CalibrationKey("m1", CritiqueRole.CODE_REVIEWER))
assertEquals(1, stats.dismissed)
assertEquals(0.0, stats.precision)
}
@Test
fun `INCONCLUSIVE counts toward total but not precision`() {
val state = fold(listOf(outcome(CritiqueOutcome.INCONCLUSIVE)))
val stats = state.byKey.getValue(CalibrationKey("m1", CritiqueRole.CODE_REVIEWER))
assertEquals(1, stats.inconclusive)
assertEquals(1, stats.total)
assertEquals(0, stats.decisive)
assertNull(stats.precision, "precision is undefined with no decisive findings")
}
@Test
fun `distinct model-role keys are tracked independently`() {
val state = fold(
listOf(
outcome(CritiqueOutcome.UPHELD, modelHash = "m1", role = CritiqueRole.CODE_REVIEWER),
outcome(CritiqueOutcome.DISMISSED, modelHash = "m2", role = CritiqueRole.CODE_REVIEWER),
outcome(CritiqueOutcome.UPHELD, modelHash = "m1", role = CritiqueRole.PLAN_CRITIC),
),
)
assertEquals(3, state.byKey.size)
assertEquals(1.0, state.byKey.getValue(CalibrationKey("m1", CritiqueRole.CODE_REVIEWER)).precision)
assertEquals(0.0, state.byKey.getValue(CalibrationKey("m2", CritiqueRole.CODE_REVIEWER)).precision)
assertEquals(1.0, state.byKey.getValue(CalibrationKey("m1", CritiqueRole.PLAN_CRITIC)).precision)
}
@Test
fun `precision is computed over a mix of outcomes for one key`() {
val state = fold(
listOf(
outcome(CritiqueOutcome.UPHELD),
outcome(CritiqueOutcome.UPHELD),
outcome(CritiqueOutcome.UPHELD),
outcome(CritiqueOutcome.DISMISSED),
outcome(CritiqueOutcome.INCONCLUSIVE),
),
)
val stats = state.byKey.getValue(CalibrationKey("m1", CritiqueRole.CODE_REVIEWER))
assertEquals(3, stats.upheld)
assertEquals(1, stats.dismissed)
assertEquals(1, stats.inconclusive)
assertEquals(5, stats.total)
assertEquals(0.75, stats.precision) // 3 / (3 + 1)
}
@Test
fun `unrelated events pass through unchanged`() {
val state = fold(
listOf(
stored(
StageCompletedEvent(
sessionId = SessionId("s1"),
stageId = StageId("plan"),
transitionId = TransitionId("t1"),
),
),
outcome(CritiqueOutcome.UPHELD),
stored(
StageCompletedEvent(
sessionId = SessionId("s1"),
stageId = StageId("review"),
transitionId = TransitionId("t2"),
),
),
),
)
assertEquals(1, state.byKey.size)
assertEquals(1, state.byKey.getValue(CalibrationKey("m1", CritiqueRole.CODE_REVIEWER)).upheld)
}
}
@@ -0,0 +1,33 @@
package com.correx.core.events.events
import com.correx.core.events.types.SessionId
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
/**
* How a prior [CritiqueFinding] actually turned out once the surrounding loop resolved.
* UPHELD = the finding was confirmed/actioned; DISMISSED = rejected as a false positive;
* INCONCLUSIVE = no signal either way.
*/
@Serializable
enum class CritiqueOutcome { UPHELD, DISMISSED, INCONCLUSIVE }
/**
* Correlates one prior [CritiqueFinding] (by [findingId]) with how it actually turned out, so a
* critic's precision can be tracked per model and per role over time (the calibration projection
* in `:core:critique`). [modelHash] identifies the model that produced the finding; [severity]
* is carried so calibration can be sliced by severity band without re-reading the finding.
*
* This is the recording half only — who decides UPHELD/DISMISSED lives in the reviewer-loop
* runtime and is wired separately.
*/
@Serializable
@SerialName("CritiqueOutcomeCorrelated")
data class CritiqueOutcomeCorrelatedEvent(
val sessionId: SessionId,
val findingId: String,
val role: CritiqueRole,
val modelHash: String,
val severity: CritiqueSeverity,
val outcome: CritiqueOutcome,
) : EventPayload
@@ -0,0 +1,34 @@
package com.correx.core.events.events
import kotlinx.serialization.Serializable
/** Which critic role produced a [CritiqueFinding]. */
@Serializable
enum class CritiqueRole { PLAN_CRITIC, CODE_REVIEWER }
/** Severity of a [CritiqueFinding], highest-impact first. */
@Serializable
enum class CritiqueSeverity { BLOCKER, MAJOR, MINOR, NIT }
/**
* One structured finding emitted by a critic (plan critic) or reviewer (code reviewer),
* replacing the free-text verdict prose those roles produced before. Shared by both roles so
* findings can be counted, ranked, and calibrated uniformly.
*
* [id] is a stable identifier for the finding, used to correlate it with its eventual outcome
* (see [CritiqueOutcomeCorrelatedEvent]). [category] is a free-form classification
* (e.g. "correctness", "missing_requirement", "style"). [location] optionally points at a
* `file:line` or an artifact/stage reference the finding concerns.
*
* Distinct from `core.validation.ValidationIssue`, which models *structural* validation
* (graph/schema) rather than a critic's judgement.
*/
@Serializable
data class CritiqueFinding(
val id: String,
val role: CritiqueRole,
val severity: CritiqueSeverity,
val category: String,
val message: String,
val location: String? = null,
)
@@ -13,6 +13,7 @@ import com.correx.core.events.events.ChatSessionStartedEvent
import com.correx.core.events.events.ClarificationAnsweredEvent import com.correx.core.events.events.ClarificationAnsweredEvent
import com.correx.core.events.events.ClarificationRequestedEvent import com.correx.core.events.events.ClarificationRequestedEvent
import com.correx.core.events.events.ChatTurnEvent import com.correx.core.events.events.ChatTurnEvent
import com.correx.core.events.events.CritiqueOutcomeCorrelatedEvent
import com.correx.core.events.events.RouterNarrationEvent import com.correx.core.events.events.RouterNarrationEvent
import com.correx.core.events.events.OperatorProfileBoundEvent import com.correx.core.events.events.OperatorProfileBoundEvent
import com.correx.core.events.events.ProjectProfileBoundEvent import com.correx.core.events.events.ProjectProfileBoundEvent
@@ -127,6 +128,7 @@ val eventModule = SerializersModule {
subclass(WorkflowProposedEvent::class) subclass(WorkflowProposedEvent::class)
subclass(IdeaCapturedEvent::class) subclass(IdeaCapturedEvent::class)
subclass(IdeaDiscardedEvent::class) subclass(IdeaDiscardedEvent::class)
subclass(CritiqueOutcomeCorrelatedEvent::class)
} }
} }
@@ -0,0 +1,33 @@
package com.correx.core.events.serialization
import com.correx.core.events.events.CritiqueOutcome
import com.correx.core.events.events.CritiqueOutcomeCorrelatedEvent
import com.correx.core.events.events.CritiqueRole
import com.correx.core.events.events.CritiqueSeverity
import com.correx.core.events.events.EventPayload
import com.correx.core.events.types.SessionId
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class CritiqueCalibrationEventSerializationTest {
@Test
fun `CritiqueOutcomeCorrelatedEvent round-trips as polymorphic EventPayload`() {
val sample: EventPayload = CritiqueOutcomeCorrelatedEvent(
sessionId = SessionId("s"),
findingId = "f-1",
role = CritiqueRole.CODE_REVIEWER,
modelHash = "sha256:abc",
severity = CritiqueSeverity.MAJOR,
outcome = CritiqueOutcome.UPHELD,
)
val encoded = eventJson.encodeToString(EventPayload.serializer(), sample)
assertTrue(
encoded.contains("\"type\":\"CritiqueOutcomeCorrelated\""),
"SerialName must be present: $encoded",
)
val decoded = eventJson.decodeFromString(EventPayload.serializer(), encoded)
assertEquals(sample, decoded)
}
}
+1
View File
@@ -28,6 +28,7 @@ include ':core:sessions'
include ':core:config' include ':core:config'
include ':core:risk' include ':core:risk'
include ':core:journal' include ':core:journal'
include ':core:critique'
include ':infrastructure:persistence' include ':infrastructure:persistence'
include ':infrastructure:inference' include ':infrastructure:inference'