feat: carry validation rationale through risk assessment

Risk tier is now derived from the worst validation issue severity and
the issues that drove it travel with the summary, so consumers can show
operators why a tier was assigned instead of an opaque level.

- Add RiskSummary.rationale (defaulted, backward-compatible) holding
  human-readable '[code] message' lines from the validation report
- Pin severity -> risk mapping: ERROR->HIGH, WARNING->MEDIUM, INFO->LOW
- DefaultRiskAssessor folds the validation-driven level into the summary
This commit is contained in:
2026-05-30 13:06:39 +04:00
parent 32d15de034
commit ee3f9b8aaa
5 changed files with 73 additions and 9 deletions
@@ -4,4 +4,5 @@ data class RiskSummary(
val level: RiskLevel,
val signals: List<RiskSignal>,
val recommendedAction: RiskAction,
val rationale: List<String> = emptyList(),
)
@@ -11,15 +11,14 @@ class DefaultRiskAssessor : RiskAssessor {
val (report, state, inferenceState) = context
val signals = mutableListOf<RiskSignal>()
val errorCount = report.sections.sumOf { s ->
s.issues.count { it.severity == ValidationSeverity.ERROR }
}
val allIssues = report.sections.flatMap { it.issues }
val errorCount = allIssues.count { it.severity == ValidationSeverity.ERROR }
if (errorCount > 0) {
signals += RiskSignal.ValidationErrors(errorCount)
}
report.sections.flatMap { it.issues }
.firstOrNull { it.code.contains("CYCLE", ignoreCase = true) }
allIssues.firstOrNull { it.code.contains("CYCLE", ignoreCase = true) }
?.let { signals += RiskSignal.CycleWithoutExit(it.code) }
state.retryPolicy?.let { policy ->
@@ -37,12 +36,23 @@ class DefaultRiskAssessor : RiskAssessor {
?.let { signals += RiskSignal.InferenceTimeout(it.latencyMs ?: 0L) }
}
val level = signals.fold(RiskLevel.LOW) { acc, signal -> maxOf(acc, signal.toRiskLevel()) }
val validationDrivenLevel = when {
allIssues.any { it.severity == ValidationSeverity.ERROR } -> RiskLevel.HIGH
allIssues.any { it.severity == ValidationSeverity.WARNING } -> RiskLevel.MEDIUM
allIssues.any { it.severity == ValidationSeverity.INFO } -> RiskLevel.LOW
else -> RiskLevel.LOW
}
var level = signals.fold(RiskLevel.LOW) { acc, signal -> maxOf(acc, signal.toRiskLevel()) }
level = maxOf(level, validationDrivenLevel)
val rationale = allIssues.map { "[${it.code}] ${it.message}" }
return RiskSummary(
level = level,
signals = signals,
recommendedAction = level.toRiskAction(),
rationale = rationale,
)
}
}
@@ -14,5 +14,6 @@ class NoOpRiskAssessor : RiskAssessor {
level = RiskLevel.LOW,
signals = emptyList(),
recommendedAction = RiskAction.PROCEED,
rationale = emptyList(),
)
}
@@ -20,7 +20,7 @@ internal fun RiskLevel.toRiskAction(): RiskAction = when (this) {
}
internal fun RiskSignal.toRiskLevel(): RiskLevel = when (this) {
is RiskSignal.ValidationErrors -> RiskLevel.MEDIUM
is RiskSignal.ValidationErrors -> RiskLevel.HIGH
is RiskSignal.CycleWithoutExit -> RiskLevel.MEDIUM
is RiskSignal.InferenceTimeout -> RiskLevel.MEDIUM
is RiskSignal.RepeatedFailure -> RiskLevel.HIGH
@@ -51,7 +51,7 @@ class DefaultRiskAssessorTest {
}
@Test
fun `validation errors produce ValidationErrors signal at MEDIUM`() {
fun `validation errors produce ValidationErrors signal at HIGH`() {
val report = ValidationReport(
sections = listOf(
ValidationSection(
@@ -68,12 +68,32 @@ class DefaultRiskAssessorTest {
val summary = assessor.assess(ctx(report = report))
assertEquals(RiskLevel.MEDIUM, summary.level)
assertEquals(RiskLevel.HIGH, summary.level)
assertEquals(RiskAction.PROMPT_USER, summary.recommendedAction)
val signal = summary.signals.filterIsInstance<RiskSignal.ValidationErrors>().single()
assertEquals(2, signal.errorCount)
}
@Test
fun `validation warnings produce MEDIUM risk when no errors present`() {
val report = ValidationReport(
sections = listOf(
ValidationSection(
name = "schema",
issues = listOf(
ValidationIssue(code = "MISSING_CONTEXT", message = "context incomplete",
severity = ValidationSeverity.WARNING),
),
),
),
)
val summary = assessor.assess(ctx(report = report))
assertEquals(RiskLevel.MEDIUM, summary.level)
assertEquals(RiskAction.PROMPT_USER, summary.recommendedAction)
}
@Test
fun `cycle issue in validation report produces CycleWithoutExit signal at MEDIUM`() {
val report = ValidationReport(
@@ -172,6 +192,38 @@ class DefaultRiskAssessorTest {
assertTrue(summary.signals.filterIsInstance<RiskSignal.InferenceTimeout>().isEmpty())
}
@Test
fun `rationale contains formatted issue codes and messages`() {
val report = ValidationReport(
sections = listOf(
ValidationSection(
name = "schema",
issues = listOf(
ValidationIssue(code = "MISSING_FIELD", message = "field required",
severity = ValidationSeverity.ERROR),
ValidationIssue(code = "INVALID_TYPE", message = "type mismatch",
severity = ValidationSeverity.WARNING),
),
),
),
)
val summary = assessor.assess(ctx(report = report))
assertTrue(summary.rationale.contains("[MISSING_FIELD] field required"))
assertTrue(summary.rationale.contains("[INVALID_TYPE] type mismatch"))
assertEquals(2, summary.rationale.size)
}
@Test
fun `empty validation report yields empty rationale`() {
val report = ValidationReport(sections = emptyList())
val summary = assessor.assess(ctx(report = report))
assertTrue(summary.rationale.isEmpty())
}
@Test
fun `multiple signals fold to highest risk level`() {
val report = ValidationReport(