feat: carry validation rationale through risk assessment
Risk tier is now derived from the worst validation issue severity and the issues that drove it travel with the summary, so consumers can show operators why a tier was assigned instead of an opaque level. - Add RiskSummary.rationale (defaulted, backward-compatible) holding human-readable '[code] message' lines from the validation report - Pin severity -> risk mapping: ERROR->HIGH, WARNING->MEDIUM, INFO->LOW - DefaultRiskAssessor folds the validation-driven level into the summary
This commit is contained in:
@@ -4,4 +4,5 @@ data class RiskSummary(
|
|||||||
val level: RiskLevel,
|
val level: RiskLevel,
|
||||||
val signals: List<RiskSignal>,
|
val signals: List<RiskSignal>,
|
||||||
val recommendedAction: RiskAction,
|
val recommendedAction: RiskAction,
|
||||||
|
val rationale: List<String> = emptyList(),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -11,15 +11,14 @@ class DefaultRiskAssessor : RiskAssessor {
|
|||||||
val (report, state, inferenceState) = context
|
val (report, state, inferenceState) = context
|
||||||
val signals = mutableListOf<RiskSignal>()
|
val signals = mutableListOf<RiskSignal>()
|
||||||
|
|
||||||
val errorCount = report.sections.sumOf { s ->
|
val allIssues = report.sections.flatMap { it.issues }
|
||||||
s.issues.count { it.severity == ValidationSeverity.ERROR }
|
|
||||||
}
|
val errorCount = allIssues.count { it.severity == ValidationSeverity.ERROR }
|
||||||
if (errorCount > 0) {
|
if (errorCount > 0) {
|
||||||
signals += RiskSignal.ValidationErrors(errorCount)
|
signals += RiskSignal.ValidationErrors(errorCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
report.sections.flatMap { it.issues }
|
allIssues.firstOrNull { it.code.contains("CYCLE", ignoreCase = true) }
|
||||||
.firstOrNull { it.code.contains("CYCLE", ignoreCase = true) }
|
|
||||||
?.let { signals += RiskSignal.CycleWithoutExit(it.code) }
|
?.let { signals += RiskSignal.CycleWithoutExit(it.code) }
|
||||||
|
|
||||||
state.retryPolicy?.let { policy ->
|
state.retryPolicy?.let { policy ->
|
||||||
@@ -37,12 +36,23 @@ class DefaultRiskAssessor : RiskAssessor {
|
|||||||
?.let { signals += RiskSignal.InferenceTimeout(it.latencyMs ?: 0L) }
|
?.let { signals += RiskSignal.InferenceTimeout(it.latencyMs ?: 0L) }
|
||||||
}
|
}
|
||||||
|
|
||||||
val level = signals.fold(RiskLevel.LOW) { acc, signal -> maxOf(acc, signal.toRiskLevel()) }
|
val validationDrivenLevel = when {
|
||||||
|
allIssues.any { it.severity == ValidationSeverity.ERROR } -> RiskLevel.HIGH
|
||||||
|
allIssues.any { it.severity == ValidationSeverity.WARNING } -> RiskLevel.MEDIUM
|
||||||
|
allIssues.any { it.severity == ValidationSeverity.INFO } -> RiskLevel.LOW
|
||||||
|
else -> RiskLevel.LOW
|
||||||
|
}
|
||||||
|
|
||||||
|
var level = signals.fold(RiskLevel.LOW) { acc, signal -> maxOf(acc, signal.toRiskLevel()) }
|
||||||
|
level = maxOf(level, validationDrivenLevel)
|
||||||
|
|
||||||
|
val rationale = allIssues.map { "[${it.code}] ${it.message}" }
|
||||||
|
|
||||||
return RiskSummary(
|
return RiskSummary(
|
||||||
level = level,
|
level = level,
|
||||||
signals = signals,
|
signals = signals,
|
||||||
recommendedAction = level.toRiskAction(),
|
recommendedAction = level.toRiskAction(),
|
||||||
|
rationale = rationale,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,5 +14,6 @@ class NoOpRiskAssessor : RiskAssessor {
|
|||||||
level = RiskLevel.LOW,
|
level = RiskLevel.LOW,
|
||||||
signals = emptyList(),
|
signals = emptyList(),
|
||||||
recommendedAction = RiskAction.PROCEED,
|
recommendedAction = RiskAction.PROCEED,
|
||||||
|
rationale = emptyList(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ internal fun RiskLevel.toRiskAction(): RiskAction = when (this) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
internal fun RiskSignal.toRiskLevel(): RiskLevel = when (this) {
|
internal fun RiskSignal.toRiskLevel(): RiskLevel = when (this) {
|
||||||
is RiskSignal.ValidationErrors -> RiskLevel.MEDIUM
|
is RiskSignal.ValidationErrors -> RiskLevel.HIGH
|
||||||
is RiskSignal.CycleWithoutExit -> RiskLevel.MEDIUM
|
is RiskSignal.CycleWithoutExit -> RiskLevel.MEDIUM
|
||||||
is RiskSignal.InferenceTimeout -> RiskLevel.MEDIUM
|
is RiskSignal.InferenceTimeout -> RiskLevel.MEDIUM
|
||||||
is RiskSignal.RepeatedFailure -> RiskLevel.HIGH
|
is RiskSignal.RepeatedFailure -> RiskLevel.HIGH
|
||||||
|
|||||||
@@ -51,7 +51,7 @@ class DefaultRiskAssessorTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun `validation errors produce ValidationErrors signal at MEDIUM`() {
|
fun `validation errors produce ValidationErrors signal at HIGH`() {
|
||||||
val report = ValidationReport(
|
val report = ValidationReport(
|
||||||
sections = listOf(
|
sections = listOf(
|
||||||
ValidationSection(
|
ValidationSection(
|
||||||
@@ -68,12 +68,32 @@ class DefaultRiskAssessorTest {
|
|||||||
|
|
||||||
val summary = assessor.assess(ctx(report = report))
|
val summary = assessor.assess(ctx(report = report))
|
||||||
|
|
||||||
assertEquals(RiskLevel.MEDIUM, summary.level)
|
assertEquals(RiskLevel.HIGH, summary.level)
|
||||||
assertEquals(RiskAction.PROMPT_USER, summary.recommendedAction)
|
assertEquals(RiskAction.PROMPT_USER, summary.recommendedAction)
|
||||||
val signal = summary.signals.filterIsInstance<RiskSignal.ValidationErrors>().single()
|
val signal = summary.signals.filterIsInstance<RiskSignal.ValidationErrors>().single()
|
||||||
assertEquals(2, signal.errorCount)
|
assertEquals(2, signal.errorCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `validation warnings produce MEDIUM risk when no errors present`() {
|
||||||
|
val report = ValidationReport(
|
||||||
|
sections = listOf(
|
||||||
|
ValidationSection(
|
||||||
|
name = "schema",
|
||||||
|
issues = listOf(
|
||||||
|
ValidationIssue(code = "MISSING_CONTEXT", message = "context incomplete",
|
||||||
|
severity = ValidationSeverity.WARNING),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
val summary = assessor.assess(ctx(report = report))
|
||||||
|
|
||||||
|
assertEquals(RiskLevel.MEDIUM, summary.level)
|
||||||
|
assertEquals(RiskAction.PROMPT_USER, summary.recommendedAction)
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun `cycle issue in validation report produces CycleWithoutExit signal at MEDIUM`() {
|
fun `cycle issue in validation report produces CycleWithoutExit signal at MEDIUM`() {
|
||||||
val report = ValidationReport(
|
val report = ValidationReport(
|
||||||
@@ -172,6 +192,38 @@ class DefaultRiskAssessorTest {
|
|||||||
assertTrue(summary.signals.filterIsInstance<RiskSignal.InferenceTimeout>().isEmpty())
|
assertTrue(summary.signals.filterIsInstance<RiskSignal.InferenceTimeout>().isEmpty())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `rationale contains formatted issue codes and messages`() {
|
||||||
|
val report = ValidationReport(
|
||||||
|
sections = listOf(
|
||||||
|
ValidationSection(
|
||||||
|
name = "schema",
|
||||||
|
issues = listOf(
|
||||||
|
ValidationIssue(code = "MISSING_FIELD", message = "field required",
|
||||||
|
severity = ValidationSeverity.ERROR),
|
||||||
|
ValidationIssue(code = "INVALID_TYPE", message = "type mismatch",
|
||||||
|
severity = ValidationSeverity.WARNING),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
val summary = assessor.assess(ctx(report = report))
|
||||||
|
|
||||||
|
assertTrue(summary.rationale.contains("[MISSING_FIELD] field required"))
|
||||||
|
assertTrue(summary.rationale.contains("[INVALID_TYPE] type mismatch"))
|
||||||
|
assertEquals(2, summary.rationale.size)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `empty validation report yields empty rationale`() {
|
||||||
|
val report = ValidationReport(sections = emptyList())
|
||||||
|
|
||||||
|
val summary = assessor.assess(ctx(report = report))
|
||||||
|
|
||||||
|
assertTrue(summary.rationale.isEmpty())
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
fun `multiple signals fold to highest risk level`() {
|
fun `multiple signals fold to highest risk level`() {
|
||||||
val report = ValidationReport(
|
val report = ValidationReport(
|
||||||
|
|||||||
Reference in New Issue
Block a user