feat: add MissingToolRule semantic validation
Flags when a workflow stage's allowedTools references a tool name that is not registered in the tool registry (config drift), surfaced as a non-blocking WARNING. - Add ValidationContext.availableTools (nullable; null = registry unavailable, skip the check) - MissingToolRule emits one MISSING_TOOL warning per (stage, tool) pair, deterministically ordered - Populate availableTools from the ToolRegistry at the orchestrator construction site; wire the rule into the prod SemanticValidator
This commit is contained in:
@@ -37,6 +37,7 @@ import com.correx.core.validation.artifact.ArtifactPayloadValidator
|
|||||||
import com.correx.core.validation.pipeline.ValidationPipeline
|
import com.correx.core.validation.pipeline.ValidationPipeline
|
||||||
import com.correx.core.validation.semantic.SemanticValidator
|
import com.correx.core.validation.semantic.SemanticValidator
|
||||||
import com.correx.core.validation.semantic.rules.CycleExitRule
|
import com.correx.core.validation.semantic.rules.CycleExitRule
|
||||||
|
import com.correx.core.validation.semantic.rules.MissingToolRule
|
||||||
import com.correx.infrastructure.InfrastructureModule
|
import com.correx.infrastructure.InfrastructureModule
|
||||||
import com.correx.infrastructure.artifactscas.DefaultMaterializingArtifactWriter
|
import com.correx.infrastructure.artifactscas.DefaultMaterializingArtifactWriter
|
||||||
import com.correx.infrastructure.inference.DefaultProviderRegistry
|
import com.correx.infrastructure.inference.DefaultProviderRegistry
|
||||||
@@ -114,6 +115,7 @@ fun main() {
|
|||||||
SemanticValidator(
|
SemanticValidator(
|
||||||
rules = listOf(
|
rules = listOf(
|
||||||
CycleExitRule(requirePolicyForCycles = false),
|
CycleExitRule(requirePolicyForCycles = false),
|
||||||
|
MissingToolRule(),
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
|||||||
+5
-1
@@ -304,7 +304,11 @@ abstract class SessionOrchestrator(
|
|||||||
is InferenceResult.Failed -> StageExecutionResult.Failure(inferenceResult.reason, retryable = true)
|
is InferenceResult.Failed -> StageExecutionResult.Failure(inferenceResult.reason, retryable = true)
|
||||||
is InferenceResult.Success -> {
|
is InferenceResult.Success -> {
|
||||||
if (isCancelled(sessionId)) return StageExecutionResult.Failure("CANCELLED", retryable = false)
|
if (isCancelled(sessionId)) return StageExecutionResult.Failure("CANCELLED", retryable = false)
|
||||||
val validationCtx = ValidationContext(graph = graph, sessionState = session.state)
|
val validationCtx = ValidationContext(
|
||||||
|
graph = graph,
|
||||||
|
sessionState = session.state,
|
||||||
|
availableTools = toolRegistry?.all()?.map { it.name }?.toSet()
|
||||||
|
)
|
||||||
when (val outcome = mapValidationOutcome(sessionId, stageId, validationCtx)) {
|
when (val outcome = mapValidationOutcome(sessionId, stageId, validationCtx)) {
|
||||||
is StageExecutionResult.Success -> {
|
is StageExecutionResult.Success -> {
|
||||||
emitToolArtifacts(sessionId, stageId, stageConfig)
|
emitToolArtifacts(sessionId, stageId, stageConfig)
|
||||||
|
|||||||
@@ -10,4 +10,5 @@ data class ValidationContext(
|
|||||||
val detectedCycles: List<DetectedCycle> = emptyList(),
|
val detectedCycles: List<DetectedCycle> = emptyList(),
|
||||||
val cyclePolicies: Set<CyclePolicyBinding> = emptySet(),
|
val cyclePolicies: Set<CyclePolicyBinding> = emptySet(),
|
||||||
val sessionState: SessionState? = null,
|
val sessionState: SessionState? = null,
|
||||||
|
val availableTools: Set<String>? = null,
|
||||||
)
|
)
|
||||||
|
|||||||
+34
@@ -0,0 +1,34 @@
|
|||||||
|
package com.correx.core.validation.semantic.rules
|
||||||
|
|
||||||
|
import com.correx.core.validation.model.ValidationContext
|
||||||
|
import com.correx.core.validation.model.ValidationIssue
|
||||||
|
import com.correx.core.validation.model.ValidationLocation
|
||||||
|
import com.correx.core.validation.model.ValidationSeverity
|
||||||
|
import com.correx.core.validation.semantic.SemanticRule
|
||||||
|
|
||||||
|
class MissingToolRule : SemanticRule {
|
||||||
|
|
||||||
|
override fun validate(context: ValidationContext): List<ValidationIssue> {
|
||||||
|
val available = context.availableTools ?: return emptyList()
|
||||||
|
|
||||||
|
val issues = mutableListOf<ValidationIssue>()
|
||||||
|
|
||||||
|
for (stageId in context.graph.stageIds.sortedBy { it.value }) {
|
||||||
|
val stageConfig = context.graph.stages[stageId] ?: continue
|
||||||
|
for (toolName in stageConfig.allowedTools.sorted()) {
|
||||||
|
if (toolName !in available) {
|
||||||
|
issues.add(
|
||||||
|
ValidationIssue(
|
||||||
|
code = "MISSING_TOOL",
|
||||||
|
message = "Stage '${stageId.value}' allows unregistered tool '$toolName'",
|
||||||
|
severity = ValidationSeverity.WARNING,
|
||||||
|
location = ValidationLocation.Graph(stageId = stageId)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return issues
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -11,12 +11,14 @@ import com.correx.core.validation.pipeline.ValidationOutcome
|
|||||||
import com.correx.core.validation.pipeline.ValidationPipeline
|
import com.correx.core.validation.pipeline.ValidationPipeline
|
||||||
import com.correx.core.validation.semantic.SemanticValidator
|
import com.correx.core.validation.semantic.SemanticValidator
|
||||||
import com.correx.core.validation.semantic.rules.CycleExitRule
|
import com.correx.core.validation.semantic.rules.CycleExitRule
|
||||||
|
import com.correx.core.validation.semantic.rules.MissingToolRule
|
||||||
import com.correx.core.validation.transition.TransitionValidator
|
import com.correx.core.validation.transition.TransitionValidator
|
||||||
import com.correx.testing.fixtures.CycleFixtures
|
import com.correx.testing.fixtures.CycleFixtures
|
||||||
import com.correx.testing.fixtures.WorkflowFixtures
|
import com.correx.testing.fixtures.WorkflowFixtures
|
||||||
import kotlinx.coroutines.runBlocking
|
import kotlinx.coroutines.runBlocking
|
||||||
import org.junit.jupiter.api.Assertions.assertFalse
|
import org.junit.jupiter.api.Assertions.assertFalse
|
||||||
import org.junit.jupiter.api.Assertions.assertInstanceOf
|
import org.junit.jupiter.api.Assertions.assertInstanceOf
|
||||||
|
import org.junit.jupiter.api.Assertions.assertEquals
|
||||||
import org.junit.jupiter.api.Test
|
import org.junit.jupiter.api.Test
|
||||||
|
|
||||||
class ValidationPipelineIntegrationTest {
|
class ValidationPipelineIntegrationTest {
|
||||||
@@ -101,4 +103,105 @@ class ValidationPipelineIntegrationTest {
|
|||||||
// so the outcome should be Passed (not NeedsApproval).
|
// so the outcome should be Passed (not NeedsApproval).
|
||||||
assertInstanceOf(ValidationOutcome.Passed::class.java, outcome)
|
assertInstanceOf(ValidationOutcome.Passed::class.java, outcome)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `MissingToolRule detects unregistered tool in stage`(): Unit = runBlocking {
|
||||||
|
// A stage allows tools "bash" and "ghost", but only "bash" is registered.
|
||||||
|
// The rule should emit exactly one MISSING_TOOL WARNING for "ghost".
|
||||||
|
val stageA = StageId("A")
|
||||||
|
val graph = WorkflowGraph(
|
||||||
|
id = "workflow-test",
|
||||||
|
stages = mapOf(stageA to StageConfig(allowedTools = setOf("bash", "ghost"))),
|
||||||
|
transitions = emptySet(),
|
||||||
|
start = stageA,
|
||||||
|
)
|
||||||
|
val validator = SemanticValidator(rules = listOf(MissingToolRule()))
|
||||||
|
val context = ValidationContext(
|
||||||
|
graph = graph,
|
||||||
|
availableTools = setOf("bash"),
|
||||||
|
)
|
||||||
|
|
||||||
|
val section = validator.validate(context)
|
||||||
|
|
||||||
|
assertEquals(1, section.issues.size)
|
||||||
|
val issue = section.issues.first()
|
||||||
|
assertEquals("MISSING_TOOL", issue.code)
|
||||||
|
assertEquals("Stage 'A' allows unregistered tool 'ghost'", issue.message)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `MissingToolRule returns empty when all allowed tools are registered`(): Unit = runBlocking {
|
||||||
|
val stageA = StageId("A")
|
||||||
|
val graph = WorkflowGraph(
|
||||||
|
id = "workflow-test",
|
||||||
|
stages = mapOf(stageA to StageConfig(allowedTools = setOf("bash", "python"))),
|
||||||
|
transitions = emptySet(),
|
||||||
|
start = stageA,
|
||||||
|
)
|
||||||
|
val validator = SemanticValidator(rules = listOf(MissingToolRule()))
|
||||||
|
val context = ValidationContext(
|
||||||
|
graph = graph,
|
||||||
|
availableTools = setOf("bash", "python"),
|
||||||
|
)
|
||||||
|
|
||||||
|
val section = validator.validate(context)
|
||||||
|
|
||||||
|
assertEquals(0, section.issues.size)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `MissingToolRule returns empty when availableTools is null`(): Unit = runBlocking {
|
||||||
|
// When tool registry is not available (null), the rule should skip validation.
|
||||||
|
val stageA = StageId("A")
|
||||||
|
val graph = WorkflowGraph(
|
||||||
|
id = "workflow-test",
|
||||||
|
stages = mapOf(stageA to StageConfig(allowedTools = setOf("ghost"))),
|
||||||
|
transitions = emptySet(),
|
||||||
|
start = stageA,
|
||||||
|
)
|
||||||
|
val validator = SemanticValidator(rules = listOf(MissingToolRule()))
|
||||||
|
val context = ValidationContext(
|
||||||
|
graph = graph,
|
||||||
|
availableTools = null,
|
||||||
|
)
|
||||||
|
|
||||||
|
val section = validator.validate(context)
|
||||||
|
|
||||||
|
assertEquals(0, section.issues.size)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun `MissingToolRule detects missing tools across multiple stages`(): Unit = runBlocking {
|
||||||
|
val stageA = StageId("A")
|
||||||
|
val stageB = StageId("B")
|
||||||
|
val graph = WorkflowGraph(
|
||||||
|
id = "workflow-test",
|
||||||
|
stages = mapOf(
|
||||||
|
stageA to StageConfig(allowedTools = setOf("bash", "ghost")),
|
||||||
|
stageB to StageConfig(allowedTools = setOf("phantom", "bash")),
|
||||||
|
),
|
||||||
|
transitions = setOf(TransitionEdge(TransitionId("t1"), from = stageA, to = stageB) { true }),
|
||||||
|
start = stageA,
|
||||||
|
)
|
||||||
|
val validator = SemanticValidator(rules = listOf(MissingToolRule()))
|
||||||
|
val context = ValidationContext(
|
||||||
|
graph = graph,
|
||||||
|
availableTools = setOf("bash"),
|
||||||
|
)
|
||||||
|
|
||||||
|
val section = validator.validate(context)
|
||||||
|
|
||||||
|
// Should emit two issues: one for "ghost" in stage A, one for "phantom" in stage B.
|
||||||
|
assertEquals(2, section.issues.size)
|
||||||
|
val codes = section.issues.map { it.code }
|
||||||
|
assertEquals(listOf("MISSING_TOOL", "MISSING_TOOL"), codes)
|
||||||
|
val messages = section.issues.map { it.message }
|
||||||
|
assertEquals(
|
||||||
|
setOf(
|
||||||
|
"Stage 'A' allows unregistered tool 'ghost'",
|
||||||
|
"Stage 'B' allows unregistered tool 'phantom'"
|
||||||
|
),
|
||||||
|
messages.toSet()
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user