feat(kernel): emit_artifact tool + actionable validation feedback
Two robustness fixes for LLM-emitted artifact stages (e.g. freestyle analyst/ architect) that were exhausting their retry budget on weak models: 1. emit_artifact meta-tool: stages with an LLM-emitted slot now expose an emit_artifact tool whose parameters ARE the artifact's JSON schema, giving the model a structured tool-call channel to produce the artifact instead of a free-form final message. Sidesteps llama.cpp's grammar+tools incompatibility (the artifact can't be grammar-constrained while tools are present). The call's arguments are captured as the artifact content. 2. Actionable retry feedback: validation rejection now surfaces the concrete error messages (e.g. 'required property summary missing') as the stage failure reason instead of a bare 'validation failed', so the retry-feedback entry tells the model what to fix.
This commit is contained in:
+67
-8
@@ -129,6 +129,8 @@ import com.correx.core.transitions.graph.WorkflowGraph
|
||||
import com.correx.core.transitions.resolution.TransitionDecision
|
||||
import com.correx.core.transitions.resolution.TransitionResolver
|
||||
import com.correx.core.validation.model.ValidationContext
|
||||
import com.correx.core.validation.model.ValidationReport
|
||||
import com.correx.core.validation.model.ValidationSeverity
|
||||
import com.correx.core.validation.pipeline.ValidationOutcome
|
||||
import com.correx.core.validation.pipeline.ValidationPipeline
|
||||
import kotlinx.coroutines.CompletableDeferred
|
||||
@@ -142,6 +144,7 @@ import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonArray
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
import kotlinx.serialization.json.jsonObject
|
||||
import com.correx.core.journal.DecisionJournalRenderer
|
||||
import com.correx.core.journal.DefaultDecisionJournalRepository
|
||||
import com.correx.core.kernel.orchestration.subagent.SubagentRunner
|
||||
@@ -154,7 +157,9 @@ import java.util.concurrent.atomic.*
|
||||
import kotlin.coroutines.cancellation.CancellationException
|
||||
|
||||
private const val MAX_TOOL_ROUNDS = 10
|
||||
private const val MAX_FEEDBACK_ISSUES = 3
|
||||
private const val STAGE_COMPLETE_TOOL = "stage_complete"
|
||||
private const val EMIT_ARTIFACT_TOOL = "emit_artifact"
|
||||
private const val SCOPE_PROPOSAL_TOOL = "propose_scope"
|
||||
private const val READ_BEFORE_WRITE_CODE = "READ_BEFORE_WRITE"
|
||||
private const val OUTPUT_SUMMARY_LIMIT = 500
|
||||
@@ -465,6 +470,9 @@ abstract class SessionOrchestrator(
|
||||
sessionId, stageId, currentContext, stageConfig, config.stageTimeoutMs, responseFormat, effectives,
|
||||
)
|
||||
var toolRounds = 0
|
||||
// Set when the model produces its artifact via the emit_artifact tool instead of a final
|
||||
// JSON message; overrides the post-loop capture of the (then-empty) assistant text.
|
||||
var llmArtifactOverride: String? = null
|
||||
|
||||
val fileWrittenSlots = stageConfig.produces.filter { it.kind.id == "file_written" }
|
||||
fun owesFileWrite(): Boolean = fileWrittenSlots.isNotEmpty() &&
|
||||
@@ -533,6 +541,14 @@ abstract class SessionOrchestrator(
|
||||
inferenceResult = pushBack(writeNudge)
|
||||
continue
|
||||
}
|
||||
// emit_artifact is a meta-tool: the LLM calls it with the artifact's fields as arguments.
|
||||
// Capture those arguments as the artifact content and exit the loop straight to validation
|
||||
// (the post-loop capture honours this override instead of the empty assistant text).
|
||||
val emitCall = inferenceResult.response.toolCalls.firstOrNull { it.function.name == EMIT_ARTIFACT_TOOL }
|
||||
if (emitCall != null && llmEmittedSlots.isNotEmpty()) {
|
||||
llmArtifactOverride = emitCall.function.arguments
|
||||
break
|
||||
}
|
||||
// stage_complete is a meta-tool: the LLM calls it to signal the stage's goal is met.
|
||||
// Skip tool dispatch — the loop exits and normal validation/transition follows
|
||||
// (emitToolArtifacts + verifyProduces run on the success path after the loop).
|
||||
@@ -547,8 +563,9 @@ abstract class SessionOrchestrator(
|
||||
when {
|
||||
owedLlm != null -> inferenceResult = pushBack(
|
||||
"ERROR: stage_complete is not allowed yet — you have not emitted the " +
|
||||
"required '${owedLlm.name.value}' artifact. Respond with the artifact as " +
|
||||
"a single JSON message (no tool call) matching the provided schema.",
|
||||
"required '${owedLlm.name.value}' artifact. Call the emit_artifact tool with " +
|
||||
"the fields filled in (or respond with the artifact as a single JSON message " +
|
||||
"matching the provided schema).",
|
||||
)
|
||||
owesFileWrite() -> inferenceResult = pushBack(writeNudge)
|
||||
owesTaskDecompose() -> inferenceResult = pushBack(
|
||||
@@ -596,11 +613,16 @@ abstract class SessionOrchestrator(
|
||||
is InferenceResult.Success -> {
|
||||
if (isCancelled(sessionId)) return StageExecutionResult.Failure("CANCELLED", retryable = false)
|
||||
// Capture the LLM-emitted artifact's JSON so transition conditions
|
||||
// (e.g. ArtifactFieldEquals on a reviewer's verdict) can read it.
|
||||
// (e.g. ArtifactFieldEquals on a reviewer's verdict) can read it. An emit_artifact
|
||||
// tool call (llmArtifactOverride) supplies the content directly; otherwise the
|
||||
// artifact is the final assistant message text.
|
||||
val artifactText = llmArtifactOverride ?: inferenceResult.response.text
|
||||
val artifactHash = llmArtifactOverride
|
||||
?.let { artifactStore.put(it.toByteArray()) }
|
||||
?: inferenceResult.responseArtifactId
|
||||
llmEmittedSlots.firstOrNull()?.let { slot ->
|
||||
artifactContentCache["${sessionId.value}:${slot.name.value}"] =
|
||||
inferenceResult.response.text
|
||||
inferenceResult.responseArtifactId?.let { hash ->
|
||||
artifactContentCache["${sessionId.value}:${slot.name.value}"] = artifactText
|
||||
artifactHash?.let { hash ->
|
||||
emit(
|
||||
sessionId,
|
||||
ArtifactContentStoredEvent(
|
||||
@@ -1780,11 +1802,48 @@ abstract class SessionOrchestrator(
|
||||
): StageExecutionResult = when (val outcome = validationPipeline.validate(context)) {
|
||||
is ValidationOutcome.Passed -> StageExecutionResult.Success(emptyList())
|
||||
is ValidationOutcome.Rejected ->
|
||||
StageExecutionResult.Failure("validation failed", retryable = outcome.retryable)
|
||||
StageExecutionResult.Failure(summarizeRejection(outcome.report), retryable = outcome.retryable)
|
||||
|
||||
is ValidationOutcome.NeedsApproval -> handleApproval(sessionId, stageId, outcome)
|
||||
}
|
||||
|
||||
// emit_artifact gives the model a structured channel to produce its required artifact: it calls
|
||||
// the tool with the fields filled in (parameter schema = the artifact's own schema) instead of
|
||||
// emitting free-form JSON as a final message. Tool-calling models handle this far more reliably,
|
||||
// and it sidesteps llama.cpp's grammar+tools incompatibility (the artifact can't be grammar-
|
||||
// constrained while tools are present). Empty when the stage has no LLM-emitted slot.
|
||||
private fun emitArtifactTool(stageConfig: StageConfig): List<ToolDefinition> {
|
||||
val slot = stageConfig.produces.firstOrNull { it.kind.llmEmitted } ?: return emptyList()
|
||||
val schema = Json.encodeToJsonElement(JsonSchema.serializer(), slot.kind.deriveJsonSchema()).jsonObject
|
||||
return listOf(
|
||||
ToolDefinition(
|
||||
function = ToolFunction(
|
||||
name = EMIT_ARTIFACT_TOOL,
|
||||
description = "Emit the required '${slot.name.value}' artifact by filling in the fields " +
|
||||
"below. Prefer this over writing the JSON as a plain message.",
|
||||
parameters = schema,
|
||||
),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
// Collapse a validation report into a concise, actionable reason. This string becomes the
|
||||
// RetryAttemptedEvent.failureReason that the retry-feedback entry feeds back to the model, so a
|
||||
// weak model is told *what* was wrong (e.g. "required property 'summary' missing") instead of a
|
||||
// bare "validation failed" it can't act on. Capped to keep the feedback prompt small.
|
||||
private fun summarizeRejection(report: ValidationReport): String {
|
||||
val errors = report.sections.flatMap { section ->
|
||||
section.issues
|
||||
.filter { it.severity == ValidationSeverity.ERROR }
|
||||
.map { it.message }
|
||||
}.distinct()
|
||||
return if (errors.isEmpty()) {
|
||||
"validation failed"
|
||||
} else {
|
||||
"validation failed: " + errors.take(MAX_FEEDBACK_ISSUES).joinToString("; ")
|
||||
}
|
||||
}
|
||||
|
||||
@Suppress("LongParameterList")
|
||||
internal open suspend fun runInference(
|
||||
sessionId: SessionId,
|
||||
@@ -1834,7 +1893,7 @@ abstract class SessionOrchestrator(
|
||||
"The orchestrator will proceed to the next stage.",
|
||||
parameters = JsonObject(emptyMap()),
|
||||
),
|
||||
),
|
||||
) + emitArtifactTool(stageConfig),
|
||||
)
|
||||
|
||||
val renderedMessages = PromptRenderer.render(contextPack)
|
||||
|
||||
Reference in New Issue
Block a user