feat(kernel): emit_artifact tool + actionable validation feedback

Two robustness fixes for LLM-emitted artifact stages (e.g. freestyle analyst/
architect) that were exhausting their retry budget on weak models:

1. emit_artifact meta-tool: stages with an LLM-emitted slot now expose an
   emit_artifact tool whose parameters ARE the artifact's JSON schema, giving
   the model a structured tool-call channel to produce the artifact instead of
   a free-form final message. Sidesteps llama.cpp's grammar+tools
   incompatibility (the artifact can't be grammar-constrained while tools are
   present). The call's arguments are captured as the artifact content.

2. Actionable retry feedback: validation rejection now surfaces the concrete
   error messages (e.g. 'required property summary missing') as the stage
   failure reason instead of a bare 'validation failed', so the retry-feedback
   entry tells the model what to fix.
This commit is contained in:
2026-06-30 20:53:40 +04:00
parent 0652d87aca
commit e72051a9ea
@@ -129,6 +129,8 @@ import com.correx.core.transitions.graph.WorkflowGraph
import com.correx.core.transitions.resolution.TransitionDecision
import com.correx.core.transitions.resolution.TransitionResolver
import com.correx.core.validation.model.ValidationContext
import com.correx.core.validation.model.ValidationReport
import com.correx.core.validation.model.ValidationSeverity
import com.correx.core.validation.pipeline.ValidationOutcome
import com.correx.core.validation.pipeline.ValidationPipeline
import kotlinx.coroutines.CompletableDeferred
@@ -142,6 +144,7 @@ import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.jsonObject
import com.correx.core.journal.DecisionJournalRenderer
import com.correx.core.journal.DefaultDecisionJournalRepository
import com.correx.core.kernel.orchestration.subagent.SubagentRunner
@@ -154,7 +157,9 @@ import java.util.concurrent.atomic.*
import kotlin.coroutines.cancellation.CancellationException
private const val MAX_TOOL_ROUNDS = 10
private const val MAX_FEEDBACK_ISSUES = 3
private const val STAGE_COMPLETE_TOOL = "stage_complete"
private const val EMIT_ARTIFACT_TOOL = "emit_artifact"
private const val SCOPE_PROPOSAL_TOOL = "propose_scope"
private const val READ_BEFORE_WRITE_CODE = "READ_BEFORE_WRITE"
private const val OUTPUT_SUMMARY_LIMIT = 500
@@ -465,6 +470,9 @@ abstract class SessionOrchestrator(
sessionId, stageId, currentContext, stageConfig, config.stageTimeoutMs, responseFormat, effectives,
)
var toolRounds = 0
// Set when the model produces its artifact via the emit_artifact tool instead of a final
// JSON message; overrides the post-loop capture of the (then-empty) assistant text.
var llmArtifactOverride: String? = null
val fileWrittenSlots = stageConfig.produces.filter { it.kind.id == "file_written" }
fun owesFileWrite(): Boolean = fileWrittenSlots.isNotEmpty() &&
@@ -533,6 +541,14 @@ abstract class SessionOrchestrator(
inferenceResult = pushBack(writeNudge)
continue
}
// emit_artifact is a meta-tool: the LLM calls it with the artifact's fields as arguments.
// Capture those arguments as the artifact content and exit the loop straight to validation
// (the post-loop capture honours this override instead of the empty assistant text).
val emitCall = inferenceResult.response.toolCalls.firstOrNull { it.function.name == EMIT_ARTIFACT_TOOL }
if (emitCall != null && llmEmittedSlots.isNotEmpty()) {
llmArtifactOverride = emitCall.function.arguments
break
}
// stage_complete is a meta-tool: the LLM calls it to signal the stage's goal is met.
// Skip tool dispatch — the loop exits and normal validation/transition follows
// (emitToolArtifacts + verifyProduces run on the success path after the loop).
@@ -547,8 +563,9 @@ abstract class SessionOrchestrator(
when {
owedLlm != null -> inferenceResult = pushBack(
"ERROR: stage_complete is not allowed yet — you have not emitted the " +
"required '${owedLlm.name.value}' artifact. Respond with the artifact as " +
"a single JSON message (no tool call) matching the provided schema.",
"required '${owedLlm.name.value}' artifact. Call the emit_artifact tool with " +
"the fields filled in (or respond with the artifact as a single JSON message " +
"matching the provided schema).",
)
owesFileWrite() -> inferenceResult = pushBack(writeNudge)
owesTaskDecompose() -> inferenceResult = pushBack(
@@ -596,11 +613,16 @@ abstract class SessionOrchestrator(
is InferenceResult.Success -> {
if (isCancelled(sessionId)) return StageExecutionResult.Failure("CANCELLED", retryable = false)
// Capture the LLM-emitted artifact's JSON so transition conditions
// (e.g. ArtifactFieldEquals on a reviewer's verdict) can read it.
// (e.g. ArtifactFieldEquals on a reviewer's verdict) can read it. An emit_artifact
// tool call (llmArtifactOverride) supplies the content directly; otherwise the
// artifact is the final assistant message text.
val artifactText = llmArtifactOverride ?: inferenceResult.response.text
val artifactHash = llmArtifactOverride
?.let { artifactStore.put(it.toByteArray()) }
?: inferenceResult.responseArtifactId
llmEmittedSlots.firstOrNull()?.let { slot ->
artifactContentCache["${sessionId.value}:${slot.name.value}"] =
inferenceResult.response.text
inferenceResult.responseArtifactId?.let { hash ->
artifactContentCache["${sessionId.value}:${slot.name.value}"] = artifactText
artifactHash?.let { hash ->
emit(
sessionId,
ArtifactContentStoredEvent(
@@ -1780,11 +1802,48 @@ abstract class SessionOrchestrator(
): StageExecutionResult = when (val outcome = validationPipeline.validate(context)) {
is ValidationOutcome.Passed -> StageExecutionResult.Success(emptyList())
is ValidationOutcome.Rejected ->
StageExecutionResult.Failure("validation failed", retryable = outcome.retryable)
StageExecutionResult.Failure(summarizeRejection(outcome.report), retryable = outcome.retryable)
is ValidationOutcome.NeedsApproval -> handleApproval(sessionId, stageId, outcome)
}
// emit_artifact gives the model a structured channel to produce its required artifact: it calls
// the tool with the fields filled in (parameter schema = the artifact's own schema) instead of
// emitting free-form JSON as a final message. Tool-calling models handle this far more reliably,
// and it sidesteps llama.cpp's grammar+tools incompatibility (the artifact can't be grammar-
// constrained while tools are present). Empty when the stage has no LLM-emitted slot.
private fun emitArtifactTool(stageConfig: StageConfig): List<ToolDefinition> {
val slot = stageConfig.produces.firstOrNull { it.kind.llmEmitted } ?: return emptyList()
val schema = Json.encodeToJsonElement(JsonSchema.serializer(), slot.kind.deriveJsonSchema()).jsonObject
return listOf(
ToolDefinition(
function = ToolFunction(
name = EMIT_ARTIFACT_TOOL,
description = "Emit the required '${slot.name.value}' artifact by filling in the fields " +
"below. Prefer this over writing the JSON as a plain message.",
parameters = schema,
),
),
)
}
// Collapse a validation report into a concise, actionable reason. This string becomes the
// RetryAttemptedEvent.failureReason that the retry-feedback entry feeds back to the model, so a
// weak model is told *what* was wrong (e.g. "required property 'summary' missing") instead of a
// bare "validation failed" it can't act on. Capped to keep the feedback prompt small.
private fun summarizeRejection(report: ValidationReport): String {
val errors = report.sections.flatMap { section ->
section.issues
.filter { it.severity == ValidationSeverity.ERROR }
.map { it.message }
}.distinct()
return if (errors.isEmpty()) {
"validation failed"
} else {
"validation failed: " + errors.take(MAX_FEEDBACK_ISSUES).joinToString("; ")
}
}
@Suppress("LongParameterList")
internal open suspend fun runInference(
sessionId: SessionId,
@@ -1834,7 +1893,7 @@ abstract class SessionOrchestrator(
"The orchestrator will proceed to the next stage.",
parameters = JsonObject(emptyMap()),
),
),
) + emitArtifactTool(stageConfig),
)
val renderedMessages = PromptRenderer.render(contextPack)