feat(validation): artifact-emission repair ladder + empty-schema fix

Deterministic repair pipeline (extraction, classification, policy, gated LLM-repair rung) with ArtifactRepairAttempted/Failed events. Also fix JsonSchemaValidator: a schema with no declared properties describes 'any object' — unknown-property enforcement now only applies against a declared shape (empty-properties schemas were rejecting every real artifact).
2026-07-05 18:04:36 +04:00
parent 24f3b843bb
commit f90f2ab39d
11 changed files with 462 additions and 1 deletions
@@ -50,3 +50,40 @@ data class ArtifactContentStoredEvent(
    val sessionId: SessionId,
    val stageId: StageId,
 ) : EventPayload
+
+/**
+ * A near-miss producer output was routed through the artifact-emission repair ladder. Only the
+ * nondeterministic rung (`rung = "LLM"`) is replay-relevant — the deterministic rungs recompute —
+ * but both are recorded for observability. See docs/specs/2026-07-04-artifact-emission-pipeline.md.
+ */
+@Serializable
+@SerialName("ArtifactRepairAttempted")
+data class ArtifactRepairAttemptedEvent(
+    val artifactId: ArtifactId,
+    val classification: String, // ArtifactFailure name
+    val rung: String,           // "DETERMINISTIC" | "LLM"
+    val sessionId: SessionId,
+    val stageId: StageId,
+) : EventPayload
+
+/** The repair ladder produced a schema-valid artifact. [contentHash] feeds the normal store path. */
+@Serializable
+@SerialName("ArtifactRepairResolved")
+data class ArtifactRepairResolvedEvent(
+    val artifactId: ArtifactId,
+    val contentHash: ArtifactId,
+    val sessionId: SessionId,
+    val stageId: StageId,
+) : EventPayload
+
+/** The repair ladder gave up; [decision] records the policy dispatch (retry/regenerate/abort/escalate). */
+@Serializable
+@SerialName("ArtifactRepairFailed")
+data class ArtifactRepairFailedEvent(
+    val artifactId: ArtifactId,
+    val classification: String,
+    val decision: String,
+    val reason: String,
+    val sessionId: SessionId,
+    val stageId: StageId,
+) : EventPayload
@@ -7,6 +7,9 @@ import com.correx.core.events.events.ApprovalGrantExpiredEvent
 import com.correx.core.events.events.ApprovalRequestedEvent
 import com.correx.core.events.events.ArtifactContentStoredEvent
 import com.correx.core.events.events.ArtifactCreatedEvent
+import com.correx.core.events.events.ArtifactRepairAttemptedEvent
+import com.correx.core.events.events.ArtifactRepairFailedEvent
+import com.correx.core.events.events.ArtifactRepairResolvedEvent
 import com.correx.core.events.events.ArtifactValidatedEvent
 import com.correx.core.events.events.ArtifactValidatingEvent
 import com.correx.core.events.events.BriefEchoMismatchEvent
@@ -120,6 +123,9 @@ val eventModule = SerializersModule {
        subclass(ApprovalGrantCreatedEvent::class)
        subclass(ApprovalGrantExpiredEvent::class)
        subclass(ArtifactContentStoredEvent::class)
+        subclass(ArtifactRepairAttemptedEvent::class)
+        subclass(ArtifactRepairResolvedEvent::class)
+        subclass(ArtifactRepairFailedEvent::class)
        subclass(ArtifactCreatedEvent::class)
        subclass(ArtifactValidatingEvent::class)
        subclass(ArtifactValidatedEvent::class)
@@ -0,0 +1,33 @@
+package com.correx.core.events.serialization
+
+import com.correx.core.events.events.ArtifactRepairAttemptedEvent
+import com.correx.core.events.events.ArtifactRepairFailedEvent
+import com.correx.core.events.events.ArtifactRepairResolvedEvent
+import com.correx.core.events.events.EventPayload
+import com.correx.core.events.types.ArtifactId
+import com.correx.core.events.types.SessionId
+import com.correx.core.events.types.StageId
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class ArtifactRepairEventSerializationTest {
+
+    private fun roundTrip(sample: EventPayload) =
+        assertEquals(sample, eventJson.decodeFromString(EventPayload.serializer(),
+            eventJson.encodeToString(EventPayload.serializer(), sample)))
+
+    @Test
+    fun `ArtifactRepairAttempted round-trips`() {
+        roundTrip(ArtifactRepairAttemptedEvent(ArtifactId("a"), "SCHEMA", "LLM", SessionId("s"), StageId("st")))
+    }
+
+    @Test
+    fun `ArtifactRepairResolved round-trips`() {
+        roundTrip(ArtifactRepairResolvedEvent(ArtifactId("a"), ArtifactId("hash"), SessionId("s"), StageId("st")))
+    }
+
+    @Test
+    fun `ArtifactRepairFailed round-trips`() {
+        roundTrip(ArtifactRepairFailedEvent(ArtifactId("a"), "UNSAFE", "Abort", "path escapes", SessionId("s"), StageId("st")))
+    }
+}
@@ -0,0 +1,44 @@
+package com.correx.core.kernel.orchestration
+
+import com.correx.core.validation.artifact.ArtifactFailure
+
+/**
+ * What to do when the artifact-emission repair ladder (deterministic + one LLM rung) still can't
+ * produce a schema-valid artifact. Each maps onto machinery the orchestrator already has.
+ * See docs/specs/2026-07-04-artifact-emission-pipeline.md §3.4.
+ */
+sealed interface ArtifactPolicyDecision {
+    data object RetryProducer : ArtifactPolicyDecision           // → retryStageOrFail (retryable failure)
+    data object RegenerateArtifactOnly : ArtifactPolicyDecision  // → re-run the tools-less emission nudge
+    data object Abort : ArtifactPolicyDecision                   // → non-retryable failure
+    data object EscalateHuman : ArtifactPolicyDecision           // → approval gate / OrchestrationPaused
+}
+
+/**
+ * Pure default policy. `attemptsSoFar` is the repair attempts already spent this stage.
+ * Mapping (spec §3.4): UNSAFE→Abort; CONTRADICTORY→RetryProducer; MISSING_INFO→RegenerateArtifactOnly
+ * first, then RetryProducer; FORMATTING/SCHEMA→RetryProducer (deterministic + LLM rung already tried);
+ * exhausted→EscalateHuman when an approver is attached, else Abort.
+ */
+fun decide(
+    failure: ArtifactFailure,
+    attemptsSoFar: Int,
+    maxAttempts: Int,
+    hasApprover: Boolean,
+): ArtifactPolicyDecision {
+    if (failure == ArtifactFailure.UNSAFE) return ArtifactPolicyDecision.Abort
+    val base = when (failure) {
+        ArtifactFailure.MISSING_INFO -> if (attemptsSoFar == 0) {
+            ArtifactPolicyDecision.RegenerateArtifactOnly
+        } else {
+            ArtifactPolicyDecision.RetryProducer
+        }
+        else -> ArtifactPolicyDecision.RetryProducer // FORMATTING, SCHEMA, CONTRADICTORY
+    }
+    // Exhaustion guard: a retry-flavoured decision with no budget left escalates (with an approver) or aborts.
+    return if (attemptsSoFar >= maxAttempts) {
+        if (hasApprover) ArtifactPolicyDecision.EscalateHuman else ArtifactPolicyDecision.Abort
+    } else {
+        base
+    }
+}
@@ -0,0 +1,34 @@
+package com.correx.core.kernel.orchestration
+
+import com.correx.core.validation.artifact.ArtifactFailure
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+
+class ArtifactPolicyDecisionTest {
+
+    private fun decideFresh(f: ArtifactFailure) = decide(f, attemptsSoFar = 0, maxAttempts = Int.MAX_VALUE, hasApprover = false)
+
+    @Test
+    fun `unsafe always aborts, even with budget and approver`() {
+        assertEquals(ArtifactPolicyDecision.Abort, decide(ArtifactFailure.UNSAFE, 0, 5, hasApprover = true))
+    }
+
+    @Test
+    fun `schema and formatting retry the producer`() {
+        assertEquals(ArtifactPolicyDecision.RetryProducer, decideFresh(ArtifactFailure.SCHEMA))
+        assertEquals(ArtifactPolicyDecision.RetryProducer, decideFresh(ArtifactFailure.FORMATTING))
+        assertEquals(ArtifactPolicyDecision.RetryProducer, decideFresh(ArtifactFailure.CONTRADICTORY))
+    }
+
+    @Test
+    fun `missing info regenerates artifact-only on first attempt then retries producer`() {
+        assertEquals(ArtifactPolicyDecision.RegenerateArtifactOnly, decide(ArtifactFailure.MISSING_INFO, 0, 5, false))
+        assertEquals(ArtifactPolicyDecision.RetryProducer, decide(ArtifactFailure.MISSING_INFO, 1, 5, false))
+    }
+
+    @Test
+    fun `exhausted budget escalates with approver, aborts without`() {
+        assertEquals(ArtifactPolicyDecision.EscalateHuman, decide(ArtifactFailure.SCHEMA, 3, 3, hasApprover = true))
+        assertEquals(ArtifactPolicyDecision.Abort, decide(ArtifactFailure.SCHEMA, 3, 3, hasApprover = false))
+    }
+}
@@ -0,0 +1,74 @@
+package com.correx.core.validation.artifact
+
+import com.correx.core.artifacts.kind.JsonSchema
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonPrimitive
+
+/**
+ * Pure, deterministic extraction ladder for LLM artifact output. Turns near-miss text
+ * (prose-wrapped JSON, code fences, trailing commas, unquoted keys) into a canonical, schema-shaped
+ * [JsonObject], or reports why it couldn't. No coroutines, no inference — so replay recomputes it
+ * (CLAUDE.md #8) and it records no events.
+ *
+ * The rare LLM-repair rung (spec §5/§8) is intentionally NOT here: it touches inference and is
+ * driven by the orchestrator when this returns [Unresolved].
+ *
+ * See docs/specs/2026-07-04-artifact-emission-pipeline.md.
+ */
+class ArtifactExtractionPipeline {
+
+    sealed interface ExtractionResult {
+        /** [repaired] = the raw text was NOT already clean; a repair transform was needed to resolve it. */
+        data class Resolved(val canonicalJson: JsonObject, val repaired: Boolean) : ExtractionResult
+        data class Unresolved(
+            val classification: ArtifactFailure,
+            val bestCandidate: String?, // most-repaired text, handed to the LLM-repair rung
+            val detail: String,
+        ) : ExtractionResult
+    }
+
+    fun run(raw: String, schema: JsonSchema): ExtractionResult {
+        val trimmed = raw.trim()
+        // Fast path: already-clean, schema-valid JSON needs no repair (and emits no repair event).
+        parse(trimmed)?.let { direct ->
+            val canonical = canonicalize(direct, schema)
+            if (JsonSchemaValidator.validate(schema, canonical).isEmpty()) {
+                return ExtractionResult.Resolved(canonical, repaired = false)
+            }
+        }
+        var candidate = trimmed
+        for (repair in ArtifactRepairs.ladder) {
+            candidate = repair(candidate)
+            val parsed = parse(candidate) ?: continue
+            val canonical = canonicalize(parsed, schema)
+            if (JsonSchemaValidator.validate(schema, canonical).isEmpty()) {
+                return ExtractionResult.Resolved(canonical, repaired = true)
+            }
+        }
+        val best = parse(candidate) ?: parse(trimmed)
+        val issues = best?.let { JsonSchemaValidator.validate(schema, canonicalize(it, schema)) } ?: emptyList()
+        return ExtractionResult.Unresolved(
+            classification = classify(candidate, best, issues),
+            bestCandidate = candidate,
+            detail = issues.takeIf { it.isNotEmpty() }?.joinToString("; ")
+                ?: "could not extract a JSON object from the output",
+        )
+    }
+
+    /** Fill in schema-declared defaults for absent properties. */
+    private fun canonicalize(obj: JsonObject, schema: JsonSchema): JsonObject {
+        val filled = obj.toMutableMap()
+        for ((key, prop) in schema.properties) {
+            if (key !in filled && prop.default != null) filled[key] = JsonPrimitive(prop.default)
+        }
+        return JsonObject(filled)
+    }
+
+    private fun parse(text: String): JsonObject? =
+        runCatching { JSON.parseToJsonElement(text) as? JsonObject }.getOrNull()
+
+    private companion object {
+        val JSON = Json { ignoreUnknownKeys = true; isLenient = true }
+    }
+}
@@ -0,0 +1,38 @@
+package com.correx.core.validation.artifact
+
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.JsonPrimitive
+
+/**
+ * Why a producer's artifact output could not be resolved deterministically. Drives the policy
+ * decision and gates whether the (expensive) LLM-repair rung is worth attempting.
+ * See docs/specs/2026-07-04-artifact-emission-pipeline.md §3.2.
+ */
+enum class ArtifactFailure {
+    FORMATTING,     // couldn't parse to a JSON object at all → deterministic repair (already tried)
+    SCHEMA,         // parsed, wrong shape: unknown/extra property or type mismatch → LLM repair
+    MISSING_INFO,   // parsed & shaped, but required content absent/empty → LLM repair
+    CONTRADICTORY,  // internally inconsistent → regenerate producer
+    UNSAFE,         // forbidden output (path traversal, secret) → reject immediately
+}
+
+/**
+ * Pure classifier. `UNSAFE` is checked first (cheapest + terminal). Ordering matters: this runs
+ * before the LLM-repair rung so that rung is only reached for `SCHEMA` / `MISSING_INFO`.
+ *
+ * ponytail: UNSAFE here is a lightweight `..` path-traversal heuristic. Full workspace-escape /
+ * denied-tool / secret-scanning lives in the orchestrator's policy layer, which has the workspace
+ * context this pure function does not. CONTRADICTORY needs semantic cross-field checks and is not
+ * auto-detected yet — the enum value exists for the policy mapping.
+ */
+fun classify(raw: String, parsed: JsonObject?, schemaIssues: List<String>): ArtifactFailure = when {
+    isUnsafe(parsed) -> ArtifactFailure.UNSAFE
+    parsed == null -> ArtifactFailure.FORMATTING
+    schemaIssues.any { it.startsWith("missing required") } -> ArtifactFailure.MISSING_INFO
+    else -> ArtifactFailure.SCHEMA
+}
+
+private fun isUnsafe(parsed: JsonObject?): Boolean {
+    val path = (parsed?.get("path") as? JsonPrimitive)?.takeIf { it.isString }?.content ?: return false
+    return path.split('/', '\\').any { it == ".." }
+}
@@ -0,0 +1,78 @@
+package com.correx.core.validation.artifact
+
+/**
+ * Deterministic, pure text repairs for near-miss LLM artifact output. Each is `(String) -> String`
+ * and safe to apply to already-valid JSON (idempotent / pass-through). The pipeline applies them in
+ * order, re-parsing after each, and stops at the first that yields parseable JSON.
+ *
+ * Motivated by weak local models that emit prose-wrapped JSON, code-fenced JSON, trailing commas,
+ * or code-as-content — see docs/specs/2026-07-04-artifact-emission-pipeline.md.
+ *
+ * ponytail: JSON-only ladder. json5/yaml/toml/xml deferred — no artifact kind emits them and each
+ * pulls a new dep. Add a parser to the pipeline when a kind actually needs it.
+ */
+object ArtifactRepairs {
+
+    val ladder: List<(String) -> String> = listOf(
+        ::stripCodeFences,
+        ::stripSurroundingProse,
+        ::removeTrailingCommas,
+        ::quoteUnquotedKeys,
+        ::fixCommonEscapes,
+    )
+
+    /** Drop a markdown code fence anywhere in the text, keeping the first fenced block's body. */
+    fun stripCodeFences(text: String): String {
+        val open = text.indexOf("```")
+        if (open < 0) return text
+        val bodyStart = text.indexOf('\n', open).let { if (it < 0) return text else it + 1 }
+        val close = text.indexOf("```", bodyStart)
+        if (close < 0) return text
+        return text.substring(bodyStart, close).trim()
+    }
+
+    /**
+     * Keep only the first balanced `{...}` (or `[...]`) object, discarding surrounding prose.
+     * String-aware so braces inside quoted values don't confuse the matcher. This is the workhorse
+     * for prose-wrapped JSON and code-as-content.
+     */
+    fun stripSurroundingProse(text: String): String {
+        val start = text.indexOfFirst { it == '{' || it == '[' }
+        if (start < 0) return text
+        val open = text[start]
+        val close = if (open == '{') '}' else ']'
+        var depth = 0
+        var inString = false
+        var escaped = false
+        for (i in start until text.length) {
+            val c = text[i]
+            when {
+                escaped -> escaped = false
+                c == '\\' && inString -> escaped = true
+                c == '"' -> inString = !inString
+                inString -> {}
+                c == open -> depth++
+                c == close -> {
+                    depth--
+                    if (depth == 0) return text.substring(start, i + 1)
+                }
+            }
+        }
+        return text
+    }
+
+    /** Remove commas that immediately precede a closing `}` or `]` (ignoring whitespace). */
+    fun removeTrailingCommas(text: String): String =
+        Regex(""",(\s*[}\]])""").replace(text) { it.groupValues[1] }
+
+    /** Quote bare identifier keys: `{ key: 1 }` → `{ "key": 1 }`. Skips already-quoted keys. */
+    fun quoteUnquotedKeys(text: String): String =
+        Regex("""([{,]\s*)([A-Za-z_][A-Za-z0-9_]*)(\s*):""")
+            .replace(text) { "${it.groupValues[1]}\"${it.groupValues[2]}\"${it.groupValues[3]}:" }
+
+    /** Normalize smart quotes and collapse doubled backslashes that break JSON string escapes. */
+    fun fixCommonEscapes(text: String): String =
+        text.replace('“', '"').replace('”', '"')
+            .replace('‘', '\'').replace('’', '\'')
+            .replace("\\\\", "\\")
+}
@@ -35,7 +35,12 @@ object JsonSchemaValidator {
        for ((key, element) in obj) {
            val property = schema.properties[key]
            if (property == null) {
-                if (!schema.additionalProperties) errors += "unknown property '$key'"
+                // A schema with no declared properties describes "any object" — flagging every
+                // key there would reject all real content. Unknown-property enforcement only
+                // makes sense against a declared shape.
+                if (!schema.additionalProperties && schema.properties.isNotEmpty()) {
+                    errors += "unknown property '$key'"
+                }
            } else {
                errors += checkProperty(key, property, element)
            }
@@ -0,0 +1,72 @@
+package com.correx.core.validation.artifact
+
+import com.correx.core.artifacts.kind.JsonSchema
+import com.correx.core.artifacts.kind.JsonSchemaProperty
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.assertInstanceOf
+import org.junit.jupiter.api.Test
+
+class ArtifactExtractionPipelineTest {
+
+    private val pipeline = ArtifactExtractionPipeline()
+    private val fileWritten = JsonSchema(
+        type = "object",
+        properties = mapOf(
+            "path" to JsonSchemaProperty(type = "string"),
+            "content" to JsonSchemaProperty(type = "string"),
+            "mode" to JsonSchemaProperty(type = "string", default = "0644"),
+        ),
+        required = listOf("path", "content"),
+    )
+
+    private fun resolved(raw: String) =
+        assertInstanceOf(ArtifactExtractionPipeline.ExtractionResult.Resolved::class.java, pipeline.run(raw, fileWritten))
+
+    @Test
+    fun `clean json passes through`() {
+        val r = resolved("""{"path":"a.kt","content":"x"}""")
+        assertEquals("a.kt", (r.canonicalJson["path"] as kotlinx.serialization.json.JsonPrimitive).content)
+    }
+
+    @Test
+    fun `prose-wrapped json is rescued`() {
+        resolved("""Sure! Here is the file:\n{"path":"a.kt","content":"x"} Let me know if you need changes.""")
+    }
+
+    @Test
+    fun `code-fenced json is rescued`() {
+        resolved("```json\n{\"path\":\"a.kt\",\"content\":\"x\"}\n```")
+    }
+
+    @Test
+    fun `trailing comma is repaired`() {
+        resolved("""{"path":"a.kt","content":"x",}""")
+    }
+
+    @Test
+    fun `unquoted keys are repaired`() {
+        resolved("""{path:"a.kt",content:"x"}""")
+    }
+
+    @Test
+    fun `missing default is filled`() {
+        val r = resolved("""{"path":"a.kt","content":"x"}""")
+        assertEquals("0644", (r.canonicalJson["mode"] as kotlinx.serialization.json.JsonPrimitive).content)
+    }
+
+    @Test
+    fun `schema-shaped but missing required is unresolved`() {
+        assertInstanceOf(
+            ArtifactExtractionPipeline.ExtractionResult.Unresolved::class.java,
+            pipeline.run("""{"path":"a.kt"}""", fileWritten),
+        )
+    }
+
+    @Test
+    fun `pure prose with no json is unresolved`() {
+        assertInstanceOf(
+            ArtifactExtractionPipeline.ExtractionResult.Unresolved::class.java,
+            pipeline.run("I could not complete this task.", fileWritten),
+        )
+    }
+}
@@ -0,0 +1,40 @@
+package com.correx.core.validation.artifact
+
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+
+class ArtifactFailureTest {
+
+    private fun obj(json: String) = Json.parseToJsonElement(json) as JsonObject
+
+    @Test
+    fun `unquoted-but-repairable never reaches here — unparsed is FORMATTING`() {
+        assertEquals(ArtifactFailure.FORMATTING, classify("not json at all", null, emptyList()))
+    }
+
+    @Test
+    fun `missing required field is MISSING_INFO`() {
+        assertEquals(
+            ArtifactFailure.MISSING_INFO,
+            classify("{}", obj("""{"path":"a"}"""), listOf("missing required property 'content'")),
+        )
+    }
+
+    @Test
+    fun `wrong shape without missing-required is SCHEMA`() {
+        assertEquals(
+            ArtifactFailure.SCHEMA,
+            classify("{}", obj("""{"path":1}"""), listOf("property 'path': expected string, got primitive")),
+        )
+    }
+
+    @Test
+    fun `path traversal is UNSAFE and wins over other signals`() {
+        assertEquals(
+            ArtifactFailure.UNSAFE,
+            classify("{}", obj("""{"path":"../../etc/passwd"}"""), listOf("missing required property 'content'")),
+        )
+    }
+}