feat(validation): artifact-emission repair ladder + empty-schema fix
Deterministic repair pipeline (extraction, classification, policy, gated LLM-repair rung) with ArtifactRepairAttempted/Failed events. Also fix JsonSchemaValidator: a schema with no declared properties describes 'any object' — unknown-property enforcement now only applies against a declared shape (empty-properties schemas were rejecting every real artifact).
This commit is contained in:
+74
@@ -0,0 +1,74 @@
|
||||
package com.correx.core.validation.artifact
|
||||
|
||||
import com.correx.core.artifacts.kind.JsonSchema
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
|
||||
/**
|
||||
* Pure, deterministic extraction ladder for LLM artifact output. Turns near-miss text
|
||||
* (prose-wrapped JSON, code fences, trailing commas, unquoted keys) into a canonical, schema-shaped
|
||||
* [JsonObject], or reports why it couldn't. No coroutines, no inference — so replay recomputes it
|
||||
* (CLAUDE.md #8) and it records no events.
|
||||
*
|
||||
* The rare LLM-repair rung (spec §5/§8) is intentionally NOT here: it touches inference and is
|
||||
* driven by the orchestrator when this returns [Unresolved].
|
||||
*
|
||||
* See docs/specs/2026-07-04-artifact-emission-pipeline.md.
|
||||
*/
|
||||
class ArtifactExtractionPipeline {
|
||||
|
||||
sealed interface ExtractionResult {
|
||||
/** [repaired] = the raw text was NOT already clean; a repair transform was needed to resolve it. */
|
||||
data class Resolved(val canonicalJson: JsonObject, val repaired: Boolean) : ExtractionResult
|
||||
data class Unresolved(
|
||||
val classification: ArtifactFailure,
|
||||
val bestCandidate: String?, // most-repaired text, handed to the LLM-repair rung
|
||||
val detail: String,
|
||||
) : ExtractionResult
|
||||
}
|
||||
|
||||
fun run(raw: String, schema: JsonSchema): ExtractionResult {
|
||||
val trimmed = raw.trim()
|
||||
// Fast path: already-clean, schema-valid JSON needs no repair (and emits no repair event).
|
||||
parse(trimmed)?.let { direct ->
|
||||
val canonical = canonicalize(direct, schema)
|
||||
if (JsonSchemaValidator.validate(schema, canonical).isEmpty()) {
|
||||
return ExtractionResult.Resolved(canonical, repaired = false)
|
||||
}
|
||||
}
|
||||
var candidate = trimmed
|
||||
for (repair in ArtifactRepairs.ladder) {
|
||||
candidate = repair(candidate)
|
||||
val parsed = parse(candidate) ?: continue
|
||||
val canonical = canonicalize(parsed, schema)
|
||||
if (JsonSchemaValidator.validate(schema, canonical).isEmpty()) {
|
||||
return ExtractionResult.Resolved(canonical, repaired = true)
|
||||
}
|
||||
}
|
||||
val best = parse(candidate) ?: parse(trimmed)
|
||||
val issues = best?.let { JsonSchemaValidator.validate(schema, canonicalize(it, schema)) } ?: emptyList()
|
||||
return ExtractionResult.Unresolved(
|
||||
classification = classify(candidate, best, issues),
|
||||
bestCandidate = candidate,
|
||||
detail = issues.takeIf { it.isNotEmpty() }?.joinToString("; ")
|
||||
?: "could not extract a JSON object from the output",
|
||||
)
|
||||
}
|
||||
|
||||
/** Fill in schema-declared defaults for absent properties. */
|
||||
private fun canonicalize(obj: JsonObject, schema: JsonSchema): JsonObject {
|
||||
val filled = obj.toMutableMap()
|
||||
for ((key, prop) in schema.properties) {
|
||||
if (key !in filled && prop.default != null) filled[key] = JsonPrimitive(prop.default)
|
||||
}
|
||||
return JsonObject(filled)
|
||||
}
|
||||
|
||||
private fun parse(text: String): JsonObject? =
|
||||
runCatching { JSON.parseToJsonElement(text) as? JsonObject }.getOrNull()
|
||||
|
||||
private companion object {
|
||||
val JSON = Json { ignoreUnknownKeys = true; isLenient = true }
|
||||
}
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
package com.correx.core.validation.artifact
|
||||
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import kotlinx.serialization.json.JsonPrimitive
|
||||
|
||||
/**
|
||||
* Why a producer's artifact output could not be resolved deterministically. Drives the policy
|
||||
* decision and gates whether the (expensive) LLM-repair rung is worth attempting.
|
||||
* See docs/specs/2026-07-04-artifact-emission-pipeline.md §3.2.
|
||||
*/
|
||||
enum class ArtifactFailure {
|
||||
FORMATTING, // couldn't parse to a JSON object at all → deterministic repair (already tried)
|
||||
SCHEMA, // parsed, wrong shape: unknown/extra property or type mismatch → LLM repair
|
||||
MISSING_INFO, // parsed & shaped, but required content absent/empty → LLM repair
|
||||
CONTRADICTORY, // internally inconsistent → regenerate producer
|
||||
UNSAFE, // forbidden output (path traversal, secret) → reject immediately
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure classifier. `UNSAFE` is checked first (cheapest + terminal). Ordering matters: this runs
|
||||
* before the LLM-repair rung so that rung is only reached for `SCHEMA` / `MISSING_INFO`.
|
||||
*
|
||||
* ponytail: UNSAFE here is a lightweight `..` path-traversal heuristic. Full workspace-escape /
|
||||
* denied-tool / secret-scanning lives in the orchestrator's policy layer, which has the workspace
|
||||
* context this pure function does not. CONTRADICTORY needs semantic cross-field checks and is not
|
||||
* auto-detected yet — the enum value exists for the policy mapping.
|
||||
*/
|
||||
fun classify(raw: String, parsed: JsonObject?, schemaIssues: List<String>): ArtifactFailure = when {
|
||||
isUnsafe(parsed) -> ArtifactFailure.UNSAFE
|
||||
parsed == null -> ArtifactFailure.FORMATTING
|
||||
schemaIssues.any { it.startsWith("missing required") } -> ArtifactFailure.MISSING_INFO
|
||||
else -> ArtifactFailure.SCHEMA
|
||||
}
|
||||
|
||||
private fun isUnsafe(parsed: JsonObject?): Boolean {
|
||||
val path = (parsed?.get("path") as? JsonPrimitive)?.takeIf { it.isString }?.content ?: return false
|
||||
return path.split('/', '\\').any { it == ".." }
|
||||
}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.correx.core.validation.artifact
|
||||
|
||||
/**
|
||||
* Deterministic, pure text repairs for near-miss LLM artifact output. Each is `(String) -> String`
|
||||
* and safe to apply to already-valid JSON (idempotent / pass-through). The pipeline applies them in
|
||||
* order, re-parsing after each, and stops at the first that yields parseable JSON.
|
||||
*
|
||||
* Motivated by weak local models that emit prose-wrapped JSON, code-fenced JSON, trailing commas,
|
||||
* or code-as-content — see docs/specs/2026-07-04-artifact-emission-pipeline.md.
|
||||
*
|
||||
* ponytail: JSON-only ladder. json5/yaml/toml/xml deferred — no artifact kind emits them and each
|
||||
* pulls a new dep. Add a parser to the pipeline when a kind actually needs it.
|
||||
*/
|
||||
object ArtifactRepairs {
|
||||
|
||||
val ladder: List<(String) -> String> = listOf(
|
||||
::stripCodeFences,
|
||||
::stripSurroundingProse,
|
||||
::removeTrailingCommas,
|
||||
::quoteUnquotedKeys,
|
||||
::fixCommonEscapes,
|
||||
)
|
||||
|
||||
/** Drop a markdown code fence anywhere in the text, keeping the first fenced block's body. */
|
||||
fun stripCodeFences(text: String): String {
|
||||
val open = text.indexOf("```")
|
||||
if (open < 0) return text
|
||||
val bodyStart = text.indexOf('\n', open).let { if (it < 0) return text else it + 1 }
|
||||
val close = text.indexOf("```", bodyStart)
|
||||
if (close < 0) return text
|
||||
return text.substring(bodyStart, close).trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep only the first balanced `{...}` (or `[...]`) object, discarding surrounding prose.
|
||||
* String-aware so braces inside quoted values don't confuse the matcher. This is the workhorse
|
||||
* for prose-wrapped JSON and code-as-content.
|
||||
*/
|
||||
fun stripSurroundingProse(text: String): String {
|
||||
val start = text.indexOfFirst { it == '{' || it == '[' }
|
||||
if (start < 0) return text
|
||||
val open = text[start]
|
||||
val close = if (open == '{') '}' else ']'
|
||||
var depth = 0
|
||||
var inString = false
|
||||
var escaped = false
|
||||
for (i in start until text.length) {
|
||||
val c = text[i]
|
||||
when {
|
||||
escaped -> escaped = false
|
||||
c == '\\' && inString -> escaped = true
|
||||
c == '"' -> inString = !inString
|
||||
inString -> {}
|
||||
c == open -> depth++
|
||||
c == close -> {
|
||||
depth--
|
||||
if (depth == 0) return text.substring(start, i + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
return text
|
||||
}
|
||||
|
||||
/** Remove commas that immediately precede a closing `}` or `]` (ignoring whitespace). */
|
||||
fun removeTrailingCommas(text: String): String =
|
||||
Regex(""",(\s*[}\]])""").replace(text) { it.groupValues[1] }
|
||||
|
||||
/** Quote bare identifier keys: `{ key: 1 }` → `{ "key": 1 }`. Skips already-quoted keys. */
|
||||
fun quoteUnquotedKeys(text: String): String =
|
||||
Regex("""([{,]\s*)([A-Za-z_][A-Za-z0-9_]*)(\s*):""")
|
||||
.replace(text) { "${it.groupValues[1]}\"${it.groupValues[2]}\"${it.groupValues[3]}:" }
|
||||
|
||||
/** Normalize smart quotes and collapse doubled backslashes that break JSON string escapes. */
|
||||
fun fixCommonEscapes(text: String): String =
|
||||
text.replace('“', '"').replace('”', '"')
|
||||
.replace('‘', '\'').replace('’', '\'')
|
||||
.replace("\\\\", "\\")
|
||||
}
|
||||
+6
-1
@@ -35,7 +35,12 @@ object JsonSchemaValidator {
|
||||
for ((key, element) in obj) {
|
||||
val property = schema.properties[key]
|
||||
if (property == null) {
|
||||
if (!schema.additionalProperties) errors += "unknown property '$key'"
|
||||
// A schema with no declared properties describes "any object" — flagging every
|
||||
// key there would reject all real content. Unknown-property enforcement only
|
||||
// makes sense against a declared shape.
|
||||
if (!schema.additionalProperties && schema.properties.isNotEmpty()) {
|
||||
errors += "unknown property '$key'"
|
||||
}
|
||||
} else {
|
||||
errors += checkProperty(key, property, element)
|
||||
}
|
||||
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
package com.correx.core.validation.artifact
|
||||
|
||||
import com.correx.core.artifacts.kind.JsonSchema
|
||||
import com.correx.core.artifacts.kind.JsonSchemaProperty
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Assertions.assertInstanceOf
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class ArtifactExtractionPipelineTest {
|
||||
|
||||
private val pipeline = ArtifactExtractionPipeline()
|
||||
private val fileWritten = JsonSchema(
|
||||
type = "object",
|
||||
properties = mapOf(
|
||||
"path" to JsonSchemaProperty(type = "string"),
|
||||
"content" to JsonSchemaProperty(type = "string"),
|
||||
"mode" to JsonSchemaProperty(type = "string", default = "0644"),
|
||||
),
|
||||
required = listOf("path", "content"),
|
||||
)
|
||||
|
||||
private fun resolved(raw: String) =
|
||||
assertInstanceOf(ArtifactExtractionPipeline.ExtractionResult.Resolved::class.java, pipeline.run(raw, fileWritten))
|
||||
|
||||
@Test
|
||||
fun `clean json passes through`() {
|
||||
val r = resolved("""{"path":"a.kt","content":"x"}""")
|
||||
assertEquals("a.kt", (r.canonicalJson["path"] as kotlinx.serialization.json.JsonPrimitive).content)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `prose-wrapped json is rescued`() {
|
||||
resolved("""Sure! Here is the file:\n{"path":"a.kt","content":"x"} Let me know if you need changes.""")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `code-fenced json is rescued`() {
|
||||
resolved("```json\n{\"path\":\"a.kt\",\"content\":\"x\"}\n```")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `trailing comma is repaired`() {
|
||||
resolved("""{"path":"a.kt","content":"x",}""")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `unquoted keys are repaired`() {
|
||||
resolved("""{path:"a.kt",content:"x"}""")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `missing default is filled`() {
|
||||
val r = resolved("""{"path":"a.kt","content":"x"}""")
|
||||
assertEquals("0644", (r.canonicalJson["mode"] as kotlinx.serialization.json.JsonPrimitive).content)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `schema-shaped but missing required is unresolved`() {
|
||||
assertInstanceOf(
|
||||
ArtifactExtractionPipeline.ExtractionResult.Unresolved::class.java,
|
||||
pipeline.run("""{"path":"a.kt"}""", fileWritten),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `pure prose with no json is unresolved`() {
|
||||
assertInstanceOf(
|
||||
ArtifactExtractionPipeline.ExtractionResult.Unresolved::class.java,
|
||||
pipeline.run("I could not complete this task.", fileWritten),
|
||||
)
|
||||
}
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package com.correx.core.validation.artifact
|
||||
|
||||
import kotlinx.serialization.json.Json
|
||||
import kotlinx.serialization.json.JsonObject
|
||||
import org.junit.jupiter.api.Assertions.assertEquals
|
||||
import org.junit.jupiter.api.Test
|
||||
|
||||
class ArtifactFailureTest {
|
||||
|
||||
private fun obj(json: String) = Json.parseToJsonElement(json) as JsonObject
|
||||
|
||||
@Test
|
||||
fun `unquoted-but-repairable never reaches here — unparsed is FORMATTING`() {
|
||||
assertEquals(ArtifactFailure.FORMATTING, classify("not json at all", null, emptyList()))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `missing required field is MISSING_INFO`() {
|
||||
assertEquals(
|
||||
ArtifactFailure.MISSING_INFO,
|
||||
classify("{}", obj("""{"path":"a"}"""), listOf("missing required property 'content'")),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `wrong shape without missing-required is SCHEMA`() {
|
||||
assertEquals(
|
||||
ArtifactFailure.SCHEMA,
|
||||
classify("{}", obj("""{"path":1}"""), listOf("property 'path': expected string, got primitive")),
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `path traversal is UNSAFE and wins over other signals`() {
|
||||
assertEquals(
|
||||
ArtifactFailure.UNSAFE,
|
||||
classify("{}", obj("""{"path":"../../etc/passwd"}"""), listOf("missing required property 'content'")),
|
||||
)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user