fix(weak-model-gates): stop three gates from stalling weak stage models
Diagnosed from session 508c8d58 (frontend freestyle run, WorkflowFailed): three independent weak-model-hostile gates, none a model-capability problem. - shell: split a collapsed single-string command line (["npm create vite …"]) into tokens instead of rejecting it as a "collapsed array". The model reliably re-emits this shape; rejecting looped bootstrap_frontend until stage_loop_break. JSON-escape mangles (quotes/commas in argv[0]) stay rejected. - recovery: a stage_loop_break route now gets a "Stuck-loop ticket", not the "Contract arbitration ticket". The arbitration prompt told the model to read and reconcile "the files named below" — but a tool-syntax loop names zero files, sending the recovery agent grepping the repo for 40+ turns until the repair ladder exhausted. - plan lint: H3 (unreferenced_prompt_artifact) demoted from hard failure to soft finding, and seeds excluded from it. It word-matches artifact IDs in free prose and cannot tell a forgotten dep from a descriptive mention, so as a hard gate it burned architect retries on words it couldn't reword away (analysis/dod). Hard tier is now deterministic graph facts only (H1/H2). Tests: ShellToolTest, PlanLinterTest, RecoveryRoutingTest green; detekt clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015Ly2mMnt9TCZbvhcC1JfuV
This commit is contained in:
+19
-9
@@ -41,10 +41,13 @@ object PlanLinter {
|
||||
|
||||
fun lint(graph: WorkflowGraph, seeds: Set<String> = seedArtifacts): PlanLintResult =
|
||||
PlanLintResult(
|
||||
hardFailures = unproducedNeeds(graph, seeds) +
|
||||
trapStates(graph) +
|
||||
// HARD = deterministic graph facts, provably broken regardless of wording. SOFT = heuristics
|
||||
// (incl. H3's prose word-match, which cannot tell a forgotten dep from a descriptive mention)
|
||||
// — they score the plan but must NOT block it, or a weak architect burns retries it can't
|
||||
// reword its way out of.
|
||||
hardFailures = unproducedNeeds(graph, seeds) + trapStates(graph),
|
||||
softFindings = stageCount(graph) + fanOut(graph) + emptyBriefs(graph) + duplicateBriefs(graph) +
|
||||
unreferencedPromptArtifacts(graph, seeds),
|
||||
softFindings = stageCount(graph) + fanOut(graph) + emptyBriefs(graph) + duplicateBriefs(graph),
|
||||
)
|
||||
|
||||
/** H1: a stage `needs` an artifact that neither a plan stage `produces` nor a seed provides. */
|
||||
@@ -92,14 +95,21 @@ object PlanLinter {
|
||||
}
|
||||
|
||||
/**
|
||||
* H3: a stage's inline prompt mentions an artifact ID that is produced by the plan or
|
||||
* provided by a seed, but the stage does not declare it in its `needs`. The stage will
|
||||
* not have access to that artifact at runtime, so any reference in the prompt is a dangling
|
||||
* assumption that will confuse or stall the model — force the architect to either add the
|
||||
* artifact to `needs` or reword the prompt.
|
||||
* H3 (SOFT): a stage's inline prompt word-matches a plan-produced artifact ID the stage does not
|
||||
* declare in `needs` — possibly a dangling reference the model can't resolve at runtime. This is a
|
||||
* prose heuristic, not a graph fact: it cannot distinguish a forgotten dependency from a
|
||||
* descriptive mention ("unlike the `report` stage"), so it only penalises the plan's score and
|
||||
* never blocks it. Seeds are excluded (see below); a genuine missing consumption is still caught
|
||||
* structurally by H1 when the stage actually declares the `needs`.
|
||||
*/
|
||||
private fun unreferencedPromptArtifacts(graph: WorkflowGraph, seeds: Set<String>): List<PlanLintFinding> {
|
||||
val produced = graph.stages.values.flatMap { it.produces }.map { it.name.value }.toSet() + seeds
|
||||
// Only plan-PRODUCED artifacts are checked — a prompt that references a distinctly-named one
|
||||
// (e.g. `server_api_analysis`) but omits it from `needs` is a real dangling dependency. Seed
|
||||
// IDs (`analysis`, `dod`) are EXCLUDED: they double as ordinary words, so a prose mention
|
||||
// ("refine the analysis", "meets the DoD") is not a dependency signal — flagging it just churns
|
||||
// architect retries on a false positive the model cannot word around. (A stage that genuinely
|
||||
// consumes a seed still declares it in `needs`; H1 keeps that path honest.)
|
||||
val produced = graph.stages.values.flatMap { it.produces }.map { it.name.value }.toSet() - seeds
|
||||
return graph.stages.entries.flatMap { (id, stage) ->
|
||||
val prompt = briefOf(stage)
|
||||
val needIds = stage.needs.map { it.value }.toSet()
|
||||
|
||||
+19
-3
@@ -107,7 +107,23 @@ class PlanLinterTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a prompt mentioning another stage's artifact not in needs is a hard failure`() {
|
||||
fun `a prompt using a seed name as an ordinary word is not a failure`() {
|
||||
// Repro: architect burned 3 retries because briefs mentioning "analysis"/"dod" as prose kept
|
||||
// tripping H3 on the seed IDs — words the model cannot reword away.
|
||||
val g = graph(
|
||||
stages = mapOf(
|
||||
"a" to stage(produces = listOf("scope"), prompt = "refine the analysis and meet the dod"),
|
||||
"b" to stage(needs = listOf("scope"), prompt = "build"),
|
||||
),
|
||||
edges = listOf("a" to "b", "b" to "done"),
|
||||
start = "a",
|
||||
)
|
||||
val result = PlanLinter.lint(g)
|
||||
assertTrue(result.passed, "seed names in prose must not fire H3, hard=${result.hardFailures}")
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `a prompt mentioning another stage's artifact not in needs is a soft finding, not a block`() {
|
||||
val g = graph(
|
||||
stages = mapOf(
|
||||
"a" to stage(produces = listOf("plan"), prompt = "write the plan artifact"),
|
||||
@@ -117,8 +133,8 @@ class PlanLinterTest {
|
||||
start = "a",
|
||||
)
|
||||
val result = PlanLinter.lint(g)
|
||||
assertFalse(result.passed)
|
||||
val h = result.hardFailures.single { it.code == "unreferenced_prompt_artifact" }
|
||||
assertTrue(result.passed, "H3 is a prose heuristic — it scores but must not block the plan")
|
||||
val h = result.softFindings.single { it.code == "unreferenced_prompt_artifact" }
|
||||
assertEquals("b", h.stageId)
|
||||
assertTrue(h.detail.contains("plan"))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user