Files
orchestra/internal/workphase/workphase_test.go
kami 822f086451 Make research findings citable
The brief at agentctx.go:167 advertised findings[].id and findings[].confidence
to every research session. The struct carried neither, so encoding/json dropped
both on every seal, silently, for as long as the schema has existed. A plan
phase had nothing stable to cite and no way to tell an observation from an
assumption.

Finding gains ID and Confidence. Ids are unique within an artifact and shaped
so "research:<id>" is unambiguous in plan prose. Confidence is fact, inference,
or assumption, matching the labels the output style already uses.

DecodeStoredResearch reads what is already in the CAS and backfills both.
Refusing an artifact sealed before this change would block every task whose
research predates it, including at rotation, where the agent that could fix it
is already gone. A backfilled finding is labelled inference rather than fact:
the old schema required evidence and made no verification claim, so upgrading
it on the way in would be the same class of lie this commit removes.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CVbaKucEYBjMqVeUgJUsc1
2026-08-28 11:21:29 +04:00

146 lines
5.0 KiB
Go

package workphase
import (
"strings"
"testing"
)
func research() Research {
return Research{
Findings: []Finding{{ID: "r1", Confidence: Fact, Claim: "attribution runs per figure", Evidence: "internal/attr/attr.go:88"}},
Code: []CodePath{{Path: "internal/attr/attr.go", Why: "aggregation happens here"}},
Invariants: []string{"identity semantics must not change"},
DeadEnds: []DeadEnd{{Tried: "figure plurality", WhyFailed: "no measured gain"}},
}
}
func plan() Plan {
return Plan{
Changes: []Change{{Target: "internal/attr/attr.go", Intent: "aggregate per person"}},
Verification: []string{"go test ./internal/attr/"},
}
}
func TestRoundTrip(t *testing.T) {
b, err := Encode(research())
if err != nil {
t.Fatal(err)
}
got, err := DecodeResearch(b)
if err != nil {
t.Fatal(err)
}
if got.Findings[0].Claim != "attribution runs per figure" || got.DeadEnds[0].Tried != "figure plurality" {
t.Fatalf("round trip lost content: %+v", got)
}
pb, err := Encode(plan())
if err != nil {
t.Fatal(err)
}
gotPlan, err := DecodePlan(pb)
if err != nil {
t.Fatal(err)
}
if gotPlan.Changes[0].Target != "internal/attr/attr.go" {
t.Fatalf("round trip lost content: %+v", gotPlan)
}
}
// The bound is the point. An artifact that can hold a transcript is a
// transcript, and the next phase pays for reading it.
func TestBoundsRejectUnboundedArtifacts(t *testing.T) {
cases := map[string]func() error{
"no findings": func() error { return Research{}.Validate() },
"no evidence": func() error { return Research{Findings: []Finding{{Claim: "x"}}}.Validate() },
"multiline claim": func() error { return Research{Findings: []Finding{{Claim: "a\nb", Evidence: "e"}}}.Validate() },
"long claim": func() error {
return Research{Findings: []Finding{{Claim: strings.Repeat("x", 501), Evidence: "e"}}}.Validate()
},
"too many findings": func() error {
r := Research{}
for i := 0; i < 65; i++ {
r.Findings = append(r.Findings, Finding{ID: "r1", Confidence: Fact, Claim: "c", Evidence: "e"})
}
return r.Validate()
},
"absolute path": func() error {
r := research()
r.Code = []CodePath{{Path: "/etc/passwd", Why: "no"}}
return r.Validate()
},
"blank invariant": func() error {
r := research()
r.Invariants = []string{" "}
return r.Validate()
},
"no changes": func() error { return Plan{}.Validate() },
"no change intent": func() error { return Plan{Changes: []Change{{Target: "x"}}}.Validate() },
"multiline risk": func() error {
p := plan()
p.Risks = []string{"a\nb"}
return p.Validate()
},
}
for name, fn := range cases {
if err := fn(); err == nil {
t.Fatalf("%s: expected rejection", name)
}
}
}
func TestEncodeRejectsInvalid(t *testing.T) {
if _, err := Encode(Research{}); err == nil {
t.Fatal("Encode must validate before sealing")
}
if _, err := DecodeResearch([]byte(`{"findings":[]}`)); err == nil {
t.Fatal("Decode must validate")
}
if _, err := DecodePlan([]byte(`not json`)); err == nil {
t.Fatal("Decode must reject non-JSON")
}
}
// The advertised schema promised findings[].id and findings[].confidence while
// the struct carried neither, so both were dropped silently on every seal. A
// plan cannot cite what was never stored.
func TestFindingIdentityIsRequiredAndUnique(t *testing.T) {
cases := map[string]Research{
"no id": {Findings: []Finding{{Confidence: Fact, Claim: "c", Evidence: "e"}}},
"upper case id": {Findings: []Finding{{ID: "R1", Confidence: Fact, Claim: "c", Evidence: "e"}}},
"spaced id": {Findings: []Finding{{ID: "r 1", Confidence: Fact, Claim: "c", Evidence: "e"}}},
"no confidence": {Findings: []Finding{{ID: "r1", Claim: "c", Evidence: "e"}}},
"bad confidence": {Findings: []Finding{{ID: "r1", Confidence: "certain", Claim: "c", Evidence: "e"}}},
"duplicate id": {Findings: []Finding{
{ID: "r1", Confidence: Fact, Claim: "c", Evidence: "e"},
{ID: "r1", Confidence: Inference, Claim: "d", Evidence: "f"},
}},
}
for name, r := range cases {
if err := r.Validate(); err == nil {
t.Errorf("%s: accepted, want a refusal", name)
}
}
}
// Research sealed before ids existed must stay readable. Refusing it would
// block every task whose research predates the change, including at rotation,
// where the agent that could fix it is already gone.
func TestStoredResearchWithoutIdentityStillDecodes(t *testing.T) {
legacy := []byte(`{"findings":[{"claim":"c","evidence":"e"},{"claim":"d","evidence":"f"}]}`)
if _, err := DecodeResearch(legacy); err == nil {
t.Fatal("a new seal without ids must be refused")
}
r, err := DecodeStoredResearch(legacy)
if err != nil {
t.Fatalf("stored decode: %v", err)
}
if r.Findings[0].ID != "legacy-1" || r.Findings[1].ID != "legacy-2" {
t.Fatalf("backfilled ids are not stable: %+v", r.Findings)
}
// Inference, not fact: the old schema required evidence and made no
// verification claim, so calling it fact would upgrade it on the way in.
if r.Findings[0].Confidence != Inference {
t.Fatalf("backfilled confidence is %q, want inference", r.Findings[0].Confidence)
}
}