orchestra/internal/operations/planprogress_test.go

package operations

import (
	"errors"
	"strings"
	"testing"

	"orchestra/internal/authz"
	"orchestra/internal/domain"
	"orchestra/internal/registry"
	"orchestra/internal/store"
)

const shaOne = "1111111111111111111111111111111111111111"
const shaTwo = "2222222222222222222222222222222222222222"
const shaThree = "3333333333333333333333333333333333333333"

func planProject() registry.Project {
	p := registry.Project{
		ID: "demo", MachineAffinity: []string{"m"},
		WorkPhases: []domain.WorkPhase{domain.WorkPhaseFrame, domain.WorkPhaseResearch, domain.WorkPhasePlan, domain.WorkPhaseImplement, domain.WorkPhaseReview},
	}
	p.Verification.Allowed = [][]string{{"go", "test", "./internal/..."}, {"go", "build", "./..."}}
	return p
}

// planWith seals research and a plan, leaving the task in implement.
func planWith(t *testing.T, markdown string) (*store.Store, registry.Project, string) {
	t.Helper()
	s, id := phaseStore(t)
	// Leased, because everything these tests drive comes from a live implement
	// session. Skipping it hid F65: two coordinator-side stops omitted the
	// fencing fields Store.Append requires on a leased task, and every test
	// passed because no test ever leased one.
	lease(t, s, id)
	project := planProject()
	if _, err := AdvanceWorkPhase(s, project, id, nil); err != nil {
		t.Fatal(err)
	}
	if _, err := AdvanceWorkPhase(s, project, id, sealed(t, research)); err != nil {
		t.Fatal(err)
	}
	if _, err := AdvanceWorkPhase(s, project, id, []byte(markdown)); err != nil {
		t.Fatal(err)
	}
	return s, project, id
}

const twoPhasePlan = "# Two phase plan\n" + `
## Overview
Two phases.

## Current state
Nothing, per research:r1.

## Desired end state
Both phases done.

## Non-goals
None.

## Approach
Straightforward.

## Phase 1: Build

### Files
- a.go

### Changes
Add a.

### Verification

#### Automated
- run: ["go", "build", "./..."]

## Phase 2: Test

### Files
- b.go

### Changes
Add b.

### Verification

#### Automated
- run: ["go", "test", "./internal/..."]

#### Manual
- Confirm the output by eye.

## Testing strategy
Per phase.

## Risks and edge cases
None.

## Migration
None.

## References
- research:r1
`

func TestPassingCommandsVerifyThePhase(t *testing.T) {
	s, project, id := planWith(t, twoPhasePlan)
	if _, err := RecordPlanPhaseVerification(s, project, id, "phase-1", shaOne,
		[]VerificationRun{{Command: []string{"go", "build", "./..."}, ExitCode: 0}}); err != nil {
		t.Fatal(err)
	}
	task, _ := s.Task(id)
	rec, ok := task.PlanPhase("phase-1")
	if !ok || rec.Status != domain.PlanPhaseVerified {
		t.Fatalf("phase-1 = %+v", rec)
	}
	if rec.AtSHA != shaOne || rec.PlanRef != task.PlanRef {
		t.Fatalf("record is not bound to the plan and the commit: %+v", rec)
	}
}

// A failing command leaves the phase where it was. "Verified" is a conclusion
// about the commands, never about the request that asked for them.
func TestFailingCommandLeavesThePhaseInProgress(t *testing.T) {
	s, project, id := planWith(t, twoPhasePlan)
	if _, err := RecordPlanPhaseVerification(s, project, id, "phase-1", shaOne,
		[]VerificationRun{{Command: []string{"go", "build", "./..."}, ExitCode: 2}}); err != nil {
		t.Fatal(err)
	}
	task, _ := s.Task(id)
	rec, _ := task.PlanPhase("phase-1")
	if rec.Status != domain.PlanPhaseInProgress {
		t.Fatalf("status = %q, want in_progress", rec.Status)
	}
	if len(rec.ExitCodes) != 1 || rec.ExitCodes[0] != 2 {
		t.Fatalf("the exit code was not recorded: %+v", rec)
	}
}

// Automated checks passing is not the whole phase when manual steps exist.
func TestManualStepsHoldThePhaseUntilAHumanSignsOff(t *testing.T) {
	s, project, id := planWith(t, twoPhasePlan)
	if _, err := RecordPlanPhaseVerification(s, project, id, "phase-2", shaOne,
		[]VerificationRun{{Command: []string{"go", "test", "./internal/..."}, ExitCode: 0}}); err != nil {
		t.Fatal(err)
	}
	task, _ := s.Task(id)
	rec, _ := task.PlanPhase("phase-2")
	if rec.Status != domain.PlanPhaseAwaitingManual {
		t.Fatalf("status = %q, want awaiting_manual_verification", rec.Status)
	}

	// A generic later comment must not satisfy the gate.
	humanReply(t, s, id, "d-generic", "looks good")
	task, _ = s.Task(id)
	rec, _ = task.PlanPhase("phase-2")
	if rec.Status != domain.PlanPhaseAwaitingManual {
		t.Fatal("an unrelated comment satisfied a manual verification gate")
	}

	// The sign-off names the plan and the phase it approves.
	signOff(t, s, id, domain.PlanPhaseSubject(task.PlanRef, "phase-2"))
	task, _ = s.Task(id)
	rec, _ = task.PlanPhase("phase-2")
	if rec.Status != domain.PlanPhaseVerified {
		t.Fatalf("status = %q after a bound sign-off, want verified", rec.Status)
	}
}

// A sign-off is bound to one plan. Replanning does not inherit it.
func TestSignOffForAnotherPlanDoesNotVerify(t *testing.T) {
	s, project, id := planWith(t, twoPhasePlan)
	if _, err := RecordPlanPhaseVerification(s, project, id, "phase-2", shaOne,
		[]VerificationRun{{Command: []string{"go", "test", "./internal/..."}, ExitCode: 0}}); err != nil {
		t.Fatal(err)
	}
	signOff(t, s, id, domain.PlanPhaseSubject("some-other-plan-ref", "phase-2"))
	task, _ := s.Task(id)
	rec, _ := task.PlanPhase("phase-2")
	if rec.Status != domain.PlanPhaseAwaitingManual {
		t.Fatalf("a sign-off naming another plan verified this one: %q", rec.Status)
	}
}

// Verified at X, code moves to Y: the record stands as provenance and must
// read as stale, never as a claim about the current tree.
func TestVerificationGoesStaleWhenTheTreeMoves(t *testing.T) {
	s, project, id := planWith(t, twoPhasePlan)
	if _, err := RecordPlanPhaseVerification(s, project, id, "phase-1", shaOne,
		[]VerificationRun{{Command: []string{"go", "build", "./..."}, ExitCode: 0}}); err != nil {
		t.Fatal(err)
	}
	task, _ := s.Task(id)
	rec, _ := task.PlanPhase("phase-1")
	if rec.Stale(shaOne) {
		t.Fatal("a verification at the current head reported stale")
	}
	if !rec.Stale(shaTwo) {
		t.Fatal("a verification at an older commit did not report stale")
	}
}

// The commands come from the accepted plan. A request cannot substitute one.
func TestReportedResultsMustMatchThePlansCommands(t *testing.T) {
	s, project, id := planWith(t, twoPhasePlan)
	_, err := RecordPlanPhaseVerification(s, project, id, "phase-1", shaOne,
		[]VerificationRun{{Command: []string{"echo", "ok"}, ExitCode: 0}})
	if !errors.Is(err, ErrPlanPhase) {
		t.Fatalf("a substituted command was accepted: %v", err)
	}
}

// A command outside project policy is refused before anything runs, and the
// refusal names the project so the planner learns its real reach.
func TestPolicyRefusalHappensBeforeExecution(t *testing.T) {
	s, project, id := planWith(t, twoPhasePlan)
	project.Verification.Allowed = [][]string{{"go", "build", "./..."}}
	_, err := PlanPhaseCommands(s, project, id, "phase-2")
	if !errors.Is(err, ErrPlanPhase) {
		t.Fatalf("an unauthorised command was resolved: %v", err)
	}
	if !strings.Contains(err.Error(), "demo") || !strings.Contains(err.Error(), "go test") {
		t.Fatalf("the refusal does not name the project and the command: %v", err)
	}
	task, _ := s.Task(id)
	if _, ok := task.PlanPhase("phase-2"); ok {
		t.Fatal("a refused phase produced a record")
	}
}

// A plan sealed before plan.md declares no executable unit, and saying so
// beats inventing a phase it never had.
func TestLegacyPlanIsExplicitlyNonProgressable(t *testing.T) {
	s, id := phaseStore(t)
	project := planProject()
	if _, err := AdvanceWorkPhase(s, project, id, nil); err != nil {
		t.Fatal(err)
	}
	if _, err := AdvanceWorkPhase(s, project, id, sealed(t, research)); err != nil {
		t.Fatal(err)
	}
	// A pre-markdown plan, appended straight to the CAS the way an old ref
	// would already be there.
	ref, err := s.PutArtifact([]byte(`{"changes":[{"target":"a.go","intent":"do a thing"}]}`))
	if err != nil {
		t.Fatal(err)
	}
	task, _ := s.Task(id)
	task.PlanRef = ref
	_, err = PlanPhaseCommands(s, project, id, "phase-1")
	if err == nil {
		t.Fatal("a legacy plan resolved a phase")
	}
}

// signOff records a human decision bound to one phase of one plan, which is
// the only thing that satisfies a manual verification gate.
func signOff(t *testing.T, s *store.Store, taskID, subject string) {
	t.Helper()
	signOffFrom(t, s, taskID, subject, "signoff-"+subject)
}

// signOffFrom names the comment the sign-off came from. Two sign-offs on one
// subject are a real sequence once a rerun sends a phase back to the human,
// and provenance is unique per comment.
func signOffFrom(t *testing.T, s *store.Store, taskID, subject, externalID string) {
	t.Helper()
	task, _ := s.Task(taskID)
	if err := s.Append(domain.Event{
		ID: domain.NewID(), Type: domain.EventHumanDecisionRecorded, TaskID: taskID,
		Version: task.Version + 1, Surface: string(authz.System),
		Payload: mustJSONBytes(t, map[string]any{
			"decision_id": domain.NewID(), "kind": "answer", "subject": subject,
			"value":  "manual steps confirmed",
			"source": map[string]any{"provider": "gitea", "external_id": externalID},
		}),
	}); err != nil {
		t.Fatalf("sign off: %v", err)
	}
}

// F63, found live on run 19. A manual sign-off says a human read what this
// code prints. An edit afterwards can change exactly that, so rerunning the
// automated half at a new commit must not carry the human half with it.
func TestASignOffDoesNotSurviveTheTreeItWasGivenAgainst(t *testing.T) {
	s, project, id := planWith(t, twoPhasePlan)
	run := []VerificationRun{{Command: []string{"go", "test", "./internal/..."}, ExitCode: 0}}
	if _, err := RecordPlanPhaseVerification(s, project, id, "phase-2", shaOne, run); err != nil {
		t.Fatal(err)
	}
	task, _ := s.Task(id)
	signOff(t, s, id, domain.PlanPhaseSubject(task.PlanRef, "phase-2"))
	task, _ = s.Task(id)
	rec, _ := task.PlanPhase("phase-2")
	if rec.Status != domain.PlanPhaseVerified || rec.ManualAtSHA != shaOne {
		t.Fatalf("sign-off did not bind to the tree it read: %+v", rec)
	}

	// The tree moves and the phase is re-verified. The commands pass again;
	// the human has not seen the new output.
	if _, err := RecordPlanPhaseVerification(s, project, id, "phase-2", shaTwo, run); err != nil {
		t.Fatal(err)
	}
	task, _ = s.Task(id)
	rec, _ = task.PlanPhase("phase-2")
	if rec.Status != domain.PlanPhaseAwaitingManual {
		t.Fatalf("status = %q at a tree the human never saw, want awaiting_manual_verification", rec.Status)
	}
	if rec.ManualAtSHA != shaOne {
		t.Fatalf("the confirmed tree was lost: %+v", rec)
	}

	// A second rerun must not re-inherit it either, which is what carrying
	// ManualAtSHA forward is for.
	if _, err := RecordPlanPhaseVerification(s, project, id, "phase-2", shaThree, run); err != nil {
		t.Fatal(err)
	}
	task, _ = s.Task(id)
	rec, _ = task.PlanPhase("phase-2")
	if rec.Status != domain.PlanPhaseAwaitingManual {
		t.Fatalf("a second rerun re-inherited the sign-off: %q", rec.Status)
	}

	// Signing off again, on the tree that is now current, verifies it.
	signOffFrom(t, s, id, domain.PlanPhaseSubject(task.PlanRef, "phase-2"), "signoff-second")
	task, _ = s.Task(id)
	rec, _ = task.PlanPhase("phase-2")
	if rec.Status != domain.PlanPhaseVerified || rec.ManualAtSHA != shaThree {
		t.Fatalf("a fresh sign-off did not verify the current tree: %+v", rec)
	}
}

// Run 20: a replan reopened the plan phase, the implementer's leftover
// verification request outlived its session, and the planning session that
// replaced it executed the request. Orchestra recorded a verified phase of the
// plan it was in the middle of replacing.
func TestVerificationIsRefusedOutsideImplement(t *testing.T) {
	s, project, id := planWith(t, twoPhasePlan)
	task, _ := s.Task(id)
	m := mismatch(task.PlanRef)
	m.RequestedAction = domain.PlanMismatchReplan
	if _, err := RecordPlanMismatch(s, project, id, m, shaOne); err != nil {
		t.Fatal(err)
	}
	assertPhase(t, s, id, domain.WorkPhasePlan)

	_, err := RecordPlanPhaseVerification(s, project, id, "phase-1", shaOne,
		[]VerificationRun{{Command: []string{"go", "build", "./..."}, ExitCode: 0}})
	if !errors.Is(err, ErrPlanPhase) {
		t.Fatalf("a reopened task verified a phase of the plan being replaced: %v", err)
	}
	if !strings.Contains(err.Error(), "implement") {
		t.Fatalf("the refusal does not say which phase owns verification: %v", err)
	}
	if after, _ := s.Task(id); len(after.PlanPhases()) != 0 {
		t.Fatalf("progress was recorded anyway: %+v", after.PlanPhases())
	}
}