ops: migrate legacy event sequences safely

internal/store/migration_test.go

@@ -0,0 +1,58 @@
+package store
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"orchestra/internal/domain"
+)
+
+func TestNormalizeLegacyEventSequence(t *testing.T) {
+	dir := t.TempDir()
+	events := []domain.Event{
+		{SchemaVersion: 2, Seq: 1, ID: "created", Type: "TaskCreated", TaskID: "t", Version: 1, At: time.Now().UTC(), Payload: json.RawMessage(`{"source":"test","external_id":"1","project":"p"}`), Surface: "system"},
+		{SchemaVersion: 2, Seq: 1, ID: "leased", Type: "TaskLeased", TaskID: "t", Version: 2, At: time.Now().UTC(), Payload: json.RawMessage(`{"harness_id":"h","ttl":60,"until_ns":2000000000000000000,"expected_version":1}`), Surface: "system"},
+		{SchemaVersion: 2, Seq: 2, ID: "recovered-lease", Type: "TaskLeased", TaskID: "t", Version: 2, At: time.Now().UTC(), Payload: json.RawMessage(`{"harness_id":"h","ttl":60,"until_ns":2000000000000000000,"expected_version":1}`), Surface: "system"},
+	}
+	var raw []byte
+	for _, e := range events {
+		b, err := json.Marshal(e)
+		if err != nil {
+			t.Fatal(err)
+		}
+		raw = append(raw, append(b, '\n')...)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "events.jsonl"), raw, 0644); err != nil {
+		t.Fatal(err)
+	}
+	changed, err := NormalizeLegacyEventSequence(dir)
+	if err != nil || !changed {
+		t.Fatalf("NormalizeLegacyEventSequence = %v, %v", changed, err)
+	}
+	if backups, err := filepath.Glob(filepath.Join(dir, "events.jsonl.legacy-*")); err != nil || len(backups) != 1 {
+		t.Fatalf("backups = %v, %v", backups, err)
+	}
+	s, err := Open(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := len(s.Events(0)); got != 2 {
+		t.Fatalf("events = %d, want 2", got)
+	}
+	if got := s.Events(0)[1].ID; got != "recovered-lease" {
+		t.Fatalf("second event = %q, want recovered suffix", got)
+	}
+}
+
+func TestNormalizeLegacyEventSequenceRejectsOtherCorruption(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "events.jsonl"), []byte(`{"seq":2}`+"\n"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := NormalizeLegacyEventSequence(dir); err == nil {
+		t.Fatal("NormalizeLegacyEventSequence accepted non-legacy corruption")
+	}
+}

internal/store/store.go

@@ -2,6 +2,7 @@ package store
 
 import (
 	"bufio"
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -94,6 +95,125 @@ func Open(dir string) (*Store, error) {
 	}
 	return s, sc.Err()
 }
+
+// NormalizeLegacyEventSequence repairs one explicitly recognized pre-v2 log
+// shape: a prefix of two or more records all numbered seq=1, followed by a
+// contiguous suffix numbered 2, 3, ... that restarts the first task at
+// version 2. Early Orchestra releases emitted precisely that shape: records
+// after the first seq=1 were never included in the compatibility snapshot,
+// then were replayed again after restart. The migration retains the initial
+// TaskCreated plus the contiguous suffix, discarding only the provably
+// abandoned duplicate prefix. It is not a general corruption repair tool:
+// any other gap or duplicate is rejected so Open's fail-closed recovery
+// guarantee remains intact.
+//
+// The caller must stop every coordinator using dir first. The original log is
+// durably copied to events.jsonl.legacy-<unix-nano> before an fsync+rename
+// replacement is installed. The return value reports whether a migration was
+// needed.
+func NormalizeLegacyEventSequence(dir string) (bool, error) {
+	path := filepath.Join(dir, "events.jsonl")
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		return false, err
+	}
+	lines := bytes.Split(bytes.TrimSpace(raw), []byte{'\n'})
+	if len(lines) == 0 || (len(lines) == 1 && len(lines[0]) == 0) {
+		return false, nil
+	}
+	events := make([]domain.Event, len(lines))
+	for i, line := range lines {
+		if err := json.Unmarshal(line, &events[i]); err != nil {
+			return false, fmt.Errorf("event %d: %w", i+1, err)
+		}
+	}
+	canonical := true
+	for i, e := range events {
+		if e.Seq != uint64(i+1) {
+			canonical = false
+			break
+		}
+	}
+	if canonical {
+		return false, nil
+	}
+	prefix := 0
+	for prefix < len(events) && events[prefix].Seq == 1 {
+		prefix++
+	}
+	if prefix < 2 {
+		return false, fmt.Errorf("refusing non-legacy event sequence")
+	}
+	if events[0].Type != "TaskCreated" || events[0].Version != 1 {
+		return false, fmt.Errorf("refusing legacy sequence without an initial task creation")
+	}
+	for i := 1; i < prefix; i++ {
+		if events[i].TaskID != events[0].TaskID || events[i].Version < 2 {
+			return false, fmt.Errorf("refusing non-legacy duplicate prefix at record %d", i+1)
+		}
+	}
+	if prefix == len(events) || events[prefix].Type != "TaskLeased" || events[prefix].TaskID != events[0].TaskID || events[prefix].Version != 2 {
+		return false, fmt.Errorf("refusing legacy sequence without a task-version-2 restart")
+	}
+	for i := prefix; i < len(events); i++ {
+		want := uint64(i - prefix + 2)
+		if events[i].Seq != want {
+			return false, fmt.Errorf("refusing non-legacy event sequence at record %d: got %d, want %d", i+1, events[i].Seq, want)
+		}
+	}
+	backup := fmt.Sprintf("%s.legacy-%d", path, time.Now().UTC().UnixNano())
+	backupFile, err := os.OpenFile(backup, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+	if err != nil {
+		return false, err
+	}
+	if _, err = backupFile.Write(raw); err == nil {
+		err = backupFile.Sync()
+	}
+	if closeErr := backupFile.Close(); err == nil {
+		err = closeErr
+	}
+	if err != nil {
+		return false, err
+	}
+	tmp := path + ".sequence-migration.tmp"
+	out, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return false, err
+	}
+	kept := append(events[:1:1], events[prefix:]...)
+	for i := range kept {
+		kept[i].Seq = uint64(i + 1)
+		line, marshalErr := json.Marshal(kept[i])
+		if marshalErr != nil {
+			err = marshalErr
+			break
+		}
+		if _, err = out.Write(append(line, '\n')); err != nil {
+			break
+		}
+	}
+	if err == nil {
+		err = out.Sync()
+	}
+	if closeErr := out.Close(); err == nil {
+		err = closeErr
+	}
+	if err != nil {
+		return false, err
+	}
+	if err := os.Rename(tmp, path); err != nil {
+		return false, err
+	}
+	d, err := os.Open(dir)
+	if err != nil {
+		return false, err
+	}
+	defer d.Close()
+	if err := d.Sync(); err != nil {
+		return false, err
+	}
+	return true, nil
+}
 func (s *Store) apply(e domain.Event) error {
 	var p map[string]any
 	if err := json.Unmarshal(e.Payload, &p); err != nil {