perf: index routing snapshots and quota usage

This commit is contained in:
kami
2026-07-30 15:36:23 +04:00
parent e8fadfc998
commit fbb13c89d2
8 changed files with 577 additions and 63 deletions
+11 -5
View File
@@ -64,11 +64,17 @@ released agent, or reject a valid completion.
## P2 — performance ## P2 — performance
- Cache/parallelise health probes; schedule from one task/worker snapshot. - **Closed 2026-07-30.** Each scheduling pass takes one atomic task/lease
Current routing repeatedly scans tasks and probes candidates per queued task. snapshot, batches cached (TTL) reachability probes concurrently, and
- Index active leases and quota windows. Do not scan the whole event log per evaluates candidate availability once. It no longer probes candidates or
availability check or rewrite the full task snapshot after every event. scans active tasks once per queued task.
- Add 1k/10k-task benchmarks with assignment and append p95 budgets. - **Closed 2026-07-30.** Active leases and per-harness, time-ordered quota
receipts are projection indexes. Availability uses indexed rolling-window
sums rather than decoding the event log; the task snapshot is a one-time
disposable compatibility cache rather than a full rewrite on every append.
- **Closed 2026-07-30.** `BenchmarkAssignPending{1K,10K}` and
`BenchmarkAppend{1K,10K}` report and enforce p95 budgets, with durable
fsync cost included in their respective paths.
## Delivery order ## Delivery order
+30 -11
View File
@@ -227,6 +227,20 @@ func (TCPReachability) Reachable(address string, timeout time.Duration) bool {
return true return true
} }
func (r Registry) Candidates(project string, check Reachability, timeout time.Duration) ([]Herdr, error) { func (r Registry) Candidates(project string, check Reachability, timeout time.Duration) ([]Herdr, error) {
return r.candidates(project, func(h Herdr) bool {
address := r.Endpoint(h)
return check == nil || check.Reachable(address, timeout)
})
}
// CandidatesWithHealth filters candidates using a health snapshot gathered by
// the router. Keeping probe execution outside this method lets a scheduling
// pass probe each herdr once, in parallel, instead of once per queued task.
func (r Registry) CandidatesWithHealth(project string, healthy map[string]bool) ([]Herdr, error) {
return r.candidates(project, func(h Herdr) bool { return healthy[h.ID] })
}
func (r Registry) candidates(project string, include func(Herdr) bool) ([]Herdr, error) {
p, ok := r.projects[project] p, ok := r.projects[project]
if !ok { if !ok {
return nil, ErrUnknownProject return nil, ErrUnknownProject
@@ -237,20 +251,25 @@ func (r Registry) Candidates(project string, check Reachability, timeout time.Du
} }
out := []Herdr{} out := []Herdr{}
for _, h := range r.herdrs { for _, h := range r.herdrs {
if !allowed[h.MachineID] { if !allowed[h.MachineID] || !include(h) {
continue continue
} }
addr := h.Address out = append(out, h)
if addr == "" {
addr = r.machines[h.MachineID].Address
if host, _, err := net.SplitHostPort(addr); err == nil {
addr = net.JoinHostPort(host, defaultHerdrPort)
}
}
if check == nil || check.Reachable(addr, timeout) {
out = append(out, h)
}
} }
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
return out, nil return out, nil
} }
// Endpoint resolves the herdr-specific address or its machine's default
// herdr endpoint. It is exposed so health checks can be batched independently
// from project routing.
func (r Registry) Endpoint(h Herdr) string {
if h.Address != "" {
return h.Address
}
address := r.machines[h.MachineID].Address
if host, _, err := net.SplitHostPort(address); err == nil {
return net.JoinHostPort(host, defaultHerdrPort)
}
return address
}
+82
View File
@@ -0,0 +1,82 @@
package router
import (
"encoding/json"
"fmt"
"orchestra/internal/authz"
"orchestra/internal/domain"
"orchestra/internal/registry"
"orchestra/internal/store"
"sort"
"testing"
"time"
)
// These are p95 budgets for a complete scheduling pass on an otherwise idle
// coordinator. The benchmark deliberately excludes fixture construction and
// disk population, which is measured separately by store's append benchmark.
const (
// Assignment retains one fsynced TaskLeased event per task; these budgets
// include that durability cost on CI-backed storage.
assignmentP95Budget1K = 6 * time.Second
assignmentP95Budget10K = time.Minute
)
func BenchmarkAssignPending1K(b *testing.B) { benchmarkAssignPending(b, 1_000, assignmentP95Budget1K) }
func BenchmarkAssignPending10K(b *testing.B) {
benchmarkAssignPending(b, 10_000, assignmentP95Budget10K)
}
func benchmarkAssignPending(b *testing.B, tasks int, budget time.Duration) {
b.Helper()
samples := make([]time.Duration, 0, b.N)
b.ResetTimer()
for sample := 0; sample < b.N; sample++ {
b.StopTimer()
s, err := store.Open(b.TempDir())
if err != nil {
b.Fatal(err)
}
for i := 0; i < tasks; i++ {
id := fmt.Sprintf("task-%d", i)
payload, _ := json.Marshal(map[string]any{"source": "benchmark", "external_id": id, "project": "p"})
if err := s.Append(domain.Event{ID: id, Type: "TaskCreated", TaskID: id, Version: 1, Payload: payload, Surface: string(authz.System)}); err != nil {
b.Fatal(err)
}
}
r, err := registry.New(registry.Config{
Projects: []registry.Project{{ID: "p", MachineAffinity: []string{"m"}}},
Machines: []registry.Machine{{ID: "m", Address: "unused"}},
Herdrs: []registry.Herdr{{ID: "h", MachineID: "m"}}, // unlimited concurrency
})
if err != nil {
b.Fatal(err)
}
router := Router{Store: s, Registry: r}
b.StartTimer()
started := time.Now()
assigned, err := router.AssignPending()
samples = append(samples, time.Since(started))
b.StopTimer()
if err != nil {
b.Fatal(err)
}
if len(assigned) != tasks {
b.Fatalf("assigned %d tasks, want %d", len(assigned), tasks)
}
}
p95 := durationP95(samples)
b.ReportMetric(float64(p95.Microseconds()), "assign_p95_us")
if p95 > budget {
b.Fatalf("assignment p95 %s exceeds budget %s for %d tasks", p95, budget, tasks)
}
}
func durationP95(samples []time.Duration) time.Duration {
if len(samples) == 0 {
return 0
}
sorted := append([]time.Duration(nil), samples...)
sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
return sorted[(len(sorted)*95+99)/100-1]
}
+107 -39
View File
@@ -10,6 +10,7 @@ import (
"orchestra/internal/store" "orchestra/internal/store"
"sort" "sort"
"strings" "strings"
"sync"
"time" "time"
) )
@@ -54,26 +55,7 @@ type QuotaAvailability struct {
} }
func (q QuotaAvailability) sumSince(harnessID string, since time.Time) (float64, bool) { func (q QuotaAvailability) sumSince(harnessID string, since time.Time) (float64, bool) {
var consumed float64 return q.Store.QuotaSince(harnessID, since)
known := false
for _, e := range q.Store.Events(0) {
if e.Type != "QuotaReported" || e.At.Before(since) {
continue
}
var p struct {
HarnessID string `json:"harness_id"`
Consumed float64 `json:"consumed"`
Known *bool `json:"known"`
}
if json.Unmarshal(e.Payload, &p) == nil && p.HarnessID == harnessID && p.Consumed >= 0 {
if p.Known != nil && !*p.Known {
return 0, false
}
known = true
consumed += p.Consumed
}
}
return consumed, known
} }
func (q QuotaAvailability) Available(h registry.Herdr) bool { func (q QuotaAvailability) Available(h registry.Herdr) bool {
@@ -116,6 +98,18 @@ type Router struct {
Retry RetryPolicy Retry RetryPolicy
Now func() time.Time Now func() time.Time
OnLease func(domain.Event) error OnLease func(domain.Event) error
// HealthTTL bounds re-use of a successful or failed reachability probe.
// A scheduling pass always has a coherent health snapshot; this cache also
// prevents bursts of TaskCreated events from repeatedly dialing the same
// herdr between passes.
HealthTTL time.Duration
healthMu sync.Mutex
health map[string]healthProbe
}
type healthProbe struct {
reachable bool
until time.Time
} }
func (r *Router) init() { func (r *Router) init() {
@@ -125,6 +119,9 @@ func (r *Router) init() {
if r.Now == nil { if r.Now == nil {
r.Now = time.Now r.Now = time.Now
} }
if r.HealthTTL <= 0 {
r.HealthTTL = 5 * time.Second
}
} }
// HandleEvent evaluates the sink after creation and after a lease is freed. // HandleEvent evaluates the sink after creation and after a lease is freed.
@@ -141,13 +138,22 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
if r.Store == nil { if r.Store == nil {
return nil, errors.New("router: store required") return nil, errors.New("router: store required")
} }
now := r.Now()
snapshot := r.Store.SchedulingSnapshot()
var queued []domain.Task var queued []domain.Task
for _, t := range r.Store.Tasks() { for _, t := range snapshot.Tasks {
if t.State == domain.StateQueued && (t.NextRetryAt.IsZero() || !r.Now().Before(t.NextRetryAt)) { if t.State == domain.StateQueued && (t.NextRetryAt.IsZero() || !now.Before(t.NextRetryAt)) {
queued = append(queued, t) queued = append(queued, t)
} }
} }
sort.SliceStable(queued, func(i, j int) bool { return importance(queued[i], r.Now()).Before(importance(queued[j], r.Now())) }) sort.SliceStable(queued, func(i, j int) bool { return importance(queued[i], now).Before(importance(queued[j], now)) })
health := r.healthSnapshot(now)
candidates := make(map[string][]registry.Herdr)
availability := make(map[string]bool)
availabilityKnown := make(map[string]bool)
projectSupport := make(map[string]bool)
projectSupportKnown := make(map[string]bool)
activeLeases := snapshot.ActiveLeases
var out []domain.Event var out []domain.Event
for _, t := range queued { for _, t := range queued {
if r.Retry.MaxAttempts > 0 && t.Attempt >= r.Retry.MaxAttempts { if r.Retry.MaxAttempts > 0 && t.Attempt >= r.Retry.MaxAttempts {
@@ -158,16 +164,34 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
out = append(out, e) out = append(out, e)
continue continue
} }
cs, err := r.Registry.Candidates(t.Project, r.Reachability, r.Timeout) cs, found := candidates[t.Project]
if err != nil { if !found {
continue var err error
cs, err = r.Registry.CandidatesWithHealth(t.Project, health)
if err != nil {
continue
}
candidates[t.Project] = cs
} }
for _, h := range cs { for _, h := range cs {
projectOK := true projectKey := h.ID + "\x00" + t.Project
if projects, ok := r.Availability.(ProjectAvailability); ok { projectOK, checked := projectSupport[projectKey], projectSupportKnown[projectKey]
projectOK = projects.Supports(h, t.Project) if !checked {
projectOK = true
if projects, ok := r.Availability.(ProjectAvailability); ok {
projectOK = projects.Supports(h, t.Project)
}
projectSupport[projectKey], projectSupportKnown[projectKey] = projectOK, true
} }
if !projectOK || !matches(t.Capability, h.Capabilities) || !r.Availability.Available(h) || occupied(r.Store, h.ID, h.Concurrency) { if !projectOK || !matches(t.Capability, h.Capabilities) {
continue
}
available, checked := availability[h.ID], availabilityKnown[h.ID]
if !checked {
available = r.Availability.Available(h)
availability[h.ID], availabilityKnown[h.ID] = available, true
}
if !available || occupiedCount(activeLeases[h.ID], h.Concurrency) {
continue continue
} }
e, err := r.Store.Lease(t.ID, h.ID, 30*time.Minute) e, err := r.Store.Lease(t.ID, h.ID, 30*time.Minute)
@@ -175,6 +199,7 @@ func (r *Router) AssignPending() ([]domain.Event, error) {
continue continue
} }
out = append(out, e) out = append(out, e)
activeLeases[h.ID]++
if r.OnLease != nil { if r.OnLease != nil {
if err := r.OnLease(e); err != nil { if err := r.OnLease(e); err != nil {
return out, err return out, err
@@ -198,17 +223,60 @@ func matches(need, have []string) bool {
} }
return true return true
} }
func occupied(s *store.Store, id string, limit int) bool { func occupiedCount(count, limit int) bool {
if limit <= 0 { return limit > 0 && count >= limit
return false }
}
n := 0 func (r *Router) healthSnapshot(now time.Time) map[string]bool {
for _, t := range s.Tasks() { herdrs := r.Registry.Herdrs()
if (t.State == domain.StateLeased || t.State == domain.StateNeedsAttention) && t.Lease != nil && t.Lease.HarnessID == id { health := make(map[string]bool, len(herdrs))
n++ if r.Reachability == nil {
for _, h := range herdrs {
health[h.ID] = true
} }
return health
} }
return n >= limit type target struct {
key string
id string
address string
}
var probes []target
r.healthMu.Lock()
if r.health == nil {
r.health = make(map[string]healthProbe)
}
for _, h := range herdrs {
key := h.ID + "\x00" + r.Registry.Endpoint(h)
if cached, ok := r.health[key]; ok && now.Before(cached.until) {
health[h.ID] = cached.reachable
continue
}
probes = append(probes, target{key: key, id: h.ID, address: r.Registry.Endpoint(h)})
}
r.healthMu.Unlock()
var wg sync.WaitGroup
var mu sync.Mutex
for _, target := range probes {
target := target
wg.Add(1)
go func() {
defer wg.Done()
reachable := r.Reachability.Reachable(target.address, r.Timeout)
mu.Lock()
health[target.id] = reachable
mu.Unlock()
}()
}
wg.Wait()
if len(probes) > 0 {
r.healthMu.Lock()
for _, target := range probes {
r.health[target.key] = healthProbe{reachable: health[target.id], until: now.Add(r.HealthTTL)}
}
r.healthMu.Unlock()
}
return health
} }
func importance(t domain.Task, now time.Time) time.Time { func importance(t domain.Task, now time.Time) time.Time {
if t.Due != nil { if t.Due != nil {
+94
View File
@@ -6,6 +6,7 @@ import (
"orchestra/internal/domain" "orchestra/internal/domain"
"orchestra/internal/registry" "orchestra/internal/registry"
"orchestra/internal/store" "orchestra/internal/store"
"sync/atomic"
"testing" "testing"
"time" "time"
) )
@@ -14,6 +15,26 @@ type reachable struct{}
func (reachable) Reachable(string, time.Duration) bool { return true } func (reachable) Reachable(string, time.Duration) bool { return true }
type countedReachability struct {
calls atomic.Int32
delay time.Duration
}
func (r *countedReachability) Reachable(string, time.Duration) bool {
r.calls.Add(1)
if r.delay > 0 {
time.Sleep(r.delay)
}
return true
}
type countedAvailability struct{ calls atomic.Int32 }
func (a *countedAvailability) Available(registry.Herdr) bool {
a.calls.Add(1)
return true
}
type projectAvailability struct{ projects map[string]bool } type projectAvailability struct{ projects map[string]bool }
func (p projectAvailability) Available(registry.Herdr) bool { return true } func (p projectAvailability) Available(registry.Herdr) bool { return true }
@@ -63,6 +84,79 @@ func TestAssignsByAffinityCapabilityAndConcurrency(t *testing.T) {
} }
} }
func TestAssignPendingSnapshotsAndCachesHealth(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
r, err := registry.New(registry.Config{
Projects: []registry.Project{{ID: "p", MachineAffinity: []string{"m"}}},
Machines: []registry.Machine{{ID: "m", Address: "unused"}},
Herdrs: []registry.Herdr{
{ID: "h1", MachineID: "m", Address: "h1:1"},
{ID: "h2", MachineID: "m", Address: "h2:1"},
{ID: "h3", MachineID: "m", Address: "h3:1"},
},
})
if err != nil {
t.Fatal(err)
}
for i := 0; i < 10; i++ {
id := domain.NewID()
payload, _ := json.Marshal(map[string]any{"source": "test", "external_id": id, "project": "p"})
if err := s.Append(domain.Event{ID: id, TaskID: id, Type: "TaskCreated", Version: 1, Payload: payload, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
}
reach := &countedReachability{}
availability := &countedAvailability{}
rt := Router{Store: s, Registry: r, Reachability: reach, Availability: availability, HealthTTL: time.Minute}
if got, err := rt.AssignPending(); err != nil || len(got) != 10 {
t.Fatalf("assigned=%d, err=%v", len(got), err)
}
if got := reach.calls.Load(); got != 3 {
t.Fatalf("health probes=%d, want one per herdr rather than one per task", got)
}
if got := availability.calls.Load(); got != 1 {
t.Fatalf("availability checks=%d, want one for selected harness", got)
}
// The next pass has no eligible work, but still obtains a health snapshot.
// It must use the TTL cache rather than re-dial all three herdrs.
if _, err := rt.AssignPending(); err != nil {
t.Fatal(err)
}
if got := reach.calls.Load(); got != 3 {
t.Fatalf("health cache missed: probes=%d, want 3", got)
}
}
func TestHealthProbesRunInParallel(t *testing.T) {
s, err := store.Open(t.TempDir())
if err != nil {
t.Fatal(err)
}
r, err := registry.New(registry.Config{
Projects: []registry.Project{{ID: "p", MachineAffinity: []string{"m"}}},
Machines: []registry.Machine{{ID: "m", Address: "unused"}},
Herdrs: []registry.Herdr{{ID: "h1", MachineID: "m", Address: "h1:1"}, {ID: "h2", MachineID: "m", Address: "h2:1"}, {ID: "h3", MachineID: "m", Address: "h3:1"}},
})
if err != nil {
t.Fatal(err)
}
payload, _ := json.Marshal(map[string]any{"source": "test", "external_id": "parallel", "project": "p"})
if err := s.Append(domain.Event{ID: "parallel", TaskID: "parallel", Type: "TaskCreated", Version: 1, Payload: payload, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
reach := &countedReachability{delay: 100 * time.Millisecond}
started := time.Now()
if _, err := (&Router{Store: s, Registry: r, Reachability: reach}).AssignPending(); err != nil {
t.Fatal(err)
}
if elapsed := time.Since(started); elapsed > 250*time.Millisecond {
t.Fatalf("health probes took %s; expected parallel probes, not ~300ms serial", elapsed)
}
}
func TestAssignPendingRequiresWorkerProjectSupport(t *testing.T) { func TestAssignPendingRequiresWorkerProjectSupport(t *testing.T) {
s, err := store.Open(t.TempDir()) s, err := store.Open(t.TempDir())
if err != nil { if err != nil {
+60
View File
@@ -0,0 +1,60 @@
package store
import (
"encoding/json"
"fmt"
"orchestra/internal/authz"
"orchestra/internal/domain"
"sort"
"testing"
"time"
)
// Append includes the fsynced event-log write and sparse projection-cache
// checkpoint. The budgets catch accidental return to an O(tasks) snapshot
// rewrite on every event while remaining practical on CI-backed storage.
const (
appendP95Budget1K = 10 * time.Second
appendP95Budget10K = 90 * time.Second
)
func BenchmarkAppend1K(b *testing.B) { benchmarkAppend(b, 1_000, appendP95Budget1K) }
func BenchmarkAppend10K(b *testing.B) { benchmarkAppend(b, 10_000, appendP95Budget10K) }
func benchmarkAppend(b *testing.B, events int, budget time.Duration) {
b.Helper()
samples := make([]time.Duration, 0, b.N)
b.ResetTimer()
for sample := 0; sample < b.N; sample++ {
b.StopTimer()
s, err := Open(b.TempDir())
if err != nil {
b.Fatal(err)
}
b.StartTimer()
started := time.Now()
for i := 0; i < events; i++ {
id := fmt.Sprintf("task-%d", i)
payload, _ := json.Marshal(map[string]any{"source": "benchmark", "external_id": id, "project": "p"})
if err := s.Append(domain.Event{ID: id, Type: "TaskCreated", TaskID: id, Version: 1, Payload: payload, Surface: string(authz.System)}); err != nil {
b.Fatal(err)
}
}
samples = append(samples, time.Since(started))
b.StopTimer()
}
p95 := appendDurationP95(samples)
b.ReportMetric(float64(p95.Microseconds()), "append_p95_us")
if p95 > budget {
b.Fatalf("append p95 %s exceeds budget %s for %d events", p95, budget, events)
}
}
func appendDurationP95(samples []time.Duration) time.Duration {
if len(samples) == 0 {
return 0
}
sorted := append([]time.Duration(nil), samples...)
sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
return sorted[(len(sorted)*95+99)/100-1]
}
+147 -8
View File
@@ -9,10 +9,26 @@ import (
"orchestra/internal/domain" "orchestra/internal/domain"
"os" "os"
"path/filepath" "path/filepath"
"sort"
"sync" "sync"
"time" "time"
) )
// QuotaUsage is an indexed native-usage receipt. It is deliberately kept
// separate from the event log: event history remains authoritative, while
// routing must not decode every historical event for each availability check.
type QuotaUsage struct {
At time.Time
Consumed float64
Known bool
}
type quotaIndex struct {
records []QuotaUsage // ordered by At
prefix []float64
unknownPrefix []int
}
type Store struct { type Store struct {
mu sync.Mutex mu sync.Mutex
path string path string
@@ -20,15 +36,19 @@ type Store struct {
events []domain.Event events []domain.Event
tasks map[string]domain.Task tasks map[string]domain.Task
external map[string]string external map[string]string
snapshot string // activeLeases is the routing occupancy index. Needs-attention retains a
seq uint64 // fenced owner, so it counts as active until that lease is released.
activeLeases map[string]map[string]struct{}
quota map[string]quotaIndex
snapshot string
seq uint64
} }
func Open(dir string) (*Store, error) { func Open(dir string) (*Store, error) {
if err := os.MkdirAll(dir, 0755); err != nil { if err := os.MkdirAll(dir, 0755); err != nil {
return nil, err return nil, err
} }
s := &Store{path: filepath.Join(dir, "events.jsonl"), cas: filepath.Join(dir, "cas"), snapshot: filepath.Join(dir, "snapshot.json"), tasks: map[string]domain.Task{}, external: map[string]string{}} s := &Store{path: filepath.Join(dir, "events.jsonl"), cas: filepath.Join(dir, "cas"), snapshot: filepath.Join(dir, "snapshot.json"), tasks: map[string]domain.Task{}, external: map[string]string{}, activeLeases: map[string]map[string]struct{}{}, quota: map[string]quotaIndex{}}
if err := os.MkdirAll(s.cas, 0755); err != nil { if err := os.MkdirAll(s.cas, 0755); err != nil {
return nil, err return nil, err
} }
@@ -58,7 +78,8 @@ func Open(dir string) (*Store, error) {
if t, ok := s.tasks[e.TaskID]; ok && e.Version != t.Version+1 { if t, ok := s.tasks[e.TaskID]; ok && e.Version != t.Version+1 {
return nil, domain.ErrConflict return nil, domain.ErrConflict
} }
if _, ok := s.tasks[e.TaskID]; !ok && e.Type != "TaskCreated" { global := e.Type == "QuotaReported" || e.Type == "StandupAdvisory" || e.Type == "ApprovalGranted" || e.Type == "ApprovalDenied"
if _, ok := s.tasks[e.TaskID]; !ok && e.Type != "TaskCreated" && !global {
return nil, domain.ErrNotFound return nil, domain.ErrNotFound
} }
s.events = append(s.events, e) s.events = append(s.events, e)
@@ -79,7 +100,17 @@ func (s *Store) apply(e domain.Event) error {
return err return err
} }
t := s.tasks[e.TaskID] t := s.tasks[e.TaskID]
if e.Type == "QuotaReported" || e.Type == "StandupAdvisory" || e.Type == "ApprovalGranted" || e.Type == "ApprovalDenied" { if e.Type == "QuotaReported" {
known := true
if v, ok := p["known"].(bool); ok {
known = v
}
harness, _ := p["harness_id"].(string)
consumed, _ := p["consumed"].(float64)
s.addQuotaUsage(harness, QuotaUsage{At: e.At, Consumed: consumed, Known: known})
return nil
}
if e.Type == "StandupAdvisory" || e.Type == "ApprovalGranted" || e.Type == "ApprovalDenied" {
return nil return nil
} }
switch e.Type { switch e.Type {
@@ -266,10 +297,72 @@ func (s *Store) apply(e domain.Event) error {
} }
} }
t.Version = e.Version t.Version = e.Version
s.tasks[e.TaskID] = t s.replaceTask(e.TaskID, t)
return nil return nil
} }
func activeLease(t domain.Task) (string, bool) {
if (t.State != domain.StateLeased && t.State != domain.StateNeedsAttention) || t.Lease == nil {
return "", false
}
return t.Lease.HarnessID, t.Lease.HarnessID != ""
}
func (s *Store) replaceTask(id string, next domain.Task) {
if previous, ok := s.tasks[id]; ok {
if harness, active := activeLease(previous); active {
delete(s.activeLeases[harness], id)
if len(s.activeLeases[harness]) == 0 {
delete(s.activeLeases, harness)
}
}
}
if harness, active := activeLease(next); active {
if s.activeLeases[harness] == nil {
s.activeLeases[harness] = map[string]struct{}{}
}
s.activeLeases[harness][id] = struct{}{}
}
s.tasks[id] = next
}
func (s *Store) addQuotaUsage(harness string, usage QuotaUsage) {
index := s.quota[harness]
// Receipts normally arrive in timestamp order, making index maintenance
// O(1). Keep the out-of-order path correct for replay and delayed worker
// reports without imposing its rebuild cost on the common append path.
at := sort.Search(len(index.records), func(i int) bool { return usage.At.Before(index.records[i].At) })
if at == len(index.records) {
if len(index.prefix) == 0 {
index.prefix = []float64{0}
index.unknownPrefix = []int{0}
}
prefix := index.prefix[len(index.prefix)-1]
unknown := index.unknownPrefix[len(index.unknownPrefix)-1]
index.records = append(index.records, usage)
index.prefix = append(index.prefix, prefix+usage.Consumed)
if !usage.Known {
unknown++
}
index.unknownPrefix = append(index.unknownPrefix, unknown)
s.quota[harness] = index
return
}
index.records = append(index.records, QuotaUsage{})
copy(index.records[at+1:], index.records[at:])
index.records[at] = usage
index.prefix = make([]float64, len(index.records)+1)
index.unknownPrefix = make([]int, len(index.records)+1)
for i, receipt := range index.records {
index.prefix[i+1] = index.prefix[i] + receipt.Consumed
index.unknownPrefix[i+1] = index.unknownPrefix[i]
if !receipt.Known {
index.unknownPrefix[i+1]++
}
}
s.quota[harness] = index
}
func retryBackoff(attempt int) time.Duration { func retryBackoff(attempt int) time.Duration {
if attempt < 1 { if attempt < 1 {
attempt = 1 attempt = 1
@@ -392,8 +485,13 @@ func (s *Store) Append(e domain.Event) error {
s.events = append(s.events, e) s.events = append(s.events, e)
s.seq = e.Seq s.seq = e.Seq
// Snapshot failure does not roll back a committed event. Open always // Snapshot failure does not roll back a committed event. Open always
// rebuilds from the log, so leaving a stale cache is safe. // rebuilds from the log, so leaving a stale cache is safe. Keep only the
_ = s.writeSnapshot() // initial compatibility cache: repeatedly serializing the full projection
// turns an otherwise O(1) append into O(tasks) work and is never used for
// recovery.
if s.seq == 1 {
_ = s.writeSnapshot()
}
return nil return nil
} }
@@ -491,6 +589,47 @@ func (s *Store) Tasks() []domain.Task {
} }
return out return out
} }
// SchedulingSnapshot captures the task projection and lease occupancy under
// one lock. A routing pass uses this immutable view, then updates its local
// occupancy as it issues leases; it never repeatedly scan-locks the store.
type SchedulingSnapshot struct {
Tasks []domain.Task
ActiveLeases map[string]int
}
func (s *Store) SchedulingSnapshot() SchedulingSnapshot {
s.mu.Lock()
defer s.mu.Unlock()
snapshot := SchedulingSnapshot{
Tasks: make([]domain.Task, 0, len(s.tasks)),
ActiveLeases: make(map[string]int, len(s.activeLeases)),
}
for _, task := range s.tasks {
snapshot.Tasks = append(snapshot.Tasks, task)
}
for harness, leases := range s.activeLeases {
snapshot.ActiveLeases[harness] = len(leases)
}
return snapshot
}
// QuotaSince answers a rolling-window usage query from the per-harness index
// instead of walking events.jsonl. known is false when the interval has no
// native receipt or any receipt explicitly reports unknown usage.
func (s *Store) QuotaSince(harness string, since time.Time) (consumed float64, known bool) {
s.mu.Lock()
defer s.mu.Unlock()
index, ok := s.quota[harness]
if !ok {
return 0, false
}
start := sort.Search(len(index.records), func(i int) bool { return !index.records[i].At.Before(since) })
if start == len(index.records) {
return 0, false
}
return index.prefix[len(index.records)] - index.prefix[start], index.unknownPrefix[len(index.records)] == index.unknownPrefix[start]
}
func (s *Store) Events(since uint64) []domain.Event { func (s *Store) Events(since uint64) []domain.Event {
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
+46
View File
@@ -212,6 +212,52 @@ func TestOpenRebuildsOnlyFromLogAndIgnoresCorruptSnapshot(t *testing.T) {
} }
} }
func TestSchedulingAndQuotaIndexesReplayFromEvents(t *testing.T) {
dir := t.TempDir()
s, err := Open(dir)
if err != nil {
t.Fatal(err)
}
if err := s.Append(created("indexed")); err != nil {
t.Fatal(err)
}
if _, err := s.Lease("task-1", "h1", time.Hour); err != nil {
t.Fatal(err)
}
snapshot := s.SchedulingSnapshot()
if got := snapshot.ActiveLeases["h1"]; got != 1 {
t.Fatalf("active lease index=%d, want 1", got)
}
now := time.Now().UTC()
report := func(at time.Time, consumed float64, known bool) {
payload, _ := json.Marshal(map[string]any{"harness_id": "h1", "consumed": consumed, "known": known})
if err := s.Append(domain.Event{ID: domain.NewID(), Type: "QuotaReported", TaskID: "quota", Version: 1, At: at, Payload: payload, Surface: string(authz.System)}); err != nil {
t.Fatal(err)
}
}
// Deliberately append out of timestamp order: the index must answer the
// rolling query by receipt time, not event sequence.
report(now, 4, true)
report(now.Add(-2*time.Hour), 3, true)
report(now.Add(-time.Hour), 1, false)
if used, known := s.QuotaSince("h1", now.Add(-90*time.Minute)); used != 5 || known {
t.Fatalf("indexed quota=(%v,%v), want (5,false)", used, known)
}
if used, known := s.QuotaSince("h1", now.Add(-30*time.Minute)); used != 4 || !known {
t.Fatalf("recent indexed quota=(%v,%v), want (4,true)", used, known)
}
reopened, err := Open(dir)
if err != nil {
t.Fatal(err)
}
if got := reopened.SchedulingSnapshot().ActiveLeases["h1"]; got != 1 {
t.Fatalf("replayed active lease index=%d, want 1", got)
}
if used, known := reopened.QuotaSince("h1", now.Add(-90*time.Minute)); used != 5 || known {
t.Fatalf("replayed indexed quota=(%v,%v), want (5,false)", used, known)
}
}
func TestReplayLegacyLeaseDerivesNonRenewableFence(t *testing.T) { func TestReplayLegacyLeaseDerivesNonRenewableFence(t *testing.T) {
dir := t.TempDir() dir := t.TempDir()
until := time.Now().Add(time.Hour).UnixNano() until := time.Now().Add(time.Hour).UnixNano()