close worktree transport and lifecycle contract gaps
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"orchestra/internal/continuity"
|
||||
"orchestra/internal/domain"
|
||||
"orchestra/internal/herdr"
|
||||
"orchestra/internal/store"
|
||||
@@ -28,8 +29,9 @@ type Adapters interface {
|
||||
// to be a clone containing the project's remote; callers may set a separate
|
||||
// root per deployment.
|
||||
type GitWorktrees struct {
|
||||
Root string
|
||||
Repo string
|
||||
Root string
|
||||
Repo string
|
||||
TaskFileSHA string
|
||||
}
|
||||
|
||||
func (w GitWorktrees) Create(ctx context.Context, t domain.Task) (string, error) {
|
||||
@@ -41,6 +43,11 @@ func (w GitWorktrees) Create(ctx context.Context, t domain.Task) (string, error)
|
||||
}
|
||||
p := filepath.Join(w.Root, t.ID)
|
||||
if _, err := os.Stat(p); err == nil {
|
||||
if w.TaskFileSHA != "" {
|
||||
if err := continuity.VerifyTaskFile(p, w.TaskFileSHA); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
branch := "orchestra/" + t.ID
|
||||
@@ -48,6 +55,11 @@ func (w GitWorktrees) Create(ctx context.Context, t domain.Task) (string, error)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return "", fmt.Errorf("%s: %w", string(out), err)
|
||||
}
|
||||
if w.TaskFileSHA != "" {
|
||||
if err := continuity.VerifyTaskFile(p, w.TaskFileSHA); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
@@ -69,6 +81,33 @@ type Coordinator struct {
|
||||
mu sync.Mutex
|
||||
sessions map[string]herdr.Session
|
||||
loaded bool
|
||||
healthMu sync.RWMutex
|
||||
health MonitorHealth
|
||||
}
|
||||
|
||||
type MonitorHealth struct {
|
||||
Running bool `json:"running"`
|
||||
LastRun time.Time `json:"last_run"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
Expired int `json:"expired"`
|
||||
}
|
||||
|
||||
func (c *Coordinator) MonitorHealth() MonitorHealth {
|
||||
c.healthMu.RLock()
|
||||
defer c.healthMu.RUnlock()
|
||||
return c.health
|
||||
}
|
||||
func (c *Coordinator) setMonitorHealth(err error, expired int) {
|
||||
c.healthMu.Lock()
|
||||
defer c.healthMu.Unlock()
|
||||
c.health.Running = err == nil
|
||||
c.health.LastRun = time.Now().UTC()
|
||||
c.health.Expired += expired
|
||||
if err != nil {
|
||||
c.health.LastError = err.Error()
|
||||
} else {
|
||||
c.health.LastError = ""
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Coordinator) loadSessions() {
|
||||
@@ -128,6 +167,7 @@ func (c *Coordinator) Reconcile(ctx context.Context) error {
|
||||
// and frees the lease for pickup by the router.
|
||||
func (c *Coordinator) Monitor(ctx context.Context, hard float64, interval time.Duration) error {
|
||||
if err := c.Reconcile(ctx); err != nil {
|
||||
c.setMonitorHealth(err, 0)
|
||||
return err
|
||||
}
|
||||
if interval <= 0 {
|
||||
@@ -138,12 +178,58 @@ func (c *Coordinator) Monitor(ctx context.Context, hard float64, interval time.D
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
c.healthMu.Lock()
|
||||
c.health.Running = false
|
||||
c.healthMu.Unlock()
|
||||
return ctx.Err()
|
||||
case <-t.C:
|
||||
expired, err := c.expire(ctx)
|
||||
c.setMonitorHealth(err, len(expired))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
c.rotate(ctx, hard)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Coordinator) expire(ctx context.Context) ([]domain.Event, error) {
|
||||
// pane.exited is the low-latency path; lease expiry below remains the
|
||||
// authoritative backstop when herdr misses an exit notification.
|
||||
c.loadSessions()
|
||||
c.mu.Lock()
|
||||
for taskID, s := range c.sessions {
|
||||
if t, ok := c.Store.Task(taskID); ok && t.State == domain.StateLeased {
|
||||
if a, ae := c.Adapters.Adapter(s.Harness); ae == nil {
|
||||
if p, ok := a.(herdr.PaneExit); ok {
|
||||
if exited, ee := p.PaneExited(ctx, s); ee == nil && exited {
|
||||
b, _ := json.Marshal(map[string]string{"reason": "pane_exited", "harness_id": s.Harness})
|
||||
_ = c.Store.Append(domain.Event{ID: domain.NewID(), Type: "TaskReleased", TaskID: taskID, Version: t.Version + 1, Payload: b})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
c.mu.Unlock()
|
||||
events, err := c.Store.ExpireLeases(time.Now())
|
||||
if err != nil {
|
||||
return events, err
|
||||
}
|
||||
for _, e := range events {
|
||||
c.loadSessions()
|
||||
c.mu.Lock()
|
||||
s, ok := c.sessions[e.TaskID]
|
||||
delete(c.sessions, e.TaskID)
|
||||
if ok {
|
||||
if a, ae := c.Adapters.Adapter(s.Harness); ae == nil {
|
||||
_ = a.Kill(ctx, s)
|
||||
}
|
||||
}
|
||||
_ = c.saveSessionsLocked()
|
||||
c.mu.Unlock()
|
||||
}
|
||||
return events, nil
|
||||
}
|
||||
func (c *Coordinator) rotate(ctx context.Context, hard float64) {
|
||||
c.loadSessions()
|
||||
c.mu.Lock()
|
||||
|
||||
Reference in New Issue
Block a user