checkpoint: multi-repo Gitea ingestion, per-project repos, rotation anchor_sha fix

Pre-existing uncommitted work found at session start: rotation now emits
anchor_sha on TaskReleased (previously silently dropped by store.Append
validation), multi-repo Gitea provider support, per-project git worktree
roots, and associated test coverage. Committing as a checkpoint before
starting remediation work tracked in AUDIT.md.
This commit is contained in:
kami
2026-07-27 18:15:02 +04:00
parent 325c684eb0
commit ce6f02f9e6
31 changed files with 2717 additions and 320 deletions
+166 -34
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"io"
"log"
"net"
"net/http"
"orchestra/internal/admin"
"orchestra/internal/authz"
@@ -26,6 +27,24 @@ import (
)
func id() string { return domain.NewID() }
const defaultHerdrPort = "9245"
func herdrAddress(rr registry.Registry, h registry.Herdr) string {
if h.Address != "" {
return h.Address
}
m, ok := rr.Machine(h.MachineID)
if !ok {
return ""
}
host, _, err := net.SplitHostPort(m.Address)
if err != nil {
return m.Address
}
return net.JoinHostPort(host, defaultHerdrPort)
}
func main() {
dir := os.Getenv("ORCHESTRA_DATA")
if dir == "" {
@@ -37,27 +56,34 @@ func main() {
}
var rr registry.Registry
var rt *router.Router
var coordinator *orchestrator.Coordinator
if config := os.Getenv("ORCHESTRA_CONFIG"); config != "" {
if rr, err = registry.Load(config); err != nil {
log.Fatalf("load orchestra config: %v", err)
}
rt = &router.Router{Store: s, Registry: rr, Reachability: registry.TCPReachability{}, Timeout: time.Second, Retry: router.RetryPolicy{MaxAttempts: 3, Backoff: time.Minute}}
limits := map[string]float64{}
limits := map[string]router.QuotaWindowLimits{}
for _, h := range rr.Herdrs() {
if h.QuotaLimit > 0 {
limits[h.ID] = h.QuotaLimit
w := router.QuotaWindowLimits{FiveHour: h.QuotaLimit5h, Weekly: h.QuotaLimitWeekly}
if w.Weekly <= 0 && h.QuotaLimit > 0 {
// Back-compat: the old single-window field meant weekly.
w.Weekly = h.QuotaLimit
}
if w.FiveHour > 0 || w.Weekly > 0 {
limits[h.ID] = w
}
}
if len(limits) > 0 {
rt.Availability = router.QuotaAvailability{Store: s, Limits: limits, Window: 7 * 24 * time.Hour}
rt.Availability = router.QuotaAvailability{Store: s, Limits: limits}
}
if repo, root := os.Getenv("ORCHESTRA_REPO"), os.Getenv("ORCHESTRA_WORKTREE_ROOT"); repo != "" && root != "" {
adapters := map[string]herdr.Adapter{}
for _, h := range rr.Herdrs() {
if h.Address == "" {
address := herdrAddress(rr, h)
if address == "" {
continue
}
client := herdr.New(h.Address)
client := herdr.New(address)
protocol := h.Protocol
if protocol == "" {
protocol = os.Getenv("ORCHESTRA_HERDR_PROTOCOL")
@@ -77,7 +103,17 @@ func main() {
log.Printf("herdr %s has unsupported harness %q", h.ID, h.Harness)
}
}
coordinator := &orchestrator.Coordinator{Store: s, StatePath: filepath.Join(dir, "runtime-sessions.json"), Worktrees: orchestrator.GitWorktrees{Root: root, Repo: repo}, Adapters: orchestrator.AdapterFactory{Herdrs: adapters}}
projectRepos := map[string]orchestrator.ProjectRepo{}
for _, p := range rr.Projects() {
if p.Repo != "" && p.WorktreeRoot != "" {
projectRepos[p.ID] = orchestrator.ProjectRepo{Repo: p.Repo, WorktreeRoot: p.WorktreeRoot}
}
}
worktrees := orchestrator.PerProjectGitWorktrees{
Projects: projectRepos,
Default: orchestrator.GitWorktrees{Root: root, Repo: repo},
}
coordinator = &orchestrator.Coordinator{Store: s, StatePath: filepath.Join(dir, "runtime-sessions.json"), Worktrees: worktrees, Adapters: orchestrator.AdapterFactory{Herdrs: adapters}}
rt.OnLease = func(e domain.Event) error { return coordinator.Start(context.Background(), e) }
hard := 0.75
if v, parseErr := strconv.ParseFloat(os.Getenv("ORCHESTRA_OCCUPANCY_HARD"), 64); parseErr == nil && v > 0 && v < 1 {
@@ -96,7 +132,7 @@ func main() {
for _, t := range s.Tasks() {
if t.State == domain.StateLeased && t.Lease != nil && t.Lease.HarnessID == w.ID {
p, _ := json.Marshal(map[string]any{"reason": "worker_offline", "harness_id": w.ID})
e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: t.ID, Version: t.Version + 1, Payload: p}
e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: t.ID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err == nil && rt != nil {
_, _ = rt.HandleEvent(e)
}
@@ -126,7 +162,7 @@ func main() {
return
}
b, _ := json.Marshal(p)
e := domain.Event{ID: id(), Type: "TaskCreated", TaskID: id(), Version: 1, Payload: b}
e := domain.Event{ID: id(), Type: "TaskCreated", TaskID: id(), Version: 1, Payload: b, Surface: string(surface(r))}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 400)
return
@@ -147,6 +183,24 @@ func main() {
}
json.NewEncoder(w).Encode(s.Events(n))
})
mux.HandleFunc("/v1/handoffs", func(w http.ResponseWriter, r *http.Request) {
out := make([]domain.Event, 0)
for _, e := range s.Events(0) {
if e.Type == "TaskReleased" {
out = append(out, e)
}
}
json.NewEncoder(w).Encode(out)
})
mux.HandleFunc("/v1/quotas", func(w http.ResponseWriter, r *http.Request) {
out := make([]domain.Event, 0)
for _, e := range s.Events(0) {
if e.Type == "QuotaReported" {
out = append(out, e)
}
}
json.NewEncoder(w).Encode(out)
})
mux.HandleFunc("/v1/artifacts", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
@@ -223,7 +277,7 @@ func main() {
return
}
b, _ := json.Marshal(map[string]any{"subject_ref": p.AdvisoryID})
e := domain.Event{ID: id(), Type: "ApprovalGranted", TaskID: "system", Version: 0, Payload: b}
e := domain.Event{ID: id(), Type: "ApprovalGranted", TaskID: "system", Version: 0, Payload: b, Surface: string(surface(r))}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 400)
return
@@ -256,7 +310,16 @@ func main() {
adminServer := &admin.Server{Store: s, RouterReady: rt != nil, Providers: providerHealth, Probes: map[string]admin.ProbeFunc{
"router": func() (bool, string) { return rt != nil, "configured router" },
"gitea": func() (bool, string) {
return os.Getenv("ORCHESTRA_GITEA_URL") == "" || providerHealth["gitea"] != nil, "configured provider"
configured := os.Getenv("ORCHESTRA_GITEA_URL") != "" || os.Getenv("ORCHESTRA_GITEA_CONFIG") != ""
if !configured {
return true, "configured provider"
}
for name := range providerHealth {
if name == "gitea" || strings.HasPrefix(name, "gitea:") {
return true, "configured provider"
}
}
return false, "configured provider"
},
"jsonl": func() (bool, string) {
return os.Getenv("ORCHESTRA_JSONL") == "" || providerHealth["jsonl"] != nil, "configured provider"
@@ -267,6 +330,26 @@ func main() {
mux.HandleFunc("/v1/admin/diagnostics", adminServer.Diagnostics)
mux.HandleFunc("/v1/tasks/", func(w http.ResponseWriter, r *http.Request) {
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
if r.Method == http.MethodGet && coordinator != nil && len(parts) == 4 {
taskID, view := parts[2], parts[3]
if view == "health" {
if h, ok := coordinator.MonitorHealth().Sessions[taskID]; ok {
json.NewEncoder(w).Encode(h)
return
}
http.Error(w, "session health not found", 404)
return
}
if view == "capture" {
body, err := coordinator.Capture(r.Context(), taskID, r.URL.Query().Get("source"))
if err != nil {
http.Error(w, err.Error(), 404)
return
}
json.NewEncoder(w).Encode(map[string]string{"task_id": taskID, "source": r.URL.Query().Get("source"), "text": body})
return
}
}
if len(parts) < 4 || len(parts) > 5 || r.Method != "POST" {
http.Error(w, "not found", http.StatusNotFound)
return
@@ -300,7 +383,7 @@ func main() {
by = "surface"
}
b, _ := json.Marshal(map[string]any{"subject_ref": taskID, "by": by})
e := domain.Event{ID: id(), Type: typ, TaskID: taskID, Version: t.Version + 1, Payload: b}
e := domain.Event{ID: id(), Type: typ, TaskID: taskID, Version: t.Version + 1, Payload: b, Surface: string(surface(r))}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 409)
return
@@ -321,7 +404,7 @@ func main() {
}
b, _ := json.Marshal(map[string]any{"subject_ref": taskID, "options": p.Options})
t, _ := s.Task(taskID)
e := domain.Event{ID: id(), Type: "ApprovalRequested", TaskID: taskID, Version: t.Version + 1, Payload: b}
e := domain.Event{ID: id(), Type: "ApprovalRequested", TaskID: taskID, Version: t.Version + 1, Payload: b, Surface: string(surface(r))}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 400)
return
@@ -377,7 +460,7 @@ func main() {
return
}
ePayload, _ := json.Marshal(p)
e = domain.Event{ID: id(), Type: types[action], TaskID: taskID, Version: t.Version + 1, Payload: ePayload}
e = domain.Event{ID: id(), Type: types[action], TaskID: taskID, Version: t.Version + 1, Payload: ePayload, Surface: string(surface(r))}
err = s.Append(e)
default:
http.Error(w, "unknown action", 404)
@@ -416,6 +499,13 @@ func main() {
}
json.NewEncoder(w).Encode(out)
})
mux.HandleFunc("/v1/orchestrator/health", func(w http.ResponseWriter, r *http.Request) {
if coordinator == nil {
http.Error(w, "orchestrator unavailable", http.StatusServiceUnavailable)
return
}
json.NewEncoder(w).Encode(coordinator.MonitorHealth())
})
mux.HandleFunc("/v1/federation/workers", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
json.NewEncoder(w).Encode(workers.Snapshot())
@@ -521,6 +611,7 @@ func main() {
TaskID string `json:"task_id"`
TTLSeconds int `json:"ttl_seconds"`
HandoffRef string `json:"handoff_ref"`
AnchorSHA string `json:"anchor_sha"`
}
if json.NewDecoder(r.Body).Decode(&b) != nil || b.TaskID == "" {
http.Error(w, "invalid lease body", 400)
@@ -551,8 +642,12 @@ func main() {
http.Error(w, "handoff_ref required", 400)
return
}
p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3]})
e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: b.TaskID, Version: t.Version + 1, Payload: p}
if len(b.AnchorSHA) != 40 {
http.Error(w, "anchor_sha required", 400)
return
}
p, _ := json.Marshal(map[string]any{"handoff_ref": b.HandoffRef, "harness_id": parts[3], "anchor_sha": b.AnchorSHA})
e := domain.Event{ID: id(), Type: "TaskReleased", TaskID: b.TaskID, Version: t.Version + 1, Payload: p, Surface: string(authz.System)}
if err := s.Append(e); err != nil {
http.Error(w, err.Error(), 409)
return
@@ -562,25 +657,57 @@ func main() {
}
json.NewEncoder(w).Encode(e)
})
if base := os.Getenv("ORCHESTRA_GITEA_URL"); base != "" {
g := provider.Gitea{BaseURL: base, Token: os.Getenv("ORCHESTRA_GITEA_TOKEN"), WebhookSecret: os.Getenv("ORCHESTRA_GITEA_WEBHOOK_SECRET"), Owner: os.Getenv("ORCHESTRA_GITEA_OWNER"), Repo: os.Getenv("ORCHESTRA_GITEA_REPO")}
reflecting := provider.ReflectingSink{Sink: s, Tasks: s, Reflector: g}
mux.Handle("/v1/providers/gitea/webhook", g.WebhookHandler(reflecting))
sup := &provider.Supervisor{Name: "gitea", Run: func(ctx context.Context) error {
pollCtx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
_, err := g.Poll(pollCtx, reflecting)
if err == nil {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Minute):
}
}
return err
var giteaSources []provider.GiteaSourceConfig
if path := os.Getenv("ORCHESTRA_GITEA_CONFIG"); path != "" {
cfgs, err := provider.LoadGiteaConfigs(path)
if err != nil {
log.Fatalf("load gitea config: %v", err)
}
giteaSources = cfgs
} else if base := os.Getenv("ORCHESTRA_GITEA_URL"); base != "" {
// Legacy single-repo configuration: project defaults to the repo
// name, matching the historical (pre-multi-source) behavior.
giteaSources = []provider.GiteaSourceConfig{{
Project: os.Getenv("ORCHESTRA_GITEA_REPO"), BaseURL: base,
Owner: os.Getenv("ORCHESTRA_GITEA_OWNER"), Repo: os.Getenv("ORCHESTRA_GITEA_REPO"),
Token: os.Getenv("ORCHESTRA_GITEA_TOKEN"), WebhookSecret: os.Getenv("ORCHESTRA_GITEA_WEBHOOK_SECRET"),
}}
sup.Start(context.Background())
providerHealth["gitea"] = sup
}
if len(giteaSources) > 0 {
reflectors := map[string]provider.Gitea{}
for _, c := range giteaSources {
g := provider.Gitea{BaseURL: c.BaseURL, Token: c.Token, WebhookSecret: c.WebhookSecret, Owner: c.Owner, Repo: c.Repo, Project: c.Project}
reflectors[g.SourceName()] = g
}
reflecting := provider.ReflectingSink{Sink: s, Tasks: s, Reflector: provider.MultiGitea{Sources: reflectors}}
for _, c := range giteaSources {
g := provider.Gitea{BaseURL: c.BaseURL, Token: c.Token, WebhookSecret: c.WebhookSecret, Owner: c.Owner, Repo: c.Repo, Project: c.Project}
name := "gitea:" + c.Project
webhookPath := "/v1/providers/gitea/webhook/" + c.Project
if len(giteaSources) == 1 && os.Getenv("ORCHESTRA_GITEA_CONFIG") == "" {
// Preserve the legacy unprefixed webhook path when running
// the single-source (env-var) configuration, so existing
// Gitea webhook configs don't need to be re-pointed.
webhookPath = "/v1/providers/gitea/webhook"
name = "gitea"
}
mux.Handle(webhookPath, g.WebhookHandler(reflecting))
sup := &provider.Supervisor{Name: name, Run: func(ctx context.Context) error {
pollCtx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
_, err := g.Poll(pollCtx, reflecting)
if err == nil {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Minute):
}
}
return err
}}
sup.Start(context.Background())
providerHealth[name] = sup
}
}
if path := os.Getenv("ORCHESTRA_JSONL"); path != "" {
sup := &provider.Supervisor{Name: "jsonl", Run: func(ctx context.Context) error {
@@ -620,6 +747,11 @@ func main() {
}
}
}()
if rt != nil {
if _, err := rt.AssignPending(); err != nil {
log.Printf("startup task assignment: %v", err)
}
}
port := os.Getenv("ORCHESTRA_PORT")
if port == "" {
port = "9145"