Files
hexis/internal/execution/engine_test.go
T
kami be08938f1f Thread context through Provider.Execute and stop reporting read failures as unknown
Finding 6 of REVIEW-2026-07-30.md. runWithTimeout could not cancel anything,
because Provider.Execute took no context: the goroutine ran on to the HTTP
client's 60s timeout, outliving the 30s capability timeout. Provider.Execute
now takes a context carrying that timeout, and the workspace provider issues
its tool call with http.NewRequestWithContext, so a timed-out execution
actually tears the request down.

A read-only capability whose provider call times out now resolves to failed
rather than unknown. Spec §4.3 reserves unknown for executions whose side
effect may or may not have landed, and never retries them — which made read
failures both unretryable and indistinguishable from genuinely ambiguous
mutations, for calls that by definition have no side effect. Mutating
capabilities still resolve to unknown.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Uea55zaiWuEByEDC4UBSdd
2026-07-30 23:39:50 +04:00

360 lines
11 KiB
Go

package execution_test
import (
"context"
"errors"
"path/filepath"
"testing"
"time"
"github.com/kami/hexis/internal/domain"
"github.com/kami/hexis/internal/execution"
"github.com/kami/hexis/internal/provider"
"github.com/kami/hexis/internal/storage"
)
type fakeProvider struct {
name string
delay time.Duration
err error
// returned closes once Execute has actually returned, letting a test
// prove the provider call was cancelled rather than left running.
returned chan struct{}
// ctxErr records the context error observed by Execute, if any.
ctxErr error
}
func (p *fakeProvider) Name() string { return p.name }
func (p *fakeProvider) Execute(ctx context.Context, capability *domain.Capability, req *domain.ExecuteRequest) (map[string]any, error) {
if p.returned != nil {
defer close(p.returned)
}
if p.delay > 0 {
// A well-behaved provider aborts as soon as its context is done —
// this stands in for tearing down an in-flight HTTP request.
select {
case <-time.After(p.delay):
case <-ctx.Done():
p.ctxErr = ctx.Err()
return nil, ctx.Err()
}
}
if p.err != nil {
return nil, p.err
}
return map[string]any{"ok": true}, nil
}
func newTestEngine(t *testing.T, prov *fakeProvider) (*execution.Engine, *storage.Store) {
t.Helper()
dir := t.TempDir()
store, err := storage.Open(filepath.Join(dir, "hexis.db"))
if err != nil {
t.Fatalf("open storage: %v", err)
}
t.Cleanup(func() { store.Close() })
reg := provider.NewRegistry()
reg.Register(prov)
return execution.New(store, reg), store
}
func mustCreateCapability(t *testing.T, store storage.Interface, mutate func(*domain.Capability)) *domain.Capability {
t.Helper()
now := time.Now().UTC()
cap := &domain.Capability{
ID: domain.NewCapabilityID(),
Name: "test.capability",
Provider: "fake",
Operation: "noop",
Risk: "low",
ReadOnly: false,
Enabled: true,
CreatedAt: now,
UpdatedAt: now,
Version: 1,
TargetTypes: []string{},
Attributes: map[string]any{},
}
if mutate != nil {
mutate(cap)
}
if err := store.CreateCapability(cap); err != nil {
t.Fatalf("create capability: %v", err)
}
return cap
}
func TestExecute_FreeTextTargetRejectedWhenBound(t *testing.T) {
prov := &fakeProvider{name: "fake"}
eng, store := newTestEngine(t, prov)
cap := mustCreateCapability(t, store, func(c *domain.Capability) {
c.TargetEntityID = "ent_bound_only"
})
_, err := eng.Execute(&domain.ExecuteRequest{
CapabilityID: cap.ID,
TargetEntityID: "ent_someone_else",
})
if !errors.Is(err, domain.ErrCapabilityNotBound) {
t.Fatalf("expected ErrCapabilityNotBound, got %v", err)
}
}
func TestExecute_DisabledCapabilityRejected(t *testing.T) {
prov := &fakeProvider{name: "fake"}
eng, store := newTestEngine(t, prov)
cap := mustCreateCapability(t, store, func(c *domain.Capability) {
c.Enabled = false
})
_, err := eng.Execute(&domain.ExecuteRequest{CapabilityID: cap.ID, TargetEntityID: "ent_x"})
if !errors.Is(err, domain.ErrCapabilityDisabled) {
t.Fatalf("expected ErrCapabilityDisabled, got %v", err)
}
}
func TestExecute_ConfirmationRequiredButMissing(t *testing.T) {
prov := &fakeProvider{name: "fake"}
eng, store := newTestEngine(t, prov)
cap := mustCreateCapability(t, store, func(c *domain.Capability) {
c.RequiresConfirmation = true
})
_, err := eng.Execute(&domain.ExecuteRequest{CapabilityID: cap.ID, TargetEntityID: "ent_x"})
if !errors.Is(err, domain.ErrConfirmationRequired) {
t.Fatalf("expected ErrConfirmationRequired, got %v", err)
}
}
func TestExecute_ConfirmationValidSucceedsAndIsConsumed(t *testing.T) {
prov := &fakeProvider{name: "fake"}
eng, store := newTestEngine(t, prov)
cap := mustCreateCapability(t, store, func(c *domain.Capability) {
c.RequiresConfirmation = true
})
args := map[string]any{"foo": "bar"}
conf, err := eng.CreateConfirmation(cap.ID, "ent_x", "kami", args)
if err != nil {
t.Fatalf("create confirmation: %v", err)
}
res, err := eng.Execute(&domain.ExecuteRequest{
CapabilityID: cap.ID,
TargetEntityID: "ent_x",
Arguments: args,
ConfirmationID: conf.ID,
})
if err != nil {
t.Fatalf("execute: %v", err)
}
if res.Execution.Status != domain.ExecutionSucceeded {
t.Fatalf("expected succeeded, got %s", res.Execution.Status)
}
got, err := store.GetConfirmation(conf.ID)
if err != nil {
t.Fatalf("get confirmation: %v", err)
}
if got.State != domain.ConfirmationConsumed {
t.Fatalf("expected confirmation consumed, got %s", got.State)
}
// Reusing the same confirmation must fail.
_, err = eng.Execute(&domain.ExecuteRequest{
CapabilityID: cap.ID,
TargetEntityID: "ent_x",
Arguments: args,
ConfirmationID: conf.ID,
})
if !errors.Is(err, domain.ErrConfirmationConsumed) {
t.Fatalf("expected ErrConfirmationConsumed on replay, got %v", err)
}
}
func TestExecute_ConfirmationExpired(t *testing.T) {
prov := &fakeProvider{name: "fake"}
eng, store := newTestEngine(t, prov)
cap := mustCreateCapability(t, store, func(c *domain.Capability) {
c.RequiresConfirmation = true
})
conf, err := eng.CreateConfirmation(cap.ID, "ent_x", "kami", nil)
if err != nil {
t.Fatalf("create confirmation: %v", err)
}
// Directly backdate the row's expiry in storage to simulate TTL elapse
// without waiting out the real 120s TTL in a test.
if _, err := store.DB().Exec(`UPDATE confirmations SET expires_at = ? WHERE id = ?`,
time.Now().UTC().Add(-time.Minute).Format("2006-01-02T15:04:05.999999999Z07:00"), conf.ID); err != nil {
t.Fatalf("backdate confirmation: %v", err)
}
_, err = eng.Execute(&domain.ExecuteRequest{
CapabilityID: cap.ID,
TargetEntityID: "ent_x",
ConfirmationID: conf.ID,
})
if !errors.Is(err, domain.ErrConfirmationExpired) {
t.Fatalf("expected ErrConfirmationExpired, got %v", err)
}
got, err := store.GetConfirmation(conf.ID)
if err != nil {
t.Fatalf("get confirmation: %v", err)
}
if got.State != domain.ConfirmationExpired {
t.Fatalf("expected confirmation state expired, got %s", got.State)
}
}
func TestExecute_ArgsHashMismatchRejected(t *testing.T) {
prov := &fakeProvider{name: "fake"}
eng, store := newTestEngine(t, prov)
cap := mustCreateCapability(t, store, func(c *domain.Capability) {
c.RequiresConfirmation = true
})
conf, err := eng.CreateConfirmation(cap.ID, "ent_x", "kami", map[string]any{"n": 1})
if err != nil {
t.Fatalf("create confirmation: %v", err)
}
_, err = eng.Execute(&domain.ExecuteRequest{
CapabilityID: cap.ID,
TargetEntityID: "ent_x",
Arguments: map[string]any{"n": 2},
ConfirmationID: conf.ID,
})
if !errors.Is(err, domain.ErrConfirmationInvalid) {
t.Fatalf("expected ErrConfirmationInvalid on args mismatch, got %v", err)
}
}
func TestExecute_UnknownCapabilityVersionRejected(t *testing.T) {
prov := &fakeProvider{name: "fake"}
eng, store := newTestEngine(t, prov)
cap := mustCreateCapability(t, store, func(c *domain.Capability) {
c.RequiresConfirmation = true
})
conf, err := eng.CreateConfirmation(cap.ID, "ent_x", "kami", nil)
if err != nil {
t.Fatalf("create confirmation: %v", err)
}
// Bump the capability version (e.g. via update) so the confirmation now
// targets a stale version.
cap.Description = "changed"
if err := store.UpdateCapability(cap); err != nil {
t.Fatalf("update capability: %v", err)
}
_, err = eng.Execute(&domain.ExecuteRequest{
CapabilityID: cap.ID,
TargetEntityID: "ent_x",
ConfirmationID: conf.ID,
})
if !errors.Is(err, domain.ErrConfirmationInvalid) {
t.Fatalf("expected ErrConfirmationInvalid on stale version, got %v", err)
}
}
func TestExecute_ConcurrentInFlightRejected(t *testing.T) {
prov := &fakeProvider{name: "fake", delay: 300 * time.Millisecond}
eng, store := newTestEngine(t, prov)
cap := mustCreateCapability(t, store, nil)
done := make(chan struct{})
go func() {
eng.Execute(&domain.ExecuteRequest{CapabilityID: cap.ID, TargetEntityID: "ent_x"})
close(done)
}()
// Give the first execution time to register as "started".
time.Sleep(50 * time.Millisecond)
_, err := eng.Execute(&domain.ExecuteRequest{CapabilityID: cap.ID, TargetEntityID: "ent_x"})
if !errors.Is(err, domain.ErrExecutionInFlight) {
t.Fatalf("expected ErrExecutionInFlight, got %v", err)
}
<-done
}
func TestExecute_TimeoutYieldsUnknownNotFailed(t *testing.T) {
// Smallest supported capability timeout is 1s; use a provider slower
// than that so the engine's context deadline fires first.
prov := &fakeProvider{name: "slow", delay: 1500 * time.Millisecond}
eng, store := newTestEngine(t, prov)
cap := mustCreateCapability(t, store, func(c *domain.Capability) {
c.TimeoutSeconds = 1
c.Provider = "slow"
})
res, err := eng.Execute(&domain.ExecuteRequest{CapabilityID: cap.ID, TargetEntityID: "ent_x"})
if err != nil {
t.Fatalf("execute: %v", err)
}
if res.Execution.Status != domain.ExecutionUnknown {
t.Fatalf("expected status unknown on timeout, got %s", res.Execution.Status)
}
}
// A read_only capability has no side effect, so a timeout is an unambiguous
// failure: retryable, and distinguishable from a mutation that may or may not
// have landed. ECOSYSTEM-SPEC.md §4.3's "unknown" outcome must not be used
// for reads.
func TestExecute_ReadOnlyTimeoutYieldsFailedNotUnknown(t *testing.T) {
prov := &fakeProvider{name: "slow", delay: 1500 * time.Millisecond}
eng, store := newTestEngine(t, prov)
cap := mustCreateCapability(t, store, func(c *domain.Capability) {
c.TimeoutSeconds = 1
c.Provider = "slow"
c.Risk = "read"
c.ReadOnly = true
})
res, err := eng.Execute(&domain.ExecuteRequest{CapabilityID: cap.ID, TargetEntityID: "ent_x"})
if err != nil {
t.Fatalf("execute: %v", err)
}
if res.Execution.Status != domain.ExecutionFailed {
t.Fatalf("expected status failed on read-only timeout, got %s", res.Execution.Status)
}
if res.Execution.Error == "" {
t.Fatal("expected a timeout error message on the execution")
}
}
// The capability timeout must actually cancel the provider call. Before the
// context was threaded through Provider.Execute, the goroutine ran on to the
// provider's own (much longer) client timeout; here it must observe
// cancellation and return promptly after the 1s capability timeout, well
// before its nominal 30s delay.
func TestExecute_TimeoutCancelsProviderCall(t *testing.T) {
prov := &fakeProvider{name: "slow", delay: 30 * time.Second, returned: make(chan struct{})}
eng, store := newTestEngine(t, prov)
cap := mustCreateCapability(t, store, func(c *domain.Capability) {
c.TimeoutSeconds = 1
c.Provider = "slow"
})
if _, err := eng.Execute(&domain.ExecuteRequest{CapabilityID: cap.ID, TargetEntityID: "ent_x"}); err != nil {
t.Fatalf("execute: %v", err)
}
select {
case <-prov.returned:
case <-time.After(5 * time.Second):
t.Fatal("provider call was not cancelled: it outlived the capability timeout")
}
if !errors.Is(prov.ctxErr, context.DeadlineExceeded) {
t.Fatalf("expected provider to observe context.DeadlineExceeded, got %v", prov.ctxErr)
}
}