feat(work): make agent execution recovery durable
This commit is contained in:
@@ -2,6 +2,8 @@ package application
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/work"
|
||||
@@ -33,16 +35,75 @@ func ExecuteAgentDispatch(ctx context.Context, store ports.ExecutionStore, execu
|
||||
if err := store.AcceptAgentCallback(ctx, heartbeat); err != nil {
|
||||
return work.AgentExecutionResult{}, err
|
||||
}
|
||||
result, err := executor.Execute(ctx, work.AgentExecutionRequest{Intent: intent, WorkerID: workerID, FencingToken: token, LeaseExpiresAt: lease.ExpiresAt})
|
||||
executionCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
var stateMu sync.Mutex
|
||||
currentLease := lease
|
||||
lastSequence := heartbeat.Sequence
|
||||
heartbeatDone := make(chan struct{})
|
||||
heartbeatStopped := make(chan struct{})
|
||||
heartbeatFailure := make(chan error, 1)
|
||||
interval := ttl / 3
|
||||
if interval <= 0 {
|
||||
return work.AgentExecutionResult{}, fmt.Errorf("invalid heartbeat interval")
|
||||
}
|
||||
go func() {
|
||||
defer close(heartbeatStopped)
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-heartbeatDone:
|
||||
return
|
||||
case observedAt := <-ticker.C:
|
||||
stateMu.Lock()
|
||||
nextLease, renewErr := domain.RenewAgentLease(currentLease, workerID, token, observedAt, ttl)
|
||||
if renewErr == nil {
|
||||
renewErr = store.RenewAgentLease(executionCtx, nextLease, nextLease.ExpiresAt)
|
||||
}
|
||||
if renewErr == nil {
|
||||
lastSequence++
|
||||
heartbeat = work.AgentCallback{RunID: intent.RunID, WorkerID: workerID, FencingToken: token, Sequence: lastSequence, Kind: work.AgentCallbackHeartbeat, ObservedAt: observedAt}
|
||||
renewErr = domain.ValidateAgentCallback(heartbeat, nextLease, lastSequence-1)
|
||||
}
|
||||
if renewErr == nil {
|
||||
renewErr = store.AcceptAgentCallback(executionCtx, heartbeat)
|
||||
}
|
||||
if renewErr == nil {
|
||||
currentLease = nextLease
|
||||
}
|
||||
stateMu.Unlock()
|
||||
if renewErr != nil {
|
||||
select {
|
||||
case heartbeatFailure <- renewErr:
|
||||
default:
|
||||
}
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
result, err := executor.Execute(executionCtx, work.AgentExecutionRequest{Intent: intent, WorkerID: workerID, FencingToken: token, LeaseExpiresAt: lease.ExpiresAt})
|
||||
close(heartbeatDone)
|
||||
<-heartbeatStopped
|
||||
select {
|
||||
case heartbeatErr := <-heartbeatFailure:
|
||||
return work.AgentExecutionResult{}, heartbeatErr
|
||||
default:
|
||||
}
|
||||
if err != nil {
|
||||
return work.AgentExecutionResult{}, err
|
||||
}
|
||||
stateMu.Lock()
|
||||
lease = currentLease
|
||||
stateMu.Unlock()
|
||||
observedAt := time.UnixMilli(result.ObservedAtUnixMillis)
|
||||
callback := work.AgentCallback{RunID: intent.RunID, WorkerID: workerID, FencingToken: token, Sequence: 2, Kind: work.AgentCallbackResult, ObservedAt: observedAt}
|
||||
callback := work.AgentCallback{RunID: intent.RunID, WorkerID: workerID, FencingToken: token, Sequence: lastSequence + 1, Kind: work.AgentCallbackResult, ObservedAt: observedAt}
|
||||
if result.State == work.AgentExecutionWaiting {
|
||||
callback.Kind = work.AgentCallbackCheckpoint
|
||||
}
|
||||
if err := domain.ValidateAgentCallback(callback, lease, heartbeat.Sequence); err != nil {
|
||||
if err := domain.ValidateAgentCallback(callback, lease, lastSequence); err != nil {
|
||||
return work.AgentExecutionResult{}, err
|
||||
}
|
||||
if err := store.AcceptAgentCallback(ctx, callback); err != nil {
|
||||
|
||||
@@ -18,6 +18,10 @@ func (s *bridgeStore) SaveAgentLease(_ context.Context, lease work.AgentLease) e
|
||||
s.lease = lease
|
||||
return nil
|
||||
}
|
||||
func (s *bridgeStore) RenewAgentLease(_ context.Context, lease work.AgentLease, _ time.Time) error {
|
||||
s.lease = lease
|
||||
return nil
|
||||
}
|
||||
func (s *bridgeStore) AcceptAgentCallback(_ context.Context, callback work.AgentCallback) error {
|
||||
s.callbacks = append(s.callbacks, callback)
|
||||
return nil
|
||||
@@ -29,6 +33,17 @@ func (bridgeExecutor) Execute(context.Context, work.AgentExecutionRequest) (work
|
||||
return work.AgentExecutionResult{State: work.AgentExecutionCompleted, Summary: "done", ObservedAtUnixMillis: 2}, nil
|
||||
}
|
||||
|
||||
type slowBridgeExecutor struct{}
|
||||
|
||||
func (slowBridgeExecutor) Execute(ctx context.Context, request work.AgentExecutionRequest) (work.AgentExecutionResult, error) {
|
||||
select {
|
||||
case <-time.After(45 * time.Millisecond):
|
||||
return work.AgentExecutionResult{State: work.AgentExecutionCompleted, Summary: "done", ObservedAtUnixMillis: time.Now().UnixMilli()}, nil
|
||||
case <-ctx.Done():
|
||||
return work.AgentExecutionResult{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAgentDispatchPersistsLeaseAndBridgeCallbacks(t *testing.T) {
|
||||
store := &bridgeStore{}
|
||||
now := time.UnixMilli(1)
|
||||
@@ -40,3 +55,20 @@ func TestExecuteAgentDispatchPersistsLeaseAndBridgeCallbacks(t *testing.T) {
|
||||
t.Fatalf("callbacks=%+v", store.callbacks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteAgentDispatchRenewsLeaseAndOrdersHeartbeats(t *testing.T) {
|
||||
store := &bridgeStore{}
|
||||
now := time.Now().UTC()
|
||||
_, err := ExecuteAgentDispatch(context.Background(), store, slowBridgeExecutor{}, work.AgentDispatchIntent{DispatchID: "d-slow", IdempotencyKey: "i-slow", DelegationID: "del-slow", RunID: "run-slow", ImplementationID: "impl", ImplementationRevision: 1, SourceRevision: "source", WorktreeID: "tree", VerificationPlanRef: "verify", CreatedAtUnixMillis: now.UnixMilli()}, "worker", 1, now, 30*time.Millisecond)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(store.callbacks) < 3 {
|
||||
t.Fatalf("expected heartbeat callbacks, got %d", len(store.callbacks))
|
||||
}
|
||||
for i, callback := range store.callbacks {
|
||||
if callback.Sequence != uint64(i+1) {
|
||||
t.Fatalf("callback sequence is not ordered: %#v", store.callbacks)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,12 @@ type WorkerCycleResult struct {
|
||||
Idle bool
|
||||
}
|
||||
|
||||
const transientDispatchRetryDelay = time.Second
|
||||
|
||||
func retryClaimedDispatch(ctx context.Context, store ports.WorkerStore, dispatchID, workerID string, now time.Time) error {
|
||||
return store.RetryDispatch(ctx, dispatchID, workerID, now.UnixMilli(), transientDispatchRetryDelay)
|
||||
}
|
||||
|
||||
// RecoverOnlyWorkerCycle keeps durable recovery alive while the real executor
|
||||
// capability is absent. It intentionally does not claim a dispatch: a missing
|
||||
// executor must never be represented as successful work.
|
||||
@@ -37,11 +43,11 @@ func ExecuteWorkerCycle(ctx context.Context, store ports.WorkerStore, workerID s
|
||||
}
|
||||
intent, err := store.LoadDispatchIntent(ctx, item.DispatchID)
|
||||
if err != nil {
|
||||
_ = store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchFailed, now.UnixMilli())
|
||||
_ = retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now)
|
||||
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
|
||||
}
|
||||
if _, err := ExecuteDryRunDispatch(ctx, store, intent, workerID, token, now, leaseTTL); err != nil {
|
||||
_ = store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchFailed, now.UnixMilli())
|
||||
_ = retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now)
|
||||
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
|
||||
}
|
||||
if err := store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchCompleted, now.UnixMilli()); err != nil {
|
||||
@@ -64,15 +70,18 @@ func ExecuteWorkerCycleWithExecutor(ctx context.Context, store ports.WorkerStore
|
||||
}
|
||||
intent, err := store.LoadDispatchIntent(ctx, item.DispatchID)
|
||||
if err != nil {
|
||||
_ = store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchFailed, now.UnixMilli())
|
||||
_ = retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now)
|
||||
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
|
||||
}
|
||||
result, err := ExecuteAgentDispatch(ctx, store, executor, intent, workerID, token, now, leaseTTL)
|
||||
if err != nil {
|
||||
_ = store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchFailed, now.UnixMilli())
|
||||
_ = retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now)
|
||||
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
|
||||
}
|
||||
if result.State != work.AgentExecutionCompleted {
|
||||
if err := retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now); err != nil {
|
||||
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
|
||||
}
|
||||
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, nil
|
||||
}
|
||||
if err := store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchCompleted, now.UnixMilli()); err != nil {
|
||||
|
||||
@@ -26,10 +26,16 @@ func (s *recoveryOnlyStore) LoadDispatchIntent(context.Context, string) (work.Ag
|
||||
func (s *recoveryOnlyStore) CompleteDispatch(context.Context, string, string, work.AgentDispatchQueueState, int64) error {
|
||||
return nil
|
||||
}
|
||||
func (s *recoveryOnlyStore) RetryDispatch(context.Context, string, string, int64, time.Duration) error {
|
||||
return nil
|
||||
}
|
||||
func (s *recoveryOnlyStore) SaveDispatchIntent(context.Context, work.AgentDispatchIntent) error {
|
||||
return nil
|
||||
}
|
||||
func (s *recoveryOnlyStore) SaveAgentLease(context.Context, work.AgentLease) error { return nil }
|
||||
func (s *recoveryOnlyStore) RenewAgentLease(context.Context, work.AgentLease, time.Time) error {
|
||||
return nil
|
||||
}
|
||||
func (s *recoveryOnlyStore) AcceptAgentCallback(context.Context, work.AgentCallback) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package ports
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/work"
|
||||
)
|
||||
@@ -13,5 +14,6 @@ type AgentExecutor interface {
|
||||
type ExecutionStore interface {
|
||||
SaveDispatchIntent(context.Context, work.AgentDispatchIntent) error
|
||||
SaveAgentLease(context.Context, work.AgentLease) error
|
||||
RenewAgentLease(context.Context, work.AgentLease, time.Time) error
|
||||
AcceptAgentCallback(context.Context, work.AgentCallback) error
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package ports
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"git.cin.su/m/Assistant/contracts/work"
|
||||
)
|
||||
@@ -14,5 +15,6 @@ type WorkerStore interface {
|
||||
LoadDispatchIntent(context.Context, string) (work.AgentDispatchIntent, error)
|
||||
ClaimNextDispatch(context.Context, string, int64) (work.AgentDispatchQueueItem, error)
|
||||
CompleteDispatch(context.Context, string, string, work.AgentDispatchQueueState, int64) error
|
||||
RetryDispatch(context.Context, string, string, int64, time.Duration) error
|
||||
RecoverStaleClaims(context.Context, int64, int64) (int64, error)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user