feat(work): make agent execution recovery durable
CI / test (push) Failing after 33s
CI / test (pull_request) Failing after 42s

This commit is contained in:
m
2026-08-23 13:10:18 +03:00
parent f9b9765100
commit 284ea88553
14 changed files with 214 additions and 13 deletions
+64 -3
View File
@@ -2,6 +2,8 @@ package application
import (
"context"
"fmt"
"sync"
"time"
"git.cin.su/m/Assistant/contracts/work"
@@ -33,16 +35,75 @@ func ExecuteAgentDispatch(ctx context.Context, store ports.ExecutionStore, execu
if err := store.AcceptAgentCallback(ctx, heartbeat); err != nil {
return work.AgentExecutionResult{}, err
}
result, err := executor.Execute(ctx, work.AgentExecutionRequest{Intent: intent, WorkerID: workerID, FencingToken: token, LeaseExpiresAt: lease.ExpiresAt})
executionCtx, cancel := context.WithCancel(ctx)
defer cancel()
var stateMu sync.Mutex
currentLease := lease
lastSequence := heartbeat.Sequence
heartbeatDone := make(chan struct{})
heartbeatStopped := make(chan struct{})
heartbeatFailure := make(chan error, 1)
interval := ttl / 3
if interval <= 0 {
return work.AgentExecutionResult{}, fmt.Errorf("invalid heartbeat interval")
}
go func() {
defer close(heartbeatStopped)
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-heartbeatDone:
return
case observedAt := <-ticker.C:
stateMu.Lock()
nextLease, renewErr := domain.RenewAgentLease(currentLease, workerID, token, observedAt, ttl)
if renewErr == nil {
renewErr = store.RenewAgentLease(executionCtx, nextLease, nextLease.ExpiresAt)
}
if renewErr == nil {
lastSequence++
heartbeat = work.AgentCallback{RunID: intent.RunID, WorkerID: workerID, FencingToken: token, Sequence: lastSequence, Kind: work.AgentCallbackHeartbeat, ObservedAt: observedAt}
renewErr = domain.ValidateAgentCallback(heartbeat, nextLease, lastSequence-1)
}
if renewErr == nil {
renewErr = store.AcceptAgentCallback(executionCtx, heartbeat)
}
if renewErr == nil {
currentLease = nextLease
}
stateMu.Unlock()
if renewErr != nil {
select {
case heartbeatFailure <- renewErr:
default:
}
cancel()
return
}
}
}
}()
result, err := executor.Execute(executionCtx, work.AgentExecutionRequest{Intent: intent, WorkerID: workerID, FencingToken: token, LeaseExpiresAt: lease.ExpiresAt})
close(heartbeatDone)
<-heartbeatStopped
select {
case heartbeatErr := <-heartbeatFailure:
return work.AgentExecutionResult{}, heartbeatErr
default:
}
if err != nil {
return work.AgentExecutionResult{}, err
}
stateMu.Lock()
lease = currentLease
stateMu.Unlock()
observedAt := time.UnixMilli(result.ObservedAtUnixMillis)
callback := work.AgentCallback{RunID: intent.RunID, WorkerID: workerID, FencingToken: token, Sequence: 2, Kind: work.AgentCallbackResult, ObservedAt: observedAt}
callback := work.AgentCallback{RunID: intent.RunID, WorkerID: workerID, FencingToken: token, Sequence: lastSequence + 1, Kind: work.AgentCallbackResult, ObservedAt: observedAt}
if result.State == work.AgentExecutionWaiting {
callback.Kind = work.AgentCallbackCheckpoint
}
if err := domain.ValidateAgentCallback(callback, lease, heartbeat.Sequence); err != nil {
if err := domain.ValidateAgentCallback(callback, lease, lastSequence); err != nil {
return work.AgentExecutionResult{}, err
}
if err := store.AcceptAgentCallback(ctx, callback); err != nil {
@@ -18,6 +18,10 @@ func (s *bridgeStore) SaveAgentLease(_ context.Context, lease work.AgentLease) e
s.lease = lease
return nil
}
func (s *bridgeStore) RenewAgentLease(_ context.Context, lease work.AgentLease, _ time.Time) error {
s.lease = lease
return nil
}
func (s *bridgeStore) AcceptAgentCallback(_ context.Context, callback work.AgentCallback) error {
s.callbacks = append(s.callbacks, callback)
return nil
@@ -29,6 +33,17 @@ func (bridgeExecutor) Execute(context.Context, work.AgentExecutionRequest) (work
return work.AgentExecutionResult{State: work.AgentExecutionCompleted, Summary: "done", ObservedAtUnixMillis: 2}, nil
}
type slowBridgeExecutor struct{}
func (slowBridgeExecutor) Execute(ctx context.Context, request work.AgentExecutionRequest) (work.AgentExecutionResult, error) {
select {
case <-time.After(45 * time.Millisecond):
return work.AgentExecutionResult{State: work.AgentExecutionCompleted, Summary: "done", ObservedAtUnixMillis: time.Now().UnixMilli()}, nil
case <-ctx.Done():
return work.AgentExecutionResult{}, ctx.Err()
}
}
func TestExecuteAgentDispatchPersistsLeaseAndBridgeCallbacks(t *testing.T) {
store := &bridgeStore{}
now := time.UnixMilli(1)
@@ -40,3 +55,20 @@ func TestExecuteAgentDispatchPersistsLeaseAndBridgeCallbacks(t *testing.T) {
t.Fatalf("callbacks=%+v", store.callbacks)
}
}
func TestExecuteAgentDispatchRenewsLeaseAndOrdersHeartbeats(t *testing.T) {
store := &bridgeStore{}
now := time.Now().UTC()
_, err := ExecuteAgentDispatch(context.Background(), store, slowBridgeExecutor{}, work.AgentDispatchIntent{DispatchID: "d-slow", IdempotencyKey: "i-slow", DelegationID: "del-slow", RunID: "run-slow", ImplementationID: "impl", ImplementationRevision: 1, SourceRevision: "source", WorktreeID: "tree", VerificationPlanRef: "verify", CreatedAtUnixMillis: now.UnixMilli()}, "worker", 1, now, 30*time.Millisecond)
if err != nil {
t.Fatal(err)
}
if len(store.callbacks) < 3 {
t.Fatalf("expected heartbeat callbacks, got %d", len(store.callbacks))
}
for i, callback := range store.callbacks {
if callback.Sequence != uint64(i+1) {
t.Fatalf("callback sequence is not ordered: %#v", store.callbacks)
}
}
}
+13 -4
View File
@@ -15,6 +15,12 @@ type WorkerCycleResult struct {
Idle bool
}
const transientDispatchRetryDelay = time.Second
func retryClaimedDispatch(ctx context.Context, store ports.WorkerStore, dispatchID, workerID string, now time.Time) error {
return store.RetryDispatch(ctx, dispatchID, workerID, now.UnixMilli(), transientDispatchRetryDelay)
}
// RecoverOnlyWorkerCycle keeps durable recovery alive while the real executor
// capability is absent. It intentionally does not claim a dispatch: a missing
// executor must never be represented as successful work.
@@ -37,11 +43,11 @@ func ExecuteWorkerCycle(ctx context.Context, store ports.WorkerStore, workerID s
}
intent, err := store.LoadDispatchIntent(ctx, item.DispatchID)
if err != nil {
_ = store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchFailed, now.UnixMilli())
_ = retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now)
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
}
if _, err := ExecuteDryRunDispatch(ctx, store, intent, workerID, token, now, leaseTTL); err != nil {
_ = store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchFailed, now.UnixMilli())
_ = retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now)
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
}
if err := store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchCompleted, now.UnixMilli()); err != nil {
@@ -64,15 +70,18 @@ func ExecuteWorkerCycleWithExecutor(ctx context.Context, store ports.WorkerStore
}
intent, err := store.LoadDispatchIntent(ctx, item.DispatchID)
if err != nil {
_ = store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchFailed, now.UnixMilli())
_ = retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now)
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
}
result, err := ExecuteAgentDispatch(ctx, store, executor, intent, workerID, token, now, leaseTTL)
if err != nil {
_ = store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchFailed, now.UnixMilli())
_ = retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now)
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
}
if result.State != work.AgentExecutionCompleted {
if err := retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now); err != nil {
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
}
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, nil
}
if err := store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchCompleted, now.UnixMilli()); err != nil {
+6
View File
@@ -26,10 +26,16 @@ func (s *recoveryOnlyStore) LoadDispatchIntent(context.Context, string) (work.Ag
func (s *recoveryOnlyStore) CompleteDispatch(context.Context, string, string, work.AgentDispatchQueueState, int64) error {
return nil
}
func (s *recoveryOnlyStore) RetryDispatch(context.Context, string, string, int64, time.Duration) error {
return nil
}
func (s *recoveryOnlyStore) SaveDispatchIntent(context.Context, work.AgentDispatchIntent) error {
return nil
}
func (s *recoveryOnlyStore) SaveAgentLease(context.Context, work.AgentLease) error { return nil }
func (s *recoveryOnlyStore) RenewAgentLease(context.Context, work.AgentLease, time.Time) error {
return nil
}
func (s *recoveryOnlyStore) AcceptAgentCallback(context.Context, work.AgentCallback) error {
return nil
}
+2
View File
@@ -2,6 +2,7 @@ package ports
import (
"context"
"time"
"git.cin.su/m/Assistant/contracts/work"
)
@@ -13,5 +14,6 @@ type AgentExecutor interface {
type ExecutionStore interface {
SaveDispatchIntent(context.Context, work.AgentDispatchIntent) error
SaveAgentLease(context.Context, work.AgentLease) error
RenewAgentLease(context.Context, work.AgentLease, time.Time) error
AcceptAgentCallback(context.Context, work.AgentCallback) error
}
+2
View File
@@ -3,6 +3,7 @@ package ports
import (
"context"
"errors"
"time"
"git.cin.su/m/Assistant/contracts/work"
)
@@ -14,5 +15,6 @@ type WorkerStore interface {
LoadDispatchIntent(context.Context, string) (work.AgentDispatchIntent, error)
ClaimNextDispatch(context.Context, string, int64) (work.AgentDispatchQueueItem, error)
CompleteDispatch(context.Context, string, string, work.AgentDispatchQueueState, int64) error
RetryDispatch(context.Context, string, string, int64, time.Duration) error
RecoverStaleClaims(context.Context, int64, int64) (int64, error)
}