feat(work): make agent execution recovery durable
This commit is contained in:
@@ -176,7 +176,7 @@ func (r *WorkRepository) ClaimNextDispatch(ctx context.Context, workerID string,
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
var item work.AgentDispatchQueueItem
|
||||
err = tx.QueryRow(ctx, `SELECT dispatch_id,state,available_at_unix_millis,attempts,claimed_by,updated_at_unix_millis FROM work.agent_dispatch_queue WHERE state='QUEUED' AND available_at_unix_millis <= $1 ORDER BY available_at_unix_millis,dispatch_id FOR UPDATE SKIP LOCKED LIMIT 1`, nowUnixMillis).Scan(&item.DispatchID, &item.State, &item.AvailableAtUnixMillis, &item.Attempts, &item.ClaimedBy, &item.UpdatedAtUnixMillis)
|
||||
err = tx.QueryRow(ctx, `SELECT dispatch_id,state,available_at_unix_millis,attempts,claimed_by,updated_at_unix_millis FROM work.agent_dispatch_queue WHERE state='QUEUED' AND available_at_unix_millis <= $1 AND attempts < retry_limit ORDER BY available_at_unix_millis,dispatch_id FOR UPDATE SKIP LOCKED LIMIT 1`, nowUnixMillis).Scan(&item.DispatchID, &item.State, &item.AvailableAtUnixMillis, &item.Attempts, &item.ClaimedBy, &item.UpdatedAtUnixMillis)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return work.AgentDispatchQueueItem{}, nil
|
||||
@@ -219,6 +219,21 @@ func (r *WorkRepository) CompleteDispatch(ctx context.Context, dispatchID, worke
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *WorkRepository) RetryDispatch(ctx context.Context, dispatchID, workerID string, nowUnixMillis int64, delay time.Duration) error {
|
||||
if delay < 0 {
|
||||
return fmt.Errorf("invalid retry delay")
|
||||
}
|
||||
availableAt := nowUnixMillis + delay.Milliseconds()
|
||||
command, err := r.pool.Exec(ctx, `UPDATE work.agent_dispatch_queue SET state=CASE WHEN attempts >= retry_limit THEN 'FAILED' ELSE 'QUEUED' END,available_at_unix_millis=$1,claimed_by='',updated_at_unix_millis=$2 WHERE dispatch_id=$3 AND state='CLAIMED' AND claimed_by=$4`, availableAt, nowUnixMillis, dispatchID, workerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if command.RowsAffected() != 1 {
|
||||
return ErrAgentCallbackRejected
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *WorkRepository) SaveWorkGraph(ctx context.Context, graph work.WorkGraph) error {
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
@@ -314,10 +329,21 @@ func (r *WorkRepository) SaveGraphSynthesis(ctx context.Context, conflict work.C
|
||||
}
|
||||
|
||||
func (r *WorkRepository) RecoverStaleClaims(ctx context.Context, nowUnixMillis, staleAfterMillis int64) (int64, error) {
|
||||
command, err := r.pool.Exec(ctx, `UPDATE work.agent_dispatch_queue SET state='QUEUED',claimed_by='',updated_at_unix_millis=$1 WHERE state='CLAIMED' AND updated_at_unix_millis <= $2`, nowUnixMillis, nowUnixMillis-staleAfterMillis)
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
if _, err := tx.Exec(ctx, `UPDATE work.agent_leases AS l SET state='EXPIRED' FROM work.agent_dispatch_queue AS q JOIN work.agent_dispatch_intents AS i ON i.dispatch_id=q.dispatch_id WHERE q.state='CLAIMED' AND q.updated_at_unix_millis <= $1 AND l.run_id=i.run_id AND l.state='ACTIVE'`, nowUnixMillis-staleAfterMillis); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
command, err := tx.Exec(ctx, `UPDATE work.agent_dispatch_queue SET state=CASE WHEN attempts >= retry_limit THEN 'FAILED' ELSE 'QUEUED' END,available_at_unix_millis=$1,claimed_by='',updated_at_unix_millis=$2 WHERE state='CLAIMED' AND updated_at_unix_millis <= $3`, nowUnixMillis, nowUnixMillis, nowUnixMillis-staleAfterMillis)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return command.RowsAffected(), nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user