Compare commits

...
Author SHA1 Message Date
m 54958d0902 feat(work): persist governed development candidates
CI / test (push) Successful in 54s
CI / test (pull_request) Successful in 58s
2026-08-23 13:26:41 +03:00
m 8d3e7f7cac feat(work): make agent execution recovery durable
CI / test (push) Successful in 1m4s
DEV-000214 durable Agent Execution recovery
2026-08-23 10:15:19 +00:00
m c9eb398857 chore(governance): declare canary scope for recovery
CI / test (push) Successful in 59s
CI / test (pull_request) Successful in 1m8s
2026-08-23 13:12:25 +03:00
m 284ea88553 feat(work): make agent execution recovery durable
CI / test (push) Failing after 33s
CI / test (pull_request) Failing after 42s
2026-08-23 13:10:18 +03:00
m f9b9765100 feat(work): link task evaluation worker execution
CI / test (push) Successful in 1m7s
DEV-000213 worker evaluator link
2026-08-23 09:59:00 +00:00
19 changed files with 401 additions and 32 deletions
+55 -6
View File
@@ -18,10 +18,11 @@ import (
type WorkRepository struct{ pool *pgxpool.Pool }
var (
ErrDispatchIntentConflict = errors.New("dispatch intent idempotency conflict")
ErrAgentLeaseUnavailable = errors.New("agent lease unavailable")
ErrAgentCallbackRejected = errors.New("agent callback rejected")
ErrDevelopmentGateConflict = errors.New("development gate revision conflict")
ErrDispatchIntentConflict = errors.New("dispatch intent idempotency conflict")
ErrAgentLeaseUnavailable = errors.New("agent lease unavailable")
ErrAgentCallbackRejected = errors.New("agent callback rejected")
ErrDevelopmentGateConflict = errors.New("development gate revision conflict")
ErrDevelopmentCandidateConflict = errors.New("development candidate conflict")
)
func OpenWork(ctx context.Context, config ContinuityConfig) (*WorkRepository, error) {
@@ -104,6 +105,28 @@ func (r *WorkRepository) SaveDevelopmentGate(ctx context.Context, gate work.Deve
return nil
}
func (r *WorkRepository) SaveExtensionProposal(ctx context.Context, value work.ExtensionProposal) error {
result, err := r.pool.Exec(ctx, `INSERT INTO work.extension_proposals(proposal_id,requested_by_principal_id,stage,evidence_refs,deduplicated_signal_keys,desired_outcome,smallest_change_layer,affected_owners,affected_contracts,risk_tier,evaluation_plan,alternatives,rollback_plan,assumptions,created_at_unix_millis) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT(proposal_id) DO NOTHING`, value.ID, value.RequestedByPrincipalID, value.Stage, value.EvidenceRefs, value.DeduplicatedSignalKeys, value.DesiredOutcome, value.SmallestChangeLayer, value.AffectedOwners, value.AffectedContracts, value.RiskTier, value.EvaluationPlan, value.Alternatives, value.RollbackPlan, value.Assumptions, value.CreatedAtUnixMillis)
if err != nil {
return err
}
if result.RowsAffected() != 1 {
return ErrDevelopmentCandidateConflict
}
return nil
}
func (r *WorkRepository) SaveDevelopmentScaffold(ctx context.Context, value work.DevelopmentScaffold) error {
result, err := r.pool.Exec(ctx, `INSERT INTO work.development_scaffolds(scaffold_id,requested_by_principal_id,stage,proposal_ref,base_repo_revision,worktree_ref,task_intent,modules,allowed_paths,forbidden_paths,required_tests,artifact_lineage_refs,protected_main_access,production_access,secret_access,created_at_unix_millis) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) ON CONFLICT(scaffold_id) DO NOTHING`, value.ID, value.RequestedByPrincipalID, value.Stage, value.ProposalRef, value.BaseRepoRevision, value.WorktreeRef, value.TaskIntent, value.Modules, value.AllowedPaths, value.ForbiddenPaths, value.RequiredTests, value.ArtifactLineageRefs, value.ProtectedMainAccess, value.ProductionAccess, value.SecretAccess, value.CreatedAtUnixMillis)
if err != nil {
return err
}
if result.RowsAffected() != 1 {
return ErrDevelopmentCandidateConflict
}
return nil
}
func (r *WorkRepository) SaveDispatchIntent(ctx context.Context, intent work.AgentDispatchIntent) error {
tx, err := r.pool.Begin(ctx)
if err != nil {
@@ -176,7 +199,7 @@ func (r *WorkRepository) ClaimNextDispatch(ctx context.Context, workerID string,
}
defer tx.Rollback(ctx)
var item work.AgentDispatchQueueItem
err = tx.QueryRow(ctx, `SELECT dispatch_id,state,available_at_unix_millis,attempts,claimed_by,updated_at_unix_millis FROM work.agent_dispatch_queue WHERE state='QUEUED' AND available_at_unix_millis <= $1 ORDER BY available_at_unix_millis,dispatch_id FOR UPDATE SKIP LOCKED LIMIT 1`, nowUnixMillis).Scan(&item.DispatchID, &item.State, &item.AvailableAtUnixMillis, &item.Attempts, &item.ClaimedBy, &item.UpdatedAtUnixMillis)
err = tx.QueryRow(ctx, `SELECT dispatch_id,state,available_at_unix_millis,attempts,claimed_by,updated_at_unix_millis FROM work.agent_dispatch_queue WHERE state='QUEUED' AND available_at_unix_millis <= $1 AND attempts < retry_limit ORDER BY available_at_unix_millis,dispatch_id FOR UPDATE SKIP LOCKED LIMIT 1`, nowUnixMillis).Scan(&item.DispatchID, &item.State, &item.AvailableAtUnixMillis, &item.Attempts, &item.ClaimedBy, &item.UpdatedAtUnixMillis)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return work.AgentDispatchQueueItem{}, nil
@@ -219,6 +242,21 @@ func (r *WorkRepository) CompleteDispatch(ctx context.Context, dispatchID, worke
return nil
}
func (r *WorkRepository) RetryDispatch(ctx context.Context, dispatchID, workerID string, nowUnixMillis int64, delay time.Duration) error {
if delay < 0 {
return fmt.Errorf("invalid retry delay")
}
availableAt := nowUnixMillis + delay.Milliseconds()
command, err := r.pool.Exec(ctx, `UPDATE work.agent_dispatch_queue SET state=CASE WHEN attempts >= retry_limit THEN 'FAILED' ELSE 'QUEUED' END,available_at_unix_millis=$1,claimed_by='',updated_at_unix_millis=$2 WHERE dispatch_id=$3 AND state='CLAIMED' AND claimed_by=$4`, availableAt, nowUnixMillis, dispatchID, workerID)
if err != nil {
return err
}
if command.RowsAffected() != 1 {
return ErrAgentCallbackRejected
}
return nil
}
func (r *WorkRepository) SaveWorkGraph(ctx context.Context, graph work.WorkGraph) error {
tx, err := r.pool.Begin(ctx)
if err != nil {
@@ -314,10 +352,21 @@ func (r *WorkRepository) SaveGraphSynthesis(ctx context.Context, conflict work.C
}
func (r *WorkRepository) RecoverStaleClaims(ctx context.Context, nowUnixMillis, staleAfterMillis int64) (int64, error) {
command, err := r.pool.Exec(ctx, `UPDATE work.agent_dispatch_queue SET state='QUEUED',claimed_by='',updated_at_unix_millis=$1 WHERE state='CLAIMED' AND updated_at_unix_millis <= $2`, nowUnixMillis, nowUnixMillis-staleAfterMillis)
tx, err := r.pool.Begin(ctx)
if err != nil {
return 0, err
}
defer tx.Rollback(ctx)
if _, err := tx.Exec(ctx, `UPDATE work.agent_leases AS l SET state='EXPIRED' FROM work.agent_dispatch_queue AS q JOIN work.agent_dispatch_intents AS i ON i.dispatch_id=q.dispatch_id WHERE q.state='CLAIMED' AND q.updated_at_unix_millis <= $1 AND l.run_id=i.run_id AND l.state='ACTIVE'`, nowUnixMillis-staleAfterMillis); err != nil {
return 0, err
}
command, err := tx.Exec(ctx, `UPDATE work.agent_dispatch_queue SET state=CASE WHEN attempts >= retry_limit THEN 'FAILED' ELSE 'QUEUED' END,available_at_unix_millis=$1,claimed_by='',updated_at_unix_millis=$2 WHERE state='CLAIMED' AND updated_at_unix_millis <= $3`, nowUnixMillis, nowUnixMillis, nowUnixMillis-staleAfterMillis)
if err != nil {
return 0, err
}
if err := tx.Commit(ctx); err != nil {
return 0, err
}
return command.RowsAffected(), nil
}
+2 -1
View File
@@ -24,7 +24,8 @@ func (s *memoryStore) SaveDispatchIntent(context.Context, work.AgentDispatchInte
s.writes++
return nil
}
func (s *memoryStore) SaveAgentLease(context.Context, work.AgentLease) error { s.writes++; return nil }
func (s *memoryStore) SaveAgentLease(context.Context, work.AgentLease) error { s.writes++; return nil }
func (s *memoryStore) RenewAgentLease(context.Context, work.AgentLease, time.Time) error { return nil }
func (s *memoryStore) AcceptAgentCallback(context.Context, work.AgentCallback) error {
s.writes++
return nil
+55
View File
@@ -178,6 +178,61 @@ func main() {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(session)
})
mux.HandleFunc("POST /v1/creator/development/proposals", func(w http.ResponseWriter, r *http.Request) {
session, ok := authenticateCreator(w, r)
if !ok {
return
}
var request struct {
Evidence []work.ExtensionEvidence `json:"evidence"`
Proposal work.ExtensionProposalRequest `json:"proposal"`
}
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&request); err != nil {
http.Error(w, "invalid development proposal", http.StatusBadRequest)
return
}
proposal, err := workdomain.BuildExtensionProposal(request.Evidence, request.Proposal)
if err != nil {
http.Error(w, "development proposal rejected", http.StatusUnprocessableEntity)
return
}
proposal.RequestedByPrincipalID = string(session.PrincipalID)
if err := workRepository.SaveExtensionProposal(r.Context(), proposal); err != nil {
http.Error(w, "development proposal conflict", http.StatusConflict)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusCreated)
_ = json.NewEncoder(w).Encode(proposal)
})
mux.HandleFunc("POST /v1/creator/development/scaffolds", func(w http.ResponseWriter, r *http.Request) {
session, ok := authenticateCreator(w, r)
if !ok {
return
}
var request work.DevelopmentScaffoldRequest
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&request); err != nil {
http.Error(w, "invalid development scaffold", http.StatusBadRequest)
return
}
scaffold, err := workdomain.BuildDevelopmentScaffold(request)
if err != nil {
http.Error(w, "development scaffold rejected", http.StatusUnprocessableEntity)
return
}
scaffold.RequestedByPrincipalID = string(session.PrincipalID)
if err := workRepository.SaveDevelopmentScaffold(r.Context(), scaffold); err != nil {
http.Error(w, "development scaffold conflict", http.StatusConflict)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusCreated)
_ = json.NewEncoder(w).Encode(scaffold)
})
mux.HandleFunc("GET /v1/creator/focus", func(w http.ResponseWriter, r *http.Request) {
session, ok := authenticateCreator(w, r)
if !ok {
+17 -15
View File
@@ -70,6 +70,7 @@ type ExtensionProposalRequest struct {
// authority grant, implementation body or promotion decision.
type ExtensionProposal struct {
ID string
RequestedByPrincipalID string
Stage ExtensionProposalStage
EvidenceRefs []string
DeduplicatedSignalKeys []string
@@ -113,19 +114,20 @@ type DevelopmentScaffoldRequest struct {
// DevelopmentScaffold is a provenance-linked task-packet candidate. It has
// no executable instruction, credential, checkout handle or authority grant.
type DevelopmentScaffold struct {
ID string
Stage DevelopmentScaffoldStage
ProposalRef string
BaseRepoRevision string
WorktreeRef string
TaskIntent string
Modules []string
AllowedPaths []string
ForbiddenPaths []string
RequiredTests []string
ArtifactLineageRefs []string
ProtectedMainAccess string
ProductionAccess string
SecretAccess string
CreatedAtUnixMillis int64
ID string
RequestedByPrincipalID string
Stage DevelopmentScaffoldStage
ProposalRef string
BaseRepoRevision string
WorktreeRef string
TaskIntent string
Modules []string
AllowedPaths []string
ForbiddenPaths []string
RequiredTests []string
ArtifactLineageRefs []string
ProtectedMainAccess string
ProductionAccess string
SecretAccess string
CreatedAtUnixMillis int64
}
@@ -0,0 +1,15 @@
BEGIN;
SET ROLE work_role;
ALTER TABLE work.agent_dispatch_queue
ADD COLUMN IF NOT EXISTS retry_limit INTEGER NOT NULL DEFAULT 3;
ALTER TABLE work.agent_dispatch_queue
DROP CONSTRAINT IF EXISTS agent_dispatch_queue_retry_limit_check;
ALTER TABLE work.agent_dispatch_queue
ADD CONSTRAINT agent_dispatch_queue_retry_limit_check CHECK (retry_limit > 0);
RESET ROLE;
COMMIT;
@@ -0,0 +1,45 @@
BEGIN;
SET ROLE work_role;
CREATE TABLE IF NOT EXISTS work.extension_proposals (
proposal_id TEXT PRIMARY KEY,
requested_by_principal_id TEXT NOT NULL,
stage TEXT NOT NULL,
evidence_refs TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
deduplicated_signal_keys TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
desired_outcome TEXT NOT NULL,
smallest_change_layer TEXT NOT NULL,
affected_owners TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
affected_contracts TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
risk_tier TEXT NOT NULL,
evaluation_plan TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
alternatives TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
rollback_plan TEXT NOT NULL,
assumptions TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
created_at_unix_millis BIGINT NOT NULL,
CHECK (proposal_id <> ''), CHECK (requested_by_principal_id <> ''), CHECK (stage = 'PROPOSED'), CHECK (desired_outcome <> ''), CHECK (smallest_change_layer <> ''), CHECK (risk_tier <> ''), CHECK (rollback_plan <> ''), CHECK (created_at_unix_millis > 0)
);
CREATE TABLE IF NOT EXISTS work.development_scaffolds (
scaffold_id TEXT PRIMARY KEY,
requested_by_principal_id TEXT NOT NULL,
stage TEXT NOT NULL,
proposal_ref TEXT NOT NULL,
base_repo_revision TEXT NOT NULL,
worktree_ref TEXT NOT NULL,
task_intent TEXT NOT NULL,
modules TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
allowed_paths TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
forbidden_paths TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
required_tests TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
artifact_lineage_refs TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
protected_main_access TEXT NOT NULL,
production_access TEXT NOT NULL,
secret_access TEXT NOT NULL,
created_at_unix_millis BIGINT NOT NULL,
CHECK (scaffold_id <> ''), CHECK (requested_by_principal_id <> ''), CHECK (stage = 'SANDBOXED'), CHECK (proposal_ref <> ''), CHECK (base_repo_revision <> ''), CHECK (worktree_ref <> ''), CHECK (task_intent <> ''), CHECK (protected_main_access = 'DENIED'), CHECK (production_access = 'DENIED'), CHECK (secret_access = 'DENIED'), CHECK (created_at_unix_millis > 0)
);
RESET ROLE;
COMMIT;
+2 -2
View File
@@ -1,5 +1,5 @@
{
"task_file": "dev/tasks/DEV-000213-worker-evaluator-link.json",
"task_id": "DEV-000213",
"task_file": "dev/tasks/DEV-000215-governed-development-control-plane.json",
"task_id": "DEV-000215",
"task_revision": 1
}
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000214",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "f9b9765100df02bead16280d2db9c1af478a4589",
"risk_tier": "E",
"intent": "Make Agent Execution recovery restart-safe: renew active leases with managed heartbeats, fence stale workers, cap retries, and requeue transient execution failures without losing durable dispatch state.",
"requirement_ids": ["durable heartbeat", "lease fencing", "bounded retry", "restart recovery", "transient failure requeue"],
"allowed_change_scope": {"modules": ["work", "agent-worker"], "paths": ["contracts/work/**", "modules/work/**", "adapters/postgres/work_repository.go", "cmd/agent-worker/**", "cmd/agent-canary/**", "database/migrations/042_agent_execution_recovery.sql", "tests/agent_execution/**", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000214-agent-execution-recovery.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "deploy/**", "cmd/devctl/**", "modules/identity/**", "modules/capability/**", "LLM-local/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 1},
"contracts": {"consume": ["existing AgentExecutor", "existing lease/callback contract", "durable dispatch queue"], "may_change": ["WorkerStore retry/recovery port"]},
"semantic_owner": "work owns dispatch, lease, callback and retry lifecycle",
"canonical_state": ["work agent dispatch queue and lease tables remain canonical; heartbeat is durable evidence of liveness"],
"state_transitions": ["CLAIMED -> QUEUED on transient failure or stale recovery", "CLAIMED -> FAILED after retry limit", "ACTIVE lease -> EXPIRED before stale claim requeue", "heartbeat renews lease and callback sequence"],
"side_effects_and_privileges": ["work_role writes only work-owned queue and lease state; no external mutation"],
"persistence_and_migrations": ["042 adds bounded retry limit to dispatch queue"],
"failure_retry_restart": ["stale recovery expires the old lease before requeue", "transient execution failure uses durable delayed retry", "retry limit prevents hot-loop poison dispatch", "heartbeat failure cancels execution and leaves a retryable dispatch"],
"assumptions": ["operational heartbeat ticker is lifecycle-managed by the bounded worker call and durable state remains authoritative"],
"acceptance_scenarios": ["long execution renews lease and emits ordered heartbeats", "stale claim is fenced and requeued", "third failed attempt becomes FAILED", "waiting/error dispatch is not left CLAIMED", "legacy dispatch behavior remains compatible"],
"required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "migration/schema checks", "CI", "production public health check"],
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY"]
}
@@ -0,0 +1,23 @@
{
"task_id": "DEV-000215",
"task_revision": 1,
"architecture_revision": "ADR-0018",
"base_repo_revision": "8d3e7f7cac236aa50919bd1cd854012dcc8500a5",
"risk_tier": "E",
"intent": "Expose a durable, authenticated control-plane boundary for governed self-development proposals and isolated scaffold candidates without executing source mutation, bypassing verification, or granting promotion authority.",
"requirement_ids": ["durable proposal intake", "isolated scaffold contract", "authenticated operator boundary", "no implicit promotion"],
"allowed_change_scope": {"modules": ["work", "core-runtime"], "paths": ["contracts/work/self_development.go", "modules/work/domain/**", "cmd/core-runtime/main.go", "adapters/postgres/work_repository.go", "database/migrations/043_governed_development_candidates.sql", "tests/semantic_evolution/**", "tests/work/**", "dev/ACTIVE_TASK.json", "dev/tasks/DEV-000215-governed-development-control-plane.json"]},
"forbidden_change_scope": {"paths": ["architecture/**", "security/**", "deploy/**", "cmd/devctl/**", "modules/identity/**", "modules/capability/**", "LLM-local/**"]},
"allowed_architecture_delta": {"new_cross_module_dependencies": 0, "new_canonical_owners": 0, "new_privileges": 0, "new_third_party_dependencies": 0, "new_database_owner_roles": 1},
"contracts": {"consume": ["ADR-0022 governed self-development boundary", "creator authentication", "work development gate"], "may_change": ["work-owned proposal/scaffold persistence and authenticated control-plane endpoints"]},
"semantic_owner": "work owns proposal, scaffold and development gate lineage",
"canonical_state": ["work-owned extension proposal and scaffold candidates are durable; source tree and production remain outside this task"],
"state_transitions": ["repeated evidence -> PROPOSED", "proposal -> SANDBOXED scaffold candidate", "gate remains explicit and cannot be promoted by these endpoints"],
"side_effects_and_privileges": ["authenticated creator session required; database writes stay within work owner; no checkout, source mutation, deployment or secret access"],
"persistence_and_migrations": ["043 adds durable work-owned proposal/scaffold candidate records"],
"failure_retry_restart": ["idempotent candidate IDs preserve lineage; duplicate IDs are rejected; no endpoint advances promotion stage"],
"assumptions": ["an independent executor and verifier remain separate future capabilities"],
"acceptance_scenarios": ["unauthenticated requests are rejected", "repeated evidence creates a durable proposal", "protected paths cannot create a scaffold", "valid scaffold is durable and remains SANDBOXED", "no promotion or source mutation occurs"],
"required_tests": ["go test", "go vet", "preflight", "check-diff", "check-architecture", "independent", "git diff --check", "migration/schema checks", "CI", "production public health check"],
"terminal_gap_states": ["BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", "BLOCKED_BY_MISSING_CAPABILITY"]
}
+64 -3
View File
@@ -2,6 +2,8 @@ package application
import (
"context"
"fmt"
"sync"
"time"
"git.cin.su/m/Assistant/contracts/work"
@@ -33,16 +35,75 @@ func ExecuteAgentDispatch(ctx context.Context, store ports.ExecutionStore, execu
if err := store.AcceptAgentCallback(ctx, heartbeat); err != nil {
return work.AgentExecutionResult{}, err
}
result, err := executor.Execute(ctx, work.AgentExecutionRequest{Intent: intent, WorkerID: workerID, FencingToken: token, LeaseExpiresAt: lease.ExpiresAt})
executionCtx, cancel := context.WithCancel(ctx)
defer cancel()
var stateMu sync.Mutex
currentLease := lease
lastSequence := heartbeat.Sequence
heartbeatDone := make(chan struct{})
heartbeatStopped := make(chan struct{})
heartbeatFailure := make(chan error, 1)
interval := ttl / 3
if interval <= 0 {
return work.AgentExecutionResult{}, fmt.Errorf("invalid heartbeat interval")
}
go func() {
defer close(heartbeatStopped)
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-heartbeatDone:
return
case observedAt := <-ticker.C:
stateMu.Lock()
nextLease, renewErr := domain.RenewAgentLease(currentLease, workerID, token, observedAt, ttl)
if renewErr == nil {
renewErr = store.RenewAgentLease(executionCtx, nextLease, nextLease.ExpiresAt)
}
if renewErr == nil {
lastSequence++
heartbeat = work.AgentCallback{RunID: intent.RunID, WorkerID: workerID, FencingToken: token, Sequence: lastSequence, Kind: work.AgentCallbackHeartbeat, ObservedAt: observedAt}
renewErr = domain.ValidateAgentCallback(heartbeat, nextLease, lastSequence-1)
}
if renewErr == nil {
renewErr = store.AcceptAgentCallback(executionCtx, heartbeat)
}
if renewErr == nil {
currentLease = nextLease
}
stateMu.Unlock()
if renewErr != nil {
select {
case heartbeatFailure <- renewErr:
default:
}
cancel()
return
}
}
}
}()
result, err := executor.Execute(executionCtx, work.AgentExecutionRequest{Intent: intent, WorkerID: workerID, FencingToken: token, LeaseExpiresAt: lease.ExpiresAt})
close(heartbeatDone)
<-heartbeatStopped
select {
case heartbeatErr := <-heartbeatFailure:
return work.AgentExecutionResult{}, heartbeatErr
default:
}
if err != nil {
return work.AgentExecutionResult{}, err
}
stateMu.Lock()
lease = currentLease
stateMu.Unlock()
observedAt := time.UnixMilli(result.ObservedAtUnixMillis)
callback := work.AgentCallback{RunID: intent.RunID, WorkerID: workerID, FencingToken: token, Sequence: 2, Kind: work.AgentCallbackResult, ObservedAt: observedAt}
callback := work.AgentCallback{RunID: intent.RunID, WorkerID: workerID, FencingToken: token, Sequence: lastSequence + 1, Kind: work.AgentCallbackResult, ObservedAt: observedAt}
if result.State == work.AgentExecutionWaiting {
callback.Kind = work.AgentCallbackCheckpoint
}
if err := domain.ValidateAgentCallback(callback, lease, heartbeat.Sequence); err != nil {
if err := domain.ValidateAgentCallback(callback, lease, lastSequence); err != nil {
return work.AgentExecutionResult{}, err
}
if err := store.AcceptAgentCallback(ctx, callback); err != nil {
@@ -18,6 +18,10 @@ func (s *bridgeStore) SaveAgentLease(_ context.Context, lease work.AgentLease) e
s.lease = lease
return nil
}
func (s *bridgeStore) RenewAgentLease(_ context.Context, lease work.AgentLease, _ time.Time) error {
s.lease = lease
return nil
}
func (s *bridgeStore) AcceptAgentCallback(_ context.Context, callback work.AgentCallback) error {
s.callbacks = append(s.callbacks, callback)
return nil
@@ -29,6 +33,17 @@ func (bridgeExecutor) Execute(context.Context, work.AgentExecutionRequest) (work
return work.AgentExecutionResult{State: work.AgentExecutionCompleted, Summary: "done", ObservedAtUnixMillis: 2}, nil
}
type slowBridgeExecutor struct{}
func (slowBridgeExecutor) Execute(ctx context.Context, request work.AgentExecutionRequest) (work.AgentExecutionResult, error) {
select {
case <-time.After(45 * time.Millisecond):
return work.AgentExecutionResult{State: work.AgentExecutionCompleted, Summary: "done", ObservedAtUnixMillis: time.Now().UnixMilli()}, nil
case <-ctx.Done():
return work.AgentExecutionResult{}, ctx.Err()
}
}
func TestExecuteAgentDispatchPersistsLeaseAndBridgeCallbacks(t *testing.T) {
store := &bridgeStore{}
now := time.UnixMilli(1)
@@ -40,3 +55,20 @@ func TestExecuteAgentDispatchPersistsLeaseAndBridgeCallbacks(t *testing.T) {
t.Fatalf("callbacks=%+v", store.callbacks)
}
}
func TestExecuteAgentDispatchRenewsLeaseAndOrdersHeartbeats(t *testing.T) {
store := &bridgeStore{}
now := time.Now().UTC()
_, err := ExecuteAgentDispatch(context.Background(), store, slowBridgeExecutor{}, work.AgentDispatchIntent{DispatchID: "d-slow", IdempotencyKey: "i-slow", DelegationID: "del-slow", RunID: "run-slow", ImplementationID: "impl", ImplementationRevision: 1, SourceRevision: "source", WorktreeID: "tree", VerificationPlanRef: "verify", CreatedAtUnixMillis: now.UnixMilli()}, "worker", 1, now, 30*time.Millisecond)
if err != nil {
t.Fatal(err)
}
if len(store.callbacks) < 3 {
t.Fatalf("expected heartbeat callbacks, got %d", len(store.callbacks))
}
for i, callback := range store.callbacks {
if callback.Sequence != uint64(i+1) {
t.Fatalf("callback sequence is not ordered: %#v", store.callbacks)
}
}
}
+13 -4
View File
@@ -15,6 +15,12 @@ type WorkerCycleResult struct {
Idle bool
}
const transientDispatchRetryDelay = time.Second
func retryClaimedDispatch(ctx context.Context, store ports.WorkerStore, dispatchID, workerID string, now time.Time) error {
return store.RetryDispatch(ctx, dispatchID, workerID, now.UnixMilli(), transientDispatchRetryDelay)
}
// RecoverOnlyWorkerCycle keeps durable recovery alive while the real executor
// capability is absent. It intentionally does not claim a dispatch: a missing
// executor must never be represented as successful work.
@@ -37,11 +43,11 @@ func ExecuteWorkerCycle(ctx context.Context, store ports.WorkerStore, workerID s
}
intent, err := store.LoadDispatchIntent(ctx, item.DispatchID)
if err != nil {
_ = store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchFailed, now.UnixMilli())
_ = retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now)
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
}
if _, err := ExecuteDryRunDispatch(ctx, store, intent, workerID, token, now, leaseTTL); err != nil {
_ = store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchFailed, now.UnixMilli())
_ = retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now)
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
}
if err := store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchCompleted, now.UnixMilli()); err != nil {
@@ -64,15 +70,18 @@ func ExecuteWorkerCycleWithExecutor(ctx context.Context, store ports.WorkerStore
}
intent, err := store.LoadDispatchIntent(ctx, item.DispatchID)
if err != nil {
_ = store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchFailed, now.UnixMilli())
_ = retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now)
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
}
result, err := ExecuteAgentDispatch(ctx, store, executor, intent, workerID, token, now, leaseTTL)
if err != nil {
_ = store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchFailed, now.UnixMilli())
_ = retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now)
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
}
if result.State != work.AgentExecutionCompleted {
if err := retryClaimedDispatch(ctx, store, item.DispatchID, workerID, now); err != nil {
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, err
}
return WorkerCycleResult{RecoveredClaims: recovered, DispatchID: item.DispatchID}, nil
}
if err := store.CompleteDispatch(ctx, item.DispatchID, workerID, work.AgentDispatchCompleted, now.UnixMilli()); err != nil {
+6
View File
@@ -26,10 +26,16 @@ func (s *recoveryOnlyStore) LoadDispatchIntent(context.Context, string) (work.Ag
func (s *recoveryOnlyStore) CompleteDispatch(context.Context, string, string, work.AgentDispatchQueueState, int64) error {
return nil
}
func (s *recoveryOnlyStore) RetryDispatch(context.Context, string, string, int64, time.Duration) error {
return nil
}
func (s *recoveryOnlyStore) SaveDispatchIntent(context.Context, work.AgentDispatchIntent) error {
return nil
}
func (s *recoveryOnlyStore) SaveAgentLease(context.Context, work.AgentLease) error { return nil }
func (s *recoveryOnlyStore) RenewAgentLease(context.Context, work.AgentLease, time.Time) error {
return nil
}
func (s *recoveryOnlyStore) AcceptAgentCallback(context.Context, work.AgentCallback) error {
return nil
}
+2
View File
@@ -2,6 +2,7 @@ package ports
import (
"context"
"time"
"git.cin.su/m/Assistant/contracts/work"
)
@@ -13,5 +14,6 @@ type AgentExecutor interface {
type ExecutionStore interface {
SaveDispatchIntent(context.Context, work.AgentDispatchIntent) error
SaveAgentLease(context.Context, work.AgentLease) error
RenewAgentLease(context.Context, work.AgentLease, time.Time) error
AcceptAgentCallback(context.Context, work.AgentCallback) error
}
+2
View File
@@ -3,6 +3,7 @@ package ports
import (
"context"
"errors"
"time"
"git.cin.su/m/Assistant/contracts/work"
)
@@ -14,5 +15,6 @@ type WorkerStore interface {
LoadDispatchIntent(context.Context, string) (work.AgentDispatchIntent, error)
ClaimNextDispatch(context.Context, string, int64) (work.AgentDispatchQueueItem, error)
CompleteDispatch(context.Context, string, string, work.AgentDispatchQueueState, int64) error
RetryDispatch(context.Context, string, string, int64, time.Duration) error
RecoverStaleClaims(context.Context, int64, int64) (int64, error)
}
+2 -1
View File
@@ -15,7 +15,8 @@ func (s *canaryStore) SaveDispatchIntent(context.Context, work.AgentDispatchInte
s.writes++
return nil
}
func (s *canaryStore) SaveAgentLease(context.Context, work.AgentLease) error { s.writes++; return nil }
func (s *canaryStore) SaveAgentLease(context.Context, work.AgentLease) error { s.writes++; return nil }
func (s *canaryStore) RenewAgentLease(context.Context, work.AgentLease, time.Time) error { return nil }
func (s *canaryStore) AcceptAgentCallback(context.Context, work.AgentCallback) error {
s.writes++
return nil
@@ -21,6 +21,9 @@ func (s *recordingExecutionStore) SaveAgentLease(context.Context, work.AgentLeas
s.operations = append(s.operations, "lease")
return nil
}
func (s *recordingExecutionStore) RenewAgentLease(context.Context, work.AgentLease, time.Time) error {
return nil
}
func (s *recordingExecutionStore) AcceptAgentCallback(context.Context, work.AgentCallback) error {
s.operations = append(s.operations, "callback")
return nil
@@ -0,0 +1,20 @@
package agent_execution_test
import (
"os"
"strings"
"testing"
)
func TestRecoveryMigrationDefinesBoundedRetryState(t *testing.T) {
payload, err := os.ReadFile("../../database/migrations/042_agent_execution_recovery.sql")
if err != nil {
t.Fatal(err)
}
source := strings.ToLower(string(payload))
for _, fragment := range []string{"set role work_role", "retry_limit integer not null default 3", "retry_limit > 0"} {
if !strings.Contains(source, fragment) {
t.Fatalf("migration is missing %q", fragment)
}
}
}
@@ -0,0 +1,20 @@
package semantic_evolution_test
import (
"os"
"strings"
"testing"
)
func TestGovernedDevelopmentCandidateMigrationIsWorkOwnedAndSandboxed(t *testing.T) {
payload, err := os.ReadFile("../../database/migrations/043_governed_development_candidates.sql")
if err != nil {
t.Fatal(err)
}
source := strings.ToLower(string(payload))
for _, fragment := range []string{"set role work_role", "create table if not exists work.extension_proposals", "create table if not exists work.development_scaffolds", "stage = 'sandboxed'", "protected_main_access = 'denied'", "production_access = 'denied'", "secret_access = 'denied'"} {
if !strings.Contains(source, fragment) {
t.Fatalf("migration is missing %q", fragment)
}
}
}