Initial project snapshot
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
package worker
|
||||
|
||||
type SessionEvent struct {
|
||||
Type string `json:"type"`
|
||||
SessionID string `json:"session_id"`
|
||||
WorkerID string `json:"worker_id"`
|
||||
Payload map[string]any `json:"payload,omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
SessionEventConnected = "session_connected"
|
||||
SessionEventHeartbeat = "session_heartbeat"
|
||||
SessionEventFailed = "session_failed"
|
||||
SessionEventTerminated = "session_terminated"
|
||||
SessionEventDisplayReady = "session_display_ready"
|
||||
SessionEventRenderReady = "session_render_ready"
|
||||
SessionEventRenderDirty = "session_render_dirty"
|
||||
SessionEventRenderResized = "session_render_resized"
|
||||
SessionEventCursorUpdated = "session_cursor_updated"
|
||||
SessionEventFrame = "session_frame"
|
||||
SessionEventClipboardText = "session_clipboard_text"
|
||||
SessionEventFileUploaded = "session_file_upload_completed"
|
||||
SessionEventFileDownloadAvailable = "session_file_download_available"
|
||||
SessionEventFileDownloadChunk = "session_file_download_chunk"
|
||||
SessionEventFileDownloadProgress = "session_file_download_progress"
|
||||
SessionEventFileDownloadCompleted = "session_file_download_completed"
|
||||
SessionEventFileDownloadFailed = "session_file_download_failed"
|
||||
SessionEventFileDownloadBlocked = "session_file_download_blocked"
|
||||
)
|
||||
@@ -0,0 +1,52 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/example/remote-access-platform/backend/internal/modules/sessionbroker"
|
||||
)
|
||||
|
||||
type LeaseMonitor struct {
|
||||
service *Service
|
||||
broker *sessionbroker.Service
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
func NewLeaseMonitor(service *Service, broker *sessionbroker.Service, interval time.Duration) *LeaseMonitor {
|
||||
if interval <= 0 {
|
||||
interval = 15 * time.Second
|
||||
}
|
||||
return &LeaseMonitor{
|
||||
service: service,
|
||||
broker: broker,
|
||||
interval: interval,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *LeaseMonitor) Run(ctx context.Context) error {
|
||||
ticker := time.NewTicker(m.interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
stale, err := m.service.RecoverStaleLeases(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, lease := range stale {
|
||||
err := m.broker.MarkSessionFailed(ctx, sessionbroker.MarkSessionFailedCommand{
|
||||
SessionID: lease.SessionID,
|
||||
Reason: "worker_lease_stale_or_worker_missing",
|
||||
})
|
||||
if err != nil && !errors.Is(err, sessionbroker.ErrSessionNotFound) && !errors.Is(err, sessionbroker.ErrSessionNotTerminable) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/example/remote-access-platform/backend/internal/modules/sessionbroker"
|
||||
)
|
||||
|
||||
type EventProcessor struct {
|
||||
client *redis.Client
|
||||
broker *sessionbroker.Service
|
||||
}
|
||||
|
||||
func NewEventProcessor(client *redis.Client, broker *sessionbroker.Service) *EventProcessor {
|
||||
return &EventProcessor{client: client, broker: broker}
|
||||
}
|
||||
|
||||
func (p *EventProcessor) Run(ctx context.Context) error {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
|
||||
result, err := p.client.BLPop(ctx, 5*time.Second, "worker:events").Result()
|
||||
if err == redis.Nil {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("consume worker event: %w", err)
|
||||
}
|
||||
if len(result) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
var event SessionEvent
|
||||
if err := json.Unmarshal([]byte(result[1]), &event); err != nil {
|
||||
continue
|
||||
}
|
||||
if err := p.handleEvent(ctx, event); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *EventProcessor) handleEvent(ctx context.Context, event SessionEvent) error {
|
||||
switch event.Type {
|
||||
case SessionEventConnected, SessionEventDisplayReady:
|
||||
if err := p.broker.HandleWorkerConnected(ctx, event.SessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(event.Payload) > 0 {
|
||||
if err := p.broker.UpdateWorkerRenderTelemetry(ctx, event.SessionID, event.Payload); err != nil && !errors.Is(err, sessionbroker.ErrSessionNotFound) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
case SessionEventHeartbeat:
|
||||
return p.broker.HandleWorkerHeartbeat(ctx, event.SessionID)
|
||||
case SessionEventRenderReady, SessionEventRenderDirty, SessionEventRenderResized, SessionEventCursorUpdated, SessionEventFrame:
|
||||
if len(event.Payload) == 0 {
|
||||
return nil
|
||||
}
|
||||
if correlationID, _ := event.Payload["input_correlation_id"].(string); correlationID != "" {
|
||||
slog.Info("worker frame event received",
|
||||
"session_id", event.SessionID,
|
||||
"worker_id", event.WorkerID,
|
||||
"frame_sequence", event.Payload["frame_sequence"],
|
||||
"correlation_id", correlationID,
|
||||
"worker_frame_captured_at", event.Payload["worker_frame_captured_at"],
|
||||
"trace_stage", "backend_frame_receive")
|
||||
}
|
||||
return p.updateRenderTelemetryWithRetry(ctx, event.SessionID, event.Payload)
|
||||
case SessionEventClipboardText:
|
||||
if len(event.Payload) == 0 {
|
||||
return nil
|
||||
}
|
||||
slog.Info("worker clipboard event received",
|
||||
"session_id", event.SessionID,
|
||||
"worker_id", event.WorkerID,
|
||||
"origin", event.Payload["origin"],
|
||||
"sequence_id", event.Payload["sequence_id"],
|
||||
"content_hash", event.Payload["content_hash"])
|
||||
return p.broker.UpdateWorkerClipboardText(ctx, event.SessionID, event.Payload)
|
||||
case SessionEventFileUploaded:
|
||||
slog.Info("worker file upload completed",
|
||||
"session_id", event.SessionID,
|
||||
"worker_id", event.WorkerID,
|
||||
"transfer_id", event.Payload["transfer_id"],
|
||||
"file_name", event.Payload["file_name"],
|
||||
"file_size", event.Payload["file_size"],
|
||||
"content_hash", event.Payload["content_hash"],
|
||||
"storage_path", event.Payload["storage_path"])
|
||||
return nil
|
||||
case SessionEventFileDownloadAvailable, SessionEventFileDownloadChunk, SessionEventFileDownloadProgress,
|
||||
SessionEventFileDownloadCompleted, SessionEventFileDownloadFailed, SessionEventFileDownloadBlocked:
|
||||
slog.Info("worker file download event received",
|
||||
"session_id", event.SessionID,
|
||||
"worker_id", event.WorkerID,
|
||||
"event_type", event.Type,
|
||||
"transfer_id", event.Payload["transfer_id"],
|
||||
"file_id", event.Payload["file_id"],
|
||||
"file_name", event.Payload["file_name"],
|
||||
"status", event.Payload["status"])
|
||||
return p.broker.UpdateWorkerFileDownloadEvent(ctx, event.SessionID, event.Type, event.Payload)
|
||||
case SessionEventFailed:
|
||||
reason, _ := event.Payload["reason"].(string)
|
||||
err := p.broker.MarkSessionFailed(ctx, sessionbroker.MarkSessionFailedCommand{
|
||||
SessionID: event.SessionID,
|
||||
Reason: reason,
|
||||
})
|
||||
if errors.Is(err, sessionbroker.ErrSessionNotFound) || errors.Is(err, sessionbroker.ErrSessionNotTerminable) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
case SessionEventTerminated:
|
||||
reason, _ := event.Payload["reason"].(string)
|
||||
err := p.broker.TerminateSession(ctx, sessionbroker.TerminateSessionCommand{
|
||||
SessionID: event.SessionID,
|
||||
Reason: reason,
|
||||
})
|
||||
if errors.Is(err, sessionbroker.ErrSessionNotFound) || errors.Is(err, sessionbroker.ErrSessionNotTerminable) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (p *EventProcessor) updateRenderTelemetryWithRetry(ctx context.Context, sessionID string, payload map[string]any) error {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 10; attempt++ {
|
||||
err := p.broker.UpdateWorkerRenderTelemetry(ctx, sessionID, payload)
|
||||
if err == nil || errors.Is(err, sessionbroker.ErrSessionNotFound) {
|
||||
return nil
|
||||
}
|
||||
lastErr = err
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
workercontracts "github.com/example/remote-access-platform/backend/pkg/contracts/worker"
|
||||
)
|
||||
|
||||
type RedisStore struct {
|
||||
client *redis.Client
|
||||
}
|
||||
|
||||
func NewRedisStore(client *redis.Client) *RedisStore {
|
||||
return &RedisStore{client: client}
|
||||
}
|
||||
|
||||
func (s *RedisStore) RegisterWorker(ctx context.Context, registration workercontracts.WorkerRegistration, ttl time.Duration) error {
|
||||
payload, err := json.Marshal(registration)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal worker registration: %w", err)
|
||||
}
|
||||
pipe := s.client.TxPipeline()
|
||||
pipe.Set(ctx, workerKey(registration.WorkerID), payload, ttl)
|
||||
pipe.SAdd(ctx, workerSetKey(), registration.WorkerID)
|
||||
_, err = pipe.Exec(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("register worker: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *RedisStore) TouchWorkerHeartbeat(ctx context.Context, heartbeat workercontracts.WorkerHeartbeat, ttl time.Duration) error {
|
||||
registration, err := s.GetWorker(ctx, heartbeat.WorkerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if registration == nil {
|
||||
registration = &workercontracts.WorkerRegistration{
|
||||
WorkerID: heartbeat.WorkerID,
|
||||
Protocol: workercontracts.ProtocolRDP,
|
||||
}
|
||||
}
|
||||
registration.Status = heartbeat.Status
|
||||
registration.LastHeartbeatAt = heartbeat.LastHeartbeatAt
|
||||
return s.RegisterWorker(ctx, *registration, ttl)
|
||||
}
|
||||
|
||||
func (s *RedisStore) ListWorkers(ctx context.Context) ([]workercontracts.WorkerRegistration, error) {
|
||||
ids, err := s.client.SMembers(ctx, workerSetKey()).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list worker ids: %w", err)
|
||||
}
|
||||
workers := make([]workercontracts.WorkerRegistration, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
worker, err := s.GetWorker(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if worker != nil {
|
||||
workers = append(workers, *worker)
|
||||
}
|
||||
}
|
||||
return workers, nil
|
||||
}
|
||||
|
||||
func (s *RedisStore) GetWorker(ctx context.Context, workerID string) (*workercontracts.WorkerRegistration, error) {
|
||||
payload, err := s.client.Get(ctx, workerKey(workerID)).Result()
|
||||
if err == redis.Nil {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get worker: %w", err)
|
||||
}
|
||||
var registration workercontracts.WorkerRegistration
|
||||
if err := json.Unmarshal([]byte(payload), ®istration); err != nil {
|
||||
return nil, fmt.Errorf("decode worker registration: %w", err)
|
||||
}
|
||||
return ®istration, nil
|
||||
}
|
||||
|
||||
func (s *RedisStore) AcquireLease(ctx context.Context, lease workercontracts.WorkerLease, ttl time.Duration) error {
|
||||
payload, err := json.Marshal(lease)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal lease: %w", err)
|
||||
}
|
||||
ok, err := s.client.SetNX(ctx, leaseKey(lease.LeaseID), payload, ttl).Result()
|
||||
if err != nil {
|
||||
return fmt.Errorf("acquire lease: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("lease already exists")
|
||||
}
|
||||
pipe := s.client.TxPipeline()
|
||||
pipe.SAdd(ctx, leaseSetKey(), lease.LeaseID)
|
||||
pipe.Set(ctx, sessionLeaseKey(lease.SessionID), lease.LeaseID, ttl)
|
||||
_, err = pipe.Exec(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("index lease: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *RedisStore) GetLease(ctx context.Context, leaseID string) (*workercontracts.WorkerLease, error) {
|
||||
payload, err := s.client.Get(ctx, leaseKey(leaseID)).Result()
|
||||
if err == redis.Nil {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get lease: %w", err)
|
||||
}
|
||||
var lease workercontracts.WorkerLease
|
||||
if err := json.Unmarshal([]byte(payload), &lease); err != nil {
|
||||
return nil, fmt.Errorf("decode lease: %w", err)
|
||||
}
|
||||
return &lease, nil
|
||||
}
|
||||
|
||||
func (s *RedisStore) GetLeaseBySession(ctx context.Context, sessionID string) (*workercontracts.WorkerLease, error) {
|
||||
leaseID, err := s.client.Get(ctx, sessionLeaseKey(sessionID)).Result()
|
||||
if err == redis.Nil {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get lease by session: %w", err)
|
||||
}
|
||||
return s.GetLease(ctx, leaseID)
|
||||
}
|
||||
|
||||
func (s *RedisStore) RenewLease(ctx context.Context, lease workercontracts.WorkerLease, ttl time.Duration) error {
|
||||
payload, err := json.Marshal(lease)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal lease renewal: %w", err)
|
||||
}
|
||||
pipe := s.client.TxPipeline()
|
||||
pipe.Set(ctx, leaseKey(lease.LeaseID), payload, ttl)
|
||||
pipe.Set(ctx, sessionLeaseKey(lease.SessionID), lease.LeaseID, ttl)
|
||||
_, err = pipe.Exec(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("renew lease: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *RedisStore) ReleaseLease(ctx context.Context, leaseID string) error {
|
||||
lease, err := s.GetLease(ctx, leaseID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pipe := s.client.TxPipeline()
|
||||
pipe.Del(ctx, leaseKey(leaseID))
|
||||
pipe.SRem(ctx, leaseSetKey(), leaseID)
|
||||
if lease != nil {
|
||||
pipe.Del(ctx, sessionLeaseKey(lease.SessionID))
|
||||
}
|
||||
_, err = pipe.Exec(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("release lease: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *RedisStore) ListLeases(ctx context.Context) ([]workercontracts.WorkerLease, error) {
|
||||
ids, err := s.client.SMembers(ctx, leaseSetKey()).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list lease ids: %w", err)
|
||||
}
|
||||
leases := make([]workercontracts.WorkerLease, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
lease, err := s.GetLease(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if lease != nil {
|
||||
leases = append(leases, *lease)
|
||||
}
|
||||
}
|
||||
return leases, nil
|
||||
}
|
||||
|
||||
func (s *RedisStore) AppendEnvelope(ctx context.Context, envelope workercontracts.RoutedEnvelope) error {
|
||||
payload, err := json.Marshal(envelope)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal routed envelope: %w", err)
|
||||
}
|
||||
key := workerQueueKey(envelope.SessionID)
|
||||
if err := s.client.RPush(ctx, key, payload).Err(); err != nil {
|
||||
return fmt.Errorf("append routed envelope: %w", err)
|
||||
}
|
||||
if envelope.Type == "input" {
|
||||
correlationID, _ := envelope.Payload["correlation_id"].(string)
|
||||
if correlationID != "" {
|
||||
if length, err := s.client.LLen(ctx, key).Result(); err == nil {
|
||||
slog.Info("worker queue length after input append",
|
||||
"session_id", envelope.SessionID,
|
||||
"attachment_id", envelope.AttachmentID,
|
||||
"correlation_id", correlationID,
|
||||
"queue_key", key,
|
||||
"queue_length", length,
|
||||
"trace_stage", "redis_queue_append")
|
||||
}
|
||||
}
|
||||
}
|
||||
return s.client.Expire(ctx, key, 10*time.Minute).Err()
|
||||
}
|
||||
|
||||
func (s *RedisStore) AppendAssignment(ctx context.Context, workerID string, payload map[string]any) error {
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal worker assignment: %w", err)
|
||||
}
|
||||
if err := s.client.RPush(ctx, workerControlQueueKey(workerID), encoded).Err(); err != nil {
|
||||
return fmt.Errorf("append worker assignment: %w", err)
|
||||
}
|
||||
return s.client.Expire(ctx, workerControlQueueKey(workerID), 10*time.Minute).Err()
|
||||
}
|
||||
|
||||
func (s *RedisStore) AppendEvent(ctx context.Context, payload map[string]any) error {
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal worker event: %w", err)
|
||||
}
|
||||
if err := s.client.RPush(ctx, workerEventsKey(), encoded).Err(); err != nil {
|
||||
return fmt.Errorf("append worker event: %w", err)
|
||||
}
|
||||
return s.client.Expire(ctx, workerEventsKey(), 10*time.Minute).Err()
|
||||
}
|
||||
|
||||
func workerKey(workerID string) string {
|
||||
return "worker:registration:" + workerID
|
||||
}
|
||||
|
||||
func workerSetKey() string {
|
||||
return "worker:registrations"
|
||||
}
|
||||
|
||||
func leaseKey(leaseID string) string {
|
||||
return "worker:lease:" + leaseID
|
||||
}
|
||||
|
||||
func leaseSetKey() string {
|
||||
return "worker:leases"
|
||||
}
|
||||
|
||||
func sessionLeaseKey(sessionID string) string {
|
||||
return "worker:session-lease:" + sessionID
|
||||
}
|
||||
|
||||
func workerQueueKey(sessionID string) string {
|
||||
return "worker:queue:" + sessionID
|
||||
}
|
||||
|
||||
func workerControlQueueKey(workerID string) string {
|
||||
return "worker:control:" + workerID
|
||||
}
|
||||
|
||||
func workerEventsKey() string {
|
||||
return "worker:events"
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/example/remote-access-platform/backend/internal/modules/sessionbroker"
|
||||
"github.com/example/remote-access-platform/backend/internal/platform/module"
|
||||
workercontracts "github.com/example/remote-access-platform/backend/pkg/contracts/worker"
|
||||
)
|
||||
|
||||
var ErrNoWorkerAvailable = errors.New("no worker available")
|
||||
|
||||
type Service struct {
|
||||
cfg module.Config
|
||||
store Store
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
func NewService(deps module.Dependencies, store Store) *Service {
|
||||
return &Service{
|
||||
cfg: deps.Config,
|
||||
store: store,
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) Register(ctx context.Context, registration workercontracts.WorkerRegistration) error {
|
||||
if registration.WorkerID == "" {
|
||||
return fmt.Errorf("worker id is required")
|
||||
}
|
||||
registration.LastHeartbeatAt = s.now().UTC()
|
||||
return s.store.RegisterWorker(ctx, registration, s.cfg.Worker.HeartbeatTTL)
|
||||
}
|
||||
|
||||
func (s *Service) Heartbeat(ctx context.Context, heartbeat workercontracts.WorkerHeartbeat) error {
|
||||
heartbeat.LastHeartbeatAt = s.now().UTC()
|
||||
return s.store.TouchWorkerHeartbeat(ctx, heartbeat, s.cfg.Worker.HeartbeatTTL)
|
||||
}
|
||||
|
||||
func (s *Service) Reserve(ctx context.Context, request workercontracts.AttachRequest) (*workercontracts.WorkerLease, error) {
|
||||
registration, err := s.reserveWorker(ctx, workercontracts.ProtocolRDP, request.RequiredCapabilities)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.AcquireLease(ctx, registration.WorkerID, request)
|
||||
}
|
||||
|
||||
func (s *Service) reserveWorker(ctx context.Context, protocol workercontracts.Protocol, capabilities []string) (*workercontracts.WorkerRegistration, error) {
|
||||
workers, err := s.store.ListWorkers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
now := s.now().UTC()
|
||||
for _, worker := range workers {
|
||||
if worker.Protocol != protocol || worker.Status != workercontracts.StatusOnline {
|
||||
continue
|
||||
}
|
||||
if now.Sub(worker.LastHeartbeatAt) > s.cfg.Worker.StaleLeaseGracePeriod+s.cfg.Worker.HeartbeatTTL {
|
||||
continue
|
||||
}
|
||||
if !hasCapabilities(worker.Capabilities, capabilities) {
|
||||
continue
|
||||
}
|
||||
return &worker, nil
|
||||
}
|
||||
return nil, ErrNoWorkerAvailable
|
||||
}
|
||||
|
||||
func (s *Service) AcquireLease(ctx context.Context, workerID string, request workercontracts.AttachRequest) (*workercontracts.WorkerLease, error) {
|
||||
if request.SessionID == "" {
|
||||
request.SessionID = uuid.NewString()
|
||||
}
|
||||
now := s.now().UTC()
|
||||
lease := workercontracts.WorkerLease{
|
||||
LeaseID: uuid.NewString(),
|
||||
WorkerID: workerID,
|
||||
Protocol: workercontracts.ProtocolRDP,
|
||||
ResourceID: request.ResourceID,
|
||||
SessionID: request.SessionID,
|
||||
Capabilities: request.RequiredCapabilities,
|
||||
ControlStream: "worker://control/" + workerID,
|
||||
ExpiresAt: now.Add(s.cfg.Worker.LeaseTTL),
|
||||
RenderQualityProfile: normalizeRenderQualityProfile(request.RenderQualityProfile),
|
||||
}
|
||||
if err := s.store.AcquireLease(ctx, lease, s.cfg.Worker.LeaseTTL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &lease, nil
|
||||
}
|
||||
|
||||
func (s *Service) GetSessionLease(ctx context.Context, sessionID string) (*workercontracts.WorkerLease, error) {
|
||||
return s.store.GetLeaseBySession(ctx, sessionID)
|
||||
}
|
||||
|
||||
func (s *Service) RenewLease(ctx context.Context, leaseID string) (*workercontracts.WorkerLease, error) {
|
||||
lease, err := s.store.GetLease(ctx, leaseID)
|
||||
if err != nil || lease == nil {
|
||||
return lease, err
|
||||
}
|
||||
lease.ExpiresAt = s.now().UTC().Add(s.cfg.Worker.LeaseTTL)
|
||||
if err := s.store.RenewLease(ctx, *lease, s.cfg.Worker.LeaseTTL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return lease, nil
|
||||
}
|
||||
|
||||
func (s *Service) ReleaseLease(ctx context.Context, leaseID string) error {
|
||||
return s.store.ReleaseLease(ctx, leaseID)
|
||||
}
|
||||
|
||||
func (s *Service) ReleaseSessionLease(ctx context.Context, sessionID string) error {
|
||||
lease, err := s.store.GetLeaseBySession(ctx, sessionID)
|
||||
if err != nil || lease == nil {
|
||||
return err
|
||||
}
|
||||
return s.store.ReleaseLease(ctx, lease.LeaseID)
|
||||
}
|
||||
|
||||
func (s *Service) RecoverStaleLeases(ctx context.Context) ([]workercontracts.WorkerLease, error) {
|
||||
leases, err := s.store.ListLeases(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var stale []workercontracts.WorkerLease
|
||||
now := s.now().UTC()
|
||||
for _, lease := range leases {
|
||||
registration, err := s.store.GetWorker(ctx, lease.WorkerID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if registration == nil || now.Sub(registration.LastHeartbeatAt) > s.cfg.Worker.StaleLeaseGracePeriod+s.cfg.Worker.HeartbeatTTL {
|
||||
if err := s.store.ReleaseLease(ctx, lease.LeaseID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stale = append(stale, lease)
|
||||
}
|
||||
}
|
||||
return stale, nil
|
||||
}
|
||||
|
||||
func (s *Service) ValidateSessionRuntime(ctx context.Context, sessionID, workerID string) (bool, string, error) {
|
||||
lease, err := s.store.GetLeaseBySession(ctx, sessionID)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if lease == nil {
|
||||
return false, "worker_lease_missing", nil
|
||||
}
|
||||
if workerID != "" && lease.WorkerID != workerID {
|
||||
_ = s.store.ReleaseLease(ctx, lease.LeaseID)
|
||||
return false, "worker_binding_mismatch", nil
|
||||
}
|
||||
now := s.now().UTC()
|
||||
if !lease.ExpiresAt.After(now) {
|
||||
_ = s.store.ReleaseLease(ctx, lease.LeaseID)
|
||||
return false, "worker_lease_expired", nil
|
||||
}
|
||||
registration, err := s.store.GetWorker(ctx, lease.WorkerID)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if registration == nil {
|
||||
_ = s.store.ReleaseLease(ctx, lease.LeaseID)
|
||||
return false, "worker_registration_missing", nil
|
||||
}
|
||||
if registration.Status != workercontracts.StatusOnline {
|
||||
return false, "worker_not_online", nil
|
||||
}
|
||||
if now.Sub(registration.LastHeartbeatAt) > s.cfg.Worker.StaleLeaseGracePeriod+s.cfg.Worker.HeartbeatTTL {
|
||||
_ = s.store.ReleaseLease(ctx, lease.LeaseID)
|
||||
return false, "worker_heartbeat_stale", nil
|
||||
}
|
||||
return true, "", nil
|
||||
}
|
||||
|
||||
func (s *Service) PublishControl(ctx context.Context, envelope workercontracts.RoutedEnvelope) error {
|
||||
return s.store.AppendEnvelope(ctx, envelope)
|
||||
}
|
||||
|
||||
func (s *Service) PublishInput(ctx context.Context, envelope workercontracts.RoutedEnvelope) error {
|
||||
return s.store.AppendEnvelope(ctx, envelope)
|
||||
}
|
||||
|
||||
func hasCapabilities(workerCaps, required []string) bool {
|
||||
for _, capability := range required {
|
||||
if !slices.Contains(workerCaps, capability) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Service) PrepareAttachment(ctx context.Context, session sessionbroker.RemoteSession, attachment sessionbroker.SessionAttachment, runtimeMetadata map[string]any) error {
|
||||
renderQualityProfile := normalizeRenderQualityProfile(session.RenderQualityProfile)
|
||||
if renderQualityProfile == "balanced" {
|
||||
renderQualityProfile = renderQualityProfileFromMetadata(session.Metadata)
|
||||
}
|
||||
if runtimeMetadata == nil {
|
||||
runtimeMetadata = decodeMetadata(session.Metadata)
|
||||
}
|
||||
return s.store.AppendAssignment(ctx, session.WorkerID, map[string]any{
|
||||
"type": "session_assignment",
|
||||
"session_id": session.ID,
|
||||
"worker_id": session.WorkerID,
|
||||
"attachment_id": attachment.ID,
|
||||
"user_id": attachment.UserID,
|
||||
"device_id": attachment.DeviceID,
|
||||
"takeover_of": attachment.TakeoverOf,
|
||||
"state": session.State,
|
||||
"render_quality_profile": renderQualityProfile,
|
||||
"metadata": runtimeMetadata,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Service) NotifyDetachment(ctx context.Context, session sessionbroker.RemoteSession, attachment sessionbroker.SessionAttachment) error {
|
||||
return s.PublishControl(ctx, workercontracts.RoutedEnvelope{
|
||||
SessionID: session.ID,
|
||||
AttachmentID: attachment.ID,
|
||||
Type: "control",
|
||||
Payload: map[string]any{
|
||||
"action": "detach",
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Service) TerminateRemoteSession(ctx context.Context, sessionID, attachmentID string) error {
|
||||
return s.PublishControl(ctx, workercontracts.RoutedEnvelope{
|
||||
SessionID: sessionID,
|
||||
AttachmentID: attachmentID,
|
||||
Type: "control",
|
||||
Payload: map[string]any{
|
||||
"action": "terminate",
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func decodeMetadata(payload []byte) map[string]any {
|
||||
var out map[string]any
|
||||
if len(payload) == 0 {
|
||||
return map[string]any{}
|
||||
}
|
||||
if err := json.Unmarshal(payload, &out); err != nil {
|
||||
return map[string]any{}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeRenderQualityProfile(profile string) string {
|
||||
switch profile {
|
||||
case "low_bandwidth", "balanced", "high_quality", "text_priority":
|
||||
return profile
|
||||
default:
|
||||
return "balanced"
|
||||
}
|
||||
}
|
||||
|
||||
func renderQualityProfileFromMetadata(metadata []byte) string {
|
||||
decoded := decodeMetadata(metadata)
|
||||
resource, _ := decoded["resource"].(map[string]any)
|
||||
if resource == nil {
|
||||
return "balanced"
|
||||
}
|
||||
if profile, ok := resource["render_quality_profile"].(string); ok && profile != "" {
|
||||
return normalizeRenderQualityProfile(profile)
|
||||
}
|
||||
return "balanced"
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
workercontracts "github.com/example/remote-access-platform/backend/pkg/contracts/worker"
|
||||
)
|
||||
|
||||
type Store interface {
|
||||
RegisterWorker(ctx context.Context, registration workercontracts.WorkerRegistration, ttl time.Duration) error
|
||||
TouchWorkerHeartbeat(ctx context.Context, heartbeat workercontracts.WorkerHeartbeat, ttl time.Duration) error
|
||||
ListWorkers(ctx context.Context) ([]workercontracts.WorkerRegistration, error)
|
||||
GetWorker(ctx context.Context, workerID string) (*workercontracts.WorkerRegistration, error)
|
||||
AcquireLease(ctx context.Context, lease workercontracts.WorkerLease, ttl time.Duration) error
|
||||
GetLease(ctx context.Context, leaseID string) (*workercontracts.WorkerLease, error)
|
||||
GetLeaseBySession(ctx context.Context, sessionID string) (*workercontracts.WorkerLease, error)
|
||||
RenewLease(ctx context.Context, lease workercontracts.WorkerLease, ttl time.Duration) error
|
||||
ReleaseLease(ctx context.Context, leaseID string) error
|
||||
ListLeases(ctx context.Context) ([]workercontracts.WorkerLease, error)
|
||||
AppendAssignment(ctx context.Context, workerID string, payload map[string]any) error
|
||||
AppendEnvelope(ctx context.Context, envelope workercontracts.RoutedEnvelope) error
|
||||
AppendEvent(ctx context.Context, payload map[string]any) error
|
||||
}
|
||||
Reference in New Issue
Block a user