package main import ( "crypto/sha256" "encoding/hex" "encoding/json" "errors" "flag" "fmt" "io" "net/http" "os" "os/exec" "path/filepath" "regexp" "sort" "strconv" "strings" "time" ) const ( readinessReady = "READY" readinessFailed = "BLOCKED" planVersion = 1 ) var deploymentExternalHealthEndpoints = []string{ "https://assistant.cin.su/healthz", "https://assistant.cin.su/chat", } var requiredChatOnlyInputs = map[string]struct{}{ // conservative, explicit bootstrap for production "POSTGRES_SUPERUSER": {}, "POSTGRES_SUPERUSER_PASSWORD": {}, "CONTINUITY_DB_PASSWORD": {}, "WORK_DB_PASSWORD": {}, "CAPABILITY_DB_PASSWORD": {}, "COMMIT_DB_PASSWORD": {}, "AUTHORITY_DB_PASSWORD": {}, "IDENTITY_DB_PASSWORD": {}, "SITUATION_DB_PASSWORD": {}, "CREATOR_USERNAME": {}, "CREATOR_BOOTSTRAP_TOKEN": {}, "CREATOR_SESSION_SIGNING_KEY": {}, "COMMIT_EXECUTOR_TOKEN": {}, "EPISTEMICS_POSTGRES_CREDENTIAL_REF": {}, "INTERACTION_POSTGRES_CREDENTIAL_REF": {}, } const ( durableBootstrapStateVersion = 1 durableBootstrapStatePath = "deployment-bootstrap-state.json" ) type bootstrapInputMetadata struct { CanonicalInput string `json:"canonical_input"` BackendFile string `json:"backend_file"` RuntimeFile string `json:"runtime_file"` SecretRef string `json:"secret_ref"` } type deploymentBootstrapState struct { Version int `json:"version"` Target string `json:"target"` ProtectedSecretRoot string `json:"protected_secret_backend_root"` Inputs map[string]bootstrapInputMetadata `json:"inputs"` LastMigratedAtUtc string `json:"last_migrated_at_utc"` MigrationType string `json:"migration_type"` DerivedFromDurableRef bool `json:"derived_from_durable_ref"` } var requiredDurableInputsForProduction = map[string]struct{}{ "CREATOR_BOOTSTRAP_TOKEN": {}, "CREATOR_SESSION_SIGNING_KEY": {}, "CREATOR_PASSWORD_FILE": {}, "EPISTEMICS_POSTGRES_CREDENTIAL_REF": {}, "INTERACTION_POSTGRES_CREDENTIAL_REF": {}, } var durableBootstrapCredentialDefaults = map[string]string{ "EPISTEMICS_POSTGRES_CREDENTIAL_REF": "credential://domovoy/epistemics/db-password/v1", "INTERACTION_POSTGRES_CREDENTIAL_REF": "credential://domovoy/interaction/db-password/v1", } var durableBootstrapRuntimeFileDefaults = map[string]string{ "CREATOR_BOOTSTRAP_TOKEN": "/run/secrets/creator_bootstrap_token", "CREATOR_SESSION_SIGNING_KEY": "/run/secrets/creator_session_signing_key", "CREATOR_PASSWORD_FILE": "/run/secrets/creator_password", "EPISTEMICS_POSTGRES_PASSWORD_FILE": "/run/secrets/epistemics_db_password", "INTERACTION_POSTGRES_PASSWORD_FILE": "/run/secrets/interaction_db_password", } var derivedPasswordByCredentialRef = map[string]string{ "EPISTEMICS_POSTGRES_CREDENTIAL_REF": "EPISTEMICS_POSTGRES_PASSWORD", "INTERACTION_POSTGRES_CREDENTIAL_REF": "INTERACTION_POSTGRES_PASSWORD", } var ( errBootstrapStateNotFound = errors.New("deployment bootstrap state not found") errBootstrapStateValidationFailed = errors.New("deployment bootstrap state invalid") errOneTimeMigrationNotImplemented = errors.New("NOT_IMPLEMENTED: one-time migration behavior not yet supported in this step") errBootstrapSourceEnvInvalid = errors.New("bootstrap source env is missing required canonical values") ) func defaultDurableBootstrapStatePath(args ...string) string { root := "." if len(args) > 0 && strings.TrimSpace(args[0]) != "" { root = args[0] } return filepath.Join(filepath.Clean(root), ".devctl", durableBootstrapStatePath) } func defaultProtectedSecretBackendRoot() string { if value := os.Getenv("SECRET_BACKEND_ROOT"); strings.TrimSpace(value) != "" { return filepath.Clean(strings.TrimSpace(value)) } return "/run/secrets" } type deploymentTarget struct { Name string ComposeFile string DockerContext string ProjectName string Containers []string Volumes []string PlanExpiryHours int } type deploymentPlan struct { Version int `json:"version"` PlanID string `json:"plan_id"` CreatedAt string `json:"created_at_utc"` ExpiresAt string `json:"expires_at_utc"` Readiness string `json:"readiness"` ReadinessReasons []string `json:"readiness_reasons"` Target string `json:"target"` ComposeFile string `json:"compose_file"` DockerContext string `json:"docker_context"` ProjectName string `json:"project_name"` EnvFile string `json:"env_file"` BootstrapState string `json:"bootstrap_state"` SourceRevision string `json:"source_revision"` ArtifactRevision string `json:"artifact_revision"` SourceRevisionMode string `json:"source_revision_mode"` DeploymentConfigRev string `json:"deployment_config_revision"` CurrentProductionRev string `json:"current_production_revision"` KnownGoodRevision string `json:"known_good_revision"` Migrations migrationReadiness `json:"migrations"` Inputs []deploymentInputRequirement `json:"inputs"` Classify map[string][]string `json:"input_classification"` SecretReferences []string `json:"secret_references"` ResolvedConfigHash string `json:"resolved_configuration_hash"` PlanBindingHash string `json:"plan_binding_hash"` Containers []string `json:"containers"` Volumes []string `json:"volumes"` KnownInputsSource map[string]string `json:"known_inputs_source"` } type deploymentInputRequirement struct { Name string `json:"name"` Required bool `json:"required"` SourceClass string `json:"source_class"` Resolved bool `json:"resolved"` Source string `json:"source"` IsSecret bool `json:"is_secret"` RefHash string `json:"value_ref_hash"` Error string `json:"error,omitempty"` } type migrationReadiness struct { CurrentMaxMigration string `json:"current_max_migration"` TargetMaxMigration string `json:"target_max_migration"` Status string `json:"status"` NeedKnownGood bool `json:"need_known_good"` } type deploymentManifest struct { Version int `json:"version"` Project string `json:"project"` Revision string `json:"revision"` Artifacts []struct { Container string `json:"container"` ImageID string `json:"image_id"` Revision string `json:"revision"` } `json:"artifacts"` } func deploy(args []string) error { if len(args) == 0 { return fmt.Errorf("deploy requires prepare, apply, or bootstrap") } switch args[0] { case "prepare": return deployPrepare(args[1:]) case "apply": return deployApply(args[1:]) case "bootstrap": return deployBootstrap(args[1:]) default: return fmt.Errorf("unknown deploy action %q", args[0]) } } func deployPrepare(args []string) error { fs := flag.NewFlagSet("deploy prepare", flag.ContinueOnError) fs.SetOutput(io.Discard) target := fs.String("target", "production", "deployment target") compose := fs.String("compose", "", "compose file") planPath := fs.String("plan", "", "deployment plan output path") context := fs.String("docker-context", "", "docker context") envFile := fs.String("env-file", "", "deployment bootstrap env file") bootstrapState := fs.String("bootstrap-state", "", "path to durable bootstrap state") ttl := fs.Int("plan-ttl-hours", 24, "prepare plan TTL in hours") sourceRevisionArg := fs.String("source-revision", "", "source revision to pin (full commit SHA)") knownGoodManifest := fs.String("known-good-manifest", defaultKnownGoodManifestPath(), "known-good manifest path") if err := fs.Parse(args); err != nil { return err } t, err := resolveDeploymentTarget(*target) if err != nil { return err } if *compose != "" { t.ComposeFile = *compose } if *context != "" { t.DockerContext = *context } if strings.TrimSpace(*compose) == "" { // explicit compose for clarity and reproducibility if t.ComposeFile == "" { return fmt.Errorf("compose is required") } } if !filepath.IsAbs(t.ComposeFile) { if abs, e := filepath.Abs(t.ComposeFile); e == nil { t.ComposeFile = abs } } if *ttl <= 0 || *ttl > 24*30 { return fmt.Errorf("plan-ttl-hours must be between 1 and 720") } if t.Name == "production" && strings.TrimSpace(*envFile) == "" { return fmt.Errorf("production prepare requires --env-file") } root, err := gitRepositoryRoot(".") if err != nil { return err } if strings.TrimSpace(*bootstrapState) == "" { *bootstrapState = defaultDurableBootstrapStatePath(root) } plan, err := buildDeploymentPlan(root, t, *envFile, *sourceRevisionArg, *ttl, *knownGoodManifest, *bootstrapState) if err != nil { return err } if *planPath == "" { base := filepath.Join(".devctl", "deployment-plans") if err := os.MkdirAll(base, 0o700); err != nil { return err } ts := time.Now().UTC().Format("20060102T150405Z") *planPath = filepath.Join(base, fmt.Sprintf("%s-%s-%s.json", strings.ToLower(plan.Target), strings.ReplaceAll(plan.PlanID, ":", "-"), ts)) } payload, err := json.MarshalIndent(plan, "", " ") if err != nil { return err } if err := os.WriteFile(*planPath, payload, 0o600); err != nil { return err } fmt.Printf("DeploymentPlan path=%s readiness=%s\n", *planPath, plan.Readiness) for _, reason := range plan.ReadinessReasons { fmt.Printf("readiness-blocker=%s\n", reason) } return nil } func deployBootstrap(args []string) error { fs := flag.NewFlagSet("deploy bootstrap", flag.ContinueOnError) fs.SetOutput(io.Discard) target := fs.String("target", "production", "deployment target") statePath := fs.String("state", "", "durable bootstrap state path") sourceEnv := fs.String("source-env", "", "source env containing recoverable bootstrap values") secretBackend := fs.String("secret-backend-root", "", "protected secret backend root") if err := fs.Parse(args); err != nil { return err } t, err := resolveDeploymentTarget(*target) if err != nil { return err } if t.Name != "production" { return fmt.Errorf("BREAK_GLASS/ONE_TIME_MIGRATION supports production only") } if strings.TrimSpace(*sourceEnv) == "" { return fmt.Errorf("BREAK_GLASS/ONE_TIME_MIGRATION requires --source-env") } if strings.TrimSpace(*statePath) == "" { *statePath = defaultDurableBootstrapStatePath() } if strings.TrimSpace(*statePath) == "" { return fmt.Errorf("durable bootstrap state path unavailable") } statePathAbs, err := filepath.Abs(*statePath) if err != nil { return err } *statePath = statePathAbs backend := strings.TrimSpace(*secretBackend) if backend == "" { backend = defaultProtectedSecretBackendRoot() } if backend == "" { return fmt.Errorf("protected secret backend unavailable") } backendExists := strings.TrimSpace(backend) if filepath.Clean(filepath.ToSlash(backendExists)) == "." || strings.TrimSpace(backendExists) == "" { return fmt.Errorf("protected secret backend unavailable") } migrated, missing, err := performOneTimeBootstrapMigration(*statePath, *sourceEnv, t.Name, backendExists) if err != nil { return err } if len(missing) > 0 { return fmt.Errorf("BREAK_GLASS/ONE_TIME_MIGRATION failed to recover: %s", strings.Join(missing, ", ")) } if migrated { fmt.Printf("BREAK_GLASS/ONE_TIME_MIGRATION completed state=%s backend=%s\n", *statePath, backendExists) } return nil } func performOneTimeBootstrapMigration(statePath, sourceEnvPath, target, backendRoot string) (bool, []string, error) { envValues, err := parseEnvFile(sourceEnvPath) if err != nil { return false, nil, err } migrationState := deploymentBootstrapState{ Version: durableBootstrapStateVersion, Target: target, ProtectedSecretRoot: filepath.Clean(filepath.ToSlash(backendRoot)), Inputs: map[string]bootstrapInputMetadata{}, LastMigratedAtUtc: time.Now().UTC().Format(time.RFC3339), MigrationType: "ONE_TIME_SOURCE_ENV", DerivedFromDurableRef: true, } for _, canonicalInput := range stableDurableInputs() { switch canonicalInput { case "CREATOR_BOOTSTRAP_TOKEN", "CREATOR_SESSION_SIGNING_KEY": meta := bootstrapInputMetadata{ CanonicalInput: canonicalInput, RuntimeFile: resolveCanonicalRuntimeFile(canonicalInput, envValues), BackendFile: resolveBootstrapBackendFile(canonicalInput, envValues), } if meta.RuntimeFile == "" { meta.RuntimeFile = meta.BackendFile } migrationState.Inputs[canonicalInput] = meta case "CREATOR_PASSWORD_FILE": meta := bootstrapInputMetadata{ CanonicalInput: canonicalInput, RuntimeFile: resolveCanonicalRuntimeFile(canonicalInput, envValues), BackendFile: resolveBootstrapBackendFile(canonicalInput, envValues), } if meta.RuntimeFile == "" { meta.RuntimeFile = meta.BackendFile } migrationState.Inputs[canonicalInput] = meta case "EPISTEMICS_POSTGRES_CREDENTIAL_REF", "INTERACTION_POSTGRES_CREDENTIAL_REF": passwordFile := derivedPasswordByCredentialRef[canonicalInput] meta := bootstrapInputMetadata{ CanonicalInput: canonicalInput, RuntimeFile: resolveCanonicalRuntimeFile(passwordFile, envValues), BackendFile: resolveBootstrapBackendFile(passwordFile, envValues), SecretRef: envValues[canonicalInput], } if strings.TrimSpace(meta.SecretRef) == "" { meta.SecretRef = strings.TrimSpace(durableBootstrapCredentialDefaults[canonicalInput]) } if meta.RuntimeFile == "" { meta.RuntimeFile = meta.BackendFile } migrationState.Inputs[canonicalInput] = meta } } missing := missingBootstrapInputsFromState(&migrationState) if len(missing) > 0 { return false, missing, nil } if err := os.MkdirAll(filepath.Dir(statePath), 0o700); err != nil { return false, nil, err } payload, err := json.MarshalIndent(migrationState, "", " ") if err != nil { return false, nil, err } if err := os.WriteFile(statePath, payload, 0o600); err != nil { return false, nil, err } return true, nil, nil } func loadBootstrapStateForDeployment(statePath string) (*deploymentBootstrapState, error) { if strings.TrimSpace(statePath) == "" { return nil, nil } content, err := os.ReadFile(statePath) if err != nil { return nil, fmt.Errorf("%w: %v", errBootstrapStateNotFound, err) } var state deploymentBootstrapState if err := json.Unmarshal(content, &state); err != nil { return nil, fmt.Errorf("%w: %v", errBootstrapStateValidationFailed, err) } if err := validateDeploymentBootstrapState(&state); err != nil { return nil, err } return &state, nil } func validateDeploymentBootstrapState(state *deploymentBootstrapState) error { if state == nil { return fmt.Errorf("%w: missing state", errBootstrapStateValidationFailed) } if state.Version != durableBootstrapStateVersion { return fmt.Errorf("%w: unexpected version %d", errBootstrapStateValidationFailed, state.Version) } if strings.TrimSpace(state.Target) == "" { state.Target = "production" } if strings.TrimSpace(state.ProtectedSecretRoot) == "" { state.ProtectedSecretRoot = defaultProtectedSecretBackendRoot() } missing := missingBootstrapInputsFromState(state) if len(missing) > 0 { return fmt.Errorf("%w: missing %s", errBootstrapSourceEnvInvalid, strings.Join(missing, ", ")) } if strings.TrimSpace(state.MigrationType) == "" { state.MigrationType = "UNKNOWN" } return nil } func missingBootstrapInputsFromState(state *deploymentBootstrapState) []string { missing := make([]string, 0) for _, canonicalInput := range stableDurableInputs() { meta, ok := state.Inputs[canonicalInput] if !ok || strings.TrimSpace(meta.CanonicalInput) == "" { missing = append(missing, canonicalInput) continue } switch canonicalInput { case "EPISTEMICS_POSTGRES_CREDENTIAL_REF", "INTERACTION_POSTGRES_CREDENTIAL_REF": if strings.TrimSpace(meta.SecretRef) == "" || strings.TrimSpace(meta.RuntimeFile) == "" { missing = append(missing, canonicalInput) } case "CREATOR_PASSWORD_FILE": if strings.TrimSpace(meta.RuntimeFile) == "" { missing = append(missing, canonicalInput) } default: if strings.TrimSpace(meta.RuntimeFile) == "" && strings.TrimSpace(meta.SecretRef) == "" { missing = append(missing, canonicalInput) } } } return missing } func stableDurableInputs() []string { inputs := make([]string, 0, len(requiredDurableInputsForProduction)) for name := range requiredDurableInputsForProduction { inputs = append(inputs, name) } sort.Strings(inputs) return inputs } func resolveCanonicalRuntimeFile(canonicalInput string, envValues map[string]string) string { if strings.TrimSpace(canonicalInput) == "" { return "" } if strings.HasSuffix(canonicalInput, "_FILE") { if explicit := strings.TrimSpace(envValues[canonicalInput]); explicit != "" { return explicit } if fallback := defaultDurableBootstrapRuntimeFile(canonicalInput); fallback != "" { return fallback } } if explicit := strings.TrimSpace(envValues[canonicalInput+"_FILE"]); explicit != "" { return explicit } if strings.HasSuffix(canonicalInput, "_CREDENTIAL_REF") { passwordInput := derivedPasswordInputByRuntimeFile(canonicalInput) if explicit := strings.TrimSpace(envValues[passwordInput]); explicit != "" { return explicit } if fallback := defaultDurableBootstrapRuntimeFile(passwordInput); fallback != "" { return fallback } } return "" } func resolveBootstrapBackendFile(canonicalInput string, envValues map[string]string) string { if canonicalInput == "" { return "" } explicit := strings.TrimSpace(envValues[canonicalInput+"_FILE"]) if explicit != "" { return explicit } runtimeFile := resolveCanonicalRuntimeFile(canonicalInput, envValues) if runtimeFile != "" { return runtimeFile } return "" } func derivedPasswordInputByRuntimeFile(credentialRef string) string { if credentialRef == "" { return "" } return strings.TrimSuffix(credentialRef, "_CREDENTIAL_REF") + "_PASSWORD" } func defaultPasswordFileForCredentialRef(credentialRef string) string { passwordInput := derivedPasswordInputByRuntimeFile(credentialRef) if strings.TrimSpace(passwordInput) == "" { return "" } return defaultDurableBootstrapRuntimeFile(passwordInput) } func defaultDurableBootstrapRuntimeFile(canonicalInput string) string { return durableBootstrapRuntimeFileDefaults[canonicalInput] } func resolveDeploymentInputsWithBootstrap(inputs map[string]composeInput, envFile string, state *deploymentBootstrapState, statePath string, requireDurable bool) ([]deploymentInputRequirement, map[string][]string, map[string]string, error) { envValues := map[string]string{} sources := map[string]string{} if strings.TrimSpace(envFile) != "" { loaded, err := parseEnvFile(envFile) if err != nil { return nil, nil, nil, err } for key, value := range loaded { envValues[key] = strings.TrimSpace(value) } } result := make([]deploymentInputRequirement, 0, len(inputs)) classification := map[string][]string{} for name, spec := range inputs { resolved := false source := "missing" value := "" refHash := "" bootstrapMeta, bootstrapResolved := bootstrapInputFromState(name, state) if v, ok := envValues[name]; ok && strings.TrimSpace(v) != "" { value = strings.TrimSpace(v) source = "file" resolved = true } else if requireDurable && bootstrapResolved { source = "bootstrap-state" resolved = true refHash = bootstrapInputRefHash(bootstrapMeta) } else if !requireDurable { if v, ok := os.LookupEnv(name); ok { value = strings.TrimSpace(v) source = "ambient" resolved = value != "" } } srcClass := inputSourceClass(name) if !strings.HasPrefix(srcClass, "SECRET") && strings.Contains(name, "SECRET") { srcClass = "SECRET_REFERENCE" } req := deploymentInputRequirement{ Name: name, Required: spec.required, SourceClass: srcClass, Source: source, Resolved: resolved, IsSecret: isSecretInput(name), RefHash: func() string { if source == "bootstrap-state" { if refHash != "" { return refHash } return bootstrapInputRefHash(bootstrapMeta) } if strings.TrimSpace(value) == "" { return "" } return valueFingerprint(name, value) }(), } if req.Required && !req.Resolved { if source == "ambient" { req.Error = "required production input resolved from ambient shell" } else if source == "bootstrap-state" { req.Error = "required production input unresolved in bootstrap state" } else { req.Error = "required production input unresolved" } } result = append(result, req) classification[srcClass] = append(classification[srcClass], name) if source == "file" { sources[name] = envFile } else if source == "ambient" { sources[name] = "ambient" } else if source == "bootstrap-state" { sources[name] = "bootstrap-state" if statePath != "" { sources[name] = statePath } } else { sources[name] = "missing" } } return result, classification, sources, nil } func bootstrapInputFromState(inputName string, state *deploymentBootstrapState) (bootstrapInputMetadata, bool) { var empty bootstrapInputMetadata if state == nil || state.Inputs == nil || strings.TrimSpace(inputName) == "" { return empty, false } if meta, ok := state.Inputs[inputName]; ok && deploymentBootstrapMetaIsResolved(meta) { return meta, true } if refName, ok := derivedCredentialRefFromPasswordInput(inputName); ok { meta, present := state.Inputs[refName] if present && deploymentBootstrapMetaIsResolved(meta) { return meta, true } } return empty, false } func derivedCredentialRefFromPasswordInput(passwordInput string) (string, bool) { for refInput, derived := range derivedPasswordByCredentialRef { if strings.TrimSpace(derived) == strings.TrimSpace(passwordInput) { return refInput, true } } return "", false } func deploymentBootstrapMetaIsResolved(meta bootstrapInputMetadata) bool { if strings.TrimSpace(meta.CanonicalInput) == "" { return false } if strings.HasSuffix(meta.CanonicalInput, "_CREDENTIAL_REF") { return strings.TrimSpace(meta.SecretRef) != "" && strings.TrimSpace(meta.RuntimeFile) != "" } return strings.TrimSpace(meta.RuntimeFile) != "" || strings.TrimSpace(meta.SecretRef) != "" } func bootstrapInputRefHash(meta bootstrapInputMetadata) string { if strings.TrimSpace(meta.SecretRef) == "" && strings.TrimSpace(meta.RuntimeFile) == "" && strings.TrimSpace(meta.BackendFile) == "" { return "" } return valueFingerprint(meta.CanonicalInput, strings.Join([]string{ meta.CanonicalInput, meta.SecretRef, meta.RuntimeFile, meta.BackendFile, }, "|")) } func bootstrapInputFromStateLegacy(inputName string, state *deploymentBootstrapState) bool { if state == nil || state.Inputs == nil || strings.TrimSpace(inputName) == "" { return false } meta, _ := bootstrapInputFromState(inputName, state) return meta.CanonicalInput != "" } func deployApply(args []string) error { fs := flag.NewFlagSet("deploy apply", flag.ContinueOnError) fs.SetOutput(io.Discard) planPath := fs.String("plan", "", "deployment plan path") dryRun := fs.Bool("dry-run", false, "validate plan without mutating") skipHealth := fs.Bool("skip-health", false, "skip health gate") autoRollback := fs.Bool("auto-rollback", false, "rollback to known-good revision on first health failure") if err := fs.Parse(args); err != nil { return err } if strings.TrimSpace(*planPath) == "" { return fmt.Errorf("plan path is required") } plan, err := readDeploymentPlan(*planPath) if err != nil { return err } if err := assertPlanCurrent(plan); err != nil { return err } if *dryRun { fmt.Printf("deploy apply dry-run: readiness=%s\n", plan.Readiness) return nil } if err := executeComposeDeploy(plan); err != nil { return err } if *skipHealth { fmt.Println("deploy apply: READY (health skipped)") return nil } if err := validatePostApplyHealth(plan); err != nil { if !*autoRollback { return err } if strings.TrimSpace(plan.KnownGoodRevision) == "" { return fmt.Errorf("health failed (%v) and no known-good revision in plan", err) } recoveryPlan, errR := rollbackToKnownGood(plan) if errR != nil { return fmt.Errorf("health failed (%v) and rollback failed: %v", err, errR) } if err := validatePostApplyHealth(recoveryPlan); err != nil { return fmt.Errorf("health failed and rollback did not restore health: %w", err) } fmt.Printf("deploy apply: HEALTH_ROLLBACK_SUCCESS known_good=%s\n", plan.KnownGoodRevision) return nil } fmt.Println("deploy apply: READY") return nil } func buildDeploymentPlan(root string, target deploymentTarget, envFile, sourceRevisionArg string, ttlHours int, knownGoodManifest string, bootstrapStateArgs ...string) (*deploymentPlan, error) { bootstrapStatePath := "" if len(bootstrapStateArgs) > 0 && strings.TrimSpace(bootstrapStateArgs[0]) != "" { bootstrapStatePath = strings.TrimSpace(bootstrapStateArgs[0]) } notify := []string{} bootstrapState, bootstrapErr := loadBootstrapStateForDeployment(bootstrapStatePath) if bootstrapErr != nil && strings.TrimSpace(bootstrapStatePath) != "" { notify = append(notify, fmt.Sprintf("bootstrap-state read: %v", bootstrapErr)) } inputMap, err := discoverComposeInputsFromFile(target.ComposeFile) if err != nil { return nil, err } inputResolutions, classification, inputSources, err := resolveDeploymentInputsWithBootstrap(inputMap, envFile, bootstrapState, bootstrapStatePath, target.Name == "production") if err != nil { if target.Name != "production" { return nil, err } } for _, req := range inputResolutions { if req.Required && !req.Resolved { notify = append(notify, req.Name) } } sourceRevision, mode, err := resolveSourceRevision(root, sourceRevisionArg) if err != nil { return nil, err } artifactRevision := sourceRevision configRevision := resolveDeploymentConfigRevision(target.ComposeFile) knownGood, currentProd, errKnownGood := captureKnownGood(target, knownGoodManifest) if errKnownGood != nil { notify = append(notify, fmt.Sprintf("known-good capture: %v", errKnownGood)) } migration, _ := migrationState(root, artifactRevision, knownGood) readiness := readinessReady reasons := []string{} for _, v := range notify { readiness = readinessFailed reasons = append(reasons, v) } if len(notify) == 0 && sourceRevision == "" { readiness = readinessFailed reasons = append(reasons, "source/artifact revision unresolved") } inputList := make([]deploymentInputRequirement, 0, len(inputResolutions)) for _, v := range inputResolutions { inputList = append(inputList, v) } sort.Slice(inputList, func(i, j int) bool { return inputList[i].Name < inputList[j].Name }) secretRefs := []string{} for _, input := range inputList { if input.SourceClass == "SECRET_REFERENCE" { secretRefs = append(secretRefs, input.Name) } } created := time.Now().UTC() plan := &deploymentPlan{ Version: planVersion, PlanID: genPlanID(target.Name, sourceRevision), CreatedAt: created.Format(time.RFC3339), ExpiresAt: created.Add(time.Duration(ttlHours) * time.Hour).Format(time.RFC3339), Readiness: readiness, ReadinessReasons: reasons, Target: target.Name, ComposeFile: target.ComposeFile, DockerContext: target.DockerContext, ProjectName: target.ProjectName, EnvFile: envFile, BootstrapState: bootstrapStatePath, SourceRevision: sourceRevision, SourceRevisionMode: mode, ArtifactRevision: artifactRevision, DeploymentConfigRev: configRevision, CurrentProductionRev: currentProd, KnownGoodRevision: knownGood, Migrations: migration, Inputs: inputList, Classify: classification, SecretReferences: uniqueSorted(secretRefs), ResolvedConfigHash: resolveConfigHash(target.ComposeFile), Containers: target.Containers, Volumes: target.Volumes, KnownInputsSource: inputSources, } plan.PlanBindingHash = buildPlanBindingHash(plan) if readiness == readinessFailed { plan.ReadinessReasons = dedupeAndSort(plan.ReadinessReasons) } return plan, nil } func executeComposeDeploy(plan *deploymentPlan) error { if plan == nil { return errors.New("missing plan") } if plan.Readiness != readinessReady { return fmt.Errorf("plan is not ready (%s)", strings.Join(plan.ReadinessReasons, "; ")) } if err := runCommandWithEnv(plan); err != nil { return err } return nil } func runCommandWithEnv(plan *deploymentPlan) error { args := []string{"--context", plan.DockerContext, "compose"} if plan.EnvFile != "" { args = append(args, "--env-file", plan.EnvFile) } args = append(args, "-p", plan.ProjectName, "-f", plan.ComposeFile, "up", "-d", "--no-build", "--force-recreate") cmd := exec.Command("docker", args...) cmd.Env = os.Environ() cmd.Env = append(cmd.Env, fmt.Sprintf("ASSISTANT_SOURCE_REVISION=%s", plan.ArtifactRevision)) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr fmt.Printf("deploy apply command: %s\n", strings.Join(cmd.Args, " ")) return cmd.Run() } func validatePostApplyHealth(plan *deploymentPlan) error { target, err := resolveDeploymentTarget(plan.Target) if err != nil { return err } target.ComposeFile = plan.ComposeFile target.DockerContext = plan.DockerContext target.ProjectName = plan.ProjectName if err := validateDeploymentRunningContainers(plan, target); err != nil { return err } if err := validateDeploymentVolumes(plan, target); err != nil { return err } for _, u := range deploymentExternalHealthEndpoints { if err := validateHTTPHealth(u); err != nil { return fmt.Errorf("health gate failed for %s: %w", u, err) } } return nil } func rollbackToKnownGood(plan *deploymentPlan) (*deploymentPlan, error) { if strings.TrimSpace(plan.KnownGoodRevision) == "" { return nil, fmt.Errorf("no known-good revision in plan") } recovery := *plan recovery.Readiness = readinessReady recovery.ReadinessReasons = nil recovery.ArtifactRevision = plan.KnownGoodRevision recovery.SourceRevision = plan.KnownGoodRevision recovery.PlanBindingHash = buildPlanBindingHash(&recovery) if err := executeComposeDeploy(&recovery); err != nil { return nil, err } return &recovery, nil } func assertPlanCurrent(plan *deploymentPlan) error { if plan == nil { return errors.New("missing plan") } if err := validatePlanReadiness(plan); err != nil { return err } if plan.Readiness == readinessFailed { return fmt.Errorf("plan is not ready (%s)", strings.Join(plan.ReadinessReasons, "; ")) } expiry, err := time.Parse(time.RFC3339, plan.ExpiresAt) if err != nil { return fmt.Errorf("invalid expires_at_utc") } if time.Now().UTC().After(expiry) { return fmt.Errorf("plan expired at %s", plan.ExpiresAt) } target, err := resolveDeploymentTarget(plan.Target) if err != nil { return err } if strings.TrimSpace(plan.ComposeFile) != "" { target.ComposeFile = plan.ComposeFile } planCompose := plan.ComposeFile targetCompose := target.ComposeFile if strings.TrimSpace(planCompose) == "" || strings.TrimSpace(targetCompose) == "" { return fmt.Errorf("stale plan: compose file changed") } if !filepath.IsAbs(planCompose) { if abs, e := filepath.Abs(planCompose); e == nil { planCompose = abs } } if !filepath.IsAbs(targetCompose) { if abs, e := filepath.Abs(targetCompose); e == nil { targetCompose = abs } } if filepath.Clean(planCompose) != filepath.Clean(targetCompose) { return fmt.Errorf("stale plan: compose file changed") } if target.DockerContext != "" && plan.DockerContext != target.DockerContext { return fmt.Errorf("stale plan: docker context changed") } currentConfig := resolveConfigHash(plan.ComposeFile) if plan.DeploymentConfigRev != currentConfig { return fmt.Errorf("stale plan: deployment config changed") } if plan.SourceRevisionMode == "head" { currentHead, _, err := resolveSourceRevision(filepath.Dir(plan.ComposeFile), "") if err != nil { return err } if currentHead != plan.SourceRevision { return fmt.Errorf("stale plan: source revision changed") } } inputMap, err := discoverComposeInputsFromFile(plan.ComposeFile) if err != nil { return err } currentInputs, _, _, err := resolveDeploymentInputs(inputMap, plan.EnvFile, target.Name == "production") if err != nil { return err } currentBindings := buildPlanBindingFromInputs( currentInputs, plan.SourceRevision, plan.ArtifactRevision, currentConfig, target.DockerContext, target.Containers, target.Volumes, ) if plan.PlanBindingHash != currentBindings { return fmt.Errorf("stale plan: required assumptions changed") } root, err := gitRepositoryRoot(".") if err != nil { return err } currentMigration, err := migrationState(root, plan.ArtifactRevision, plan.CurrentProductionRev) if err != nil { return fmt.Errorf("stale plan: migration state check failed: %w", err) } if !migrationReadinessEqual(plan.Migrations, currentMigration) { return fmt.Errorf("stale plan: migration state changed") } if plan.Target == "production" && currentMigration.NeedKnownGood && strings.TrimSpace(plan.KnownGoodRevision) == "" { return fmt.Errorf("migration precondition failed: %s", currentMigration.Status) } // explicit re-check of required inputs from env file ensures no ambient fallback inputResolutions, _, _, err := resolveDeploymentInputs(inputMap, plan.EnvFile, target.Name == "production") if err != nil { return err } for _, req := range inputResolutions { if req.Required && !req.Resolved { return fmt.Errorf("required input %s unresolved since prepare", req.Name) } } return nil } func migrationStateFromRevisions(root, targetRevision, currentRevision string) (migrationReadiness, error) { currentMax := "0" targetMax := "0" var err error if strings.TrimSpace(currentRevision) != "" { currentMax, err = maxMigrationPrefixForRevision(root, currentRevision) if err != nil { currentMax = "unknown" } } targetMax, err = maxMigrationPrefixForRevision(root, targetRevision) if err != nil { targetMax = "unknown" } currentMaxValue, currentParseErr := strconv.Atoi(currentMax) targetMaxValue, targetParseErr := strconv.Atoi(targetMax) status := "matched" needKnownGood := false if currentMax == "unknown" || targetMax == "unknown" || currentParseErr != nil || targetParseErr != nil { status = "needs-manual-verification" needKnownGood = true } else if currentMaxValue != targetMaxValue { if currentMaxValue > targetMaxValue { status = "target-lower-than-production" } else { status = "upgrade-required" } needKnownGood = true } return migrationReadiness{ CurrentMaxMigration: currentMax, TargetMaxMigration: targetMax, Status: status, NeedKnownGood: needKnownGood, }, nil } func migrationReadinessEqual(a, b migrationReadiness) bool { return a.CurrentMaxMigration == b.CurrentMaxMigration && a.TargetMaxMigration == b.TargetMaxMigration && a.Status == b.Status && a.NeedKnownGood == b.NeedKnownGood } func resolvePlanInputFingerprint(reqs []deploymentInputRequirement) string { parts := make([]string, 0, len(reqs)) for _, req := range reqs { parts = append(parts, fmt.Sprintf("%s=%s@%s", req.Name, req.RefHash, req.Source)) } sort.Strings(parts) return hashString(strings.Join(parts, "|")) } func buildPlanBindingHash(plan *deploymentPlan) string { raw := buildPlanBindingFromInputs( plan.Inputs, plan.SourceRevision, plan.ArtifactRevision, plan.DeploymentConfigRev, plan.DockerContext, plan.Containers, plan.Volumes, ) return hashString(raw) } func buildPlanBindingFromInputs(reqs []deploymentInputRequirement, sourceRevision, artifactRevision, configRev, context string, containers, volumes []string) string { parts := []string{ sourceRevision, artifactRevision, configRev, context, } if len(containers) > 0 { orderedContainers := append([]string{}, containers...) sort.Strings(orderedContainers) parts = append(parts, strings.Join(orderedContainers, ",")) } if len(volumes) > 0 { orderedVolumes := append([]string{}, volumes...) sort.Strings(orderedVolumes) parts = append(parts, strings.Join(orderedVolumes, ",")) } for _, req := range reqs { parts = append(parts, fmt.Sprintf("%s=%s@%s", req.Name, req.RefHash, req.Source)) } sort.Strings(parts) return hashString(strings.Join(parts, "|")) } func validatePlanReadiness(plan *deploymentPlan) error { if plan.Readiness == "" { return errors.New("plan readiness is empty") } if plan.SourceRevision == "" || plan.ArtifactRevision == "" { return fmt.Errorf("plan missing source/artifact revision") } if plan.DeploymentConfigRev == "" { return fmt.Errorf("plan missing deployment configuration revision") } if len(plan.Inputs) == 0 { return fmt.Errorf("plan has no input requirements") } for _, req := range plan.Inputs { if req.Required && req.Source == "ambient" { return fmt.Errorf("required input %s resolved from ambient source", req.Name) } if req.Required && !req.Resolved { return fmt.Errorf("required input %s unresolved", req.Name) } } return nil } func discoverComposeInputsFromFile(composeFile string) (map[string]composeInput, error) { b, err := os.ReadFile(composeFile) if err != nil { return nil, err } return discoverComposeInputs(string(b)), nil } type composeInput struct { name string required bool } func discoverComposeInputs(content string) map[string]composeInput { result := map[string]composeInput{} re := regexp.MustCompile(`\$\{([A-Z0-9_]+)(?::-[^}]*)?\}`) for _, match := range re.FindAllStringSubmatch(content, -1) { if len(match) < 2 { continue } token := match[0] name := match[1] result[name] = composeInput{name: name, required: !strings.Contains(token, ":-")} } for name := range requiredChatOnlyInputs { if _, ok := result[name]; !ok { result[name] = composeInput{name: name, required: true} } } return result } func resolveDeploymentInputs(inputs map[string]composeInput, envFile string, requireDurable bool) ([]deploymentInputRequirement, map[string][]string, map[string]string, error) { envValues := map[string]string{} sources := map[string]string{} if strings.TrimSpace(envFile) != "" { loaded, err := parseEnvFile(envFile) if err != nil { return nil, nil, nil, err } for key, value := range loaded { envValues[key] = strings.TrimSpace(value) sources[key] = "file" } } result := make([]deploymentInputRequirement, 0, len(inputs)) classification := map[string][]string{} for name, spec := range inputs { required := spec.required source := "missing" resolved := false value := "" if v, ok := envValues[name]; ok { value = v source = "file" resolved = strings.TrimSpace(v) != "" if !resolved { source = "file" } } if !resolved && os.Getenv(name) != "" { if requireDurable { source = "ambient" } else { value = os.Getenv(name) source = "ambient" resolved = true } } srcClass := inputSourceClass(name) if !strings.HasPrefix(srcClass, "SECRET") && strings.Contains(name, "SECRET") { srcClass = "SECRET_REFERENCE" } req := deploymentInputRequirement{ Name: name, Required: required, SourceClass: srcClass, Resolved: resolved, Source: source, IsSecret: isSecretInput(name), RefHash: func() string { if strings.TrimSpace(value) == "" { return "" } return valueFingerprint(name, value) }(), } if req.Required && !req.Resolved { if source == "ambient" { req.Error = "required production input resolved from ambient shell" } else { req.Error = "required production input unresolved" } } result = append(result, req) classification[srcClass] = append(classification[srcClass], name) if source == "file" { sources[name] = envFile } else if source == "ambient" { sources[name] = "ambient" } else { sources[name] = "missing" } } return result, classification, sources, nil } func runCommand(name string, args ...string) ([]byte, error) { cmd := exec.Command(name, args...) out, err := cmd.Output() if err != nil { if exitErr, ok := err.(*exec.ExitError); ok && len(exitErr.Stderr) > 0 { return nil, fmt.Errorf("%s", strings.TrimSpace(string(exitErr.Stderr))) } return nil, err } return out, nil } func parseEnvFile(path string) (map[string]string, error) { contents, err := os.ReadFile(path) if err != nil { return nil, err } result := make(map[string]string) for _, raw := range strings.Split(string(contents), "\n") { line := strings.TrimSpace(raw) if line == "" || strings.HasPrefix(line, "#") { continue } if strings.HasPrefix(line, "export ") { line = strings.TrimSpace(strings.TrimPrefix(line, "export ")) } parts := strings.SplitN(line, "=", 2) if len(parts) != 2 { return nil, fmt.Errorf("invalid env line: %q", raw) } key := strings.TrimSpace(parts[0]) value := strings.Trim(strings.TrimSpace(parts[1]), "\"'") result[key] = value } return result, nil } func resolveSourceRevision(root, override string) (string, string, error) { if strings.TrimSpace(override) != "" { out, err := runGit(root, "rev-parse", override) if err != nil { return "", "", err } rev := strings.TrimSpace(string(out)) if !isHex40(rev) { return "", "", fmt.Errorf("revision is not 40-char commit: %q", rev) } return rev, "explicit", nil } out, err := runGit(root, "rev-parse", "HEAD") if err != nil { return "", "", err } rev := strings.TrimSpace(string(out)) if !isHex40(rev) { return "", "", fmt.Errorf("HEAD is not a full commit: %q", rev) } return rev, "head", nil } func resolveDeploymentConfigRevision(composeFile string) string { b, err := os.ReadFile(composeFile) if err != nil { return "" } return hashString(string(b)) } func resolveConfigHash(composeFile string) string { b, err := os.ReadFile(composeFile) if err != nil { return "" } return hashString(string(b)) } func captureKnownGood(target deploymentTarget, manifestPath string) (string, string, error) { known := "" current := "" if manifestPath == "" { manifestPath = defaultKnownGoodManifestPath() } if manifestPath != "" { if b, err := os.ReadFile(manifestPath); err == nil { var manifest deploymentManifest if err := json.Unmarshal(b, &manifest); err == nil { if manifest.Revision != "" { known = manifest.Revision current = manifest.Revision } } } } if known != "" { return known, current, nil } // legacy fallback for explicit recovery contexts only if currentFromContainers, err := captureCurrentProductionRevision(target); err == nil && currentFromContainers != "" { return currentFromContainers, currentFromContainers, nil } return "", "", fmt.Errorf("known-good manifest unavailable") } func captureCurrentProductionRevision(target deploymentTarget) (string, error) { revisions := make([]string, 0, len(target.Containers)) for _, container := range target.Containers { label, err := dockerContainerLabel(target.DockerContext, container, "org.opencontainers.image.revision") if err != nil { return "", err } if label != "" { revisions = append(revisions, label) } } if len(revisions) == 0 { return "", fmt.Errorf("no production container labels available") } base := revisions[0] for _, revision := range revisions[1:] { if revision != base { return "", fmt.Errorf("container revision mismatch among production services") } } return base, nil } func migrationState(root, targetRevision, currentRevision string) (migrationReadiness, error) { return migrationStateFromRevisions(root, targetRevision, currentRevision) } func maxMigrationPrefixForRevision(root, revision string) (string, error) { out, err := runGit(root, "ls-tree", "-r", "--name-only", revision, "database/migrations") if err != nil { return "", err } return maxMigrationPrefixFromList(strings.Split(strings.TrimSpace(string(out)), "\n")), nil } func maxMigrationPrefixFromList(paths []string) string { re := regexp.MustCompile(`^(\d{3})_`) max := "0" maxValue := 0 for _, file := range paths { if file == "" { continue } base := filepath.Base(filepath.FromSlash(strings.TrimSpace(file))) m := re.FindStringSubmatch(base) if len(m) != 2 { continue } value := m[1] i, parseErr := strconv.Atoi(value) if parseErr != nil { continue } if i > maxValue { maxValue = i } } max = fmt.Sprintf("%d", maxValue) return max } func resolveDeploymentTarget(name string) (deploymentTarget, error) { switch name { case "", "production", "prod": return deploymentTarget{ Name: "production", ComposeFile: filepath.FromSlash("deploy/docker-compose.prod.yml"), DockerContext: "docker-prod", ProjectName: "assistant", Containers: []string{ "assistant-postgres-1", "assistant-notification-receiver-1", "assistant-core-runtime-1", "assistant-continuity-runtime-1", "assistant-commit-runtime-1", "assistant-cognitive-worker-1", "assistant-creator-web-1", }, Volumes: []string{ "assistant_pg", "assistant_nats", "assistant_core_secrets", "assistant_notification_receipts", "assistant_notification_writer_secret", "assistant_notification_reader_secret", }, PlanExpiryHours: 12, }, nil default: return deploymentTarget{}, fmt.Errorf("unknown target %q", name) } } func validateDeploymentRunningContainers(plan *deploymentPlan, target deploymentTarget) error { required := map[string]struct{}{} for _, item := range target.Containers { required[item] = struct{}{} } for container := range required { revision, err := dockerContainerLabel(target.DockerContext, container, "org.opencontainers.image.revision") if err != nil { return fmt.Errorf("container %s inspect failed: %w", container, err) } if plan.ArtifactRevision != "" && revision != "" && revision != plan.ArtifactRevision { return fmt.Errorf("container %s revision mismatch (expected=%s actual=%s)", container, plan.ArtifactRevision, revision) } state, err := dockerContainerState(target.DockerContext, container) if err != nil { return fmt.Errorf("container %s state failed: %w", container, err) } if state != "running" { return fmt.Errorf("container %s state=%s", container, state) } } return nil } func validateDeploymentVolumes(_ *deploymentPlan, target deploymentTarget) error { for _, volume := range target.Volumes { if _, err := runCommand("docker", "--context", target.DockerContext, "volume", "inspect", volume); err != nil { return fmt.Errorf("required volume missing: %s", volume) } } return nil } func validateHTTPHealth(url string) error { client := &http.Client{Timeout: 12 * time.Second} req, err := http.NewRequest(http.MethodGet, url, http.NoBody) if err != nil { return err } resp, err := client.Do(req) if err != nil { return err } if resp.StatusCode < 200 || resp.StatusCode >= 300 { defer resp.Body.Close() _, _ = io.Copy(io.Discard, resp.Body) return fmt.Errorf("unexpected status: %d", resp.StatusCode) } defer resp.Body.Close() _, _ = io.Copy(io.Discard, resp.Body) return nil } func readDeploymentPlan(path string) (*deploymentPlan, error) { b, err := os.ReadFile(path) if err != nil { return nil, err } var plan deploymentPlan if err := json.Unmarshal(b, &plan); err != nil { return nil, err } return &plan, nil } func genPlanID(target, revision string) string { if len(revision) > 12 { revision = revision[:12] } if target == "" { target = "deployment" } return fmt.Sprintf("%s-%s", target, revision) } func defaultKnownGoodManifestPath() string { if local := os.Getenv("LOCALAPPDATA"); local != "" { return filepath.Join(local, "Domovoy", "monitoring", "deployment-manifest.json") } if local := os.Getenv("LOCALAPPDATA_ALT"); local != "" { return filepath.Join(local, "Domovoy", "monitoring", "deployment-manifest.json") } return "" } func inputSourceClass(name string) string { n := strings.ToUpper(name) switch { case strings.Contains(n, "_CREDENTIAL_REF"): return "SECRET_REFERENCE" case strings.Contains(n, "_FILE"): return "DOMOVOY_MANAGED" case strings.HasSuffix(n, "_PASSWORD") || strings.HasSuffix(n, "_TOKEN") || strings.HasSuffix(n, "_KEY") || strings.Contains(n, "SECRET"): return "SECRET_REFERENCE" case n == "ASSISTANT_SOURCE_REVISION": return "DEPLOYMENT_LOCAL" default: return "BOOTSTRAP" } } func isSecretInput(name string) bool { n := strings.ToUpper(name) return strings.HasSuffix(n, "_PASSWORD") || strings.HasSuffix(n, "_TOKEN") || strings.HasSuffix(n, "_KEY") || strings.Contains(n, "SECRET") } func valueFingerprint(name, value string) string { if strings.TrimSpace(value) == "" { return "" } sum := sha256.Sum256([]byte(value)) if isSecretInput(name) { return hex.EncodeToString(sum[:])[:16] } return hex.EncodeToString(sum[:]) } func hashString(value string) string { sum := sha256.Sum256([]byte(value)) return hex.EncodeToString(sum[:]) } func isHex40(value string) bool { if len(value) != 40 { return false } for _, r := range value { if (r < '0' || r > '9') && (r < 'a' || r > 'f') && (r < 'A' || r > 'F') { return false } } return true } func dedupeAndSort(values []string) []string { seen := map[string]struct{}{} out := make([]string, 0, len(values)) for _, value := range values { if _, ok := seen[value]; ok { continue } seen[value] = struct{}{} out = append(out, value) } sort.Strings(out) return out } func uniqueSorted(values []string) []string { return dedupeAndSort(values) } func dockerContainerLabel(context, container, label string) (string, error) { out, err := runCommand("docker", "--context", context, "inspect", container, "--format", fmt.Sprintf("{{index .Config.Labels \"%s\"}}", label)) if err != nil { return "", err } return strings.TrimSpace(string(out)), nil } func dockerContainerState(context, container string) (string, error) { out, err := runCommand("docker", "--context", context, "inspect", container, "--format", "{{.State.Status}}") if err != nil { return "", err } return strings.TrimSpace(string(out)), nil }