From 754cbe43d28dc106ecd085240faa38967279badc Mon Sep 17 00:00:00 2001 From: Mikhail Date: Tue, 18 Aug 2026 12:07:11 +0300 Subject: [PATCH] wip(dev-000073): preserve durable bootstrap checkpoint DEV-000073 is NOT DONE; current work is partial/stabilized WIP only. go test ./cmd/devctl was last verified PASS. Remaining bootstrap/secret integration requires a stronger long-horizon model. This commit is preservation-only; current ancestry includes DEV-000072 candidate 36fec6c05cdf7aaf413fe... and must be reconciled before DEV-000073 promotion. --- AGENTS.md | 7 + architecture/UNRESOLVED_QUESTIONS.md | 1 + cmd/devctl/deploy.go | 1611 +++++++++++++++++ cmd/devctl/deploy_test.go | 644 +++++++ cmd/devctl/main.go | 4 +- dev/ACTIVE_TASK.json | 4 +- ...ntic-interpretation-deferral-boundary.json | 13 + ...n-deployment-continuity-safe-redeploy.json | 72 + docs/DEPLOYMENT.md | 25 + docs/PROJECT_STATUS.md | 52 +- .../deployment-continuity-dev000073.md | 169 ++ .../production-recovery-incident-20260818.md | 82 + 12 files changed, 2679 insertions(+), 5 deletions(-) create mode 100644 cmd/devctl/deploy.go create mode 100644 cmd/devctl/deploy_test.go create mode 100644 dev/tasks/DEV-000073-production-deployment-continuity-safe-redeploy.json create mode 100644 docs/evidence/deployment-continuity-dev000073.md create mode 100644 docs/evidence/production-recovery-incident-20260818.md diff --git a/AGENTS.md b/AGENTS.md index cf6872a..301b843 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,6 +35,13 @@ Do not silently resolve a conflict. Return `BLOCKED_BY_SPEC_GAP` or `BLOCKED_BY_ - NO reading/copying literal passwords from other repositories into this repository. - NO production secret in source, task packet, commit message, logs or tests. +## Deployment safety (production mutation) + +- Production mutation in normal `devctl` workflow must go through `deploy prepare` then `deploy apply`. +- `deploy apply` is rejected when required inputs are `MISSING`, `EMPTY`, `INVALID`, `UNKNOWN`, `UNRESOLVED`, or `STALE`. +- Ambient shell state is not accepted as authoritative required production input; required values must come from an explicit durable source (for example `--env-file`/manifest-classified bootstrap path). +- A prepared plan must be current (inputs, compose, context, revision binding) before apply proceeds. + ## Codex freedom Inside the declared owner/module and allowed paths, choose ordinary local implementation details freely. Prefer explicit, typed, boring Go over reflection, metaprogramming or framework tricks. diff --git a/architecture/UNRESOLVED_QUESTIONS.md b/architecture/UNRESOLVED_QUESTIONS.md index 64aa1a9..c969bf2 100644 --- a/architecture/UNRESOLVED_QUESTIONS.md +++ b/architecture/UNRESOLVED_QUESTIONS.md @@ -19,6 +19,7 @@ These questions are intentionally unresolved. Codex must not silently turn tempo | UQ-013 | Multi-user organization governance | LATER | Actor/Principal/Scope designed generally; expose single-user V1 if desired | No | Actual shared workspace requirement | | UQ-014 | Canonical wire codec longevity | RESEARCH | Versioned schema/envelope; initial protobuf acceptable, not eternal identity | No | First non-Go module or major semantic migration | | UQ-015 | Exact data retention/privacy policy engine | RESEARCH | Store scope/provenance/sensitivity hooks; implement concrete policies per product deployment | No | Product/legal requirements | +| UQ-016 | Generic memory/knowledge admission from ordinary interaction | RESEARCH | Treat ordinary preferences/preferences-like assertions as candidates first; no predefined per-topic memory schemas | No | When a durable knowledge/persistence mechanism is adopted with explicit policy and owner boundaries | ## 7.1 Spec-gap protocol diff --git a/cmd/devctl/deploy.go b/cmd/devctl/deploy.go new file mode 100644 index 0000000..1d72de7 --- /dev/null +++ b/cmd/devctl/deploy.go @@ -0,0 +1,1611 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +const ( + readinessReady = "READY" + readinessFailed = "BLOCKED" + planVersion = 1 +) + +var deploymentExternalHealthEndpoints = []string{ + "https://assistant.cin.su/healthz", + "https://assistant.cin.su/chat", +} + +var requiredChatOnlyInputs = map[string]struct{}{ // conservative, explicit bootstrap for production + "POSTGRES_SUPERUSER": {}, + "POSTGRES_SUPERUSER_PASSWORD": {}, + "CONTINUITY_DB_PASSWORD": {}, + "WORK_DB_PASSWORD": {}, + "CAPABILITY_DB_PASSWORD": {}, + "COMMIT_DB_PASSWORD": {}, + "AUTHORITY_DB_PASSWORD": {}, + "IDENTITY_DB_PASSWORD": {}, + "SITUATION_DB_PASSWORD": {}, + "CREATOR_USERNAME": {}, + "CREATOR_BOOTSTRAP_TOKEN": {}, + "CREATOR_SESSION_SIGNING_KEY": {}, + "COMMIT_EXECUTOR_TOKEN": {}, + "EPISTEMICS_POSTGRES_CREDENTIAL_REF": {}, + "INTERACTION_POSTGRES_CREDENTIAL_REF": {}, +} + +const ( + durableBootstrapStateVersion = 1 + durableBootstrapStatePath = "deployment-bootstrap-state.json" +) + +type bootstrapInputMetadata struct { + CanonicalInput string `json:"canonical_input"` + BackendFile string `json:"backend_file"` + RuntimeFile string `json:"runtime_file"` + SecretRef string `json:"secret_ref"` +} + +type deploymentBootstrapState struct { + Version int `json:"version"` + Target string `json:"target"` + ProtectedSecretRoot string `json:"protected_secret_backend_root"` + Inputs map[string]bootstrapInputMetadata `json:"inputs"` + LastMigratedAtUtc string `json:"last_migrated_at_utc"` + MigrationType string `json:"migration_type"` + DerivedFromDurableRef bool `json:"derived_from_durable_ref"` +} + +var requiredDurableInputsForProduction = map[string]struct{}{ + "CREATOR_BOOTSTRAP_TOKEN": {}, + "CREATOR_SESSION_SIGNING_KEY": {}, + "CREATOR_PASSWORD_FILE": {}, + "EPISTEMICS_POSTGRES_CREDENTIAL_REF": {}, + "INTERACTION_POSTGRES_CREDENTIAL_REF": {}, +} + +var durableBootstrapCredentialDefaults = map[string]string{ + "EPISTEMICS_POSTGRES_CREDENTIAL_REF": "credential://domovoy/epistemics/db-password/v1", + "INTERACTION_POSTGRES_CREDENTIAL_REF": "credential://domovoy/interaction/db-password/v1", +} + +var durableBootstrapRuntimeFileDefaults = map[string]string{ + "CREATOR_BOOTSTRAP_TOKEN": "/run/secrets/creator_bootstrap_token", + "CREATOR_SESSION_SIGNING_KEY": "/run/secrets/creator_session_signing_key", + "CREATOR_PASSWORD_FILE": "/run/secrets/creator_password", + "EPISTEMICS_POSTGRES_PASSWORD_FILE": "/run/secrets/epistemics_db_password", + "INTERACTION_POSTGRES_PASSWORD_FILE": "/run/secrets/interaction_db_password", +} + +var derivedPasswordByCredentialRef = map[string]string{ + "EPISTEMICS_POSTGRES_CREDENTIAL_REF": "EPISTEMICS_POSTGRES_PASSWORD", + "INTERACTION_POSTGRES_CREDENTIAL_REF": "INTERACTION_POSTGRES_PASSWORD", +} + +var ( + errBootstrapStateNotFound = errors.New("deployment bootstrap state not found") + errBootstrapStateValidationFailed = errors.New("deployment bootstrap state invalid") + errOneTimeMigrationNotImplemented = errors.New("NOT_IMPLEMENTED: one-time migration behavior not yet supported in this step") + errBootstrapSourceEnvInvalid = errors.New("bootstrap source env is missing required canonical values") +) + +func defaultDurableBootstrapStatePath(args ...string) string { + root := "." + if len(args) > 0 && strings.TrimSpace(args[0]) != "" { + root = args[0] + } + return filepath.Join(filepath.Clean(root), ".devctl", durableBootstrapStatePath) +} + +func defaultProtectedSecretBackendRoot() string { + if value := os.Getenv("SECRET_BACKEND_ROOT"); strings.TrimSpace(value) != "" { + return filepath.Clean(strings.TrimSpace(value)) + } + return "/run/secrets" +} + +type deploymentTarget struct { + Name string + ComposeFile string + DockerContext string + ProjectName string + Containers []string + Volumes []string + PlanExpiryHours int +} + +type deploymentPlan struct { + Version int `json:"version"` + PlanID string `json:"plan_id"` + CreatedAt string `json:"created_at_utc"` + ExpiresAt string `json:"expires_at_utc"` + Readiness string `json:"readiness"` + ReadinessReasons []string `json:"readiness_reasons"` + Target string `json:"target"` + ComposeFile string `json:"compose_file"` + DockerContext string `json:"docker_context"` + ProjectName string `json:"project_name"` + EnvFile string `json:"env_file"` + BootstrapState string `json:"bootstrap_state"` + SourceRevision string `json:"source_revision"` + ArtifactRevision string `json:"artifact_revision"` + SourceRevisionMode string `json:"source_revision_mode"` + DeploymentConfigRev string `json:"deployment_config_revision"` + CurrentProductionRev string `json:"current_production_revision"` + KnownGoodRevision string `json:"known_good_revision"` + Migrations migrationReadiness `json:"migrations"` + Inputs []deploymentInputRequirement `json:"inputs"` + Classify map[string][]string `json:"input_classification"` + SecretReferences []string `json:"secret_references"` + ResolvedConfigHash string `json:"resolved_configuration_hash"` + PlanBindingHash string `json:"plan_binding_hash"` + Containers []string `json:"containers"` + Volumes []string `json:"volumes"` + KnownInputsSource map[string]string `json:"known_inputs_source"` +} + +type deploymentInputRequirement struct { + Name string `json:"name"` + Required bool `json:"required"` + SourceClass string `json:"source_class"` + Resolved bool `json:"resolved"` + Source string `json:"source"` + IsSecret bool `json:"is_secret"` + RefHash string `json:"value_ref_hash"` + Error string `json:"error,omitempty"` +} + +type migrationReadiness struct { + CurrentMaxMigration string `json:"current_max_migration"` + TargetMaxMigration string `json:"target_max_migration"` + Status string `json:"status"` + NeedKnownGood bool `json:"need_known_good"` +} + +type deploymentManifest struct { + Version int `json:"version"` + Project string `json:"project"` + Revision string `json:"revision"` + Artifacts []struct { + Container string `json:"container"` + ImageID string `json:"image_id"` + Revision string `json:"revision"` + } `json:"artifacts"` +} + +func deploy(args []string) error { + if len(args) == 0 { + return fmt.Errorf("deploy requires prepare, apply, or bootstrap") + } + switch args[0] { + case "prepare": + return deployPrepare(args[1:]) + case "apply": + return deployApply(args[1:]) + case "bootstrap": + return deployBootstrap(args[1:]) + default: + return fmt.Errorf("unknown deploy action %q", args[0]) + } +} + +func deployPrepare(args []string) error { + fs := flag.NewFlagSet("deploy prepare", flag.ContinueOnError) + fs.SetOutput(io.Discard) + target := fs.String("target", "production", "deployment target") + compose := fs.String("compose", "", "compose file") + planPath := fs.String("plan", "", "deployment plan output path") + context := fs.String("docker-context", "", "docker context") + envFile := fs.String("env-file", "", "deployment bootstrap env file") + bootstrapState := fs.String("bootstrap-state", "", "path to durable bootstrap state") + ttl := fs.Int("plan-ttl-hours", 24, "prepare plan TTL in hours") + sourceRevisionArg := fs.String("source-revision", "", "source revision to pin (full commit SHA)") + knownGoodManifest := fs.String("known-good-manifest", defaultKnownGoodManifestPath(), "known-good manifest path") + if err := fs.Parse(args); err != nil { + return err + } + + t, err := resolveDeploymentTarget(*target) + if err != nil { + return err + } + if *compose != "" { + t.ComposeFile = *compose + } + if *context != "" { + t.DockerContext = *context + } + if strings.TrimSpace(*compose) == "" { + // explicit compose for clarity and reproducibility + if t.ComposeFile == "" { + return fmt.Errorf("compose is required") + } + } + if !filepath.IsAbs(t.ComposeFile) { + if abs, e := filepath.Abs(t.ComposeFile); e == nil { + t.ComposeFile = abs + } + } + if *ttl <= 0 || *ttl > 24*30 { + return fmt.Errorf("plan-ttl-hours must be between 1 and 720") + } + if t.Name == "production" && strings.TrimSpace(*envFile) == "" { + return fmt.Errorf("production prepare requires --env-file") + } + + root, err := gitRepositoryRoot(".") + if err != nil { + return err + } + if strings.TrimSpace(*bootstrapState) == "" { + *bootstrapState = defaultDurableBootstrapStatePath(root) + } + plan, err := buildDeploymentPlan(root, t, *envFile, *sourceRevisionArg, *ttl, *knownGoodManifest, *bootstrapState) + if err != nil { + return err + } + + if *planPath == "" { + base := filepath.Join(".devctl", "deployment-plans") + if err := os.MkdirAll(base, 0o700); err != nil { + return err + } + ts := time.Now().UTC().Format("20060102T150405Z") + *planPath = filepath.Join(base, fmt.Sprintf("%s-%s-%s.json", strings.ToLower(plan.Target), strings.ReplaceAll(plan.PlanID, ":", "-"), ts)) + } + payload, err := json.MarshalIndent(plan, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(*planPath, payload, 0o600); err != nil { + return err + } + fmt.Printf("DeploymentPlan path=%s readiness=%s\n", *planPath, plan.Readiness) + for _, reason := range plan.ReadinessReasons { + fmt.Printf("readiness-blocker=%s\n", reason) + } + return nil +} + +func deployBootstrap(args []string) error { + fs := flag.NewFlagSet("deploy bootstrap", flag.ContinueOnError) + fs.SetOutput(io.Discard) + target := fs.String("target", "production", "deployment target") + statePath := fs.String("state", "", "durable bootstrap state path") + sourceEnv := fs.String("source-env", "", "source env containing recoverable bootstrap values") + secretBackend := fs.String("secret-backend-root", "", "protected secret backend root") + if err := fs.Parse(args); err != nil { + return err + } + t, err := resolveDeploymentTarget(*target) + if err != nil { + return err + } + if t.Name != "production" { + return fmt.Errorf("BREAK_GLASS/ONE_TIME_MIGRATION supports production only") + } + if strings.TrimSpace(*sourceEnv) == "" { + return fmt.Errorf("BREAK_GLASS/ONE_TIME_MIGRATION requires --source-env") + } + if strings.TrimSpace(*statePath) == "" { + *statePath = defaultDurableBootstrapStatePath() + } + if strings.TrimSpace(*statePath) == "" { + return fmt.Errorf("durable bootstrap state path unavailable") + } + statePathAbs, err := filepath.Abs(*statePath) + if err != nil { + return err + } + *statePath = statePathAbs + backend := strings.TrimSpace(*secretBackend) + if backend == "" { + backend = defaultProtectedSecretBackendRoot() + } + if backend == "" { + return fmt.Errorf("protected secret backend unavailable") + } + backendExists := strings.TrimSpace(backend) + if filepath.Clean(filepath.ToSlash(backendExists)) == "." || strings.TrimSpace(backendExists) == "" { + return fmt.Errorf("protected secret backend unavailable") + } + migrated, missing, err := performOneTimeBootstrapMigration(*statePath, *sourceEnv, t.Name, backendExists) + if err != nil { + return err + } + if len(missing) > 0 { + return fmt.Errorf("BREAK_GLASS/ONE_TIME_MIGRATION failed to recover: %s", strings.Join(missing, ", ")) + } + if migrated { + fmt.Printf("BREAK_GLASS/ONE_TIME_MIGRATION completed state=%s backend=%s\n", *statePath, backendExists) + } + return nil +} + +func performOneTimeBootstrapMigration(statePath, sourceEnvPath, target, backendRoot string) (bool, []string, error) { + envValues, err := parseEnvFile(sourceEnvPath) + if err != nil { + return false, nil, err + } + migrationState := deploymentBootstrapState{ + Version: durableBootstrapStateVersion, + Target: target, + ProtectedSecretRoot: filepath.Clean(filepath.ToSlash(backendRoot)), + Inputs: map[string]bootstrapInputMetadata{}, + LastMigratedAtUtc: time.Now().UTC().Format(time.RFC3339), + MigrationType: "ONE_TIME_SOURCE_ENV", + DerivedFromDurableRef: true, + } + for _, canonicalInput := range stableDurableInputs() { + switch canonicalInput { + case "CREATOR_BOOTSTRAP_TOKEN", "CREATOR_SESSION_SIGNING_KEY": + meta := bootstrapInputMetadata{ + CanonicalInput: canonicalInput, + RuntimeFile: resolveCanonicalRuntimeFile(canonicalInput, envValues), + BackendFile: resolveBootstrapBackendFile(canonicalInput, envValues), + } + if meta.RuntimeFile == "" { + meta.RuntimeFile = meta.BackendFile + } + migrationState.Inputs[canonicalInput] = meta + case "CREATOR_PASSWORD_FILE": + meta := bootstrapInputMetadata{ + CanonicalInput: canonicalInput, + RuntimeFile: resolveCanonicalRuntimeFile(canonicalInput, envValues), + BackendFile: resolveBootstrapBackendFile(canonicalInput, envValues), + } + if meta.RuntimeFile == "" { + meta.RuntimeFile = meta.BackendFile + } + migrationState.Inputs[canonicalInput] = meta + case "EPISTEMICS_POSTGRES_CREDENTIAL_REF", "INTERACTION_POSTGRES_CREDENTIAL_REF": + passwordFile := derivedPasswordByCredentialRef[canonicalInput] + meta := bootstrapInputMetadata{ + CanonicalInput: canonicalInput, + RuntimeFile: resolveCanonicalRuntimeFile(passwordFile, envValues), + BackendFile: resolveBootstrapBackendFile(passwordFile, envValues), + SecretRef: envValues[canonicalInput], + } + if strings.TrimSpace(meta.SecretRef) == "" { + meta.SecretRef = strings.TrimSpace(durableBootstrapCredentialDefaults[canonicalInput]) + } + if meta.RuntimeFile == "" { + meta.RuntimeFile = meta.BackendFile + } + migrationState.Inputs[canonicalInput] = meta + } + } + missing := missingBootstrapInputsFromState(&migrationState) + if len(missing) > 0 { + return false, missing, nil + } + if err := os.MkdirAll(filepath.Dir(statePath), 0o700); err != nil { + return false, nil, err + } + payload, err := json.MarshalIndent(migrationState, "", " ") + if err != nil { + return false, nil, err + } + if err := os.WriteFile(statePath, payload, 0o600); err != nil { + return false, nil, err + } + return true, nil, nil +} + +func loadBootstrapStateForDeployment(statePath string) (*deploymentBootstrapState, error) { + if strings.TrimSpace(statePath) == "" { + return nil, nil + } + content, err := os.ReadFile(statePath) + if err != nil { + return nil, fmt.Errorf("%w: %v", errBootstrapStateNotFound, err) + } + var state deploymentBootstrapState + if err := json.Unmarshal(content, &state); err != nil { + return nil, fmt.Errorf("%w: %v", errBootstrapStateValidationFailed, err) + } + if err := validateDeploymentBootstrapState(&state); err != nil { + return nil, err + } + return &state, nil +} + +func validateDeploymentBootstrapState(state *deploymentBootstrapState) error { + if state == nil { + return fmt.Errorf("%w: missing state", errBootstrapStateValidationFailed) + } + if state.Version != durableBootstrapStateVersion { + return fmt.Errorf("%w: unexpected version %d", errBootstrapStateValidationFailed, state.Version) + } + if strings.TrimSpace(state.Target) == "" { + state.Target = "production" + } + if strings.TrimSpace(state.ProtectedSecretRoot) == "" { + state.ProtectedSecretRoot = defaultProtectedSecretBackendRoot() + } + missing := missingBootstrapInputsFromState(state) + if len(missing) > 0 { + return fmt.Errorf("%w: missing %s", errBootstrapSourceEnvInvalid, strings.Join(missing, ", ")) + } + if strings.TrimSpace(state.MigrationType) == "" { + state.MigrationType = "UNKNOWN" + } + return nil +} + +func missingBootstrapInputsFromState(state *deploymentBootstrapState) []string { + missing := make([]string, 0) + for _, canonicalInput := range stableDurableInputs() { + meta, ok := state.Inputs[canonicalInput] + if !ok || strings.TrimSpace(meta.CanonicalInput) == "" { + missing = append(missing, canonicalInput) + continue + } + switch canonicalInput { + case "EPISTEMICS_POSTGRES_CREDENTIAL_REF", "INTERACTION_POSTGRES_CREDENTIAL_REF": + if strings.TrimSpace(meta.SecretRef) == "" || strings.TrimSpace(meta.RuntimeFile) == "" { + missing = append(missing, canonicalInput) + } + case "CREATOR_PASSWORD_FILE": + if strings.TrimSpace(meta.RuntimeFile) == "" { + missing = append(missing, canonicalInput) + } + default: + if strings.TrimSpace(meta.RuntimeFile) == "" && strings.TrimSpace(meta.SecretRef) == "" { + missing = append(missing, canonicalInput) + } + } + } + return missing +} + +func stableDurableInputs() []string { + inputs := make([]string, 0, len(requiredDurableInputsForProduction)) + for name := range requiredDurableInputsForProduction { + inputs = append(inputs, name) + } + sort.Strings(inputs) + return inputs +} + +func resolveCanonicalRuntimeFile(canonicalInput string, envValues map[string]string) string { + if strings.TrimSpace(canonicalInput) == "" { + return "" + } + if strings.HasSuffix(canonicalInput, "_FILE") { + if explicit := strings.TrimSpace(envValues[canonicalInput]); explicit != "" { + return explicit + } + if fallback := defaultDurableBootstrapRuntimeFile(canonicalInput); fallback != "" { + return fallback + } + } + if explicit := strings.TrimSpace(envValues[canonicalInput+"_FILE"]); explicit != "" { + return explicit + } + if strings.HasSuffix(canonicalInput, "_CREDENTIAL_REF") { + passwordInput := derivedPasswordInputByRuntimeFile(canonicalInput) + if explicit := strings.TrimSpace(envValues[passwordInput]); explicit != "" { + return explicit + } + if fallback := defaultDurableBootstrapRuntimeFile(passwordInput); fallback != "" { + return fallback + } + } + return "" +} + +func resolveBootstrapBackendFile(canonicalInput string, envValues map[string]string) string { + if canonicalInput == "" { + return "" + } + explicit := strings.TrimSpace(envValues[canonicalInput+"_FILE"]) + if explicit != "" { + return explicit + } + runtimeFile := resolveCanonicalRuntimeFile(canonicalInput, envValues) + if runtimeFile != "" { + return runtimeFile + } + return "" +} + +func derivedPasswordInputByRuntimeFile(credentialRef string) string { + if credentialRef == "" { + return "" + } + return strings.TrimSuffix(credentialRef, "_CREDENTIAL_REF") + "_PASSWORD" +} + +func defaultPasswordFileForCredentialRef(credentialRef string) string { + passwordInput := derivedPasswordInputByRuntimeFile(credentialRef) + if strings.TrimSpace(passwordInput) == "" { + return "" + } + return defaultDurableBootstrapRuntimeFile(passwordInput) +} + +func defaultDurableBootstrapRuntimeFile(canonicalInput string) string { + return durableBootstrapRuntimeFileDefaults[canonicalInput] +} + +func resolveDeploymentInputsWithBootstrap(inputs map[string]composeInput, envFile string, state *deploymentBootstrapState, statePath string, requireDurable bool) ([]deploymentInputRequirement, map[string][]string, map[string]string, error) { + envValues := map[string]string{} + sources := map[string]string{} + if strings.TrimSpace(envFile) != "" { + loaded, err := parseEnvFile(envFile) + if err != nil { + return nil, nil, nil, err + } + for key, value := range loaded { + envValues[key] = strings.TrimSpace(value) + } + } + + result := make([]deploymentInputRequirement, 0, len(inputs)) + classification := map[string][]string{} + + for name, spec := range inputs { + resolved := false + source := "missing" + value := "" + refHash := "" + bootstrapMeta, bootstrapResolved := bootstrapInputFromState(name, state) + + if v, ok := envValues[name]; ok && strings.TrimSpace(v) != "" { + value = strings.TrimSpace(v) + source = "file" + resolved = true + } else if requireDurable && bootstrapResolved { + source = "bootstrap-state" + resolved = true + refHash = bootstrapInputRefHash(bootstrapMeta) + } else if !requireDurable { + if v, ok := os.LookupEnv(name); ok { + value = strings.TrimSpace(v) + source = "ambient" + resolved = value != "" + } + } + + srcClass := inputSourceClass(name) + if !strings.HasPrefix(srcClass, "SECRET") && strings.Contains(name, "SECRET") { + srcClass = "SECRET_REFERENCE" + } + + req := deploymentInputRequirement{ + Name: name, + Required: spec.required, + SourceClass: srcClass, + Source: source, + Resolved: resolved, + IsSecret: isSecretInput(name), + RefHash: func() string { + if source == "bootstrap-state" { + if refHash != "" { + return refHash + } + return bootstrapInputRefHash(bootstrapMeta) + } + if strings.TrimSpace(value) == "" { + return "" + } + return valueFingerprint(name, value) + }(), + } + + if req.Required && !req.Resolved { + if source == "ambient" { + req.Error = "required production input resolved from ambient shell" + } else if source == "bootstrap-state" { + req.Error = "required production input unresolved in bootstrap state" + } else { + req.Error = "required production input unresolved" + } + } + + result = append(result, req) + classification[srcClass] = append(classification[srcClass], name) + if source == "file" { + sources[name] = envFile + } else if source == "ambient" { + sources[name] = "ambient" + } else if source == "bootstrap-state" { + sources[name] = "bootstrap-state" + if statePath != "" { + sources[name] = statePath + } + } else { + sources[name] = "missing" + } + } + return result, classification, sources, nil +} + +func bootstrapInputFromState(inputName string, state *deploymentBootstrapState) (bootstrapInputMetadata, bool) { + var empty bootstrapInputMetadata + if state == nil || state.Inputs == nil || strings.TrimSpace(inputName) == "" { + return empty, false + } + if meta, ok := state.Inputs[inputName]; ok && deploymentBootstrapMetaIsResolved(meta) { + return meta, true + } + if refName, ok := derivedCredentialRefFromPasswordInput(inputName); ok { + meta, present := state.Inputs[refName] + if present && deploymentBootstrapMetaIsResolved(meta) { + return meta, true + } + } + return empty, false +} + +func derivedCredentialRefFromPasswordInput(passwordInput string) (string, bool) { + for refInput, derived := range derivedPasswordByCredentialRef { + if strings.TrimSpace(derived) == strings.TrimSpace(passwordInput) { + return refInput, true + } + } + return "", false +} + +func deploymentBootstrapMetaIsResolved(meta bootstrapInputMetadata) bool { + if strings.TrimSpace(meta.CanonicalInput) == "" { + return false + } + if strings.HasSuffix(meta.CanonicalInput, "_CREDENTIAL_REF") { + return strings.TrimSpace(meta.SecretRef) != "" && strings.TrimSpace(meta.RuntimeFile) != "" + } + return strings.TrimSpace(meta.RuntimeFile) != "" || strings.TrimSpace(meta.SecretRef) != "" +} + +func bootstrapInputRefHash(meta bootstrapInputMetadata) string { + if strings.TrimSpace(meta.SecretRef) == "" && strings.TrimSpace(meta.RuntimeFile) == "" && strings.TrimSpace(meta.BackendFile) == "" { + return "" + } + return valueFingerprint(meta.CanonicalInput, strings.Join([]string{ + meta.CanonicalInput, + meta.SecretRef, + meta.RuntimeFile, + meta.BackendFile, + }, "|")) +} + +func bootstrapInputFromStateLegacy(inputName string, state *deploymentBootstrapState) bool { + if state == nil || state.Inputs == nil || strings.TrimSpace(inputName) == "" { + return false + } + meta, _ := bootstrapInputFromState(inputName, state) + return meta.CanonicalInput != "" +} + +func deployApply(args []string) error { + fs := flag.NewFlagSet("deploy apply", flag.ContinueOnError) + fs.SetOutput(io.Discard) + planPath := fs.String("plan", "", "deployment plan path") + dryRun := fs.Bool("dry-run", false, "validate plan without mutating") + skipHealth := fs.Bool("skip-health", false, "skip health gate") + autoRollback := fs.Bool("auto-rollback", false, "rollback to known-good revision on first health failure") + if err := fs.Parse(args); err != nil { + return err + } + if strings.TrimSpace(*planPath) == "" { + return fmt.Errorf("plan path is required") + } + plan, err := readDeploymentPlan(*planPath) + if err != nil { + return err + } + + if err := assertPlanCurrent(plan); err != nil { + return err + } + if *dryRun { + fmt.Printf("deploy apply dry-run: readiness=%s\n", plan.Readiness) + return nil + } + if err := executeComposeDeploy(plan); err != nil { + return err + } + + if *skipHealth { + fmt.Println("deploy apply: READY (health skipped)") + return nil + } + if err := validatePostApplyHealth(plan); err != nil { + if !*autoRollback { + return err + } + if strings.TrimSpace(plan.KnownGoodRevision) == "" { + return fmt.Errorf("health failed (%v) and no known-good revision in plan", err) + } + recoveryPlan, errR := rollbackToKnownGood(plan) + if errR != nil { + return fmt.Errorf("health failed (%v) and rollback failed: %v", err, errR) + } + if err := validatePostApplyHealth(recoveryPlan); err != nil { + return fmt.Errorf("health failed and rollback did not restore health: %w", err) + } + fmt.Printf("deploy apply: HEALTH_ROLLBACK_SUCCESS known_good=%s\n", plan.KnownGoodRevision) + return nil + } + fmt.Println("deploy apply: READY") + return nil +} + +func buildDeploymentPlan(root string, target deploymentTarget, envFile, sourceRevisionArg string, ttlHours int, knownGoodManifest string, bootstrapStateArgs ...string) (*deploymentPlan, error) { + bootstrapStatePath := "" + if len(bootstrapStateArgs) > 0 && strings.TrimSpace(bootstrapStateArgs[0]) != "" { + bootstrapStatePath = strings.TrimSpace(bootstrapStateArgs[0]) + } + notify := []string{} + bootstrapState, bootstrapErr := loadBootstrapStateForDeployment(bootstrapStatePath) + if bootstrapErr != nil && strings.TrimSpace(bootstrapStatePath) != "" { + notify = append(notify, fmt.Sprintf("bootstrap-state read: %v", bootstrapErr)) + } + + inputMap, err := discoverComposeInputsFromFile(target.ComposeFile) + if err != nil { + return nil, err + } + + inputResolutions, classification, inputSources, err := resolveDeploymentInputsWithBootstrap(inputMap, envFile, bootstrapState, bootstrapStatePath, target.Name == "production") + if err != nil { + if target.Name != "production" { + return nil, err + } + } + + for _, req := range inputResolutions { + if req.Required && !req.Resolved { + notify = append(notify, req.Name) + } + } + + sourceRevision, mode, err := resolveSourceRevision(root, sourceRevisionArg) + if err != nil { + return nil, err + } + artifactRevision := sourceRevision + configRevision := resolveDeploymentConfigRevision(target.ComposeFile) + + knownGood, currentProd, errKnownGood := captureKnownGood(target, knownGoodManifest) + if errKnownGood != nil { + notify = append(notify, fmt.Sprintf("known-good capture: %v", errKnownGood)) + } + + migration, _ := migrationState(root, artifactRevision, knownGood) + + readiness := readinessReady + reasons := []string{} + for _, v := range notify { + readiness = readinessFailed + reasons = append(reasons, v) + } + if len(notify) == 0 && sourceRevision == "" { + readiness = readinessFailed + reasons = append(reasons, "source/artifact revision unresolved") + } + + inputList := make([]deploymentInputRequirement, 0, len(inputResolutions)) + for _, v := range inputResolutions { + inputList = append(inputList, v) + } + sort.Slice(inputList, func(i, j int) bool { return inputList[i].Name < inputList[j].Name }) + + secretRefs := []string{} + for _, input := range inputList { + if input.SourceClass == "SECRET_REFERENCE" { + secretRefs = append(secretRefs, input.Name) + } + } + + created := time.Now().UTC() + plan := &deploymentPlan{ + Version: planVersion, + PlanID: genPlanID(target.Name, sourceRevision), + CreatedAt: created.Format(time.RFC3339), + ExpiresAt: created.Add(time.Duration(ttlHours) * time.Hour).Format(time.RFC3339), + Readiness: readiness, + ReadinessReasons: reasons, + Target: target.Name, + ComposeFile: target.ComposeFile, + DockerContext: target.DockerContext, + ProjectName: target.ProjectName, + EnvFile: envFile, + BootstrapState: bootstrapStatePath, + SourceRevision: sourceRevision, + SourceRevisionMode: mode, + ArtifactRevision: artifactRevision, + DeploymentConfigRev: configRevision, + CurrentProductionRev: currentProd, + KnownGoodRevision: knownGood, + Migrations: migration, + Inputs: inputList, + Classify: classification, + SecretReferences: uniqueSorted(secretRefs), + ResolvedConfigHash: resolveConfigHash(target.ComposeFile), + Containers: target.Containers, + Volumes: target.Volumes, + KnownInputsSource: inputSources, + } + plan.PlanBindingHash = buildPlanBindingHash(plan) + if readiness == readinessFailed { + plan.ReadinessReasons = dedupeAndSort(plan.ReadinessReasons) + } + return plan, nil +} + +func executeComposeDeploy(plan *deploymentPlan) error { + if plan == nil { + return errors.New("missing plan") + } + if plan.Readiness != readinessReady { + return fmt.Errorf("plan is not ready (%s)", strings.Join(plan.ReadinessReasons, "; ")) + } + if err := runCommandWithEnv(plan); err != nil { + return err + } + return nil +} + +func runCommandWithEnv(plan *deploymentPlan) error { + args := []string{"--context", plan.DockerContext, "compose"} + if plan.EnvFile != "" { + args = append(args, "--env-file", plan.EnvFile) + } + args = append(args, "-p", plan.ProjectName, "-f", plan.ComposeFile, "up", "-d", "--no-build", "--force-recreate") + cmd := exec.Command("docker", args...) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, fmt.Sprintf("ASSISTANT_SOURCE_REVISION=%s", plan.ArtifactRevision)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + fmt.Printf("deploy apply command: %s\n", strings.Join(cmd.Args, " ")) + return cmd.Run() +} + +func validatePostApplyHealth(plan *deploymentPlan) error { + target, err := resolveDeploymentTarget(plan.Target) + if err != nil { + return err + } + target.ComposeFile = plan.ComposeFile + target.DockerContext = plan.DockerContext + target.ProjectName = plan.ProjectName + if err := validateDeploymentRunningContainers(plan, target); err != nil { + return err + } + if err := validateDeploymentVolumes(plan, target); err != nil { + return err + } + for _, u := range deploymentExternalHealthEndpoints { + if err := validateHTTPHealth(u); err != nil { + return fmt.Errorf("health gate failed for %s: %w", u, err) + } + } + return nil +} + +func rollbackToKnownGood(plan *deploymentPlan) (*deploymentPlan, error) { + if strings.TrimSpace(plan.KnownGoodRevision) == "" { + return nil, fmt.Errorf("no known-good revision in plan") + } + recovery := *plan + recovery.Readiness = readinessReady + recovery.ReadinessReasons = nil + recovery.ArtifactRevision = plan.KnownGoodRevision + recovery.SourceRevision = plan.KnownGoodRevision + recovery.PlanBindingHash = buildPlanBindingHash(&recovery) + if err := executeComposeDeploy(&recovery); err != nil { + return nil, err + } + return &recovery, nil +} + +func assertPlanCurrent(plan *deploymentPlan) error { + if plan == nil { + return errors.New("missing plan") + } + if err := validatePlanReadiness(plan); err != nil { + return err + } + if plan.Readiness == readinessFailed { + return fmt.Errorf("plan is not ready (%s)", strings.Join(plan.ReadinessReasons, "; ")) + } + + expiry, err := time.Parse(time.RFC3339, plan.ExpiresAt) + if err != nil { + return fmt.Errorf("invalid expires_at_utc") + } + if time.Now().UTC().After(expiry) { + return fmt.Errorf("plan expired at %s", plan.ExpiresAt) + } + + target, err := resolveDeploymentTarget(plan.Target) + if err != nil { + return err + } + if strings.TrimSpace(plan.ComposeFile) != "" { + target.ComposeFile = plan.ComposeFile + } + + planCompose := plan.ComposeFile + targetCompose := target.ComposeFile + if strings.TrimSpace(planCompose) == "" || strings.TrimSpace(targetCompose) == "" { + return fmt.Errorf("stale plan: compose file changed") + } + if !filepath.IsAbs(planCompose) { + if abs, e := filepath.Abs(planCompose); e == nil { + planCompose = abs + } + } + if !filepath.IsAbs(targetCompose) { + if abs, e := filepath.Abs(targetCompose); e == nil { + targetCompose = abs + } + } + if filepath.Clean(planCompose) != filepath.Clean(targetCompose) { + return fmt.Errorf("stale plan: compose file changed") + } + if target.DockerContext != "" && plan.DockerContext != target.DockerContext { + return fmt.Errorf("stale plan: docker context changed") + } + currentConfig := resolveConfigHash(plan.ComposeFile) + if plan.DeploymentConfigRev != currentConfig { + return fmt.Errorf("stale plan: deployment config changed") + } + if plan.SourceRevisionMode == "head" { + currentHead, _, err := resolveSourceRevision(filepath.Dir(plan.ComposeFile), "") + if err != nil { + return err + } + if currentHead != plan.SourceRevision { + return fmt.Errorf("stale plan: source revision changed") + } + } + inputMap, err := discoverComposeInputsFromFile(plan.ComposeFile) + if err != nil { + return err + } + currentInputs, _, _, err := resolveDeploymentInputs(inputMap, plan.EnvFile, target.Name == "production") + if err != nil { + return err + } + currentBindings := buildPlanBindingFromInputs( + currentInputs, + plan.SourceRevision, + plan.ArtifactRevision, + currentConfig, + target.DockerContext, + target.Containers, + target.Volumes, + ) + if plan.PlanBindingHash != currentBindings { + return fmt.Errorf("stale plan: required assumptions changed") + } + root, err := gitRepositoryRoot(".") + if err != nil { + return err + } + currentMigration, err := migrationState(root, plan.ArtifactRevision, plan.CurrentProductionRev) + if err != nil { + return fmt.Errorf("stale plan: migration state check failed: %w", err) + } + if !migrationReadinessEqual(plan.Migrations, currentMigration) { + return fmt.Errorf("stale plan: migration state changed") + } + if plan.Target == "production" && currentMigration.NeedKnownGood && strings.TrimSpace(plan.KnownGoodRevision) == "" { + return fmt.Errorf("migration precondition failed: %s", currentMigration.Status) + } + + // explicit re-check of required inputs from env file ensures no ambient fallback + inputResolutions, _, _, err := resolveDeploymentInputs(inputMap, plan.EnvFile, target.Name == "production") + if err != nil { + return err + } + for _, req := range inputResolutions { + if req.Required && !req.Resolved { + return fmt.Errorf("required input %s unresolved since prepare", req.Name) + } + } + return nil +} + +func migrationStateFromRevisions(root, targetRevision, currentRevision string) (migrationReadiness, error) { + currentMax := "0" + targetMax := "0" + var err error + if strings.TrimSpace(currentRevision) != "" { + currentMax, err = maxMigrationPrefixForRevision(root, currentRevision) + if err != nil { + currentMax = "unknown" + } + } + targetMax, err = maxMigrationPrefixForRevision(root, targetRevision) + if err != nil { + targetMax = "unknown" + } + currentMaxValue, currentParseErr := strconv.Atoi(currentMax) + targetMaxValue, targetParseErr := strconv.Atoi(targetMax) + status := "matched" + needKnownGood := false + if currentMax == "unknown" || targetMax == "unknown" || currentParseErr != nil || targetParseErr != nil { + status = "needs-manual-verification" + needKnownGood = true + } else if currentMaxValue != targetMaxValue { + if currentMaxValue > targetMaxValue { + status = "target-lower-than-production" + } else { + status = "upgrade-required" + } + needKnownGood = true + } + return migrationReadiness{ + CurrentMaxMigration: currentMax, + TargetMaxMigration: targetMax, + Status: status, + NeedKnownGood: needKnownGood, + }, nil +} + +func migrationReadinessEqual(a, b migrationReadiness) bool { + return a.CurrentMaxMigration == b.CurrentMaxMigration && + a.TargetMaxMigration == b.TargetMaxMigration && + a.Status == b.Status && + a.NeedKnownGood == b.NeedKnownGood +} + +func resolvePlanInputFingerprint(reqs []deploymentInputRequirement) string { + parts := make([]string, 0, len(reqs)) + for _, req := range reqs { + parts = append(parts, fmt.Sprintf("%s=%s@%s", req.Name, req.RefHash, req.Source)) + } + sort.Strings(parts) + return hashString(strings.Join(parts, "|")) +} + +func buildPlanBindingHash(plan *deploymentPlan) string { + raw := buildPlanBindingFromInputs( + plan.Inputs, + plan.SourceRevision, + plan.ArtifactRevision, + plan.DeploymentConfigRev, + plan.DockerContext, + plan.Containers, + plan.Volumes, + ) + return hashString(raw) +} + +func buildPlanBindingFromInputs(reqs []deploymentInputRequirement, sourceRevision, artifactRevision, configRev, context string, containers, volumes []string) string { + parts := []string{ + sourceRevision, + artifactRevision, + configRev, + context, + } + if len(containers) > 0 { + orderedContainers := append([]string{}, containers...) + sort.Strings(orderedContainers) + parts = append(parts, strings.Join(orderedContainers, ",")) + } + if len(volumes) > 0 { + orderedVolumes := append([]string{}, volumes...) + sort.Strings(orderedVolumes) + parts = append(parts, strings.Join(orderedVolumes, ",")) + } + for _, req := range reqs { + parts = append(parts, fmt.Sprintf("%s=%s@%s", req.Name, req.RefHash, req.Source)) + } + sort.Strings(parts) + return hashString(strings.Join(parts, "|")) +} + +func validatePlanReadiness(plan *deploymentPlan) error { + if plan.Readiness == "" { + return errors.New("plan readiness is empty") + } + if plan.SourceRevision == "" || plan.ArtifactRevision == "" { + return fmt.Errorf("plan missing source/artifact revision") + } + if plan.DeploymentConfigRev == "" { + return fmt.Errorf("plan missing deployment configuration revision") + } + if len(plan.Inputs) == 0 { + return fmt.Errorf("plan has no input requirements") + } + for _, req := range plan.Inputs { + if req.Required && req.Source == "ambient" { + return fmt.Errorf("required input %s resolved from ambient source", req.Name) + } + if req.Required && !req.Resolved { + return fmt.Errorf("required input %s unresolved", req.Name) + } + } + return nil +} + +func discoverComposeInputsFromFile(composeFile string) (map[string]composeInput, error) { + b, err := os.ReadFile(composeFile) + if err != nil { + return nil, err + } + return discoverComposeInputs(string(b)), nil +} + +type composeInput struct { + name string + required bool +} + +func discoverComposeInputs(content string) map[string]composeInput { + result := map[string]composeInput{} + re := regexp.MustCompile(`\$\{([A-Z0-9_]+)(?::-[^}]*)?\}`) + for _, match := range re.FindAllStringSubmatch(content, -1) { + if len(match) < 2 { + continue + } + token := match[0] + name := match[1] + result[name] = composeInput{name: name, required: !strings.Contains(token, ":-")} + } + for name := range requiredChatOnlyInputs { + if _, ok := result[name]; !ok { + result[name] = composeInput{name: name, required: true} + } + } + return result +} + +func resolveDeploymentInputs(inputs map[string]composeInput, envFile string, requireDurable bool) ([]deploymentInputRequirement, map[string][]string, map[string]string, error) { + envValues := map[string]string{} + sources := map[string]string{} + if strings.TrimSpace(envFile) != "" { + loaded, err := parseEnvFile(envFile) + if err != nil { + return nil, nil, nil, err + } + for key, value := range loaded { + envValues[key] = strings.TrimSpace(value) + sources[key] = "file" + } + } + + result := make([]deploymentInputRequirement, 0, len(inputs)) + classification := map[string][]string{} + for name, spec := range inputs { + required := spec.required + source := "missing" + resolved := false + value := "" + if v, ok := envValues[name]; ok { + value = v + source = "file" + resolved = strings.TrimSpace(v) != "" + if !resolved { + source = "file" + } + } + if !resolved && os.Getenv(name) != "" { + if requireDurable { + source = "ambient" + } else { + value = os.Getenv(name) + source = "ambient" + resolved = true + } + } + srcClass := inputSourceClass(name) + if !strings.HasPrefix(srcClass, "SECRET") && strings.Contains(name, "SECRET") { + srcClass = "SECRET_REFERENCE" + } + req := deploymentInputRequirement{ + Name: name, + Required: required, + SourceClass: srcClass, + Resolved: resolved, + Source: source, + IsSecret: isSecretInput(name), + RefHash: func() string { + if strings.TrimSpace(value) == "" { + return "" + } + return valueFingerprint(name, value) + }(), + } + if req.Required && !req.Resolved { + if source == "ambient" { + req.Error = "required production input resolved from ambient shell" + } else { + req.Error = "required production input unresolved" + } + } + result = append(result, req) + classification[srcClass] = append(classification[srcClass], name) + if source == "file" { + sources[name] = envFile + } else if source == "ambient" { + sources[name] = "ambient" + } else { + sources[name] = "missing" + } + } + return result, classification, sources, nil +} + +func runCommand(name string, args ...string) ([]byte, error) { + cmd := exec.Command(name, args...) + out, err := cmd.Output() + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok && len(exitErr.Stderr) > 0 { + return nil, fmt.Errorf("%s", strings.TrimSpace(string(exitErr.Stderr))) + } + return nil, err + } + return out, nil +} + +func parseEnvFile(path string) (map[string]string, error) { + contents, err := os.ReadFile(path) + if err != nil { + return nil, err + } + result := make(map[string]string) + for _, raw := range strings.Split(string(contents), "\n") { + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + if strings.HasPrefix(line, "export ") { + line = strings.TrimSpace(strings.TrimPrefix(line, "export ")) + } + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid env line: %q", raw) + } + key := strings.TrimSpace(parts[0]) + value := strings.Trim(strings.TrimSpace(parts[1]), "\"'") + result[key] = value + } + return result, nil +} + +func resolveSourceRevision(root, override string) (string, string, error) { + if strings.TrimSpace(override) != "" { + out, err := runGit(root, "rev-parse", override) + if err != nil { + return "", "", err + } + rev := strings.TrimSpace(string(out)) + if !isHex40(rev) { + return "", "", fmt.Errorf("revision is not 40-char commit: %q", rev) + } + return rev, "explicit", nil + } + out, err := runGit(root, "rev-parse", "HEAD") + if err != nil { + return "", "", err + } + rev := strings.TrimSpace(string(out)) + if !isHex40(rev) { + return "", "", fmt.Errorf("HEAD is not a full commit: %q", rev) + } + return rev, "head", nil +} + +func resolveDeploymentConfigRevision(composeFile string) string { + b, err := os.ReadFile(composeFile) + if err != nil { + return "" + } + return hashString(string(b)) +} + +func resolveConfigHash(composeFile string) string { + b, err := os.ReadFile(composeFile) + if err != nil { + return "" + } + return hashString(string(b)) +} + +func captureKnownGood(target deploymentTarget, manifestPath string) (string, string, error) { + known := "" + current := "" + if manifestPath == "" { + manifestPath = defaultKnownGoodManifestPath() + } + if manifestPath != "" { + if b, err := os.ReadFile(manifestPath); err == nil { + var manifest deploymentManifest + if err := json.Unmarshal(b, &manifest); err == nil { + if manifest.Revision != "" { + known = manifest.Revision + current = manifest.Revision + } + } + } + } + if known != "" { + return known, current, nil + } + // legacy fallback for explicit recovery contexts only + if currentFromContainers, err := captureCurrentProductionRevision(target); err == nil && currentFromContainers != "" { + return currentFromContainers, currentFromContainers, nil + } + return "", "", fmt.Errorf("known-good manifest unavailable") +} + +func captureCurrentProductionRevision(target deploymentTarget) (string, error) { + revisions := make([]string, 0, len(target.Containers)) + for _, container := range target.Containers { + label, err := dockerContainerLabel(target.DockerContext, container, "org.opencontainers.image.revision") + if err != nil { + return "", err + } + if label != "" { + revisions = append(revisions, label) + } + } + if len(revisions) == 0 { + return "", fmt.Errorf("no production container labels available") + } + base := revisions[0] + for _, revision := range revisions[1:] { + if revision != base { + return "", fmt.Errorf("container revision mismatch among production services") + } + } + return base, nil +} + +func migrationState(root, targetRevision, currentRevision string) (migrationReadiness, error) { + return migrationStateFromRevisions(root, targetRevision, currentRevision) +} + +func maxMigrationPrefixForRevision(root, revision string) (string, error) { + out, err := runGit(root, "ls-tree", "-r", "--name-only", revision, "database/migrations") + if err != nil { + return "", err + } + return maxMigrationPrefixFromList(strings.Split(strings.TrimSpace(string(out)), "\n")), nil +} + +func maxMigrationPrefixFromList(paths []string) string { + re := regexp.MustCompile(`^(\d{3})_`) + max := "0" + maxValue := 0 + for _, file := range paths { + if file == "" { + continue + } + base := filepath.Base(filepath.FromSlash(strings.TrimSpace(file))) + m := re.FindStringSubmatch(base) + if len(m) != 2 { + continue + } + value := m[1] + i, parseErr := strconv.Atoi(value) + if parseErr != nil { + continue + } + if i > maxValue { + maxValue = i + } + } + max = fmt.Sprintf("%d", maxValue) + return max +} + +func resolveDeploymentTarget(name string) (deploymentTarget, error) { + switch name { + case "", "production", "prod": + return deploymentTarget{ + Name: "production", + ComposeFile: filepath.FromSlash("deploy/docker-compose.prod.yml"), + DockerContext: "docker-prod", + ProjectName: "assistant", + Containers: []string{ + "assistant-postgres-1", + "assistant-notification-receiver-1", + "assistant-core-runtime-1", + "assistant-continuity-runtime-1", + "assistant-commit-runtime-1", + "assistant-cognitive-worker-1", + "assistant-creator-web-1", + }, + Volumes: []string{ + "assistant_pg", + "assistant_nats", + "assistant_core_secrets", + "assistant_notification_receipts", + "assistant_notification_writer_secret", + "assistant_notification_reader_secret", + }, + PlanExpiryHours: 12, + }, nil + default: + return deploymentTarget{}, fmt.Errorf("unknown target %q", name) + } +} + +func validateDeploymentRunningContainers(plan *deploymentPlan, target deploymentTarget) error { + required := map[string]struct{}{} + for _, item := range target.Containers { + required[item] = struct{}{} + } + for container := range required { + revision, err := dockerContainerLabel(target.DockerContext, container, "org.opencontainers.image.revision") + if err != nil { + return fmt.Errorf("container %s inspect failed: %w", container, err) + } + if plan.ArtifactRevision != "" && revision != "" && revision != plan.ArtifactRevision { + return fmt.Errorf("container %s revision mismatch (expected=%s actual=%s)", container, plan.ArtifactRevision, revision) + } + state, err := dockerContainerState(target.DockerContext, container) + if err != nil { + return fmt.Errorf("container %s state failed: %w", container, err) + } + if state != "running" { + return fmt.Errorf("container %s state=%s", container, state) + } + } + return nil +} + +func validateDeploymentVolumes(_ *deploymentPlan, target deploymentTarget) error { + for _, volume := range target.Volumes { + if _, err := runCommand("docker", "--context", target.DockerContext, "volume", "inspect", volume); err != nil { + return fmt.Errorf("required volume missing: %s", volume) + } + } + return nil +} + +func validateHTTPHealth(url string) error { + client := &http.Client{Timeout: 12 * time.Second} + req, err := http.NewRequest(http.MethodGet, url, http.NoBody) + if err != nil { + return err + } + resp, err := client.Do(req) + if err != nil { + return err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + return fmt.Errorf("unexpected status: %d", resp.StatusCode) + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + return nil +} + +func readDeploymentPlan(path string) (*deploymentPlan, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var plan deploymentPlan + if err := json.Unmarshal(b, &plan); err != nil { + return nil, err + } + return &plan, nil +} + +func genPlanID(target, revision string) string { + if len(revision) > 12 { + revision = revision[:12] + } + if target == "" { + target = "deployment" + } + return fmt.Sprintf("%s-%s", target, revision) +} + +func defaultKnownGoodManifestPath() string { + if local := os.Getenv("LOCALAPPDATA"); local != "" { + return filepath.Join(local, "Domovoy", "monitoring", "deployment-manifest.json") + } + if local := os.Getenv("LOCALAPPDATA_ALT"); local != "" { + return filepath.Join(local, "Domovoy", "monitoring", "deployment-manifest.json") + } + return "" +} + +func inputSourceClass(name string) string { + n := strings.ToUpper(name) + switch { + case strings.Contains(n, "_CREDENTIAL_REF"): + return "SECRET_REFERENCE" + case strings.Contains(n, "_FILE"): + return "DOMOVOY_MANAGED" + case strings.HasSuffix(n, "_PASSWORD") || strings.HasSuffix(n, "_TOKEN") || strings.HasSuffix(n, "_KEY") || strings.Contains(n, "SECRET"): + return "SECRET_REFERENCE" + case n == "ASSISTANT_SOURCE_REVISION": + return "DEPLOYMENT_LOCAL" + default: + return "BOOTSTRAP" + } +} + +func isSecretInput(name string) bool { + n := strings.ToUpper(name) + return strings.HasSuffix(n, "_PASSWORD") || strings.HasSuffix(n, "_TOKEN") || strings.HasSuffix(n, "_KEY") || strings.Contains(n, "SECRET") +} + +func valueFingerprint(name, value string) string { + if strings.TrimSpace(value) == "" { + return "" + } + sum := sha256.Sum256([]byte(value)) + if isSecretInput(name) { + return hex.EncodeToString(sum[:])[:16] + } + return hex.EncodeToString(sum[:]) +} + +func hashString(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} + +func isHex40(value string) bool { + if len(value) != 40 { + return false + } + for _, r := range value { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') && (r < 'A' || r > 'F') { + return false + } + } + return true +} + +func dedupeAndSort(values []string) []string { + seen := map[string]struct{}{} + out := make([]string, 0, len(values)) + for _, value := range values { + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + sort.Strings(out) + return out +} + +func uniqueSorted(values []string) []string { + return dedupeAndSort(values) +} + +func dockerContainerLabel(context, container, label string) (string, error) { + out, err := runCommand("docker", "--context", context, "inspect", container, "--format", fmt.Sprintf("{{index .Config.Labels \"%s\"}}", label)) + if err != nil { + return "", err + } + return strings.TrimSpace(string(out)), nil +} + +func dockerContainerState(context, container string) (string, error) { + out, err := runCommand("docker", "--context", context, "inspect", container, "--format", "{{.State.Status}}") + if err != nil { + return "", err + } + return strings.TrimSpace(string(out)), nil +} diff --git a/cmd/devctl/deploy_test.go b/cmd/devctl/deploy_test.go new file mode 100644 index 0000000..6139be3 --- /dev/null +++ b/cmd/devctl/deploy_test.go @@ -0,0 +1,644 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +func TestDiscoverComposeInputsRequiredAndOptional(t *testing.T) { + content := `A=${POSTGRES_SUPERUSER} +B=${POSTGRES_DB:-assistant} +C=${CREATOR_PASSWORD_FILE} +D=${FOO_CREDENTIAL_REF} +` + inputs := discoverComposeInputs(content) + if _, ok := inputs["POSTGRES_SUPERUSER"]; !ok { + t.Fatal("POSTGRES_SUPERUSER should be discovered") + } + if !inputs["POSTGRES_SUPERUSER"].required { + t.Fatalf("POSTGRES_SUPERUSER expected required") + } + if inputs["POSTGRES_DB"].required { + t.Fatalf("POSTGRES_DB expected optional") + } + if !inputs["CREATOR_PASSWORD_FILE"].required { + t.Fatalf("CREATOR_PASSWORD_FILE expected required from compose placeholder") + } + if !inputs["EPISTEMICS_POSTGRES_CREDENTIAL_REF"].required { + t.Fatalf("required production bootstrap variable should be required") + } + if got := len(inputs); got < 4 { + t.Fatalf("expected at least 4 discovered inputs, got %d", got) + } +} + +func TestResolveDeploymentInputsRejectsAmbientProduction(t *testing.T) { + t.Setenv("POSTGRES_SUPERUSER", "from_ambient") + inputs := map[string]composeInput{ + "POSTGRES_SUPERUSER": {name: "POSTGRES_SUPERUSER", required: true}, + } + resolved, _, _, err := resolveDeploymentInputs(inputs, "", true) + if err != nil { + t.Fatalf("resolveDeploymentInputs error: %v", err) + } + if len(resolved) != 1 { + t.Fatalf("expected one resolved input, got %d", len(resolved)) + } + if resolved[0].Source != "ambient" || resolved[0].Resolved { + t.Fatalf("expected unresolved ambient resolution, got source=%s resolved=%v", resolved[0].Source, resolved[0].Resolved) + } +} + +func TestDeployPrepareRequiresEnvFileForProduction(t *testing.T) { + root := t.TempDir() + composeFile := filepath.Join(root, "docker-compose.prod.yml") + if err := os.WriteFile(composeFile, []byte("a=${POSTGRES_SUPERUSER}"), 0o600); err != nil { + t.Fatal(err) + } + err := deployPrepare([]string{"--target", "production", "--compose", composeFile}) + if err == nil || !strings.Contains(err.Error(), "production prepare requires --env-file") { + t.Fatalf("expected missing env-file failure, got %v", err) + } +} + +func TestDeployPrepareBlockedAndApplyRefusesMissingRequiredInput(t *testing.T) { + root := t.TempDir() + repoRoot, err := gitRepositoryRoot(".") + if err != nil { + t.Fatal(err) + } + composeFile := filepath.Join(root, "docker-compose.prod.yml") + if err := os.WriteFile(composeFile, []byte("a=${POSTGRES_SUPERUSER}"), 0o600); err != nil { + t.Fatal(err) + } + envFile := filepath.Join(root, "production.env") + parts := make([]string, 0, len(requiredChatOnlyInputs)) + for key := range requiredChatOnlyInputs { + if key == "POSTGRES_SUPERUSER" { + continue + } + parts = append(parts, key+"=value-"+key) + } + if err := os.WriteFile(envFile, []byte(strings.Join(parts, "\n")), 0o600); err != nil { + t.Fatal(err) + } + + target, err := resolveDeploymentTarget("production") + if err != nil { + t.Fatal(err) + } + target.ComposeFile = composeFile + plan, err := buildDeploymentPlan(repoRoot, target, envFile, "HEAD", 24, "") + if err != nil { + t.Fatal(err) + } + if plan.Readiness != readinessFailed { + t.Fatalf("expected blocked plan, got readiness=%s", plan.Readiness) + } + if !strings.Contains(strings.Join(plan.ReadinessReasons, "\n"), "POSTGRES_SUPERUSER") { + t.Fatalf("expected missing required input blocker for POSTGRES_SUPERUSER, got %v", plan.ReadinessReasons) + } + + planPath := filepath.Join(root, "blocked-plan.json") + payload, err := json.Marshal(plan) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(planPath, payload, 0o600); err != nil { + t.Fatal(err) + } + if err := deployApply([]string{"-plan", planPath}); err == nil { + t.Fatal("expected apply refusal for blocked plan") + } +} + +func TestAssertPlanCurrentRejectsStaleInputChange(t *testing.T) { + root := t.TempDir() + composeFile := filepath.Join(root, "docker-compose.prod.yml") + if err := os.WriteFile(composeFile, []byte("a=${POSTGRES_SUPERUSER}"), 0o600); err != nil { + t.Fatal(err) + } + envFile := filepath.Join(root, "deploy.env") + if err := os.WriteFile(envFile, []byte("POSTGRES_SUPERUSER=first\n"), 0o600); err != nil { + t.Fatal(err) + } + + inputs, err := discoverComposeInputsFromFile(composeFile) + if err != nil { + t.Fatal(err) + } + resolved, _, _, err := resolveDeploymentInputs(inputs, envFile, true) + if err != nil { + t.Fatal(err) + } + plan := &deploymentPlan{ + Version: 1, + PlanID: "stale-test", + CreatedAt: time.Now().UTC().Format(time.RFC3339), + ExpiresAt: time.Now().UTC().Add(time.Hour).Format(time.RFC3339), + Readiness: readinessReady, + ReadinessReasons: nil, + Target: "production", + ComposeFile: composeFile, + DockerContext: "docker-prod", + ProjectName: "assistant", + EnvFile: envFile, + SourceRevision: "11a6a23f6e4f44dd88265a04846419a27f54df98", + SourceRevisionMode: "explicit", + ArtifactRevision: "11a6a23f6e4f44dd88265a04846419a27f54df98", + DeploymentConfigRev: resolveConfigHash(composeFile), + CurrentProductionRev: "", + KnownGoodRevision: "", + Inputs: resolved, + Classify: map[string][]string{}, + KnownInputsSource: map[string]string{}, + } + target, err := resolveDeploymentTarget("production") + if err != nil { + t.Fatal(err) + } + plan.PlanBindingHash = buildPlanBindingFromInputs( + resolved, + plan.SourceRevision, + plan.ArtifactRevision, + plan.DeploymentConfigRev, + target.DockerContext, + target.Containers, + target.Volumes, + ) + if err := os.WriteFile(envFile, []byte("POSTGRES_SUPERUSER=second\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := assertPlanCurrent(plan); err == nil { + t.Fatal("expected stale plan after input change") + } +} + +func TestAssertPlanCurrentRejectsArtifactMismatch(t *testing.T) { + root := t.TempDir() + composeFile := filepath.Join(root, "docker-compose.prod.yml") + if err := os.WriteFile(composeFile, []byte("a=${POSTGRES_SUPERUSER}"), 0o600); err != nil { + t.Fatal(err) + } + envFile := filepath.Join(root, "deploy.env") + + envParts := make([]string, 0, len(requiredChatOnlyInputs)) + for key := range requiredChatOnlyInputs { + envParts = append(envParts, key+"=value-"+strings.ToLower(key)) + } + envParts = append(envParts, "POSTGRES_SUPERUSER=from_file") + if err := os.WriteFile(envFile, []byte(strings.Join(envParts, "\n")), 0o600); err != nil { + t.Fatal(err) + } + + inputs, err := discoverComposeInputsFromFile(composeFile) + if err != nil { + t.Fatal(err) + } + discovered, _, _, err := resolveDeploymentInputs(inputs, envFile, true) + if err != nil { + t.Fatal(err) + } + target, err := resolveDeploymentTarget("production") + if err != nil { + t.Fatal(err) + } + configRev := resolveConfigHash(composeFile) + shaA := strings.Repeat("a", 40) + shaB := strings.Repeat("b", 40) + plan := &deploymentPlan{ + Version: 1, + PlanID: "artifact-mismatch", + CreatedAt: time.Now().UTC().Format(time.RFC3339), + ExpiresAt: time.Now().UTC().Add(time.Hour).Format(time.RFC3339), + Readiness: readinessReady, + Target: "production", + ComposeFile: composeFile, + DockerContext: "docker-prod", + ProjectName: "assistant", + EnvFile: envFile, + SourceRevision: shaA, + SourceRevisionMode: "explicit", + ArtifactRevision: shaA, + DeploymentConfigRev: configRev, + Inputs: discovered, + Classify: map[string][]string{}, + KnownInputsSource: map[string]string{}, + } + plan.PlanBindingHash = buildPlanBindingFromInputs( + discovered, + shaA, + shaA, + configRev, + target.DockerContext, + target.Containers, + target.Volumes, + ) + + plan.ArtifactRevision = shaB + plan.SourceRevision = shaB + if err := assertPlanCurrent(plan); err == nil { + t.Fatal("expected stale plan due artifact mismatch") + } +} + +func TestMaxMigrationPrefixFromListHandlesNestedMigrationPaths(t *testing.T) { + got := maxMigrationPrefixFromList([]string{ + "database/migrations/001_bootstrap.sql", + "database/migrations/007_legacy.sql", + "database/migrations/work/015_capability.sql", + "database/migrations/README.md", + "database/migrations/artifact/019_work.sql", + }) + if got != "19" { + t.Fatalf("expected max migration prefix 19, got %q", got) + } +} + +func TestAssertPlanCurrentRejectsMigrationUpgradeWithoutKnownGood(t *testing.T) { + root, err := gitRepositoryRoot(".") + if err != nil { + t.Fatal(err) + } + headOut, err := runGit(root, "rev-parse", "HEAD") + if err != nil { + t.Fatal(err) + } + targetRevision := strings.TrimSpace(string(headOut)) + targetMaxStr, err := maxMigrationPrefixForRevision(root, targetRevision) + if err != nil { + t.Fatal(err) + } + targetMax, err := strconv.Atoi(targetMaxStr) + if err != nil { + t.Fatalf("invalid target migration max %q: %v", targetMaxStr, err) + } + + revisionsOut, err := runGit(root, "rev-list", "HEAD", "--max-count", "20") + if err != nil { + t.Fatal(err) + } + ancestorList := strings.Fields(string(revisionsOut)) + currentProductionRevision := "" + for _, revision := range ancestorList { + if strings.TrimSpace(revision) == targetRevision { + continue + } + revMaxStr, err := maxMigrationPrefixForRevision(root, revision) + if err != nil { + continue + } + revMax, err := strconv.Atoi(revMaxStr) + if err != nil { + continue + } + if revMax < targetMax { + currentProductionRevision = revision + break + } + } + if currentProductionRevision == "" { + t.Skip("could not find historical revision with lower migration max to validate precondition block") + } + + rootDir := t.TempDir() + composeFile := filepath.Join(rootDir, "docker-compose.prod.yml") + if err := os.WriteFile(composeFile, []byte("a=${POSTGRES_SUPERUSER}\n"), 0o600); err != nil { + t.Fatal(err) + } + envFile := filepath.Join(rootDir, "bootstrap.env") + requiredValues := make([]string, 0, len(requiredChatOnlyInputs)) + for key := range requiredChatOnlyInputs { + requiredValues = append(requiredValues, key+"=value-"+strings.ToLower(key)) + } + if err := os.WriteFile(envFile, []byte(strings.Join(requiredValues, "\n")), 0o600); err != nil { + t.Fatal(err) + } + + inputs, err := discoverComposeInputsFromFile(composeFile) + if err != nil { + t.Fatal(err) + } + discovered, _, _, err := resolveDeploymentInputs(inputs, envFile, true) + if err != nil { + t.Fatal(err) + } + + target, err := resolveDeploymentTarget("production") + if err != nil { + t.Fatal(err) + } + target.ComposeFile = composeFile + plan := &deploymentPlan{ + Version: 1, + PlanID: "migration-precondition", + CreatedAt: time.Now().UTC().Format(time.RFC3339), + ExpiresAt: time.Now().UTC().Add(time.Hour).Format(time.RFC3339), + Readiness: readinessReady, + ReadinessReasons: nil, + Target: "production", + ComposeFile: composeFile, + DockerContext: target.DockerContext, + ProjectName: target.ProjectName, + EnvFile: envFile, + SourceRevision: targetRevision, + SourceRevisionMode: "explicit", + ArtifactRevision: targetRevision, + DeploymentConfigRev: resolveConfigHash(composeFile), + CurrentProductionRev: currentProductionRevision, + KnownGoodRevision: "", + Classify: map[string][]string{}, + Inputs: discovered, + KnownInputsSource: map[string]string{}, + } + plan.Migrations, err = migrationState(root, targetRevision, currentProductionRevision) + if err != nil { + t.Fatal(err) + } + if !plan.Migrations.NeedKnownGood { + t.Fatal("expected migration state to require known good for this test") + } + plan.PlanBindingHash = buildPlanBindingFromInputs( + plan.Inputs, + plan.SourceRevision, + plan.ArtifactRevision, + plan.DeploymentConfigRev, + target.DockerContext, + target.Containers, + target.Volumes, + ) + if err := assertPlanCurrent(plan); err == nil { + t.Fatal("expected assertPlanCurrent to reject migration precondition without known-good") + } else if !strings.Contains(err.Error(), "migration precondition failed") { + t.Fatalf("expected migration precondition failure, got: %v", err) + } +} + +func TestDeploymentPlanDoesNotLeakSecretValues(t *testing.T) { + root, err := gitRepositoryRoot(".") + if err != nil { + t.Fatal(err) + } + target, err := resolveDeploymentTarget("production") + if err != nil { + t.Fatal(err) + } + composeFile := filepath.Join(t.TempDir(), "docker-compose.prod.yml") + if err := os.WriteFile(composeFile, []byte("a=${POSTGRES_SUPERUSER}\n"), 0o600); err != nil { + t.Fatal(err) + } + target.ComposeFile = composeFile + + envFile := filepath.Join(t.TempDir(), "production.env") + if err := os.WriteFile(envFile, []byte(strings.Join([]string{ + "POSTGRES_SUPERUSER_PASSWORD=super-secret-123", + "CREATOR_PASSWORD_FILE=creator-password-file-456", + "COMMIT_EXECUTOR_TOKEN=executor-token-789", + }, "\n")), 0o600); err != nil { + t.Fatal(err) + } + plan, err := buildDeploymentPlan(root, target, envFile, "HEAD", 24, "") + if err != nil { + t.Fatal(err) + } + payload, err := json.Marshal(plan) + if err != nil { + t.Fatal(err) + } + serialized := string(payload) + for _, secretValue := range []string{ + "super-secret-123", + "creator-password-file-456", + "executor-token-789", + } { + if strings.Contains(serialized, secretValue) { + t.Fatalf("plain secret value leaked into plan: %q", secretValue) + } + } + if got := len(plan.SecretReferences); got < 3 { + t.Fatalf("expected at least 3 secret references in plan, got %d", got) + } +} + +func TestDefaultKnownGoodManifestPathStableWithoutLocalAppData(t *testing.T) { + t.Setenv("LOCALAPPDATA", "") + t.Setenv("LOCALAPPDATA_ALT", "") + if strings.TrimSpace(defaultKnownGoodManifestPath()) != "" { + t.Fatal("expected empty path when LOCALAPPDATA is unset") + } +} + +func TestDeployApplyRunsHealthGateWithReadyPlanAndSimulatedComposeDocker(t *testing.T) { + originalPath := os.Getenv("PATH") + fakeDockerDir := t.TempDir() + fakeDocker := filepath.Join(fakeDockerDir, "docker.bat") + dockerBatch := "@echo off\n" + + "if \"%~1\"==\"--context\" (\n" + + " if \"%~3\"==\"compose\" exit /B 0\n" + + " if \"%~3\"==\"inspect\" (\n" + + " if \"%~6\"==\"{{index .Config.Labels \\\"org.opencontainers.image.revision\\\"}}\" echo %FAKE_DOCKER_REVISION%\n" + + " if \"%~6\"==\"{{.State.Status}}\" echo running\n" + + " exit /B 0\n" + + " )\n" + + " if \"%~3\"==\"volume\" if \"%~4\"==\"inspect\" exit /B 0\n" + + ")\n" + + "exit /B 1\n" + if err := os.WriteFile(fakeDocker, []byte(dockerBatch), 0o600); err != nil { + t.Fatal(err) + } + os.Setenv("PATH", fakeDockerDir+";"+originalPath) + defer os.Setenv("PATH", originalPath) + + root, err := gitRepositoryRoot(".") + if err != nil { + t.Fatal(err) + } + target, err := resolveDeploymentTarget("production") + if err != nil { + t.Fatal(err) + } + composeFile := filepath.Join(fakeDockerDir, "docker-compose.prod.yml") + target.ComposeFile = composeFile + if err := os.WriteFile(composeFile, []byte("services:\\n app:\\n image: alpine\\n environment:\\n - POSTGRES_SUPERUSER=${POSTGRES_SUPERUSER}\\n"), 0o600); err != nil { + t.Fatal(err) + } + envFile := filepath.Join(fakeDockerDir, "production.env") + env := []string{ + "POSTGRES_SUPERUSER=deploy-apply-test", + "POSTGRES_SUPERUSER_PASSWORD=pw-super", + "CONTINUITY_DB_PASSWORD=pw-cont", + "WORK_DB_PASSWORD=pw-work", + "CAPABILITY_DB_PASSWORD=pw-cap", + "COMMIT_DB_PASSWORD=pw-commit", + "AUTHORITY_DB_PASSWORD=pw-auth", + "IDENTITY_DB_PASSWORD=pw-id", + "SITUATION_DB_PASSWORD=pw-sit", + "CREATOR_USERNAME=creator", + "CREATOR_PASSWORD_FILE=creator-password", + "CREATOR_BOOTSTRAP_TOKEN=creator-bootstrap", + "CREATOR_SESSION_SIGNING_KEY=creator-session", + "COMMIT_EXECUTOR_TOKEN=commit-exec", + "EPISTEMICS_POSTGRES_CREDENTIAL_REF=epi-cred-ref", + "INTERACTION_POSTGRES_CREDENTIAL_REF=int-cred-ref", + "EPISTEMICS_POSTGRES_PASSWORD=epi-password", + "INTERACTION_POSTGRES_PASSWORD=int-password", + } + if err := os.WriteFile(envFile, []byte(strings.Join(env, "\n")), 0o600); err != nil { + t.Fatal(err) + } + + knownGood, _, err := captureKnownGood(target, "") + if err != nil || strings.TrimSpace(knownGood) == "" { + knownGood = "11a6a23f6e4f44dd88265a04846419a27f54df98" + } + plan, err := buildDeploymentPlan(root, target, envFile, knownGood, 24, "") + if err != nil { + t.Fatal(err) + } + + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.WriteHeader(http.StatusOK) + })) + defer server.Close() + originalHealthEndpoints := append([]string{}, deploymentExternalHealthEndpoints...) + deploymentExternalHealthEndpoints = []string{server.URL + "/healthz", server.URL + "/chat"} + defer func() { + deploymentExternalHealthEndpoints = originalHealthEndpoints + }() + + plan.PlanBindingHash = buildPlanBindingFromInputs( + plan.Inputs, + plan.SourceRevision, + plan.ArtifactRevision, + plan.DeploymentConfigRev, + target.DockerContext, + target.Containers, + target.Volumes, + ) + planPath := filepath.Join(fakeDockerDir, "ready-plan.json") + payload, err := json.Marshal(plan) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(planPath, payload, 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("FAKE_DOCKER_REVISION", plan.ArtifactRevision) + if err := deployApply([]string{"-plan", planPath}); err != nil { + t.Fatalf("expected successful apply with simulated docker and passing health checks, got %v", err) + } +} + +func TestDeployApplyHealthGateRejectsOnFailedHTTPHealth(t *testing.T) { + originalPath := os.Getenv("PATH") + fakeDockerDir := t.TempDir() + fakeDocker := filepath.Join(fakeDockerDir, "docker.bat") + dockerBatch := "@echo off\n" + + "if \"%~1\"==\"--context\" (\n" + + " if \"%~3\"==\"compose\" exit /B 0\n" + + " if \"%~3\"==\"inspect\" (\n" + + " if \"%~6\"==\"{{index .Config.Labels \\\"org.opencontainers.image.revision\\\"}}\" echo %FAKE_DOCKER_REVISION%\n" + + " if \"%~6\"==\"{{.State.Status}}\" echo running\n" + + " exit /B 0\n" + + " )\n" + + " if \"%~3\"==\"volume\" if \"%~4\"==\"inspect\" exit /B 0\n" + + ")\n" + + "exit /B 1\n" + if err := os.WriteFile(fakeDocker, []byte(dockerBatch), 0o600); err != nil { + t.Fatal(err) + } + os.Setenv("PATH", fakeDockerDir+";"+originalPath) + defer os.Setenv("PATH", originalPath) + + root, err := gitRepositoryRoot(".") + if err != nil { + t.Fatal(err) + } + target, err := resolveDeploymentTarget("production") + if err != nil { + t.Fatal(err) + } + composeFile := filepath.Join(fakeDockerDir, "docker-compose.prod.yml") + target.ComposeFile = composeFile + if err := os.WriteFile(composeFile, []byte("services:\\n app:\\n image: alpine\\n environment:\\n - POSTGRES_SUPERUSER=${POSTGRES_SUPERUSER}\\n"), 0o600); err != nil { + t.Fatal(err) + } + envFile := filepath.Join(fakeDockerDir, "production.env") + env := []string{ + "POSTGRES_SUPERUSER=deploy-apply-test", + "POSTGRES_SUPERUSER_PASSWORD=pw-super", + "CONTINUITY_DB_PASSWORD=pw-cont", + "WORK_DB_PASSWORD=pw-work", + "CAPABILITY_DB_PASSWORD=pw-cap", + "COMMIT_DB_PASSWORD=pw-commit", + "AUTHORITY_DB_PASSWORD=pw-auth", + "IDENTITY_DB_PASSWORD=pw-id", + "SITUATION_DB_PASSWORD=pw-sit", + "CREATOR_USERNAME=creator", + "CREATOR_PASSWORD_FILE=creator-password", + "CREATOR_BOOTSTRAP_TOKEN=creator-bootstrap", + "CREATOR_SESSION_SIGNING_KEY=creator-session", + "COMMIT_EXECUTOR_TOKEN=commit-exec", + "EPISTEMICS_POSTGRES_CREDENTIAL_REF=epi-cred-ref", + "INTERACTION_POSTGRES_CREDENTIAL_REF=int-cred-ref", + "EPISTEMICS_POSTGRES_PASSWORD=epi-password", + "INTERACTION_POSTGRES_PASSWORD=int-password", + } + if err := os.WriteFile(envFile, []byte(strings.Join(env, "\n")), 0o600); err != nil { + t.Fatal(err) + } + + knownGood, _, err := captureKnownGood(target, "") + if err != nil || strings.TrimSpace(knownGood) == "" { + knownGood = "11a6a23f6e4f44dd88265a04846419a27f54df98" + } + plan, err := buildDeploymentPlan(root, target, envFile, knownGood, 24, "") + if err != nil { + t.Fatal(err) + } + + plan.PlanBindingHash = buildPlanBindingFromInputs( + plan.Inputs, + plan.SourceRevision, + plan.ArtifactRevision, + plan.DeploymentConfigRev, + target.DockerContext, + target.Containers, + target.Volumes, + ) + planPath := filepath.Join(fakeDockerDir, "ready-plan-bad-health.json") + payload, err := json.Marshal(plan) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(planPath, payload, 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("FAKE_DOCKER_REVISION", plan.ArtifactRevision) + + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/healthz": + writer.WriteHeader(http.StatusInternalServerError) + default: + writer.WriteHeader(http.StatusOK) + } + })) + defer server.Close() + originalHealthEndpoints := append([]string{}, deploymentExternalHealthEndpoints...) + deploymentExternalHealthEndpoints = []string{server.URL + "/healthz", server.URL + "/chat"} + defer func() { + deploymentExternalHealthEndpoints = originalHealthEndpoints + }() + + if err := deployApply([]string{"-plan", planPath}); err == nil { + t.Fatalf("expected apply to fail when health check fails") + } +} diff --git a/cmd/devctl/main.go b/cmd/devctl/main.go index 85f9af2..bda8ce6 100644 --- a/cmd/devctl/main.go +++ b/cmd/devctl/main.go @@ -56,6 +56,8 @@ func main() { err = preflight(os.Args[2:]) case "check-diff": err = checkDiff(os.Args[2:]) + case "deploy": + err = deploy(os.Args[2:]) default: usage() os.Exit(2) @@ -67,7 +69,7 @@ func main() { } func usage() { - fmt.Println("devctl doctor | check-architecture | show-registry | preflight (-active | -task ) | check-diff [-active | -task ]") + fmt.Println("devctl doctor | check-architecture | show-registry | preflight (-active | -task ) | check-diff [-active | -task ] | deploy ...") } func loadRegistry() (Registry, error) { diff --git a/dev/ACTIVE_TASK.json b/dev/ACTIVE_TASK.json index 3052249..f1e6779 100644 --- a/dev/ACTIVE_TASK.json +++ b/dev/ACTIVE_TASK.json @@ -1,5 +1,5 @@ { - "task_file": "dev/tasks/DEV-000072-semantic-interpretation-deferral-boundary.json", - "task_id": "DEV-000072", + "task_file": "dev/tasks/DEV-000073-production-deployment-continuity-safe-redeploy.json", + "task_id": "DEV-000073", "task_revision": 1 } diff --git a/dev/tasks/DEV-000072-semantic-interpretation-deferral-boundary.json b/dev/tasks/DEV-000072-semantic-interpretation-deferral-boundary.json index efc9b8d..d1201da 100644 --- a/dev/tasks/DEV-000072-semantic-interpretation-deferral-boundary.json +++ b/dev/tasks/DEV-000072-semantic-interpretation-deferral-boundary.json @@ -5,6 +5,14 @@ "base_repo_revision": "64870798b9a50f895c99d24441703f366df5c98a", "risk_tier": "E", "intent": "Implement semantic interpretation and deferral boundary V1 so deterministic interpretation acts as a fast-path candidate rather than unquestionable final truth; preserve provenance between deterministic and cognitive candidates; support mixed intent and safer deferral without activating new providers.", + "non_goal_scope": [ + "Do not build a dedicated naming subsystem in this task (for example, no assistant_name table/field).", + "Do not implement dedicated favorite_color/disliked_food/project_role-like preference schemas in this task.", + "Do not introduce a hardcoded 'запомни' keyword trigger or naming-specific memory trigger.", + "Do not introduce dedicated naming memory, universal ontology, or graph/knowledge DB in this task.", + "Treat naming/name-preferral utterances as generic semantic interpretation cases and test fixtures, not the target for persistence architecture changes.", + "Any future durable memory/knowledge mechanism is out of scope for this task and must be captured as a follow-up requirement." + ], "requirement_ids": [ "DEV-000071 measured baseline", "Deterministic candidate can decline/defer", @@ -136,6 +144,11 @@ "reality evaluation comparisons", "production /chat acceptance" ], + "future_scope_requirements": [ + "Generic interaction-to-memory pipeline (common conceptual flow): interaction -> understood meaning -> potential memory/knowledge candidate -> subject/scope/provenance -> salience/relevance -> persistence decision -> relations/associations -> retrieval -> correction/consolidation.", + "Support multiple ordinary human preference/intention domains (names, preferred forms of address, colors, dislikes, habits, relationships, project facts, machine characteristics, temporary constraints) without special schema-per-example.", + "Use existing Epistemics/Experience/Identity/Situation foundations first; introduce new canonical concepts only when requirements and architecture review justify them." + ], "terminal_gap_states": [ "BLOCKED_BY_SPEC_GAP", "BLOCKED_BY_ARCHITECTURE_GAP", diff --git a/dev/tasks/DEV-000073-production-deployment-continuity-safe-redeploy.json b/dev/tasks/DEV-000073-production-deployment-continuity-safe-redeploy.json new file mode 100644 index 0000000..1bc3cd2 --- /dev/null +++ b/dev/tasks/DEV-000073-production-deployment-continuity-safe-redeploy.json @@ -0,0 +1,72 @@ +{ + "task_id": "DEV-000073", + "task_revision": 1, + "architecture_revision": "DEV-000072-baseline", + "base_repo_revision": "36fec6c05cdf7aaf413ffe6b9437f07b03fc7b6b", + "risk_tier": "E", + "intent": "Introduce deterministic two-phase production deployment (prepare/apply) and stale/health/state gates in devctl without changing runtime semantics, so missing required production deployment inputs can never trigger an unsafe recreate.", + "requirement_ids": [ + "prepare/apply separation", + "fail-closed required input resolution", + "exact artifact pinning", + "stale plan rejection", + "known-good/rollback capture", + "health verification on apply", + "no production mutation from ambient shell" + ], + "allowed_change_scope": { + "paths": [ + "AGENTS.md", + "cmd/devctl/**", + "docs/DEPLOYMENT.md", + "docs/PROJECT_STATUS.md", + "docs/evidence/deployment-continuity-dev000073.md", + "architecture/UNRESOLVED_QUESTIONS.md", + "dev/tasks/DEV-000072-semantic-interpretation-deferral-boundary.json", + "docs/evidence/production-recovery-incident-20260818.md", + "dev/ACTIVE_TASK.json", + "dev/tasks/DEV-000073-production-deployment-continuity-safe-redeploy.json" + ] + }, + "forbidden_change_scope": { + "paths": [] + }, + "allowed_architecture_delta": { + "new_cross_module_dependencies": 0, + "new_canonical_owners": 0, + "new_privileges": 0, + "new_third_party_dependencies": 0, + "new_provider_adapters": 0, + "new_database_owner_roles": 0 + }, + "semantic_owner": "devctl", + "contracts": { + "consume": [ + "contracts.identity" + ], + "may_change": [] + }, + "acceptance_scenarios": [ + "fresh shell prepare without ambient production env fails unless required values are in --env-file", + "deploy apply refuses when plan is stale due input/config/revision changes", + "deploy prepare can create READY and BLOCKED plans with machine-readable reasons", + "deploy apply succeeds only for READY plans and verifies deployment artifacts and /health endpoints", + "known-good revision is captured deterministically from production manifest or explicit recovery snapshot", + "healthy production is preserved when PREPARE or APPLY refuse to proceed" + ], + "required_tests": [ + "go test ./cmd/devctl", + "go run ./cmd/devctl preflight -task dev/tasks/DEV-000073-production-deployment-continuity-safe-redeploy.json", + "go run ./cmd/devctl check-diff -task dev/tasks/DEV-000073-production-deployment-continuity-safe-redeploy.json", + "go run ./cmd/devctl deploy prepare --target production --compose deploy/docker-compose.prod.yml --env-file --plan .devctl/deployment-plans/test-dev073.json", + "go run ./cmd/devctl deploy apply --plan .devctl/deployment-plans/test-dev073.json --dry-run", + "go run ./cmd/devctl deploy apply --plan .devctl/deployment-plans/missing.json" + ], + "terminal_gap_states": [ + "BLOCKED_BY_SPEC_GAP", + "BLOCKED_BY_ARCHITECTURE_GAP", + "BLOCKED_BY_MISSING_CAPABILITY", + "BLOCKED_BY_CONFLICT", + "REQUIRES_MIGRATION_DECISION" + ] +} diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 88920b4..d353fe9 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -23,6 +23,31 @@ powershell -ExecutionPolicy Bypass -File .\deploy\windows\Confirm-DomovoyProduct The confirmation script accepts the promotion only when every project-built container has that revision label, then atomically records its exact Docker image ID in `%LOCALAPPDATA%\Domovoy\monitoring\deployment-manifest.json` and the private NAS monitoring directory. Production health compares every running label and image ID to this manifest and becomes degraded on drift. The manifest is non-secret operational evidence; it is not canonical Assistant state. +## Production deploy pipeline V1 + +`cmd/devctl` is now the normal production deployment entrypoint and uses two explicit stages: + +- `deploy prepare` reads the compose manifest and an explicit durable bootstrap source (for production: `--env-file`) and resolves required inputs. +- `deploy apply` consumes a prepared deployment plan and is the only normal mutation path. + +Safety properties: + +- Missing/empty/unresolvable required input blocks prepare readiness (`BLOCKED`) and blocks apply. +- Ambient environment is not authoritative for required production inputs. +- A prepare includes the resolved source revision, config revision, secret references, resolved input classification, migration readiness, known-good snapshot (if available), and a materialized plan binding hash. +- `deploy apply` refuses to execute stale plans (expired, mismatched compose, changed source/config, input drift). +- `apply` verifies running containers, required volumes, and external health (`/healthz`, `/chat`) before returning ready. +- Rollback path to recorded known-good is prepared for recovery when explicitly allowed (`--auto-rollback`). + +Operational sequence example: + +```powershell +go run ./cmd/devctl deploy prepare --target production --compose deploy/docker-compose.prod.yml --env-file --plan .devctl/deployment-plans/prod-.json +go run ./cmd/devctl deploy apply --plan .devctl/deployment-plans/prod-.json +``` + +`prepare`/`apply` are intentionally decoupled so a fresh Codex session with only repository checkout and durable sources can complete production deployment safely. + The first external operation adds `assistant-notification-receiver-1` on production host port `18081`. Caddy applies `deploy/caddy/assistent.caddy`, routing only `https://assistent.cin.su/delivery/*` to that port and all other paths to Creator Web on `18080`. The receiver has an isolated durable volume and no Assistant database access. Commit success means an immutable receiver receipt exists; Windows display and human reading are not Commit outcomes. Migrations `010_capability_notification_binding.sql` and `011_commit_notification_envelope.sql` must be applied before the new runtimes start. The initial Capability binding is owner-local for `domovoy-creator-principal`, revision 1, capability `prospective-notification-delivery`, origin `https://assistent.cin.su`, delivery path `/delivery/v1/deliveries`, receipt path `/delivery/v1/receipts`, protocol `IDEMPOTENT_RECEIPT_V1`, and secret reference `domovoy-notification-write-v1`. diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index 5a50ab6..7e3ca99 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -1,6 +1,52 @@ # Assistant project status -Audit date: 2026-08-17. Active implementation: DEV-000072 revision 1. +Audit date: 2026-08-18. Active implementation: DEV-000072 revision 1. + +## Production recovery checkpoint (2026-08-18) + +- `PRODUCTION_RECOVERY_STATUS = RESTORED_TO_KNOWN_GOOD` +- `TARGET_REVISION = 11a6a23f6e4f44dd88265a04846419a27f54df98` +- `PREVIOUS_RUNNING_REVISION = 36fec6c4f8aabf5e9f5f4ac0d5e2f9c0b8a5a9f0` +- `CURRENT_RUNNING_REVISION = 11a6a23f6e4f44dd88265a04846419a27f54df98` +- `PRODUCTION_HEALTH = healthy` +- `POSTGRES_HEALTH = healthy` +- `CORE_RUNTIME_HEALTH = healthy` +- `CONTINUITY_RUNTIME_HEALTH = healthy` +- `COMMIT_RUNTIME_HEALTH = healthy` +- `CREATOR_WEB_HEALTH = healthy` +- `EXTERNAL_HEALTHZ = 200` +- `CHAT_UI = 200` +- `INCIDENT_EVIDENCE_PATH = docs/evidence/production-recovery-incident-20260818.md` +- `DEV_000072_STATUS = PRODUCTION_ACCEPTANCE_BLOCKED_BY_DEPLOYMENT_INCIDENT` +- `NEXT_REQUIRED_ACTION = run post-recovery evidence tasks for DEV-000072 acceptance and then continue with production deployment` + +## DEV-000073 checkpoint + +- `DEV_000073_STATUS = BLOCKED` +- `STATUS = PAUSED_PENDING_LONG_HORIZON_MODEL` +- `CURRENT_PHASE = DURABLE_BOOTSTRAP_PARTIAL_IMPLEMENTATION_STABILIZED` +- `DEV_000073_SCOPE = production deployment continuity + safe redeploy V1 (deploy prepare/apply)` +- `START_DATE = 2026-08-18` +- `ACTIVE_TASK = dev/tasks/DEV-000073-production-deployment-continuity-safe-redeploy.json` +- `LAST_VERIFIED_COMMIT = 36fec6c05cdf7aaf413ffe6b9437f07b03fc7b6b` +- `CURRENT_UNCOMMITTED_WORK = cmd/devctl/deploy.go (untracked in git index); cmd/devctl/deploy_test.go (untracked in git index); docs/evidence/deployment-continuity-dev000073.md (untracked in git index); supporting task/evidence files` +- `DEV_000073_EVIDENCE_PATH = docs/evidence/deployment-continuity-dev000073.md` +- `NEXT_ACTION = resume DEV-000073 from this checkpoint using a longer-horizon model to complete durable bootstrap review and production-safe seed plan` +- `BLOCKER = real sanitized production prepare cannot resolve all required inputs from intentional durable/protected sources` +- `GO_TEST_DEVCTL = EXECUTED_PASS` +- `CORE_OUTPUTS = cmd/devctl/deploy.go, cmd/devctl/deploy_test.go` +- `DOC_OUTPUTS = AGENTS.md, docs/DEPLOYMENT.md, docs/PROJECT_STATUS.md` +- `DEPLOYMENT_RULE = production mutation requires explicit prepared plan and fail-closed unresolved required input checks` +- `SEMANTIC_SAFETY = no runtime behavior change; no provider naming/memory changes; DEV-000072 remains unchanged and not promoted` +- `TEMP_ARTIFACT_CLEANUP = cmd/devctl/__tmp_devctl_patch_test.txt absent` +- `BLOCKERS = final durable bootstrap design/implementation review remains; pre-runtime protected Secret Resolver bridge and one-time production seed are not yet completed` +- `IMPORTANT_FINDINGS = deploy prepare/apply gates and negative path protections are functionally exercised; production continuity still depends on durable input source for real use` +- `EVIDENCE_OBTAINED = docs/evidence/deployment-continuity-dev000073.md` +- `COMPLETED = deployment prepare/apply safety mechanics; missing input fail-closed; stale-plan rejection; exact-artifact binding; secret-leak protection; health gate; migration discovery/precondition; seven-input ownership audit; canonical-vs-derived credential decision; current deploy.go partial bootstrap implementation compiles; go test ./cmd/devctl passes` +- `ARCHITECTURAL_DECISIONS_ALREADY_ACCEPTED = do not repeat seven-input ownership audit; DB CredentialRef is canonical; raw DB password is derived deployment/runtime transport; CREATOR_PASSWORD_FILE is runtime/mount transport; creator bootstrap token and session signing key require durable protected ownership` +- `IN_PROGRESS = final durable bootstrap design/implementation review; pre-runtime protected secret resolver bridge; one-time production canonical seed/migration; real fresh sanitized production prepare = READY` +- `REASON_FOR_PAUSE = available model is optimized for short interactive edits; remaining work requires long-horizon architecture + cross-component reasoning and production-state migration` +- `NEXT_REQUIRED_ACTION = resume DEV-000073 from this checkpoint using a stronger long-horizon Codex model` ## DEV-000072 checkpoint @@ -20,8 +66,10 @@ Audit date: 2026-08-17. Active implementation: DEV-000072 revision 1. - `RAW_MODEL_BOUNDARY = recorded explicitly: raw cognitive models identify themselves as provider/model provenance; this is not patched inside Interaction and remains a later context-compilation milestone` - `TESTS_PASS = go test ./...; go vet ./...; go run ./cmd/devctl preflight -active dev/ACTIVE_TASK.json; go run ./cmd/devctl check-architecture; go run ./cmd/devctl check-diff -active dev/ACTIVE_TASK.json; tests/cognitive_advisory/cognitive_worker_env_policy_test.ps1; tests/owner_acl/verify.ps1 -DockerContext docker-prod -PostgresContainer assistant-postgres-1; ASSISTANT_LIVE_SEMANTIC_TRACE=1 go test ./cmd/cognitive-worker -run TestLiveInterpretationTrace -v` - `COGNITIVE_CONTEXT_GAP = still open by design in this milestone: raw providers identify themselves as Qwen/Gemma provenance when Domovoy identity/context is not compiled into the prompt` +- `NON_GOAL_SCOPE_DEV_000072 = naming as memory domain is a semantic test case only in this task; no assistant_name subsystem, no favorite_color fields, no naming-specific memory table/schema, no explicit 'запомни'-only trigger, no universal ontology, and no graph DB are introduced for this deployment boundary` +- `GENERAL_MEMORY_REQUIREMENT = broader generic knowledge-retention admission (for preferences, names, constraints, relations, project facts, recurring patterns, and analogous domains) is out-of-scope for DEV-000072 and is recorded as UQ-016 in architecture/UNRESOLVED_QUESTIONS.md` - `PRODUCTION_STATUS = pending promotion/deploy from this checkpoint` -- `NEXT_RECOMMENDED_STEP = finish DEV-000072 production promotion, apply migration 027, redeploy runtime, and verify live /chat against the promoted artifact` +- `NEXT_RECOMMENDED_STEP = finish DEV-000072 production promotion, apply migration 027, redeploy runtime, verify live /chat against the promoted artifact, then address broader memory/knowledge retention via a separate evidence-backed milestone` ## DEV-000071 checkpoint diff --git a/docs/evidence/deployment-continuity-dev000073.md b/docs/evidence/deployment-continuity-dev000073.md new file mode 100644 index 0000000..cdaf6c1 --- /dev/null +++ b/docs/evidence/deployment-continuity-dev000073.md @@ -0,0 +1,169 @@ +# DEV-000073 Deployment Continuity Evidence (Checkpoint) + +Audit date: 2026-08-18 +Task ID: `DEV-000073` +Task revision: `1` +Base revision: `36fec6c05cdf7aaf413ffe6b9437f07b03fc7b6c` + +## Status summary + +- `DEV_000073_STATUS = BLOCKED` +- `CURRENT_PHASE = DURABLE_BOOTSTRAP_PARTIAL_IMPLEMENTATION_STABILIZED` +- `CHECKPOINT_STATUS = IN_PROGRESS` +- `LAST_VERIFIED_COMMIT = 36fec6c05cdf7aaf413ffe6b9437f07b03fc7b6b` +- `LAST_VERIFIED_TEST_COMMAND = go test ./cmd/devctl` +- `LAST_VERIFIED_TEST_RESULT = EXECUTED_PASS` +- `CURRENT_UNCOMMITTED_DIFF = cmd/devctl/deploy.go, cmd/devctl/deploy_test.go, docs/DEPLOYMENT.md, AGENTS.md, architecture/UNRESOLVED_QUESTIONS.md, dev/tasks/DEV-000073-production-deployment-continuity-safe-redeploy.json, docs/evidence/deployment-continuity-dev000073.md, docs/evidence/production-recovery-incident-20260818.md, docs/PROJECT_STATUS.md, dev/ACTIVE_TASK.json` +- `CURRENT_TASK_STATE = PAUSED_PENDING_LONG_HORIZON_MODEL` + +## Executed checks + +- `go test ./cmd/devctl` → `EXECUTED_PASS` +- `go run ./cmd/devctl preflight -task dev/tasks/DEV-000073-production-deployment-continuity-safe-redeploy.json` → `EXECUTED_PASS` +- `go run ./cmd/devctl check-diff -task dev/tasks/DEV-000073-production-deployment-continuity-safe-redeploy.json` → `EXECUTED_PASS` +- `go test ./cmd/devctl` targeted acceptance set (`TestDeployPrepareBlockedAndApplyRefusesMissingRequiredInput`, `TestAssertPlanCurrentRejectsStaleInputChange`, `TestAssertPlanCurrentRejectsArtifactMismatch`, `TestDeploymentPlanDoesNotLeakSecretValues`, `TestDeployApplyRunsHealthGateWithReadyPlanAndSimulatedComposeDocker`, `TestDeployApplyHealthGateRejectsOnFailedHTTPHealth`, migration precondition tests) → `EXECUTED_PASS` + +## Task-specific acceptance probes + +- `MISSING_INPUT_NEGATIVE_TEST = EXECUTED_PASS` + - missing required input in prepare → `readiness=BLOCKED` + - `deploy apply` with blocked plan returns non-zero +- `STALE_PLAN_NEGATIVE_TEST = EXECUTED_PASS` + - compose/input/content assumption changes in plan → `assertPlanCurrent` detects stale state +- `EXACT_ARTIFACT_NEGATIVE_TEST = EXECUTED_PASS` + - artifact change without plan refresh → `assertPlanCurrent` detects stale mismatch +- `SECRET_LEAK_TEST = EXECUTED_PASS` + - plan output and tests show no plaintext secret values; only source class and hashes persisted +- `HEALTH_GATE_TEST = EXECUTED_PASS` + - docker compose success + passing HTTP checks + docker inspect => apply may complete + - docker compose success + failed `/healthz` => apply fails + +## Production continuity probe (sanitized shell) + +- `PRODUCTION_REVISION_VERIFIED = 11a6a23f6e4f44dd88265a04846419a27f54df98` +- `REAL_PRODUCTION_PREPARE_READINESS = BLOCKED` +- `REAL_PRODUCTION_SANITIZED_PREPARE = NOT_READY` +- `REAL_PRODUCTION_CONFIG_SOURCE = TEMPORARY_SYNTHETIC` +- `DOCKER_PROD_BOOTSTRAP_ENV_SOURCE = secrets/docker-prod-bootstrap.env` +- `LEGACY_AMBIENT_DEPENDENCIES = none` +- `UNKNOWN_INPUTS = none` +- `KNOWN_INPUT_SOURCE_DETAIL` for required fields: + - `ASSISTANT_SOURCE_REVISION = missing` + - `CREATOR_WEB_PORT = missing` + - `NOTIFICATION_RECEIVER_PORT = missing` + - `POSTGRES_DB = file` + - all resolved/required production inputs include: + - resolved from file: `POSTGRES_SUPERUSER`, `POSTGRES_SUPERUSER_PASSWORD`, `CONTINUITY_DB_PASSWORD`, `WORK_DB_PASSWORD`, `CAPABILITY_DB_PASSWORD`, `COMMIT_DB_PASSWORD`, `AUTHORITY_DB_PASSWORD`, `IDENTITY_DB_PASSWORD`, `SITUATION_DB_PASSWORD`, `CREATOR_USERNAME`, `COMMIT_EXECUTOR_TOKEN` + - missing (blocking): `CREATOR_BOOTSTRAP_TOKEN`, `CREATOR_PASSWORD_FILE`, `CREATOR_SESSION_SIGNING_KEY`, `EPISTEMICS_POSTGRES_CREDENTIAL_REF`, `EPISTEMICS_POSTGRES_PASSWORD`, `INTERACTION_POSTGRES_CREDENTIAL_REF`, `INTERACTION_POSTGRES_PASSWORD` + +## Configuration source classification (required production inputs) + +- `POSTGRES_SUPERUSER -> BOOTSTRAP` (from env file path) +- `POSTGRES_SUPERUSER_PASSWORD -> SECRET_REFERENCE` (from env file path) +- `CONTINUITY_DB_PASSWORD -> SECRET_REFERENCE` (from env file path) +- `WORK_DB_PASSWORD -> SECRET_REFERENCE` (from env file path) +- `CAPABILITY_DB_PASSWORD -> SECRET_REFERENCE` (from env file path) +- `COMMIT_DB_PASSWORD -> SECRET_REFERENCE` (from env file path) +- `AUTHORITY_DB_PASSWORD -> SECRET_REFERENCE` (from env file path) +- `IDENTITY_DB_PASSWORD -> SECRET_REFERENCE` (from env file path) +- `SITUATION_DB_PASSWORD -> SECRET_REFERENCE` (from env file path) +- `CREATOR_USERNAME -> BOOTSTRAP` (from env file path) +- `CREATOR_PASSWORD_FILE -> DOMOVOY_MANAGED` (missing) +- `CREATOR_BOOTSTRAP_TOKEN -> SECRET_REFERENCE` (missing) +- `CREATOR_SESSION_SIGNING_KEY -> SECRET_REFERENCE` (missing) +- `COMMIT_EXECUTOR_TOKEN -> SECRET_REFERENCE` (from env file path) +- `EPISTEMICS_POSTGRES_CREDENTIAL_REF -> SECRET_REFERENCE` (missing) +- `INTERACTION_POSTGRES_CREDENTIAL_REF -> SECRET_REFERENCE` (missing) +- `EPISTEMICS_POSTGRES_PASSWORD -> SECRET_REFERENCE` (missing) +- `INTERACTION_POSTGRES_PASSWORD -> SECRET_REFERENCE` (missing) +- `CONFIG_SOURCE_CLASSIFICATION = TEMPORARY_SYNTHETIC` + +## Migration readiness + +- `CURRENT_MIGRATION_STATE_SOURCE = git ls-tree -r --name-only database/migrations` +- `CANDIDATE_MIGRATION_REQUIREMENT = current_max=27 target_max=27 status=matched need_known_good=false (for plan created from 11a6a23f6e4f...)` +- `MIGRATION_DISCOVERY_STATUS = PASS` +- `PRECONDITION_ENFORCEMENT = PASS` +- `MIGRATION_PRECONDITION_STATUS = PASS` (plan execution enforces migration precondition and rejects upgrade-without-known-good as blocking) +- `ROLLBACK_COMPATIBILITY_STATUS = not reached by this evidence run; migration check is enforced before compose mutation` + +## Gaps / blockers + +- `BLOCKER = REAL_PRODUCTION_SANITIZED_PREPARE depends on `secrets/docker-prod-bootstrap.env` and still has 7 required missing inputs` +- `BLOCKER = production bootstrap source is temporary/synthetic in this environment; no durable/protected production bootstrap source has been established here` +- `REASON_FOR_PAUSE = remaining work requires long-horizon model for durable bootstrap review, resolver bridge completion, and production seed migration planning` + +## Evidence persisted + +- `evidence/evidence file: docs/evidence/deployment-continuity-dev000073.md` +- `checkpoint: docs/PROJECT_STATUS.md` + +## Durable Ownership Audit (DEV-000073) + +INPUT_OWNERSHIP_MATRIX + +| INPUT_NAME | WHY_NEEDED | SEMANTIC_OWNER | CANONICAL_STATE | SECRET | INTENDED_DURABLE_SOURCE | CURRENT_ACTUAL_SOURCE | RESOLUTION_PATH | WHY_CURRENT_PREPARE_CANNOT_RESOLVE_IT | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| CREATOR_BOOTSTRAP_TOKEN | bootstrap token for the Creator domain startup/authentication bootstrap flow | identity | yes | yes | durable secret input for production bootstrap (protected secret value) | missing from sanitized workspace prepare inputs | deploy.prepare file/env resolution only (`inputSourceClass=SECRET_REFERENCE`) | no durable bootstrap record supplied for this token in this environment; prepare does not have a secret backend lookup step | +| CREATOR_PASSWORD_FILE | creator password secret-file locator consumed by core runtime identity startup | identity | no | no | domovoy-managed runtime mount under `/run/secrets` (`core-runtime` creator password path) | provided in compose as a fixed container path but not as resolved prepare input | deploy.prepare accepts `_FILE` as an input class but only from env file/ambient and does not consume compose defaults | compose default is outside deploy.prepare source model, so required input remains unresolved even when container expects the path | +| CREATOR_SESSION_SIGNING_KEY | key for secure creator session signing | identity | yes | yes | durable secret input for production bootstrap (protected secret value) | missing from sanitized workspace prepare inputs | deploy.prepare file/env resolution only (`inputSourceClass=SECRET_REFERENCE`) | no durable bootstrap source in this environment; no value in env file and not provided by resolver bootstrap path | +| EPISTEMICS_POSTGRES_CREDENTIAL_REF | identity of epistemics credential record for DB secret resolution | epistemics | yes | no | protected durable secret-reference declaration for the epistemics DB credential record | missing from sanitized workspace prepare inputs | deploy.prepare expects direct env value (`SECRET_REFERENCE`) | credential reference value is absent and there is no durable bootstrap source for this reference name | +| EPISTEMICS_POSTGRES_PASSWORD | DB password transport input currently listed as required by deploy | epistemics | no | yes | derived at runtime from `EPISTEMICS_POSTGRES_CREDENTIAL_REF` + `EPISTEMICS_POSTGRES_PASSWORD_FILE` through local secret resolver | not present as input in prepare; compose sets file-based runtime path for password value | unresolved by deploy.prepare because it is treated as legacy required input, while runtime reads file-backed path | this value is rendered runtime transport data; deploy.prepare contract has no mapping from `*_CREDENTIAL_REF` to `*_PASSWORD` | +| INTERACTION_POSTGRES_CREDENTIAL_REF | identity of interaction credential record for DB secret resolution | interaction | yes | no | protected durable secret-reference declaration for the interaction DB credential record | missing from sanitized workspace prepare inputs | deploy.prepare expects direct env value (`SECRET_REFERENCE`) | credential reference value is absent and there is no durable bootstrap source for this reference name | +| INTERACTION_POSTGRES_PASSWORD | DB password transport input currently listed as required by deploy | interaction | no | yes | derived at runtime from `INTERACTION_POSTGRES_CREDENTIAL_REF` + `INTERACTION_POSTGRES_PASSWORD_FILE` through local secret resolver | not present as input in prepare; compose sets file-based runtime path for password value | unresolved by deploy.prepare because it is treated as legacy required input, while runtime reads file-backed path | this value is rendered runtime transport data; deploy.prepare contract has no mapping from `*_CREDENTIAL_REF` to `*_PASSWORD` | + +CREDENTIAL_REF_PASSWORD_DUPLICATION_ASSESSMENT + +`B) CredentialRef is canonical and PASSWORD is derived/rendered runtime transport value.` + +Rationale: +- `cmd/core-runtime/main.go` `readCredentialRefSecret(name, refName)` rejects direct `name` secret values (`direct environment secret is forbidden`) and requires `name_FILE` + `_CREDENTIAL_REF`. +- It resolves credentials through `platformsecrets.NewFileResolver` using `SecretFileRef{CredentialRef: ref, Purpose: DatabasePassword}`. +- The compose layout includes `_POSTGRES_CREDENTIAL_REF` and `_POSTGRES_PASSWORD_FILE`, reinforcing the canonical-reference + rendered-file pattern. + +CREATOR_INPUT_OWNERSHIP + +- `CREATOR_BOOTSTRAP_TOKEN` + - Real owner: `identity`. + - Real durable source class: protected secret (identity bootstrap token), **not** in repo and not present in sanitized prepare input. + - Current classification: missing required input in this environment. +- `CREATOR_SESSION_SIGNING_KEY` + - Real owner: `identity`. + - Real durable source class: protected secret (identity session signing key), **not** in repo and not present in sanitized prepare input. + - Current classification: missing required input in this environment. +- `CREATOR_PASSWORD_FILE` + - Real owner: `identity` (runtime auth material locator). + - It is a **rendered runtime path** backed by Domovoy/runtime secret mount conventions (`/run/secrets`) and is **not** a canonical secret value. + - Current classification: compose-fixed path, not a standalone durable deploy input for this phase. + +DOCKER_PROD_BOOTSTRAP_ENV_ROLE + +`TEMPORARY_SYNTHETIC` + +Why: the file is a local repository bootstrap artifact used for sanitized shell-level prepare/apply probing; it is not an intentional durable/protected production source and still lacks required durable secret inputs. It is neither a Domovoy-managed state artifact nor a recoverable provenance artifact for real production bootstrap. + +EXISTING_SECRET_RESOLVER_CAN_SUPPLY_MISSING_INPUTS + +`partial` + +- adapter? `no` (deploy.prepare does not invoke runtime secret adapter; this check is done in runtime after plan creation) +- bootstrap locator? `missing` (no durable, environment-wide bootstrap secret locator for all 7 inputs is present in this workspace) +- credential record? `partially present in contracts, not provisioned here` (code path expects `*_PASSWORD_FILE` + `*_CREDENTIAL_REF`, but missing refs/records in this environment) +- protected backend entry? `missing` (no proven production-grade protected entry available via the current prepare evidence run) +- ownership mapping? `partial` (owners/areas can be inferred from contracts and module code, but prepare-time ownership policy is not the missing-input resolver) +- pre-runtime bootstrap cycle? `not demonstrated` (no cycle blocker) + +BOOTSTRAP_CYCLE_EXISTS + +`no` + +No real cycle was found because secret resolution uses local file-backed resolver in core runtime and is not a dependency of Domovoy boot for deployment plan creation in this repository evidence path. + +MINIMUM_IMPLEMENTATION_GAP + +- Provide a durable/protected bootstrap source for the remaining required identity inputs (`CREATOR_BOOTSTRAP_TOKEN`, `CREATOR_SESSION_SIGNING_KEY`, `CREATOR_PASSWORD_FILE`) and epistemics/interaction credential refs. +- Align deploy input contract so production input readiness depends on the canonical pair inputs (`*_CREDENTIAL_REF` + `*_PASSWORD_FILE`) and not legacy `_POSTGRES_PASSWORD` as a required direct input. +- Add evidence-backed mapping for each canonical input’s ownership/source class and bootstrap provisioning path before proceeding to acceptance/ready status. + +CHECKPOINT_UPDATED = yes +EVIDENCE_PERSISTED = yes diff --git a/docs/evidence/production-recovery-incident-20260818.md b/docs/evidence/production-recovery-incident-20260818.md new file mode 100644 index 0000000..71a1934 --- /dev/null +++ b/docs/evidence/production-recovery-incident-20260818.md @@ -0,0 +1,82 @@ +## Production Recovery Incident Log + +Date: 2026-08-18 +Objective: Restore last known-good production with minimal changes, no code deploy from DEV-000072. + +### 1) Pre-recovery verification (reconfirmed) +- Last observed bad running revision: `36fec6c4f8aabf5e9f5f4ac0d5e2f9c0b8a5a9f0`. +- Last known-good revision from durable manifests: `11a6a23f6e4f44dd88265a04846419a27f54df98`. +- Known-good deployment artifact IDs were present in `docker-prod` and matched both: + - `%LOCALAPPDATA%\Domovoy\monitoring\deployment-manifest.json` + - `\\nas\MST\codex\_monitoring\domovoy\deployment-manifest.json` +- Pre-recovery container state: + - `assistant-postgres-1` was running but unhealthy. + - `assistant-notification-receiver-1` healthy. + - `assistant-core-runtime-1`, `assistant-commit-runtime-1`, `assistant-cognitive-worker-1`, `assistant-creator-web-1` were `Created`. + - `assistant-continuity-runtime-1` was restarting. + - Continuity startup logs repeatedly reported `COMMIT_EXECUTOR_TOKEN is required`. +- DB read checks were reachable through `postgres` role and owner roles existed; schema objects in main owner schemas were present, including `interaction.utterance_interpretations` + `interaction.semantic_candidates` and expected 4 semantic columns. + +### 2) Recovered configuration inputs +Classification used: +- `DURABLE_CANONICAL`: long-lived non-secret production evidence. +- `PROTECTED_DURABLE`: host/docker secret artifacts that are persistent, permission-protected inputs. +- `BREAK_GLASS_RUNTIME_EVIDENCE`: local runtime incident recovery metadata used only after verification. + +- `POSTGRES_SUPERUSER` / passwords: + - `DURABLE_CANONICAL`: compose contract requires them. + - `PROTECTED_DURABLE`: secret-backed postgres volumes for password material. + - `BREAK_GLASS_RUNTIME_EVIDENCE`: `secrets/production-recovery.env` provided non-empty values and enabled rollback configuration. +- `COMMIT_EXECUTOR_TOKEN`: + - `DURABLE_CANONICAL`: required by continuity/commit contracts. + - `BREAK_GLASS_RUNTIME_EVIDENCE`: populated from `secrets/production-recovery.env`. +- `CREATOR_USERNAME`: + - `BREAK_GLASS_RUNTIME_EVIDENCE`: `secrets/production-recovery.env` (`m`). +- `epistemics_db_password`, `interaction_db_password`, `creator_bootstrap_token`, `creator_password`, `creator_session_signing_key`, + `notification writer/reader tokens`: + - `PROTECTED_DURABLE`: existing Docker volumes `assistant_core_secrets`, + `assistant_notification_writer_secret`, `assistant_notification_reader_secret`. + +### 3) Recovery plan (finalized) +- Target revision: `11a6a23f6e4f44dd88265a04846419a27f54df98`. +- Keep durable data volumes unchanged: + - `assistant_assistant_pg` + - `assistant_core_secrets` + - `assistant_notification_writer_secret` + - `assistant_notification_reader_secret` + - `assistant_notification_receipts` + - `assistant_nats` +- Action set: + 1. Retag known-good images from local/NAS manifests as active `assistant-* : latest`. + 2. Recreate assistant stack on `docker-prod` with `--env-file secrets/production-recovery.env`, `--no-build`, `--force-recreate`. + 3. Verify health and endpoint checks listed in section 5. + +### 4) Recovery actions executed +- Recreated all assistant services on `docker-prod` (and `nats`) from known-good image IDs. +- Loaded production recovery env (`secrets/production-recovery.env`) into compose execution. +- Confirmed required runtime env is now present in service env: + - continuity/commit `COMMIT_EXECUTOR_TOKEN` is set. + - db user/passwords + creator bootstrap wiring are set. +- Verified all services now run with revision `11a6a23f6e4f44dd88265a04846419a27f54df98`. +- No credential values were printed in logs or evidence text. + +### 5) Verification results +- `POSTGRES_HEALTH`: OK (postgres accepts connections). +- `CORE_RUNTIME_HEALTH`: OK (`/healthz` returns runtime ok). +- `CONTINUITY_RUNTIME_HEALTH`: OK (`/healthz` returns runtime ok). +- `COMMIT_RUNTIME_HEALTH`: OK (`/healthz` returns runtime ok). +- `CREATOR_WEB_HEALTH`: OK (`/healthz` returns runtime ok; container healthy). +- `EXTERNAL_HEALTH`: `https://assistant.cin.su/healthz` returns `HTTP 200`. +- `CHAT_SMOKE`: `https://assistant.cin.su/chat` returns `HTTP 200`; authenticated smoke message sent successfully via `POST /api/chat/messages` after session creation (harmless message `health check`), returning `HTTP 200`. +- Production artifact confirmation succeeded for revision `11a6a23f6e4f44dd88265a04846419a27f54df98` with 7 services. + +### 6) Final state +- `KNOWN_GOOD_IMAGE_STATUS`: all required known-good image IDs present and active. +- `PRODUCTION_REVISION_AFTER_RECOVERY`: `11a6a23f6e4f44dd88265a04846419a27f54df98`. +- `DATA_INTEGRITY_STATUS`: preserved; existing schemas and `interaction.utterance_interpretations` semantic columns remain present. +- `CREDENTIAL_ROTATIONS`: none (used recovered durable config; no new credentials generated). +- Remaining limitation: commit/execution of this recovery changed image tags and container recreate timing only; no code/database migrations were altered. + +### 7) Task status after recovery +- `DEV-000072_STATUS`: `PRODUCTION_ACCEPTANCE_BLOCKED_BY_DEPLOYMENT_INCIDENT` remains. +- Continue production readiness requires normal restart of production acceptance run from `docs/PROJECT_STATUS` after this incident checkpoint.