CI / test (push) Failing after 19s
DEV-000073 is NOT DONE; current work is partial/stabilized WIP only. go test ./cmd/devctl was last verified PASS. Remaining bootstrap/secret integration requires a stronger long-horizon model. This commit is preservation-only; current ancestry includes DEV-000072 candidate 36fec6c05cdf7aaf413fe... and must be reconciled before DEV-000073 promotion.
1612 lines
50 KiB
Go
1612 lines
50 KiB
Go
package main
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
readinessReady = "READY"
|
|
readinessFailed = "BLOCKED"
|
|
planVersion = 1
|
|
)
|
|
|
|
var deploymentExternalHealthEndpoints = []string{
|
|
"https://assistant.cin.su/healthz",
|
|
"https://assistant.cin.su/chat",
|
|
}
|
|
|
|
var requiredChatOnlyInputs = map[string]struct{}{ // conservative, explicit bootstrap for production
|
|
"POSTGRES_SUPERUSER": {},
|
|
"POSTGRES_SUPERUSER_PASSWORD": {},
|
|
"CONTINUITY_DB_PASSWORD": {},
|
|
"WORK_DB_PASSWORD": {},
|
|
"CAPABILITY_DB_PASSWORD": {},
|
|
"COMMIT_DB_PASSWORD": {},
|
|
"AUTHORITY_DB_PASSWORD": {},
|
|
"IDENTITY_DB_PASSWORD": {},
|
|
"SITUATION_DB_PASSWORD": {},
|
|
"CREATOR_USERNAME": {},
|
|
"CREATOR_BOOTSTRAP_TOKEN": {},
|
|
"CREATOR_SESSION_SIGNING_KEY": {},
|
|
"COMMIT_EXECUTOR_TOKEN": {},
|
|
"EPISTEMICS_POSTGRES_CREDENTIAL_REF": {},
|
|
"INTERACTION_POSTGRES_CREDENTIAL_REF": {},
|
|
}
|
|
|
|
const (
|
|
durableBootstrapStateVersion = 1
|
|
durableBootstrapStatePath = "deployment-bootstrap-state.json"
|
|
)
|
|
|
|
type bootstrapInputMetadata struct {
|
|
CanonicalInput string `json:"canonical_input"`
|
|
BackendFile string `json:"backend_file"`
|
|
RuntimeFile string `json:"runtime_file"`
|
|
SecretRef string `json:"secret_ref"`
|
|
}
|
|
|
|
type deploymentBootstrapState struct {
|
|
Version int `json:"version"`
|
|
Target string `json:"target"`
|
|
ProtectedSecretRoot string `json:"protected_secret_backend_root"`
|
|
Inputs map[string]bootstrapInputMetadata `json:"inputs"`
|
|
LastMigratedAtUtc string `json:"last_migrated_at_utc"`
|
|
MigrationType string `json:"migration_type"`
|
|
DerivedFromDurableRef bool `json:"derived_from_durable_ref"`
|
|
}
|
|
|
|
var requiredDurableInputsForProduction = map[string]struct{}{
|
|
"CREATOR_BOOTSTRAP_TOKEN": {},
|
|
"CREATOR_SESSION_SIGNING_KEY": {},
|
|
"CREATOR_PASSWORD_FILE": {},
|
|
"EPISTEMICS_POSTGRES_CREDENTIAL_REF": {},
|
|
"INTERACTION_POSTGRES_CREDENTIAL_REF": {},
|
|
}
|
|
|
|
var durableBootstrapCredentialDefaults = map[string]string{
|
|
"EPISTEMICS_POSTGRES_CREDENTIAL_REF": "credential://domovoy/epistemics/db-password/v1",
|
|
"INTERACTION_POSTGRES_CREDENTIAL_REF": "credential://domovoy/interaction/db-password/v1",
|
|
}
|
|
|
|
var durableBootstrapRuntimeFileDefaults = map[string]string{
|
|
"CREATOR_BOOTSTRAP_TOKEN": "/run/secrets/creator_bootstrap_token",
|
|
"CREATOR_SESSION_SIGNING_KEY": "/run/secrets/creator_session_signing_key",
|
|
"CREATOR_PASSWORD_FILE": "/run/secrets/creator_password",
|
|
"EPISTEMICS_POSTGRES_PASSWORD_FILE": "/run/secrets/epistemics_db_password",
|
|
"INTERACTION_POSTGRES_PASSWORD_FILE": "/run/secrets/interaction_db_password",
|
|
}
|
|
|
|
var derivedPasswordByCredentialRef = map[string]string{
|
|
"EPISTEMICS_POSTGRES_CREDENTIAL_REF": "EPISTEMICS_POSTGRES_PASSWORD",
|
|
"INTERACTION_POSTGRES_CREDENTIAL_REF": "INTERACTION_POSTGRES_PASSWORD",
|
|
}
|
|
|
|
var (
|
|
errBootstrapStateNotFound = errors.New("deployment bootstrap state not found")
|
|
errBootstrapStateValidationFailed = errors.New("deployment bootstrap state invalid")
|
|
errOneTimeMigrationNotImplemented = errors.New("NOT_IMPLEMENTED: one-time migration behavior not yet supported in this step")
|
|
errBootstrapSourceEnvInvalid = errors.New("bootstrap source env is missing required canonical values")
|
|
)
|
|
|
|
func defaultDurableBootstrapStatePath(args ...string) string {
|
|
root := "."
|
|
if len(args) > 0 && strings.TrimSpace(args[0]) != "" {
|
|
root = args[0]
|
|
}
|
|
return filepath.Join(filepath.Clean(root), ".devctl", durableBootstrapStatePath)
|
|
}
|
|
|
|
func defaultProtectedSecretBackendRoot() string {
|
|
if value := os.Getenv("SECRET_BACKEND_ROOT"); strings.TrimSpace(value) != "" {
|
|
return filepath.Clean(strings.TrimSpace(value))
|
|
}
|
|
return "/run/secrets"
|
|
}
|
|
|
|
type deploymentTarget struct {
|
|
Name string
|
|
ComposeFile string
|
|
DockerContext string
|
|
ProjectName string
|
|
Containers []string
|
|
Volumes []string
|
|
PlanExpiryHours int
|
|
}
|
|
|
|
type deploymentPlan struct {
|
|
Version int `json:"version"`
|
|
PlanID string `json:"plan_id"`
|
|
CreatedAt string `json:"created_at_utc"`
|
|
ExpiresAt string `json:"expires_at_utc"`
|
|
Readiness string `json:"readiness"`
|
|
ReadinessReasons []string `json:"readiness_reasons"`
|
|
Target string `json:"target"`
|
|
ComposeFile string `json:"compose_file"`
|
|
DockerContext string `json:"docker_context"`
|
|
ProjectName string `json:"project_name"`
|
|
EnvFile string `json:"env_file"`
|
|
BootstrapState string `json:"bootstrap_state"`
|
|
SourceRevision string `json:"source_revision"`
|
|
ArtifactRevision string `json:"artifact_revision"`
|
|
SourceRevisionMode string `json:"source_revision_mode"`
|
|
DeploymentConfigRev string `json:"deployment_config_revision"`
|
|
CurrentProductionRev string `json:"current_production_revision"`
|
|
KnownGoodRevision string `json:"known_good_revision"`
|
|
Migrations migrationReadiness `json:"migrations"`
|
|
Inputs []deploymentInputRequirement `json:"inputs"`
|
|
Classify map[string][]string `json:"input_classification"`
|
|
SecretReferences []string `json:"secret_references"`
|
|
ResolvedConfigHash string `json:"resolved_configuration_hash"`
|
|
PlanBindingHash string `json:"plan_binding_hash"`
|
|
Containers []string `json:"containers"`
|
|
Volumes []string `json:"volumes"`
|
|
KnownInputsSource map[string]string `json:"known_inputs_source"`
|
|
}
|
|
|
|
type deploymentInputRequirement struct {
|
|
Name string `json:"name"`
|
|
Required bool `json:"required"`
|
|
SourceClass string `json:"source_class"`
|
|
Resolved bool `json:"resolved"`
|
|
Source string `json:"source"`
|
|
IsSecret bool `json:"is_secret"`
|
|
RefHash string `json:"value_ref_hash"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
type migrationReadiness struct {
|
|
CurrentMaxMigration string `json:"current_max_migration"`
|
|
TargetMaxMigration string `json:"target_max_migration"`
|
|
Status string `json:"status"`
|
|
NeedKnownGood bool `json:"need_known_good"`
|
|
}
|
|
|
|
type deploymentManifest struct {
|
|
Version int `json:"version"`
|
|
Project string `json:"project"`
|
|
Revision string `json:"revision"`
|
|
Artifacts []struct {
|
|
Container string `json:"container"`
|
|
ImageID string `json:"image_id"`
|
|
Revision string `json:"revision"`
|
|
} `json:"artifacts"`
|
|
}
|
|
|
|
func deploy(args []string) error {
|
|
if len(args) == 0 {
|
|
return fmt.Errorf("deploy requires prepare, apply, or bootstrap")
|
|
}
|
|
switch args[0] {
|
|
case "prepare":
|
|
return deployPrepare(args[1:])
|
|
case "apply":
|
|
return deployApply(args[1:])
|
|
case "bootstrap":
|
|
return deployBootstrap(args[1:])
|
|
default:
|
|
return fmt.Errorf("unknown deploy action %q", args[0])
|
|
}
|
|
}
|
|
|
|
func deployPrepare(args []string) error {
|
|
fs := flag.NewFlagSet("deploy prepare", flag.ContinueOnError)
|
|
fs.SetOutput(io.Discard)
|
|
target := fs.String("target", "production", "deployment target")
|
|
compose := fs.String("compose", "", "compose file")
|
|
planPath := fs.String("plan", "", "deployment plan output path")
|
|
context := fs.String("docker-context", "", "docker context")
|
|
envFile := fs.String("env-file", "", "deployment bootstrap env file")
|
|
bootstrapState := fs.String("bootstrap-state", "", "path to durable bootstrap state")
|
|
ttl := fs.Int("plan-ttl-hours", 24, "prepare plan TTL in hours")
|
|
sourceRevisionArg := fs.String("source-revision", "", "source revision to pin (full commit SHA)")
|
|
knownGoodManifest := fs.String("known-good-manifest", defaultKnownGoodManifestPath(), "known-good manifest path")
|
|
if err := fs.Parse(args); err != nil {
|
|
return err
|
|
}
|
|
|
|
t, err := resolveDeploymentTarget(*target)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if *compose != "" {
|
|
t.ComposeFile = *compose
|
|
}
|
|
if *context != "" {
|
|
t.DockerContext = *context
|
|
}
|
|
if strings.TrimSpace(*compose) == "" {
|
|
// explicit compose for clarity and reproducibility
|
|
if t.ComposeFile == "" {
|
|
return fmt.Errorf("compose is required")
|
|
}
|
|
}
|
|
if !filepath.IsAbs(t.ComposeFile) {
|
|
if abs, e := filepath.Abs(t.ComposeFile); e == nil {
|
|
t.ComposeFile = abs
|
|
}
|
|
}
|
|
if *ttl <= 0 || *ttl > 24*30 {
|
|
return fmt.Errorf("plan-ttl-hours must be between 1 and 720")
|
|
}
|
|
if t.Name == "production" && strings.TrimSpace(*envFile) == "" {
|
|
return fmt.Errorf("production prepare requires --env-file")
|
|
}
|
|
|
|
root, err := gitRepositoryRoot(".")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if strings.TrimSpace(*bootstrapState) == "" {
|
|
*bootstrapState = defaultDurableBootstrapStatePath(root)
|
|
}
|
|
plan, err := buildDeploymentPlan(root, t, *envFile, *sourceRevisionArg, *ttl, *knownGoodManifest, *bootstrapState)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if *planPath == "" {
|
|
base := filepath.Join(".devctl", "deployment-plans")
|
|
if err := os.MkdirAll(base, 0o700); err != nil {
|
|
return err
|
|
}
|
|
ts := time.Now().UTC().Format("20060102T150405Z")
|
|
*planPath = filepath.Join(base, fmt.Sprintf("%s-%s-%s.json", strings.ToLower(plan.Target), strings.ReplaceAll(plan.PlanID, ":", "-"), ts))
|
|
}
|
|
payload, err := json.MarshalIndent(plan, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := os.WriteFile(*planPath, payload, 0o600); err != nil {
|
|
return err
|
|
}
|
|
fmt.Printf("DeploymentPlan path=%s readiness=%s\n", *planPath, plan.Readiness)
|
|
for _, reason := range plan.ReadinessReasons {
|
|
fmt.Printf("readiness-blocker=%s\n", reason)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func deployBootstrap(args []string) error {
|
|
fs := flag.NewFlagSet("deploy bootstrap", flag.ContinueOnError)
|
|
fs.SetOutput(io.Discard)
|
|
target := fs.String("target", "production", "deployment target")
|
|
statePath := fs.String("state", "", "durable bootstrap state path")
|
|
sourceEnv := fs.String("source-env", "", "source env containing recoverable bootstrap values")
|
|
secretBackend := fs.String("secret-backend-root", "", "protected secret backend root")
|
|
if err := fs.Parse(args); err != nil {
|
|
return err
|
|
}
|
|
t, err := resolveDeploymentTarget(*target)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if t.Name != "production" {
|
|
return fmt.Errorf("BREAK_GLASS/ONE_TIME_MIGRATION supports production only")
|
|
}
|
|
if strings.TrimSpace(*sourceEnv) == "" {
|
|
return fmt.Errorf("BREAK_GLASS/ONE_TIME_MIGRATION requires --source-env")
|
|
}
|
|
if strings.TrimSpace(*statePath) == "" {
|
|
*statePath = defaultDurableBootstrapStatePath()
|
|
}
|
|
if strings.TrimSpace(*statePath) == "" {
|
|
return fmt.Errorf("durable bootstrap state path unavailable")
|
|
}
|
|
statePathAbs, err := filepath.Abs(*statePath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
*statePath = statePathAbs
|
|
backend := strings.TrimSpace(*secretBackend)
|
|
if backend == "" {
|
|
backend = defaultProtectedSecretBackendRoot()
|
|
}
|
|
if backend == "" {
|
|
return fmt.Errorf("protected secret backend unavailable")
|
|
}
|
|
backendExists := strings.TrimSpace(backend)
|
|
if filepath.Clean(filepath.ToSlash(backendExists)) == "." || strings.TrimSpace(backendExists) == "" {
|
|
return fmt.Errorf("protected secret backend unavailable")
|
|
}
|
|
migrated, missing, err := performOneTimeBootstrapMigration(*statePath, *sourceEnv, t.Name, backendExists)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(missing) > 0 {
|
|
return fmt.Errorf("BREAK_GLASS/ONE_TIME_MIGRATION failed to recover: %s", strings.Join(missing, ", "))
|
|
}
|
|
if migrated {
|
|
fmt.Printf("BREAK_GLASS/ONE_TIME_MIGRATION completed state=%s backend=%s\n", *statePath, backendExists)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func performOneTimeBootstrapMigration(statePath, sourceEnvPath, target, backendRoot string) (bool, []string, error) {
|
|
envValues, err := parseEnvFile(sourceEnvPath)
|
|
if err != nil {
|
|
return false, nil, err
|
|
}
|
|
migrationState := deploymentBootstrapState{
|
|
Version: durableBootstrapStateVersion,
|
|
Target: target,
|
|
ProtectedSecretRoot: filepath.Clean(filepath.ToSlash(backendRoot)),
|
|
Inputs: map[string]bootstrapInputMetadata{},
|
|
LastMigratedAtUtc: time.Now().UTC().Format(time.RFC3339),
|
|
MigrationType: "ONE_TIME_SOURCE_ENV",
|
|
DerivedFromDurableRef: true,
|
|
}
|
|
for _, canonicalInput := range stableDurableInputs() {
|
|
switch canonicalInput {
|
|
case "CREATOR_BOOTSTRAP_TOKEN", "CREATOR_SESSION_SIGNING_KEY":
|
|
meta := bootstrapInputMetadata{
|
|
CanonicalInput: canonicalInput,
|
|
RuntimeFile: resolveCanonicalRuntimeFile(canonicalInput, envValues),
|
|
BackendFile: resolveBootstrapBackendFile(canonicalInput, envValues),
|
|
}
|
|
if meta.RuntimeFile == "" {
|
|
meta.RuntimeFile = meta.BackendFile
|
|
}
|
|
migrationState.Inputs[canonicalInput] = meta
|
|
case "CREATOR_PASSWORD_FILE":
|
|
meta := bootstrapInputMetadata{
|
|
CanonicalInput: canonicalInput,
|
|
RuntimeFile: resolveCanonicalRuntimeFile(canonicalInput, envValues),
|
|
BackendFile: resolveBootstrapBackendFile(canonicalInput, envValues),
|
|
}
|
|
if meta.RuntimeFile == "" {
|
|
meta.RuntimeFile = meta.BackendFile
|
|
}
|
|
migrationState.Inputs[canonicalInput] = meta
|
|
case "EPISTEMICS_POSTGRES_CREDENTIAL_REF", "INTERACTION_POSTGRES_CREDENTIAL_REF":
|
|
passwordFile := derivedPasswordByCredentialRef[canonicalInput]
|
|
meta := bootstrapInputMetadata{
|
|
CanonicalInput: canonicalInput,
|
|
RuntimeFile: resolveCanonicalRuntimeFile(passwordFile, envValues),
|
|
BackendFile: resolveBootstrapBackendFile(passwordFile, envValues),
|
|
SecretRef: envValues[canonicalInput],
|
|
}
|
|
if strings.TrimSpace(meta.SecretRef) == "" {
|
|
meta.SecretRef = strings.TrimSpace(durableBootstrapCredentialDefaults[canonicalInput])
|
|
}
|
|
if meta.RuntimeFile == "" {
|
|
meta.RuntimeFile = meta.BackendFile
|
|
}
|
|
migrationState.Inputs[canonicalInput] = meta
|
|
}
|
|
}
|
|
missing := missingBootstrapInputsFromState(&migrationState)
|
|
if len(missing) > 0 {
|
|
return false, missing, nil
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(statePath), 0o700); err != nil {
|
|
return false, nil, err
|
|
}
|
|
payload, err := json.MarshalIndent(migrationState, "", " ")
|
|
if err != nil {
|
|
return false, nil, err
|
|
}
|
|
if err := os.WriteFile(statePath, payload, 0o600); err != nil {
|
|
return false, nil, err
|
|
}
|
|
return true, nil, nil
|
|
}
|
|
|
|
func loadBootstrapStateForDeployment(statePath string) (*deploymentBootstrapState, error) {
|
|
if strings.TrimSpace(statePath) == "" {
|
|
return nil, nil
|
|
}
|
|
content, err := os.ReadFile(statePath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%w: %v", errBootstrapStateNotFound, err)
|
|
}
|
|
var state deploymentBootstrapState
|
|
if err := json.Unmarshal(content, &state); err != nil {
|
|
return nil, fmt.Errorf("%w: %v", errBootstrapStateValidationFailed, err)
|
|
}
|
|
if err := validateDeploymentBootstrapState(&state); err != nil {
|
|
return nil, err
|
|
}
|
|
return &state, nil
|
|
}
|
|
|
|
func validateDeploymentBootstrapState(state *deploymentBootstrapState) error {
|
|
if state == nil {
|
|
return fmt.Errorf("%w: missing state", errBootstrapStateValidationFailed)
|
|
}
|
|
if state.Version != durableBootstrapStateVersion {
|
|
return fmt.Errorf("%w: unexpected version %d", errBootstrapStateValidationFailed, state.Version)
|
|
}
|
|
if strings.TrimSpace(state.Target) == "" {
|
|
state.Target = "production"
|
|
}
|
|
if strings.TrimSpace(state.ProtectedSecretRoot) == "" {
|
|
state.ProtectedSecretRoot = defaultProtectedSecretBackendRoot()
|
|
}
|
|
missing := missingBootstrapInputsFromState(state)
|
|
if len(missing) > 0 {
|
|
return fmt.Errorf("%w: missing %s", errBootstrapSourceEnvInvalid, strings.Join(missing, ", "))
|
|
}
|
|
if strings.TrimSpace(state.MigrationType) == "" {
|
|
state.MigrationType = "UNKNOWN"
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func missingBootstrapInputsFromState(state *deploymentBootstrapState) []string {
|
|
missing := make([]string, 0)
|
|
for _, canonicalInput := range stableDurableInputs() {
|
|
meta, ok := state.Inputs[canonicalInput]
|
|
if !ok || strings.TrimSpace(meta.CanonicalInput) == "" {
|
|
missing = append(missing, canonicalInput)
|
|
continue
|
|
}
|
|
switch canonicalInput {
|
|
case "EPISTEMICS_POSTGRES_CREDENTIAL_REF", "INTERACTION_POSTGRES_CREDENTIAL_REF":
|
|
if strings.TrimSpace(meta.SecretRef) == "" || strings.TrimSpace(meta.RuntimeFile) == "" {
|
|
missing = append(missing, canonicalInput)
|
|
}
|
|
case "CREATOR_PASSWORD_FILE":
|
|
if strings.TrimSpace(meta.RuntimeFile) == "" {
|
|
missing = append(missing, canonicalInput)
|
|
}
|
|
default:
|
|
if strings.TrimSpace(meta.RuntimeFile) == "" && strings.TrimSpace(meta.SecretRef) == "" {
|
|
missing = append(missing, canonicalInput)
|
|
}
|
|
}
|
|
}
|
|
return missing
|
|
}
|
|
|
|
func stableDurableInputs() []string {
|
|
inputs := make([]string, 0, len(requiredDurableInputsForProduction))
|
|
for name := range requiredDurableInputsForProduction {
|
|
inputs = append(inputs, name)
|
|
}
|
|
sort.Strings(inputs)
|
|
return inputs
|
|
}
|
|
|
|
func resolveCanonicalRuntimeFile(canonicalInput string, envValues map[string]string) string {
|
|
if strings.TrimSpace(canonicalInput) == "" {
|
|
return ""
|
|
}
|
|
if strings.HasSuffix(canonicalInput, "_FILE") {
|
|
if explicit := strings.TrimSpace(envValues[canonicalInput]); explicit != "" {
|
|
return explicit
|
|
}
|
|
if fallback := defaultDurableBootstrapRuntimeFile(canonicalInput); fallback != "" {
|
|
return fallback
|
|
}
|
|
}
|
|
if explicit := strings.TrimSpace(envValues[canonicalInput+"_FILE"]); explicit != "" {
|
|
return explicit
|
|
}
|
|
if strings.HasSuffix(canonicalInput, "_CREDENTIAL_REF") {
|
|
passwordInput := derivedPasswordInputByRuntimeFile(canonicalInput)
|
|
if explicit := strings.TrimSpace(envValues[passwordInput]); explicit != "" {
|
|
return explicit
|
|
}
|
|
if fallback := defaultDurableBootstrapRuntimeFile(passwordInput); fallback != "" {
|
|
return fallback
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func resolveBootstrapBackendFile(canonicalInput string, envValues map[string]string) string {
|
|
if canonicalInput == "" {
|
|
return ""
|
|
}
|
|
explicit := strings.TrimSpace(envValues[canonicalInput+"_FILE"])
|
|
if explicit != "" {
|
|
return explicit
|
|
}
|
|
runtimeFile := resolveCanonicalRuntimeFile(canonicalInput, envValues)
|
|
if runtimeFile != "" {
|
|
return runtimeFile
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func derivedPasswordInputByRuntimeFile(credentialRef string) string {
|
|
if credentialRef == "" {
|
|
return ""
|
|
}
|
|
return strings.TrimSuffix(credentialRef, "_CREDENTIAL_REF") + "_PASSWORD"
|
|
}
|
|
|
|
func defaultPasswordFileForCredentialRef(credentialRef string) string {
|
|
passwordInput := derivedPasswordInputByRuntimeFile(credentialRef)
|
|
if strings.TrimSpace(passwordInput) == "" {
|
|
return ""
|
|
}
|
|
return defaultDurableBootstrapRuntimeFile(passwordInput)
|
|
}
|
|
|
|
func defaultDurableBootstrapRuntimeFile(canonicalInput string) string {
|
|
return durableBootstrapRuntimeFileDefaults[canonicalInput]
|
|
}
|
|
|
|
func resolveDeploymentInputsWithBootstrap(inputs map[string]composeInput, envFile string, state *deploymentBootstrapState, statePath string, requireDurable bool) ([]deploymentInputRequirement, map[string][]string, map[string]string, error) {
|
|
envValues := map[string]string{}
|
|
sources := map[string]string{}
|
|
if strings.TrimSpace(envFile) != "" {
|
|
loaded, err := parseEnvFile(envFile)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
for key, value := range loaded {
|
|
envValues[key] = strings.TrimSpace(value)
|
|
}
|
|
}
|
|
|
|
result := make([]deploymentInputRequirement, 0, len(inputs))
|
|
classification := map[string][]string{}
|
|
|
|
for name, spec := range inputs {
|
|
resolved := false
|
|
source := "missing"
|
|
value := ""
|
|
refHash := ""
|
|
bootstrapMeta, bootstrapResolved := bootstrapInputFromState(name, state)
|
|
|
|
if v, ok := envValues[name]; ok && strings.TrimSpace(v) != "" {
|
|
value = strings.TrimSpace(v)
|
|
source = "file"
|
|
resolved = true
|
|
} else if requireDurable && bootstrapResolved {
|
|
source = "bootstrap-state"
|
|
resolved = true
|
|
refHash = bootstrapInputRefHash(bootstrapMeta)
|
|
} else if !requireDurable {
|
|
if v, ok := os.LookupEnv(name); ok {
|
|
value = strings.TrimSpace(v)
|
|
source = "ambient"
|
|
resolved = value != ""
|
|
}
|
|
}
|
|
|
|
srcClass := inputSourceClass(name)
|
|
if !strings.HasPrefix(srcClass, "SECRET") && strings.Contains(name, "SECRET") {
|
|
srcClass = "SECRET_REFERENCE"
|
|
}
|
|
|
|
req := deploymentInputRequirement{
|
|
Name: name,
|
|
Required: spec.required,
|
|
SourceClass: srcClass,
|
|
Source: source,
|
|
Resolved: resolved,
|
|
IsSecret: isSecretInput(name),
|
|
RefHash: func() string {
|
|
if source == "bootstrap-state" {
|
|
if refHash != "" {
|
|
return refHash
|
|
}
|
|
return bootstrapInputRefHash(bootstrapMeta)
|
|
}
|
|
if strings.TrimSpace(value) == "" {
|
|
return ""
|
|
}
|
|
return valueFingerprint(name, value)
|
|
}(),
|
|
}
|
|
|
|
if req.Required && !req.Resolved {
|
|
if source == "ambient" {
|
|
req.Error = "required production input resolved from ambient shell"
|
|
} else if source == "bootstrap-state" {
|
|
req.Error = "required production input unresolved in bootstrap state"
|
|
} else {
|
|
req.Error = "required production input unresolved"
|
|
}
|
|
}
|
|
|
|
result = append(result, req)
|
|
classification[srcClass] = append(classification[srcClass], name)
|
|
if source == "file" {
|
|
sources[name] = envFile
|
|
} else if source == "ambient" {
|
|
sources[name] = "ambient"
|
|
} else if source == "bootstrap-state" {
|
|
sources[name] = "bootstrap-state"
|
|
if statePath != "" {
|
|
sources[name] = statePath
|
|
}
|
|
} else {
|
|
sources[name] = "missing"
|
|
}
|
|
}
|
|
return result, classification, sources, nil
|
|
}
|
|
|
|
func bootstrapInputFromState(inputName string, state *deploymentBootstrapState) (bootstrapInputMetadata, bool) {
|
|
var empty bootstrapInputMetadata
|
|
if state == nil || state.Inputs == nil || strings.TrimSpace(inputName) == "" {
|
|
return empty, false
|
|
}
|
|
if meta, ok := state.Inputs[inputName]; ok && deploymentBootstrapMetaIsResolved(meta) {
|
|
return meta, true
|
|
}
|
|
if refName, ok := derivedCredentialRefFromPasswordInput(inputName); ok {
|
|
meta, present := state.Inputs[refName]
|
|
if present && deploymentBootstrapMetaIsResolved(meta) {
|
|
return meta, true
|
|
}
|
|
}
|
|
return empty, false
|
|
}
|
|
|
|
func derivedCredentialRefFromPasswordInput(passwordInput string) (string, bool) {
|
|
for refInput, derived := range derivedPasswordByCredentialRef {
|
|
if strings.TrimSpace(derived) == strings.TrimSpace(passwordInput) {
|
|
return refInput, true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func deploymentBootstrapMetaIsResolved(meta bootstrapInputMetadata) bool {
|
|
if strings.TrimSpace(meta.CanonicalInput) == "" {
|
|
return false
|
|
}
|
|
if strings.HasSuffix(meta.CanonicalInput, "_CREDENTIAL_REF") {
|
|
return strings.TrimSpace(meta.SecretRef) != "" && strings.TrimSpace(meta.RuntimeFile) != ""
|
|
}
|
|
return strings.TrimSpace(meta.RuntimeFile) != "" || strings.TrimSpace(meta.SecretRef) != ""
|
|
}
|
|
|
|
func bootstrapInputRefHash(meta bootstrapInputMetadata) string {
|
|
if strings.TrimSpace(meta.SecretRef) == "" && strings.TrimSpace(meta.RuntimeFile) == "" && strings.TrimSpace(meta.BackendFile) == "" {
|
|
return ""
|
|
}
|
|
return valueFingerprint(meta.CanonicalInput, strings.Join([]string{
|
|
meta.CanonicalInput,
|
|
meta.SecretRef,
|
|
meta.RuntimeFile,
|
|
meta.BackendFile,
|
|
}, "|"))
|
|
}
|
|
|
|
func bootstrapInputFromStateLegacy(inputName string, state *deploymentBootstrapState) bool {
|
|
if state == nil || state.Inputs == nil || strings.TrimSpace(inputName) == "" {
|
|
return false
|
|
}
|
|
meta, _ := bootstrapInputFromState(inputName, state)
|
|
return meta.CanonicalInput != ""
|
|
}
|
|
|
|
func deployApply(args []string) error {
|
|
fs := flag.NewFlagSet("deploy apply", flag.ContinueOnError)
|
|
fs.SetOutput(io.Discard)
|
|
planPath := fs.String("plan", "", "deployment plan path")
|
|
dryRun := fs.Bool("dry-run", false, "validate plan without mutating")
|
|
skipHealth := fs.Bool("skip-health", false, "skip health gate")
|
|
autoRollback := fs.Bool("auto-rollback", false, "rollback to known-good revision on first health failure")
|
|
if err := fs.Parse(args); err != nil {
|
|
return err
|
|
}
|
|
if strings.TrimSpace(*planPath) == "" {
|
|
return fmt.Errorf("plan path is required")
|
|
}
|
|
plan, err := readDeploymentPlan(*planPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := assertPlanCurrent(plan); err != nil {
|
|
return err
|
|
}
|
|
if *dryRun {
|
|
fmt.Printf("deploy apply dry-run: readiness=%s\n", plan.Readiness)
|
|
return nil
|
|
}
|
|
if err := executeComposeDeploy(plan); err != nil {
|
|
return err
|
|
}
|
|
|
|
if *skipHealth {
|
|
fmt.Println("deploy apply: READY (health skipped)")
|
|
return nil
|
|
}
|
|
if err := validatePostApplyHealth(plan); err != nil {
|
|
if !*autoRollback {
|
|
return err
|
|
}
|
|
if strings.TrimSpace(plan.KnownGoodRevision) == "" {
|
|
return fmt.Errorf("health failed (%v) and no known-good revision in plan", err)
|
|
}
|
|
recoveryPlan, errR := rollbackToKnownGood(plan)
|
|
if errR != nil {
|
|
return fmt.Errorf("health failed (%v) and rollback failed: %v", err, errR)
|
|
}
|
|
if err := validatePostApplyHealth(recoveryPlan); err != nil {
|
|
return fmt.Errorf("health failed and rollback did not restore health: %w", err)
|
|
}
|
|
fmt.Printf("deploy apply: HEALTH_ROLLBACK_SUCCESS known_good=%s\n", plan.KnownGoodRevision)
|
|
return nil
|
|
}
|
|
fmt.Println("deploy apply: READY")
|
|
return nil
|
|
}
|
|
|
|
func buildDeploymentPlan(root string, target deploymentTarget, envFile, sourceRevisionArg string, ttlHours int, knownGoodManifest string, bootstrapStateArgs ...string) (*deploymentPlan, error) {
|
|
bootstrapStatePath := ""
|
|
if len(bootstrapStateArgs) > 0 && strings.TrimSpace(bootstrapStateArgs[0]) != "" {
|
|
bootstrapStatePath = strings.TrimSpace(bootstrapStateArgs[0])
|
|
}
|
|
notify := []string{}
|
|
bootstrapState, bootstrapErr := loadBootstrapStateForDeployment(bootstrapStatePath)
|
|
if bootstrapErr != nil && strings.TrimSpace(bootstrapStatePath) != "" {
|
|
notify = append(notify, fmt.Sprintf("bootstrap-state read: %v", bootstrapErr))
|
|
}
|
|
|
|
inputMap, err := discoverComposeInputsFromFile(target.ComposeFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
inputResolutions, classification, inputSources, err := resolveDeploymentInputsWithBootstrap(inputMap, envFile, bootstrapState, bootstrapStatePath, target.Name == "production")
|
|
if err != nil {
|
|
if target.Name != "production" {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
for _, req := range inputResolutions {
|
|
if req.Required && !req.Resolved {
|
|
notify = append(notify, req.Name)
|
|
}
|
|
}
|
|
|
|
sourceRevision, mode, err := resolveSourceRevision(root, sourceRevisionArg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
artifactRevision := sourceRevision
|
|
configRevision := resolveDeploymentConfigRevision(target.ComposeFile)
|
|
|
|
knownGood, currentProd, errKnownGood := captureKnownGood(target, knownGoodManifest)
|
|
if errKnownGood != nil {
|
|
notify = append(notify, fmt.Sprintf("known-good capture: %v", errKnownGood))
|
|
}
|
|
|
|
migration, _ := migrationState(root, artifactRevision, knownGood)
|
|
|
|
readiness := readinessReady
|
|
reasons := []string{}
|
|
for _, v := range notify {
|
|
readiness = readinessFailed
|
|
reasons = append(reasons, v)
|
|
}
|
|
if len(notify) == 0 && sourceRevision == "" {
|
|
readiness = readinessFailed
|
|
reasons = append(reasons, "source/artifact revision unresolved")
|
|
}
|
|
|
|
inputList := make([]deploymentInputRequirement, 0, len(inputResolutions))
|
|
for _, v := range inputResolutions {
|
|
inputList = append(inputList, v)
|
|
}
|
|
sort.Slice(inputList, func(i, j int) bool { return inputList[i].Name < inputList[j].Name })
|
|
|
|
secretRefs := []string{}
|
|
for _, input := range inputList {
|
|
if input.SourceClass == "SECRET_REFERENCE" {
|
|
secretRefs = append(secretRefs, input.Name)
|
|
}
|
|
}
|
|
|
|
created := time.Now().UTC()
|
|
plan := &deploymentPlan{
|
|
Version: planVersion,
|
|
PlanID: genPlanID(target.Name, sourceRevision),
|
|
CreatedAt: created.Format(time.RFC3339),
|
|
ExpiresAt: created.Add(time.Duration(ttlHours) * time.Hour).Format(time.RFC3339),
|
|
Readiness: readiness,
|
|
ReadinessReasons: reasons,
|
|
Target: target.Name,
|
|
ComposeFile: target.ComposeFile,
|
|
DockerContext: target.DockerContext,
|
|
ProjectName: target.ProjectName,
|
|
EnvFile: envFile,
|
|
BootstrapState: bootstrapStatePath,
|
|
SourceRevision: sourceRevision,
|
|
SourceRevisionMode: mode,
|
|
ArtifactRevision: artifactRevision,
|
|
DeploymentConfigRev: configRevision,
|
|
CurrentProductionRev: currentProd,
|
|
KnownGoodRevision: knownGood,
|
|
Migrations: migration,
|
|
Inputs: inputList,
|
|
Classify: classification,
|
|
SecretReferences: uniqueSorted(secretRefs),
|
|
ResolvedConfigHash: resolveConfigHash(target.ComposeFile),
|
|
Containers: target.Containers,
|
|
Volumes: target.Volumes,
|
|
KnownInputsSource: inputSources,
|
|
}
|
|
plan.PlanBindingHash = buildPlanBindingHash(plan)
|
|
if readiness == readinessFailed {
|
|
plan.ReadinessReasons = dedupeAndSort(plan.ReadinessReasons)
|
|
}
|
|
return plan, nil
|
|
}
|
|
|
|
func executeComposeDeploy(plan *deploymentPlan) error {
|
|
if plan == nil {
|
|
return errors.New("missing plan")
|
|
}
|
|
if plan.Readiness != readinessReady {
|
|
return fmt.Errorf("plan is not ready (%s)", strings.Join(plan.ReadinessReasons, "; "))
|
|
}
|
|
if err := runCommandWithEnv(plan); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func runCommandWithEnv(plan *deploymentPlan) error {
|
|
args := []string{"--context", plan.DockerContext, "compose"}
|
|
if plan.EnvFile != "" {
|
|
args = append(args, "--env-file", plan.EnvFile)
|
|
}
|
|
args = append(args, "-p", plan.ProjectName, "-f", plan.ComposeFile, "up", "-d", "--no-build", "--force-recreate")
|
|
cmd := exec.Command("docker", args...)
|
|
cmd.Env = os.Environ()
|
|
cmd.Env = append(cmd.Env, fmt.Sprintf("ASSISTANT_SOURCE_REVISION=%s", plan.ArtifactRevision))
|
|
cmd.Stdout = os.Stdout
|
|
cmd.Stderr = os.Stderr
|
|
fmt.Printf("deploy apply command: %s\n", strings.Join(cmd.Args, " "))
|
|
return cmd.Run()
|
|
}
|
|
|
|
func validatePostApplyHealth(plan *deploymentPlan) error {
|
|
target, err := resolveDeploymentTarget(plan.Target)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
target.ComposeFile = plan.ComposeFile
|
|
target.DockerContext = plan.DockerContext
|
|
target.ProjectName = plan.ProjectName
|
|
if err := validateDeploymentRunningContainers(plan, target); err != nil {
|
|
return err
|
|
}
|
|
if err := validateDeploymentVolumes(plan, target); err != nil {
|
|
return err
|
|
}
|
|
for _, u := range deploymentExternalHealthEndpoints {
|
|
if err := validateHTTPHealth(u); err != nil {
|
|
return fmt.Errorf("health gate failed for %s: %w", u, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func rollbackToKnownGood(plan *deploymentPlan) (*deploymentPlan, error) {
|
|
if strings.TrimSpace(plan.KnownGoodRevision) == "" {
|
|
return nil, fmt.Errorf("no known-good revision in plan")
|
|
}
|
|
recovery := *plan
|
|
recovery.Readiness = readinessReady
|
|
recovery.ReadinessReasons = nil
|
|
recovery.ArtifactRevision = plan.KnownGoodRevision
|
|
recovery.SourceRevision = plan.KnownGoodRevision
|
|
recovery.PlanBindingHash = buildPlanBindingHash(&recovery)
|
|
if err := executeComposeDeploy(&recovery); err != nil {
|
|
return nil, err
|
|
}
|
|
return &recovery, nil
|
|
}
|
|
|
|
func assertPlanCurrent(plan *deploymentPlan) error {
|
|
if plan == nil {
|
|
return errors.New("missing plan")
|
|
}
|
|
if err := validatePlanReadiness(plan); err != nil {
|
|
return err
|
|
}
|
|
if plan.Readiness == readinessFailed {
|
|
return fmt.Errorf("plan is not ready (%s)", strings.Join(plan.ReadinessReasons, "; "))
|
|
}
|
|
|
|
expiry, err := time.Parse(time.RFC3339, plan.ExpiresAt)
|
|
if err != nil {
|
|
return fmt.Errorf("invalid expires_at_utc")
|
|
}
|
|
if time.Now().UTC().After(expiry) {
|
|
return fmt.Errorf("plan expired at %s", plan.ExpiresAt)
|
|
}
|
|
|
|
target, err := resolveDeploymentTarget(plan.Target)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if strings.TrimSpace(plan.ComposeFile) != "" {
|
|
target.ComposeFile = plan.ComposeFile
|
|
}
|
|
|
|
planCompose := plan.ComposeFile
|
|
targetCompose := target.ComposeFile
|
|
if strings.TrimSpace(planCompose) == "" || strings.TrimSpace(targetCompose) == "" {
|
|
return fmt.Errorf("stale plan: compose file changed")
|
|
}
|
|
if !filepath.IsAbs(planCompose) {
|
|
if abs, e := filepath.Abs(planCompose); e == nil {
|
|
planCompose = abs
|
|
}
|
|
}
|
|
if !filepath.IsAbs(targetCompose) {
|
|
if abs, e := filepath.Abs(targetCompose); e == nil {
|
|
targetCompose = abs
|
|
}
|
|
}
|
|
if filepath.Clean(planCompose) != filepath.Clean(targetCompose) {
|
|
return fmt.Errorf("stale plan: compose file changed")
|
|
}
|
|
if target.DockerContext != "" && plan.DockerContext != target.DockerContext {
|
|
return fmt.Errorf("stale plan: docker context changed")
|
|
}
|
|
currentConfig := resolveConfigHash(plan.ComposeFile)
|
|
if plan.DeploymentConfigRev != currentConfig {
|
|
return fmt.Errorf("stale plan: deployment config changed")
|
|
}
|
|
if plan.SourceRevisionMode == "head" {
|
|
currentHead, _, err := resolveSourceRevision(filepath.Dir(plan.ComposeFile), "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if currentHead != plan.SourceRevision {
|
|
return fmt.Errorf("stale plan: source revision changed")
|
|
}
|
|
}
|
|
inputMap, err := discoverComposeInputsFromFile(plan.ComposeFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
currentInputs, _, _, err := resolveDeploymentInputs(inputMap, plan.EnvFile, target.Name == "production")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
currentBindings := buildPlanBindingFromInputs(
|
|
currentInputs,
|
|
plan.SourceRevision,
|
|
plan.ArtifactRevision,
|
|
currentConfig,
|
|
target.DockerContext,
|
|
target.Containers,
|
|
target.Volumes,
|
|
)
|
|
if plan.PlanBindingHash != currentBindings {
|
|
return fmt.Errorf("stale plan: required assumptions changed")
|
|
}
|
|
root, err := gitRepositoryRoot(".")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
currentMigration, err := migrationState(root, plan.ArtifactRevision, plan.CurrentProductionRev)
|
|
if err != nil {
|
|
return fmt.Errorf("stale plan: migration state check failed: %w", err)
|
|
}
|
|
if !migrationReadinessEqual(plan.Migrations, currentMigration) {
|
|
return fmt.Errorf("stale plan: migration state changed")
|
|
}
|
|
if plan.Target == "production" && currentMigration.NeedKnownGood && strings.TrimSpace(plan.KnownGoodRevision) == "" {
|
|
return fmt.Errorf("migration precondition failed: %s", currentMigration.Status)
|
|
}
|
|
|
|
// explicit re-check of required inputs from env file ensures no ambient fallback
|
|
inputResolutions, _, _, err := resolveDeploymentInputs(inputMap, plan.EnvFile, target.Name == "production")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, req := range inputResolutions {
|
|
if req.Required && !req.Resolved {
|
|
return fmt.Errorf("required input %s unresolved since prepare", req.Name)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func migrationStateFromRevisions(root, targetRevision, currentRevision string) (migrationReadiness, error) {
|
|
currentMax := "0"
|
|
targetMax := "0"
|
|
var err error
|
|
if strings.TrimSpace(currentRevision) != "" {
|
|
currentMax, err = maxMigrationPrefixForRevision(root, currentRevision)
|
|
if err != nil {
|
|
currentMax = "unknown"
|
|
}
|
|
}
|
|
targetMax, err = maxMigrationPrefixForRevision(root, targetRevision)
|
|
if err != nil {
|
|
targetMax = "unknown"
|
|
}
|
|
currentMaxValue, currentParseErr := strconv.Atoi(currentMax)
|
|
targetMaxValue, targetParseErr := strconv.Atoi(targetMax)
|
|
status := "matched"
|
|
needKnownGood := false
|
|
if currentMax == "unknown" || targetMax == "unknown" || currentParseErr != nil || targetParseErr != nil {
|
|
status = "needs-manual-verification"
|
|
needKnownGood = true
|
|
} else if currentMaxValue != targetMaxValue {
|
|
if currentMaxValue > targetMaxValue {
|
|
status = "target-lower-than-production"
|
|
} else {
|
|
status = "upgrade-required"
|
|
}
|
|
needKnownGood = true
|
|
}
|
|
return migrationReadiness{
|
|
CurrentMaxMigration: currentMax,
|
|
TargetMaxMigration: targetMax,
|
|
Status: status,
|
|
NeedKnownGood: needKnownGood,
|
|
}, nil
|
|
}
|
|
|
|
func migrationReadinessEqual(a, b migrationReadiness) bool {
|
|
return a.CurrentMaxMigration == b.CurrentMaxMigration &&
|
|
a.TargetMaxMigration == b.TargetMaxMigration &&
|
|
a.Status == b.Status &&
|
|
a.NeedKnownGood == b.NeedKnownGood
|
|
}
|
|
|
|
func resolvePlanInputFingerprint(reqs []deploymentInputRequirement) string {
|
|
parts := make([]string, 0, len(reqs))
|
|
for _, req := range reqs {
|
|
parts = append(parts, fmt.Sprintf("%s=%s@%s", req.Name, req.RefHash, req.Source))
|
|
}
|
|
sort.Strings(parts)
|
|
return hashString(strings.Join(parts, "|"))
|
|
}
|
|
|
|
func buildPlanBindingHash(plan *deploymentPlan) string {
|
|
raw := buildPlanBindingFromInputs(
|
|
plan.Inputs,
|
|
plan.SourceRevision,
|
|
plan.ArtifactRevision,
|
|
plan.DeploymentConfigRev,
|
|
plan.DockerContext,
|
|
plan.Containers,
|
|
plan.Volumes,
|
|
)
|
|
return hashString(raw)
|
|
}
|
|
|
|
func buildPlanBindingFromInputs(reqs []deploymentInputRequirement, sourceRevision, artifactRevision, configRev, context string, containers, volumes []string) string {
|
|
parts := []string{
|
|
sourceRevision,
|
|
artifactRevision,
|
|
configRev,
|
|
context,
|
|
}
|
|
if len(containers) > 0 {
|
|
orderedContainers := append([]string{}, containers...)
|
|
sort.Strings(orderedContainers)
|
|
parts = append(parts, strings.Join(orderedContainers, ","))
|
|
}
|
|
if len(volumes) > 0 {
|
|
orderedVolumes := append([]string{}, volumes...)
|
|
sort.Strings(orderedVolumes)
|
|
parts = append(parts, strings.Join(orderedVolumes, ","))
|
|
}
|
|
for _, req := range reqs {
|
|
parts = append(parts, fmt.Sprintf("%s=%s@%s", req.Name, req.RefHash, req.Source))
|
|
}
|
|
sort.Strings(parts)
|
|
return hashString(strings.Join(parts, "|"))
|
|
}
|
|
|
|
func validatePlanReadiness(plan *deploymentPlan) error {
|
|
if plan.Readiness == "" {
|
|
return errors.New("plan readiness is empty")
|
|
}
|
|
if plan.SourceRevision == "" || plan.ArtifactRevision == "" {
|
|
return fmt.Errorf("plan missing source/artifact revision")
|
|
}
|
|
if plan.DeploymentConfigRev == "" {
|
|
return fmt.Errorf("plan missing deployment configuration revision")
|
|
}
|
|
if len(plan.Inputs) == 0 {
|
|
return fmt.Errorf("plan has no input requirements")
|
|
}
|
|
for _, req := range plan.Inputs {
|
|
if req.Required && req.Source == "ambient" {
|
|
return fmt.Errorf("required input %s resolved from ambient source", req.Name)
|
|
}
|
|
if req.Required && !req.Resolved {
|
|
return fmt.Errorf("required input %s unresolved", req.Name)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func discoverComposeInputsFromFile(composeFile string) (map[string]composeInput, error) {
|
|
b, err := os.ReadFile(composeFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return discoverComposeInputs(string(b)), nil
|
|
}
|
|
|
|
type composeInput struct {
|
|
name string
|
|
required bool
|
|
}
|
|
|
|
func discoverComposeInputs(content string) map[string]composeInput {
|
|
result := map[string]composeInput{}
|
|
re := regexp.MustCompile(`\$\{([A-Z0-9_]+)(?::-[^}]*)?\}`)
|
|
for _, match := range re.FindAllStringSubmatch(content, -1) {
|
|
if len(match) < 2 {
|
|
continue
|
|
}
|
|
token := match[0]
|
|
name := match[1]
|
|
result[name] = composeInput{name: name, required: !strings.Contains(token, ":-")}
|
|
}
|
|
for name := range requiredChatOnlyInputs {
|
|
if _, ok := result[name]; !ok {
|
|
result[name] = composeInput{name: name, required: true}
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func resolveDeploymentInputs(inputs map[string]composeInput, envFile string, requireDurable bool) ([]deploymentInputRequirement, map[string][]string, map[string]string, error) {
|
|
envValues := map[string]string{}
|
|
sources := map[string]string{}
|
|
if strings.TrimSpace(envFile) != "" {
|
|
loaded, err := parseEnvFile(envFile)
|
|
if err != nil {
|
|
return nil, nil, nil, err
|
|
}
|
|
for key, value := range loaded {
|
|
envValues[key] = strings.TrimSpace(value)
|
|
sources[key] = "file"
|
|
}
|
|
}
|
|
|
|
result := make([]deploymentInputRequirement, 0, len(inputs))
|
|
classification := map[string][]string{}
|
|
for name, spec := range inputs {
|
|
required := spec.required
|
|
source := "missing"
|
|
resolved := false
|
|
value := ""
|
|
if v, ok := envValues[name]; ok {
|
|
value = v
|
|
source = "file"
|
|
resolved = strings.TrimSpace(v) != ""
|
|
if !resolved {
|
|
source = "file"
|
|
}
|
|
}
|
|
if !resolved && os.Getenv(name) != "" {
|
|
if requireDurable {
|
|
source = "ambient"
|
|
} else {
|
|
value = os.Getenv(name)
|
|
source = "ambient"
|
|
resolved = true
|
|
}
|
|
}
|
|
srcClass := inputSourceClass(name)
|
|
if !strings.HasPrefix(srcClass, "SECRET") && strings.Contains(name, "SECRET") {
|
|
srcClass = "SECRET_REFERENCE"
|
|
}
|
|
req := deploymentInputRequirement{
|
|
Name: name,
|
|
Required: required,
|
|
SourceClass: srcClass,
|
|
Resolved: resolved,
|
|
Source: source,
|
|
IsSecret: isSecretInput(name),
|
|
RefHash: func() string {
|
|
if strings.TrimSpace(value) == "" {
|
|
return ""
|
|
}
|
|
return valueFingerprint(name, value)
|
|
}(),
|
|
}
|
|
if req.Required && !req.Resolved {
|
|
if source == "ambient" {
|
|
req.Error = "required production input resolved from ambient shell"
|
|
} else {
|
|
req.Error = "required production input unresolved"
|
|
}
|
|
}
|
|
result = append(result, req)
|
|
classification[srcClass] = append(classification[srcClass], name)
|
|
if source == "file" {
|
|
sources[name] = envFile
|
|
} else if source == "ambient" {
|
|
sources[name] = "ambient"
|
|
} else {
|
|
sources[name] = "missing"
|
|
}
|
|
}
|
|
return result, classification, sources, nil
|
|
}
|
|
|
|
func runCommand(name string, args ...string) ([]byte, error) {
|
|
cmd := exec.Command(name, args...)
|
|
out, err := cmd.Output()
|
|
if err != nil {
|
|
if exitErr, ok := err.(*exec.ExitError); ok && len(exitErr.Stderr) > 0 {
|
|
return nil, fmt.Errorf("%s", strings.TrimSpace(string(exitErr.Stderr)))
|
|
}
|
|
return nil, err
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func parseEnvFile(path string) (map[string]string, error) {
|
|
contents, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
result := make(map[string]string)
|
|
for _, raw := range strings.Split(string(contents), "\n") {
|
|
line := strings.TrimSpace(raw)
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
if strings.HasPrefix(line, "export ") {
|
|
line = strings.TrimSpace(strings.TrimPrefix(line, "export "))
|
|
}
|
|
parts := strings.SplitN(line, "=", 2)
|
|
if len(parts) != 2 {
|
|
return nil, fmt.Errorf("invalid env line: %q", raw)
|
|
}
|
|
key := strings.TrimSpace(parts[0])
|
|
value := strings.Trim(strings.TrimSpace(parts[1]), "\"'")
|
|
result[key] = value
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func resolveSourceRevision(root, override string) (string, string, error) {
|
|
if strings.TrimSpace(override) != "" {
|
|
out, err := runGit(root, "rev-parse", override)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
rev := strings.TrimSpace(string(out))
|
|
if !isHex40(rev) {
|
|
return "", "", fmt.Errorf("revision is not 40-char commit: %q", rev)
|
|
}
|
|
return rev, "explicit", nil
|
|
}
|
|
out, err := runGit(root, "rev-parse", "HEAD")
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
rev := strings.TrimSpace(string(out))
|
|
if !isHex40(rev) {
|
|
return "", "", fmt.Errorf("HEAD is not a full commit: %q", rev)
|
|
}
|
|
return rev, "head", nil
|
|
}
|
|
|
|
func resolveDeploymentConfigRevision(composeFile string) string {
|
|
b, err := os.ReadFile(composeFile)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return hashString(string(b))
|
|
}
|
|
|
|
func resolveConfigHash(composeFile string) string {
|
|
b, err := os.ReadFile(composeFile)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return hashString(string(b))
|
|
}
|
|
|
|
func captureKnownGood(target deploymentTarget, manifestPath string) (string, string, error) {
|
|
known := ""
|
|
current := ""
|
|
if manifestPath == "" {
|
|
manifestPath = defaultKnownGoodManifestPath()
|
|
}
|
|
if manifestPath != "" {
|
|
if b, err := os.ReadFile(manifestPath); err == nil {
|
|
var manifest deploymentManifest
|
|
if err := json.Unmarshal(b, &manifest); err == nil {
|
|
if manifest.Revision != "" {
|
|
known = manifest.Revision
|
|
current = manifest.Revision
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if known != "" {
|
|
return known, current, nil
|
|
}
|
|
// legacy fallback for explicit recovery contexts only
|
|
if currentFromContainers, err := captureCurrentProductionRevision(target); err == nil && currentFromContainers != "" {
|
|
return currentFromContainers, currentFromContainers, nil
|
|
}
|
|
return "", "", fmt.Errorf("known-good manifest unavailable")
|
|
}
|
|
|
|
func captureCurrentProductionRevision(target deploymentTarget) (string, error) {
|
|
revisions := make([]string, 0, len(target.Containers))
|
|
for _, container := range target.Containers {
|
|
label, err := dockerContainerLabel(target.DockerContext, container, "org.opencontainers.image.revision")
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if label != "" {
|
|
revisions = append(revisions, label)
|
|
}
|
|
}
|
|
if len(revisions) == 0 {
|
|
return "", fmt.Errorf("no production container labels available")
|
|
}
|
|
base := revisions[0]
|
|
for _, revision := range revisions[1:] {
|
|
if revision != base {
|
|
return "", fmt.Errorf("container revision mismatch among production services")
|
|
}
|
|
}
|
|
return base, nil
|
|
}
|
|
|
|
func migrationState(root, targetRevision, currentRevision string) (migrationReadiness, error) {
|
|
return migrationStateFromRevisions(root, targetRevision, currentRevision)
|
|
}
|
|
|
|
func maxMigrationPrefixForRevision(root, revision string) (string, error) {
|
|
out, err := runGit(root, "ls-tree", "-r", "--name-only", revision, "database/migrations")
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return maxMigrationPrefixFromList(strings.Split(strings.TrimSpace(string(out)), "\n")), nil
|
|
}
|
|
|
|
func maxMigrationPrefixFromList(paths []string) string {
|
|
re := regexp.MustCompile(`^(\d{3})_`)
|
|
max := "0"
|
|
maxValue := 0
|
|
for _, file := range paths {
|
|
if file == "" {
|
|
continue
|
|
}
|
|
base := filepath.Base(filepath.FromSlash(strings.TrimSpace(file)))
|
|
m := re.FindStringSubmatch(base)
|
|
if len(m) != 2 {
|
|
continue
|
|
}
|
|
value := m[1]
|
|
i, parseErr := strconv.Atoi(value)
|
|
if parseErr != nil {
|
|
continue
|
|
}
|
|
if i > maxValue {
|
|
maxValue = i
|
|
}
|
|
}
|
|
max = fmt.Sprintf("%d", maxValue)
|
|
return max
|
|
}
|
|
|
|
func resolveDeploymentTarget(name string) (deploymentTarget, error) {
|
|
switch name {
|
|
case "", "production", "prod":
|
|
return deploymentTarget{
|
|
Name: "production",
|
|
ComposeFile: filepath.FromSlash("deploy/docker-compose.prod.yml"),
|
|
DockerContext: "docker-prod",
|
|
ProjectName: "assistant",
|
|
Containers: []string{
|
|
"assistant-postgres-1",
|
|
"assistant-notification-receiver-1",
|
|
"assistant-core-runtime-1",
|
|
"assistant-continuity-runtime-1",
|
|
"assistant-commit-runtime-1",
|
|
"assistant-cognitive-worker-1",
|
|
"assistant-creator-web-1",
|
|
},
|
|
Volumes: []string{
|
|
"assistant_pg",
|
|
"assistant_nats",
|
|
"assistant_core_secrets",
|
|
"assistant_notification_receipts",
|
|
"assistant_notification_writer_secret",
|
|
"assistant_notification_reader_secret",
|
|
},
|
|
PlanExpiryHours: 12,
|
|
}, nil
|
|
default:
|
|
return deploymentTarget{}, fmt.Errorf("unknown target %q", name)
|
|
}
|
|
}
|
|
|
|
func validateDeploymentRunningContainers(plan *deploymentPlan, target deploymentTarget) error {
|
|
required := map[string]struct{}{}
|
|
for _, item := range target.Containers {
|
|
required[item] = struct{}{}
|
|
}
|
|
for container := range required {
|
|
revision, err := dockerContainerLabel(target.DockerContext, container, "org.opencontainers.image.revision")
|
|
if err != nil {
|
|
return fmt.Errorf("container %s inspect failed: %w", container, err)
|
|
}
|
|
if plan.ArtifactRevision != "" && revision != "" && revision != plan.ArtifactRevision {
|
|
return fmt.Errorf("container %s revision mismatch (expected=%s actual=%s)", container, plan.ArtifactRevision, revision)
|
|
}
|
|
state, err := dockerContainerState(target.DockerContext, container)
|
|
if err != nil {
|
|
return fmt.Errorf("container %s state failed: %w", container, err)
|
|
}
|
|
if state != "running" {
|
|
return fmt.Errorf("container %s state=%s", container, state)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateDeploymentVolumes(_ *deploymentPlan, target deploymentTarget) error {
|
|
for _, volume := range target.Volumes {
|
|
if _, err := runCommand("docker", "--context", target.DockerContext, "volume", "inspect", volume); err != nil {
|
|
return fmt.Errorf("required volume missing: %s", volume)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateHTTPHealth(url string) error {
|
|
client := &http.Client{Timeout: 12 * time.Second}
|
|
req, err := http.NewRequest(http.MethodGet, url, http.NoBody)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
defer resp.Body.Close()
|
|
_, _ = io.Copy(io.Discard, resp.Body)
|
|
return fmt.Errorf("unexpected status: %d", resp.StatusCode)
|
|
}
|
|
defer resp.Body.Close()
|
|
_, _ = io.Copy(io.Discard, resp.Body)
|
|
return nil
|
|
}
|
|
|
|
func readDeploymentPlan(path string) (*deploymentPlan, error) {
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var plan deploymentPlan
|
|
if err := json.Unmarshal(b, &plan); err != nil {
|
|
return nil, err
|
|
}
|
|
return &plan, nil
|
|
}
|
|
|
|
func genPlanID(target, revision string) string {
|
|
if len(revision) > 12 {
|
|
revision = revision[:12]
|
|
}
|
|
if target == "" {
|
|
target = "deployment"
|
|
}
|
|
return fmt.Sprintf("%s-%s", target, revision)
|
|
}
|
|
|
|
func defaultKnownGoodManifestPath() string {
|
|
if local := os.Getenv("LOCALAPPDATA"); local != "" {
|
|
return filepath.Join(local, "Domovoy", "monitoring", "deployment-manifest.json")
|
|
}
|
|
if local := os.Getenv("LOCALAPPDATA_ALT"); local != "" {
|
|
return filepath.Join(local, "Domovoy", "monitoring", "deployment-manifest.json")
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func inputSourceClass(name string) string {
|
|
n := strings.ToUpper(name)
|
|
switch {
|
|
case strings.Contains(n, "_CREDENTIAL_REF"):
|
|
return "SECRET_REFERENCE"
|
|
case strings.Contains(n, "_FILE"):
|
|
return "DOMOVOY_MANAGED"
|
|
case strings.HasSuffix(n, "_PASSWORD") || strings.HasSuffix(n, "_TOKEN") || strings.HasSuffix(n, "_KEY") || strings.Contains(n, "SECRET"):
|
|
return "SECRET_REFERENCE"
|
|
case n == "ASSISTANT_SOURCE_REVISION":
|
|
return "DEPLOYMENT_LOCAL"
|
|
default:
|
|
return "BOOTSTRAP"
|
|
}
|
|
}
|
|
|
|
func isSecretInput(name string) bool {
|
|
n := strings.ToUpper(name)
|
|
return strings.HasSuffix(n, "_PASSWORD") || strings.HasSuffix(n, "_TOKEN") || strings.HasSuffix(n, "_KEY") || strings.Contains(n, "SECRET")
|
|
}
|
|
|
|
func valueFingerprint(name, value string) string {
|
|
if strings.TrimSpace(value) == "" {
|
|
return ""
|
|
}
|
|
sum := sha256.Sum256([]byte(value))
|
|
if isSecretInput(name) {
|
|
return hex.EncodeToString(sum[:])[:16]
|
|
}
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func hashString(value string) string {
|
|
sum := sha256.Sum256([]byte(value))
|
|
return hex.EncodeToString(sum[:])
|
|
}
|
|
|
|
func isHex40(value string) bool {
|
|
if len(value) != 40 {
|
|
return false
|
|
}
|
|
for _, r := range value {
|
|
if (r < '0' || r > '9') && (r < 'a' || r > 'f') && (r < 'A' || r > 'F') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func dedupeAndSort(values []string) []string {
|
|
seen := map[string]struct{}{}
|
|
out := make([]string, 0, len(values))
|
|
for _, value := range values {
|
|
if _, ok := seen[value]; ok {
|
|
continue
|
|
}
|
|
seen[value] = struct{}{}
|
|
out = append(out, value)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
func uniqueSorted(values []string) []string {
|
|
return dedupeAndSort(values)
|
|
}
|
|
|
|
func dockerContainerLabel(context, container, label string) (string, error) {
|
|
out, err := runCommand("docker", "--context", context, "inspect", container, "--format", fmt.Sprintf("{{index .Config.Labels \"%s\"}}", label))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return strings.TrimSpace(string(out)), nil
|
|
}
|
|
|
|
func dockerContainerState(context, container string) (string, error) {
|
|
out, err := runCommand("docker", "--context", context, "inspect", container, "--format", "{{.State.Status}}")
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return strings.TrimSpace(string(out)), nil
|
|
}
|