#!/usr/bin/env bash set -Eeuo pipefail umask 077 # ============================================================ # Configuration # ============================================================ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SRC="${ROOT}/stacks/vaultwarden" LIVE="/srv/nfs/infra/vaultwarden" ENV_FILE="${LIVE}/.env" BACKUP_ROOT="/srv/nfs/infra/.deploy-backups/vaultwarden" BACKUP_KEEP=3 STAMP="$(date '+%Y%m%d-%H%M%S')" BACKUP_DIR="${BACKUP_ROOT}/${STAMP}" DB_BACKUP_DIR="${BACKUP_DIR}/database" DB_BACKUP_FILE="${DB_BACKUP_DIR}/vaultwarden.dump" START_TIMEOUT=300 CHECK_INTERVAL=5 # ============================================================ # Runtime state # ============================================================ VERSION_CHANGED=false SERVICE_STOPPED=false NEW_CONTAINER_STARTED=false # ============================================================ # Helpers # ============================================================ log() { echo echo "============================================================" echo "$*" echo "============================================================" } fail() { echo echo "ERROR: $*" >&2 exit 1 } extract_version() { grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n1 } version_less_than() { local A="$1" local B="$2" [[ "${A}" == "${B}" ]] && return 1 [[ "$( printf '%s\n%s\n' "${A}" "${B}" | sort -V | head -n1 )" == "${A}" ]] } get_container_env() { local VARIABLE="$1" docker inspect \ --format '{{range .Config.Env}}{{println .}}{{end}}' \ vaultwarden | sed -n "s/^${VARIABLE}=//p" | head -n1 } cleanup_old_backups() { echo echo "--- Backup retention ---" echo "Keeping last ${BACKUP_KEEP} pre-upgrade backups." mkdir -p "${BACKUP_ROOT}" mapfile -t BACKUPS < <( find "${BACKUP_ROOT}" \ -mindepth 1 \ -maxdepth 1 \ -type d \ -printf '%f\n' | grep -E '^[0-9]{8}-[0-9]{6}$' | sort -r ) if (( ${#BACKUPS[@]} <= BACKUP_KEEP )); then echo "Current backup count: ${#BACKUPS[@]}" echo "Nothing to delete." return 0 fi for BACKUP_NAME in "${BACKUPS[@]:${BACKUP_KEEP}}" do OLD_BACKUP="${BACKUP_ROOT}/${BACKUP_NAME}" echo echo "Removing old backup:" echo " ${OLD_BACKUP}" rm -rf -- "${OLD_BACKUP}" done } restore_old_service() { echo echo "--- Restore previous Vaultwarden service ---" if [[ -f "${BACKUP_DIR}/docker-compose.yml" ]]; then echo "Restoring previous docker-compose.yml" cp \ "${BACKUP_DIR}/docker-compose.yml" \ "${LIVE}/docker-compose.yml" fi ( cd "${LIVE}" docker compose \ up -d \ vaultwarden ) || true } wait_for_vaultwarden() { local ELAPSED=0 echo echo "Waiting for Vaultwarden health=healthy..." while (( ELAPSED < START_TIMEOUT )) do RUNNING="$( docker inspect \ -f '{{.State.Running}}' \ vaultwarden \ 2>/dev/null || echo false )" if [[ "${RUNNING}" != "true" ]]; then echo echo "--- Vaultwarden logs ---" docker logs \ --tail 200 \ vaultwarden \ || true fail "Vaultwarden container stopped during startup" fi HEALTH="$( docker inspect \ -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ vaultwarden )" echo "Vaultwarden health: ${HEALTH} (${ELAPSED}s)" if [[ "${HEALTH}" == "healthy" ]]; then echo echo "Vaultwarden is healthy." return 0 fi if [[ "${HEALTH}" == "unhealthy" ]]; then echo echo "--- Vaultwarden logs ---" docker logs \ --tail 200 \ vaultwarden \ || true fi sleep "${CHECK_INTERVAL}" ELAPSED=$((ELAPSED + CHECK_INTERVAL)) done echo echo "--- Vaultwarden logs ---" docker logs \ --tail 200 \ vaultwarden \ || true fail \ "Timeout waiting for Vaultwarden health=healthy after ${START_TIMEOUT}s" } on_error() { rc=$? trap - ERR echo echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" echo " Vaultwarden deployment FAILED" echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" if [[ -d "${BACKUP_DIR}" ]]; then echo echo "Pre-upgrade backup:" echo " ${BACKUP_DIR}" fi # # If Vaultwarden was stopped for backup but the new # container was never started, database migration # cannot have begun. # # In that case returning the old service is safe. # if [[ "${SERVICE_STOPPED}" == "true" ]] && [[ "${NEW_CONTAINER_STARTED}" != "true" ]] then echo echo "New Vaultwarden version has NOT been started." echo "Restoring previous container." restore_old_service else if [[ "${NEW_CONTAINER_STARTED}" == "true" ]]; then echo echo "Automatic rollback is NOT performed." echo echo "Reason:" echo " The new Vaultwarden version has already started." echo " Database migrations may already have been applied." echo echo "For rollback restore the PostgreSQL dump first." fi fi exit "${rc}" } trap on_error ERR # ============================================================ # Required commands # ============================================================ log "Pre-flight checks" for CMD in \ docker \ psql \ pg_dump \ pg_restore \ sha256sum \ grep \ sed \ sort \ find \ awk do command -v "${CMD}" >/dev/null 2>&1 || \ fail "Required command not found: ${CMD}" done docker compose version >/dev/null 2>&1 || \ fail "Docker Compose plugin is not available" # ============================================================ # Required files # ============================================================ [[ -f "${SRC}/docker-compose.yml" ]] || \ fail "${SRC}/docker-compose.yml not found" [[ -f "${LIVE}/docker-compose.yml" ]] || \ fail "${LIVE}/docker-compose.yml not found" [[ -f "${ENV_FILE}" ]] || \ fail "${ENV_FILE} not found" # ============================================================ # Backup directory access # ============================================================ mkdir -p "${BACKUP_ROOT}" [[ -w "${BACKUP_ROOT}" ]] || \ fail "${BACKUP_ROOT} is not writable" # ============================================================ # Validate requested Compose # ============================================================ log "Validate Docker Compose" docker compose \ -f "${SRC}/docker-compose.yml" \ config >/dev/null echo "Docker Compose configuration is valid." # ============================================================ # Detect requested Vaultwarden image # ============================================================ log "Detect requested Vaultwarden image" mapfile -t VAULTWARDEN_IMAGES < <( docker compose \ -f "${SRC}/docker-compose.yml" \ config --images | grep 'vaultwarden/server:' | sort -u ) if [[ "${#VAULTWARDEN_IMAGES[@]}" -ne 1 ]]; then echo "Detected Vaultwarden images:" if (( ${#VAULTWARDEN_IMAGES[@]} > 0 )); then printf ' %s\n' "${VAULTWARDEN_IMAGES[@]}" else echo " none" fi fail "Expected exactly one Vaultwarden image" fi NEW_IMAGE="${VAULTWARDEN_IMAGES[0]}" echo "Requested image:" echo " ${NEW_IMAGE}" # ============================================================ # Current container # ============================================================ docker inspect vaultwarden >/dev/null 2>&1 || \ fail "Current Vaultwarden container does not exist" CURRENT_RUNNING="$( docker inspect \ -f '{{.State.Running}}' \ vaultwarden )" [[ "${CURRENT_RUNNING}" == "true" ]] || \ fail "Current Vaultwarden container is not running" OLD_IMAGE="$( docker inspect \ -f '{{.Config.Image}}' \ vaultwarden )" # ============================================================ # Current actual Vaultwarden version # ============================================================ log "Detect current Vaultwarden version" CURRENT_VERSION_OUTPUT="$( docker exec \ vaultwarden \ /vaultwarden \ --version )" CURRENT_VERSION="$( printf '%s\n' "${CURRENT_VERSION_OUTPUT}" | extract_version )" [[ -n "${CURRENT_VERSION}" ]] || \ fail "Could not determine current Vaultwarden version" echo "Current image:" echo " ${OLD_IMAGE}" echo echo "Current Vaultwarden version:" echo " ${CURRENT_VERSION}" # ============================================================ # Database configuration # # Read DATABASE_URL from the ACTUALLY RUNNING container. # This prevents the backup script from accidentally backing # up some other DB because of a separate shell .env parsing # problem. # ============================================================ log "Detect Vaultwarden database configuration" DATABASE_URL="$( get_container_env DATABASE_URL )" [[ -n "${DATABASE_URL}" ]] || \ fail "DATABASE_URL is not defined in running Vaultwarden container" # # Vaultwarden supports PostgreSQL URLs such as: # # postgresql://user:password@host:5432/db # postgres://user:password@host:5432/db # # libpq keyword connection strings also work with psql/pg_dump. # if [[ "${DATABASE_URL}" == sqlite:* ]] || [[ "${DATABASE_URL}" == *.sqlite3 ]] then fail "Vaultwarden is using SQLite, not external PostgreSQL" fi # ============================================================ # Pull target image before downtime # ============================================================ log "Pull target Vaultwarden image" docker pull "${NEW_IMAGE}" # ============================================================ # Detect target version # ============================================================ log "Detect target Vaultwarden version" TARGET_VERSION_OUTPUT="$( docker run \ --rm \ --entrypoint /vaultwarden \ "${NEW_IMAGE}" \ --version )" TARGET_VERSION="$( printf '%s\n' "${TARGET_VERSION_OUTPUT}" | extract_version )" [[ -n "${TARGET_VERSION}" ]] || \ fail "Could not determine target Vaultwarden version" echo "Target image:" echo " ${NEW_IMAGE}" echo echo "Target Vaultwarden version:" echo " ${TARGET_VERSION}" # ============================================================ # Version validation # ============================================================ log "Validate version transition" if version_less_than \ "${TARGET_VERSION}" \ "${CURRENT_VERSION}" then fail \ "Vaultwarden downgrade is not allowed: ${CURRENT_VERSION} -> ${TARGET_VERSION}" fi if [[ "${CURRENT_VERSION}" != "${TARGET_VERSION}" ]]; then VERSION_CHANGED=true fi echo "Current version: ${CURRENT_VERSION}" echo "Target version: ${TARGET_VERSION}" echo "Version changed: ${VERSION_CHANGED}" # ============================================================ # VAULTWARDEN VERSION UPDATE # ============================================================ if [[ "${VERSION_CHANGED}" == "true" ]] then # ======================================================== # Prepare backup directory # ======================================================== log "Prepare Vaultwarden pre-upgrade backup" mkdir -p "${DB_BACKUP_DIR}" # # Do NOT use cp -a because NFS does not support # preserving all POSIX metadata in this environment. # cp \ "${LIVE}/docker-compose.yml" \ "${BACKUP_DIR}/docker-compose.yml" # ======================================================== # Stop Vaultwarden BEFORE DB backup # # This freezes Vaultwarden writes while the dump is made. # ======================================================== log "Stop Vaultwarden before database backup" cd "${LIVE}" docker compose \ stop \ vaultwarden SERVICE_STOPPED=true # ======================================================== # Test PostgreSQL connection # ======================================================== log "Test PostgreSQL connection" DB_TEST="$( psql \ --dbname="${DATABASE_URL}" \ -Atqc 'SELECT 1;' )" [[ "${DB_TEST}" == "1" ]] || \ fail "Cannot connect to Vaultwarden PostgreSQL database" echo "PostgreSQL connection successful." # ======================================================== # Collect non-secret PostgreSQL metadata # ======================================================== DB_NAME="$( psql \ --dbname="${DATABASE_URL}" \ -Atqc 'SELECT current_database();' )" DB_USER="$( psql \ --dbname="${DATABASE_URL}" \ -Atqc 'SELECT current_user;' )" DB_HOST="$( psql \ --dbname="${DATABASE_URL}" \ -Atqc \ "SELECT COALESCE(inet_server_addr()::text, 'local');" )" DB_PORT="$( psql \ --dbname="${DATABASE_URL}" \ -Atqc \ 'SELECT inet_server_port();' )" SERVER_VERSION="$( psql \ --dbname="${DATABASE_URL}" \ -Atqc 'SHOW server_version;' )" [[ -n "${SERVER_VERSION}" ]] || \ fail "Could not determine PostgreSQL server version" echo "${SERVER_VERSION}" \ > "${DB_BACKUP_DIR}/postgresql-version.txt" # ======================================================== # Save metadata # # DATABASE_URL/password is intentionally NOT stored. # ======================================================== cat > "${BACKUP_DIR}/metadata.txt" < "${DB_BACKUP_DIR}/vaultwarden.dump.list" [[ -s "${DB_BACKUP_DIR}/vaultwarden.dump.list" ]] || \ fail "pg_restore could not validate PostgreSQL dump" # ======================================================== # SHA256 # ======================================================== ( cd "${DB_BACKUP_DIR}" sha256sum \ vaultwarden.dump \ > vaultwarden.dump.sha256 sha256sum \ -c vaultwarden.dump.sha256 ) DB_BACKUP_SIZE="$( du -h "${DB_BACKUP_FILE}" | awk '{print $1}' )" echo echo "PostgreSQL backup successful." echo echo "Backup directory:" echo " ${BACKUP_DIR}" echo echo "Database dump:" echo " ${DB_BACKUP_FILE}" echo echo "Dump size:" echo " ${DB_BACKUP_SIZE}" echo echo "SHA256:" cat "${DB_BACKUP_DIR}/vaultwarden.dump.sha256" # ======================================================== # Retention # # Remove old backups only after the new backup has # been successfully created and validated. # ======================================================== cleanup_old_backups # ======================================================== # Deploy new Compose # ======================================================== log "Deploy new docker-compose.yml" cp \ "${SRC}/docker-compose.yml" \ "${LIVE}/docker-compose.yml" cd "${LIVE}" # ======================================================== # From this moment migration may occur # ======================================================== NEW_CONTAINER_STARTED=true log "Start new Vaultwarden" docker compose \ up -d \ vaultwarden # ======================================================== # Wait for application + DB readiness # ======================================================== wait_for_vaultwarden # ======================================================== # Verify actual running version # ======================================================== log "Validate running Vaultwarden version" RUNNING_VERSION_OUTPUT="$( docker exec \ vaultwarden \ /vaultwarden \ --version )" RUNNING_VERSION="$( printf '%s\n' "${RUNNING_VERSION_OUTPUT}" | extract_version )" [[ -n "${RUNNING_VERSION}" ]] || \ fail "Could not determine running Vaultwarden version" echo "Expected version:" echo " ${TARGET_VERSION}" echo "Running version:" echo " ${RUNNING_VERSION}" [[ "${RUNNING_VERSION}" == "${TARGET_VERSION}" ]] || \ fail \ "Running Vaultwarden version ${RUNNING_VERSION} does not match ${TARGET_VERSION}" # ============================================================ # NO VERSION CHANGE # ============================================================ else log "No Vaultwarden version change" echo "Current version:" echo " ${CURRENT_VERSION}" # # Compose itself may still have changed: # healthcheck, networks, volumes, etc. # cp \ "${SRC}/docker-compose.yml" \ "${LIVE}/docker-compose.yml" cd "${LIVE}" echo echo "--- Pull images ---" docker compose pull echo echo "--- Apply Docker Compose ---" docker compose \ up -d \ --remove-orphans wait_for_vaultwarden # ======================================================== # Validate running version # ======================================================== RUNNING_VERSION_OUTPUT="$( docker exec \ vaultwarden \ /vaultwarden \ --version )" RUNNING_VERSION="$( printf '%s\n' "${RUNNING_VERSION_OUTPUT}" | extract_version )" [[ "${RUNNING_VERSION}" == "${TARGET_VERSION}" ]] || \ fail \ "Running Vaultwarden version ${RUNNING_VERSION} does not match ${TARGET_VERSION}" fi # ============================================================ # Final container checks # ============================================================ log "Final Vaultwarden checks" FINAL_RUNNING="$( docker inspect \ -f '{{.State.Running}}' \ vaultwarden )" FINAL_HEALTH="$( docker inspect \ -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ vaultwarden )" [[ "${FINAL_RUNNING}" == "true" ]] || \ fail "Vaultwarden container is not running" [[ "${FINAL_HEALTH}" == "healthy" ]] || \ fail "Vaultwarden container is not healthy" # ============================================================ # Final version # ============================================================ FINAL_VERSION_OUTPUT="$( docker exec \ vaultwarden \ /vaultwarden \ --version )" FINAL_VERSION="$( printf '%s\n' "${FINAL_VERSION_OUTPUT}" | extract_version )" [[ -n "${FINAL_VERSION}" ]] || \ fail "Could not determine final Vaultwarden version" [[ "${FINAL_VERSION}" == "${TARGET_VERSION}" ]] || \ fail \ "Final Vaultwarden version mismatch: ${FINAL_VERSION} != ${TARGET_VERSION}" # ============================================================ # Running image # ============================================================ RUNNING_IMAGE="$( docker inspect \ -f '{{.Config.Image}}' \ vaultwarden )" # ============================================================ # Success # ============================================================ log "Vaultwarden deployment successful" echo "Previous Vaultwarden version:" echo " ${CURRENT_VERSION}" echo echo "Running Vaultwarden version:" echo " ${FINAL_VERSION}" echo echo "Running Docker image:" echo " ${RUNNING_IMAGE}" echo echo "Health:" echo " ${FINAL_HEALTH}" if [[ "${VERSION_CHANGED}" == "true" ]]; then echo echo "Pre-upgrade backup:" echo " ${BACKUP_DIR}" echo echo "PostgreSQL dump:" echo " ${DB_BACKUP_FILE}" echo echo "Backup retention:" echo " last ${BACKUP_KEEP} backups" fi # ============================================================ # Clear database connection string # ============================================================ unset DATABASE_URL