From 11ee440ad017c1e385abb3d9b1eacb845dcb5ddd Mon Sep 17 00:00:00 2001 From: Alexander Zubarev Date: Wed, 30 Sep 2026 12:31:21 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9E=D0=B1=D0=BD=D0=BE=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20scripts/deploy-vaultwarden.sh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/deploy-vaultwarden.sh | 1201 +++++++++++++++++++++++++++++---- 1 file changed, 1065 insertions(+), 136 deletions(-) diff --git a/scripts/deploy-vaultwarden.sh b/scripts/deploy-vaultwarden.sh index 5d82afb..44996be 100755 --- a/scripts/deploy-vaultwarden.sh +++ b/scripts/deploy-vaultwarden.sh @@ -2,190 +2,1084 @@ set -Eeuo pipefail +umask 077 + + +# ============================================================ +# Configuration +# ============================================================ + ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SRC="${ROOT}/stacks/vaultwarden" + LIVE="/srv/nfs/infra/vaultwarden" ENV_FILE="${LIVE}/.env" BACKUP_ROOT="/srv/nfs/infra/.deploy-backups/vaultwarden" +BACKUP_KEEP=3 + STAMP="$(date '+%Y%m%d-%H%M%S')" -BACKUP="${BACKUP_ROOT}/${STAMP}" -echo "=================================" -echo " Vaultwarden deployment" -echo "=================================" +BACKUP_DIR="${BACKUP_ROOT}/${STAMP}" +DB_BACKUP_DIR="${BACKUP_DIR}/database" +DB_BACKUP_FILE="${DB_BACKUP_DIR}/vaultwarden.dump" -echo "Source: ${SRC}" -echo "Target: ${LIVE}" - -# -# Проверки исходных файлов -# - -if [[ ! -f "${SRC}/docker-compose.yml" ]]; then - echo "ERROR: ${SRC}/docker-compose.yml not found" - exit 1 -fi - -if [[ ! -f "${ENV_FILE}" ]]; then - echo "ERROR: ${ENV_FILE} not found" - exit 1 -fi +START_TIMEOUT=300 +CHECK_INTERVAL=5 -# -# Проверка docker-compose -# +# ============================================================ +# Runtime state +# ============================================================ -echo -echo "--- Validate Docker Compose ---" - -docker compose \ - --env-file "${ENV_FILE}" \ - -f "${SRC}/docker-compose.yml" \ - config >/dev/null - -echo "Docker Compose configuration is valid." +VERSION_CHANGED=false +SERVICE_STOPPED=false +NEW_CONTAINER_STARTED=false -# -# Определяем образ -# +# ============================================================ +# Helpers +# ============================================================ -echo -echo "--- Detect Vaultwarden image ---" - -VAULTWARDEN_IMAGE="$( - docker compose \ - --env-file "${ENV_FILE}" \ - -f "${SRC}/docker-compose.yml" \ - config --images | - grep -m1 'vaultwarden/server' -)" - -if [[ -z "${VAULTWARDEN_IMAGE}" ]]; then - echo "ERROR: Vaultwarden image not found" - exit 1 -fi - -echo "Vaultwarden image: ${VAULTWARDEN_IMAGE}" - - -# -# Backup текущего compose -# - -echo -echo "--- Backup current configuration ---" - -mkdir -p "${BACKUP}" - -if [[ -f "${LIVE}/docker-compose.yml" ]]; then - cp \ - "${LIVE}/docker-compose.yml" \ - "${BACKUP}/docker-compose.yml" -fi - -echo "Backup: ${BACKUP}" - - -# -# Rollback -# - -rollback() +log() { - rc=$? - echo - echo "!!! Deployment failed" + echo "============================================================" + echo "$*" + echo "============================================================" +} - if [[ -f "${BACKUP}/docker-compose.yml" ]]; then + +fail() +{ + echo + echo "ERROR: $*" >&2 + exit 1 +} + + +extract_version() +{ + grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | + head -n1 +} + + +version_less_than() +{ + local A="$1" + local B="$2" + + [[ "${A}" == "${B}" ]] && return 1 + + [[ "$( + printf '%s\n%s\n' "${A}" "${B}" | + sort -V | + head -n1 + )" == "${A}" ]] +} + + +get_container_env() +{ + local VARIABLE="$1" + + docker inspect \ + --format '{{range .Config.Env}}{{println .}}{{end}}' \ + vaultwarden | + sed -n "s/^${VARIABLE}=//p" | + head -n1 +} + + +cleanup_old_backups() +{ + echo + echo "--- Backup retention ---" + echo "Keeping last ${BACKUP_KEEP} pre-upgrade backups." + + mkdir -p "${BACKUP_ROOT}" + + mapfile -t BACKUPS < <( + find "${BACKUP_ROOT}" \ + -mindepth 1 \ + -maxdepth 1 \ + -type d \ + -printf '%f\n' | + grep -E '^[0-9]{8}-[0-9]{6}$' | + sort -r + ) + + + if (( ${#BACKUPS[@]} <= BACKUP_KEEP )); then + + echo "Current backup count: ${#BACKUPS[@]}" + echo "Nothing to delete." + + return 0 + + fi + + + for BACKUP_NAME in "${BACKUPS[@]:${BACKUP_KEEP}}" + do + + OLD_BACKUP="${BACKUP_ROOT}/${BACKUP_NAME}" + + echo + echo "Removing old backup:" + echo " ${OLD_BACKUP}" + + rm -rf -- "${OLD_BACKUP}" + + done +} + + +restore_old_service() +{ + echo + echo "--- Restore previous Vaultwarden service ---" + + if [[ -f "${BACKUP_DIR}/docker-compose.yml" ]]; then echo "Restoring previous docker-compose.yml" cp \ - "${BACKUP}/docker-compose.yml" \ + "${BACKUP_DIR}/docker-compose.yml" \ "${LIVE}/docker-compose.yml" - ( - cd "${LIVE}" - - docker compose \ - --env-file "${ENV_FILE}" \ - up -d --remove-orphans || true - ) fi + + ( + cd "${LIVE}" + + docker compose \ + up -d \ + vaultwarden + + ) || true +} + + +wait_for_vaultwarden() +{ + local ELAPSED=0 + + echo + echo "Waiting for Vaultwarden health=healthy..." + + + while (( ELAPSED < START_TIMEOUT )) + do + + RUNNING="$( + docker inspect \ + -f '{{.State.Running}}' \ + vaultwarden \ + 2>/dev/null || + echo false + )" + + + if [[ "${RUNNING}" != "true" ]]; then + + echo + echo "--- Vaultwarden logs ---" + + docker logs \ + --tail 200 \ + vaultwarden \ + || true + + fail "Vaultwarden container stopped during startup" + + fi + + + HEALTH="$( + docker inspect \ + -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ + vaultwarden + )" + + + echo "Vaultwarden health: ${HEALTH} (${ELAPSED}s)" + + + if [[ "${HEALTH}" == "healthy" ]]; then + + echo + echo "Vaultwarden is healthy." + + return 0 + + fi + + + if [[ "${HEALTH}" == "unhealthy" ]]; then + + echo + echo "--- Vaultwarden logs ---" + + docker logs \ + --tail 200 \ + vaultwarden \ + || true + + fi + + + sleep "${CHECK_INTERVAL}" + + ELAPSED=$((ELAPSED + CHECK_INTERVAL)) + + done + + + echo + echo "--- Vaultwarden logs ---" + + docker logs \ + --tail 200 \ + vaultwarden \ + || true + + + fail \ + "Timeout waiting for Vaultwarden health=healthy after ${START_TIMEOUT}s" +} + + +on_error() +{ + rc=$? + + trap - ERR + + echo + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo " Vaultwarden deployment FAILED" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + + + if [[ -d "${BACKUP_DIR}" ]]; then + + echo + echo "Pre-upgrade backup:" + echo " ${BACKUP_DIR}" + + fi + + + # + # If Vaultwarden was stopped for backup but the new + # container was never started, database migration + # cannot have begun. + # + # In that case returning the old service is safe. + # + + if [[ "${SERVICE_STOPPED}" == "true" ]] && + [[ "${NEW_CONTAINER_STARTED}" != "true" ]] + then + + echo + echo "New Vaultwarden version has NOT been started." + echo "Restoring previous container." + + restore_old_service + + else + + if [[ "${NEW_CONTAINER_STARTED}" == "true" ]]; then + + echo + echo "Automatic rollback is NOT performed." + echo + echo "Reason:" + echo " The new Vaultwarden version has already started." + echo " Database migrations may already have been applied." + echo + echo "For rollback restore the PostgreSQL dump first." + + fi + + fi + + exit "${rc}" } -trap rollback ERR + +trap on_error ERR -# -# Deployment compose-файла -# +# ============================================================ +# Required commands +# ============================================================ -echo -echo "--- Deploy docker-compose.yml ---" +log "Pre-flight checks" -cp \ - "${SRC}/docker-compose.yml" \ - "${LIVE}/docker-compose.yml" +for CMD in \ + docker \ + psql \ + pg_dump \ + pg_restore \ + sha256sum \ + grep \ + sed \ + sort \ + find \ + awk +do + + command -v "${CMD}" >/dev/null 2>&1 || \ + fail "Required command not found: ${CMD}" + +done -# -# Pull образов -# +docker compose version >/dev/null 2>&1 || \ + fail "Docker Compose plugin is not available" -echo -echo "--- Pull images ---" -cd "${LIVE}" +# ============================================================ +# Required files +# ============================================================ + +[[ -f "${SRC}/docker-compose.yml" ]] || \ + fail "${SRC}/docker-compose.yml not found" + +[[ -f "${LIVE}/docker-compose.yml" ]] || \ + fail "${LIVE}/docker-compose.yml not found" + +[[ -f "${ENV_FILE}" ]] || \ + fail "${ENV_FILE} not found" + + +# ============================================================ +# Backup directory access +# ============================================================ + +mkdir -p "${BACKUP_ROOT}" + +[[ -w "${BACKUP_ROOT}" ]] || \ + fail "${BACKUP_ROOT} is not writable" + + +# ============================================================ +# Validate requested Compose +# ============================================================ + +log "Validate Docker Compose" + docker compose \ - --env-file "${ENV_FILE}" \ - pull + -f "${SRC}/docker-compose.yml" \ + config >/dev/null -# -# Apply -# - -echo -echo "--- Apply Docker Compose ---" - -docker compose \ - --env-file "${ENV_FILE}" \ - up -d --remove-orphans +echo "Docker Compose configuration is valid." -# -# Post-check -# +# ============================================================ +# Detect requested Vaultwarden image +# ============================================================ -echo -echo "--- Check Vaultwarden container ---" +log "Detect requested Vaultwarden image" -sleep 3 -RUNNING="$( +mapfile -t VAULTWARDEN_IMAGES < <( + docker compose \ + -f "${SRC}/docker-compose.yml" \ + config --images | + grep 'vaultwarden/server:' | + sort -u +) + + +if [[ "${#VAULTWARDEN_IMAGES[@]}" -ne 1 ]]; then + + echo "Detected Vaultwarden images:" + + if (( ${#VAULTWARDEN_IMAGES[@]} > 0 )); then + printf ' %s\n' "${VAULTWARDEN_IMAGES[@]}" + else + echo " none" + fi + + fail "Expected exactly one Vaultwarden image" + +fi + + +NEW_IMAGE="${VAULTWARDEN_IMAGES[0]}" + + +echo "Requested image:" +echo " ${NEW_IMAGE}" + + +# ============================================================ +# Current container +# ============================================================ + +docker inspect vaultwarden >/dev/null 2>&1 || \ + fail "Current Vaultwarden container does not exist" + + +CURRENT_RUNNING="$( docker inspect \ -f '{{.State.Running}}' \ vaultwarden )" -if [[ "${RUNNING}" != "true" ]]; then - echo "ERROR: Vaultwarden container is not running" - exit 1 + +[[ "${CURRENT_RUNNING}" == "true" ]] || \ + fail "Current Vaultwarden container is not running" + + +OLD_IMAGE="$( + docker inspect \ + -f '{{.Config.Image}}' \ + vaultwarden +)" + + +# ============================================================ +# Current actual Vaultwarden version +# ============================================================ + +log "Detect current Vaultwarden version" + + +CURRENT_VERSION_OUTPUT="$( + docker exec \ + vaultwarden \ + /vaultwarden \ + --version +)" + + +CURRENT_VERSION="$( + printf '%s\n' "${CURRENT_VERSION_OUTPUT}" | + extract_version +)" + + +[[ -n "${CURRENT_VERSION}" ]] || \ + fail "Could not determine current Vaultwarden version" + + +echo "Current image:" +echo " ${OLD_IMAGE}" + +echo +echo "Current Vaultwarden version:" +echo " ${CURRENT_VERSION}" + + +# ============================================================ +# Database configuration +# +# Read DATABASE_URL from the ACTUALLY RUNNING container. +# This prevents the backup script from accidentally backing +# up some other DB because of a separate shell .env parsing +# problem. +# ============================================================ + +log "Detect Vaultwarden database configuration" + + +DATABASE_URL="$( + get_container_env DATABASE_URL +)" + + +[[ -n "${DATABASE_URL}" ]] || \ + fail "DATABASE_URL is not defined in running Vaultwarden container" + + +# +# Vaultwarden supports PostgreSQL URLs such as: +# +# postgresql://user:password@host:5432/db +# postgres://user:password@host:5432/db +# +# libpq keyword connection strings also work with psql/pg_dump. +# + +if [[ "${DATABASE_URL}" == sqlite:* ]] || + [[ "${DATABASE_URL}" == *.sqlite3 ]] +then + + fail "Vaultwarden is using SQLite, not external PostgreSQL" + fi -# -# Показываем фактически запущенный image -# +# ============================================================ +# Pull target image before downtime +# ============================================================ + +log "Pull target Vaultwarden image" + + +docker pull "${NEW_IMAGE}" + + +# ============================================================ +# Detect target version +# ============================================================ + +log "Detect target Vaultwarden version" + + +TARGET_VERSION_OUTPUT="$( + docker run \ + --rm \ + --entrypoint /vaultwarden \ + "${NEW_IMAGE}" \ + --version +)" + + +TARGET_VERSION="$( + printf '%s\n' "${TARGET_VERSION_OUTPUT}" | + extract_version +)" + + +[[ -n "${TARGET_VERSION}" ]] || \ + fail "Could not determine target Vaultwarden version" + + +echo "Target image:" +echo " ${NEW_IMAGE}" + +echo +echo "Target Vaultwarden version:" +echo " ${TARGET_VERSION}" + + +# ============================================================ +# Version validation +# ============================================================ + +log "Validate version transition" + + +if version_less_than \ + "${TARGET_VERSION}" \ + "${CURRENT_VERSION}" +then + + fail \ + "Vaultwarden downgrade is not allowed: ${CURRENT_VERSION} -> ${TARGET_VERSION}" + +fi + + +if [[ "${CURRENT_VERSION}" != "${TARGET_VERSION}" ]]; then + + VERSION_CHANGED=true + +fi + + +echo "Current version: ${CURRENT_VERSION}" +echo "Target version: ${TARGET_VERSION}" +echo "Version changed: ${VERSION_CHANGED}" + + +# ============================================================ +# VAULTWARDEN VERSION UPDATE +# ============================================================ + +if [[ "${VERSION_CHANGED}" == "true" ]] +then + + # ======================================================== + # Prepare backup directory + # ======================================================== + + log "Prepare Vaultwarden pre-upgrade backup" + + + mkdir -p "${DB_BACKUP_DIR}" + + + # + # Do NOT use cp -a because NFS does not support + # preserving all POSIX metadata in this environment. + # + + cp \ + "${LIVE}/docker-compose.yml" \ + "${BACKUP_DIR}/docker-compose.yml" + + + # ======================================================== + # Stop Vaultwarden BEFORE DB backup + # + # This freezes Vaultwarden writes while the dump is made. + # ======================================================== + + log "Stop Vaultwarden before database backup" + + + cd "${LIVE}" + + + docker compose \ + stop \ + vaultwarden + + + SERVICE_STOPPED=true + + + # ======================================================== + # Test PostgreSQL connection + # ======================================================== + + log "Test PostgreSQL connection" + + + DB_TEST="$( + psql \ + --dbname="${DATABASE_URL}" \ + -Atqc 'SELECT 1;' + )" + + + [[ "${DB_TEST}" == "1" ]] || \ + fail "Cannot connect to Vaultwarden PostgreSQL database" + + + echo "PostgreSQL connection successful." + + + # ======================================================== + # Collect non-secret PostgreSQL metadata + # ======================================================== + + DB_NAME="$( + psql \ + --dbname="${DATABASE_URL}" \ + -Atqc 'SELECT current_database();' + )" + + + DB_USER="$( + psql \ + --dbname="${DATABASE_URL}" \ + -Atqc 'SELECT current_user;' + )" + + + DB_HOST="$( + psql \ + --dbname="${DATABASE_URL}" \ + -Atqc \ + "SELECT COALESCE(inet_server_addr()::text, 'local');" + )" + + + DB_PORT="$( + psql \ + --dbname="${DATABASE_URL}" \ + -Atqc \ + 'SELECT inet_server_port();' + )" + + + SERVER_VERSION="$( + psql \ + --dbname="${DATABASE_URL}" \ + -Atqc 'SHOW server_version;' + )" + + + [[ -n "${SERVER_VERSION}" ]] || \ + fail "Could not determine PostgreSQL server version" + + + echo "${SERVER_VERSION}" \ + > "${DB_BACKUP_DIR}/postgresql-version.txt" + + + # ======================================================== + # Save metadata + # + # DATABASE_URL/password is intentionally NOT stored. + # ======================================================== + + cat > "${BACKUP_DIR}/metadata.txt" < "${DB_BACKUP_DIR}/vaultwarden.dump.list" + + + [[ -s "${DB_BACKUP_DIR}/vaultwarden.dump.list" ]] || \ + fail "pg_restore could not validate PostgreSQL dump" + + + # ======================================================== + # SHA256 + # ======================================================== + + ( + cd "${DB_BACKUP_DIR}" + + sha256sum \ + vaultwarden.dump \ + > vaultwarden.dump.sha256 + + sha256sum \ + -c vaultwarden.dump.sha256 + ) + + + DB_BACKUP_SIZE="$( + du -h "${DB_BACKUP_FILE}" | + awk '{print $1}' + )" + + + echo + echo "PostgreSQL backup successful." + + echo + echo "Backup directory:" + echo " ${BACKUP_DIR}" + + echo + echo "Database dump:" + echo " ${DB_BACKUP_FILE}" + + echo + echo "Dump size:" + echo " ${DB_BACKUP_SIZE}" + + echo + echo "SHA256:" + cat "${DB_BACKUP_DIR}/vaultwarden.dump.sha256" + + + # ======================================================== + # Retention + # + # Remove old backups only after the new backup has + # been successfully created and validated. + # ======================================================== + + cleanup_old_backups + + + # ======================================================== + # Deploy new Compose + # ======================================================== + + log "Deploy new docker-compose.yml" + + + cp \ + "${SRC}/docker-compose.yml" \ + "${LIVE}/docker-compose.yml" + + + cd "${LIVE}" + + + # ======================================================== + # From this moment migration may occur + # ======================================================== + + NEW_CONTAINER_STARTED=true + + + log "Start new Vaultwarden" + + + docker compose \ + up -d \ + vaultwarden + + + # ======================================================== + # Wait for application + DB readiness + # ======================================================== + + wait_for_vaultwarden + + + # ======================================================== + # Verify actual running version + # ======================================================== + + log "Validate running Vaultwarden version" + + + RUNNING_VERSION_OUTPUT="$( + docker exec \ + vaultwarden \ + /vaultwarden \ + --version + )" + + + RUNNING_VERSION="$( + printf '%s\n' "${RUNNING_VERSION_OUTPUT}" | + extract_version + )" + + + [[ -n "${RUNNING_VERSION}" ]] || \ + fail "Could not determine running Vaultwarden version" + + + echo "Expected version:" + echo " ${TARGET_VERSION}" + + echo "Running version:" + echo " ${RUNNING_VERSION}" + + + [[ "${RUNNING_VERSION}" == "${TARGET_VERSION}" ]] || \ + fail \ + "Running Vaultwarden version ${RUNNING_VERSION} does not match ${TARGET_VERSION}" + + +# ============================================================ +# NO VERSION CHANGE +# ============================================================ + +else + + log "No Vaultwarden version change" + + + echo "Current version:" + echo " ${CURRENT_VERSION}" + + + # + # Compose itself may still have changed: + # healthcheck, networks, volumes, etc. + # + + cp \ + "${SRC}/docker-compose.yml" \ + "${LIVE}/docker-compose.yml" + + + cd "${LIVE}" + + + echo + echo "--- Pull images ---" + + + docker compose pull + + + echo + echo "--- Apply Docker Compose ---" + + + docker compose \ + up -d \ + --remove-orphans + + + wait_for_vaultwarden + + + # ======================================================== + # Validate running version + # ======================================================== + + RUNNING_VERSION_OUTPUT="$( + docker exec \ + vaultwarden \ + /vaultwarden \ + --version + )" + + + RUNNING_VERSION="$( + printf '%s\n' "${RUNNING_VERSION_OUTPUT}" | + extract_version + )" + + + [[ "${RUNNING_VERSION}" == "${TARGET_VERSION}" ]] || \ + fail \ + "Running Vaultwarden version ${RUNNING_VERSION} does not match ${TARGET_VERSION}" + +fi + + +# ============================================================ +# Final container checks +# ============================================================ + +log "Final Vaultwarden checks" + + +FINAL_RUNNING="$( + docker inspect \ + -f '{{.State.Running}}' \ + vaultwarden +)" + + +FINAL_HEALTH="$( + docker inspect \ + -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ + vaultwarden +)" + + +[[ "${FINAL_RUNNING}" == "true" ]] || \ + fail "Vaultwarden container is not running" + + +[[ "${FINAL_HEALTH}" == "healthy" ]] || \ + fail "Vaultwarden container is not healthy" + + +# ============================================================ +# Final version +# ============================================================ + +FINAL_VERSION_OUTPUT="$( + docker exec \ + vaultwarden \ + /vaultwarden \ + --version +)" + + +FINAL_VERSION="$( + printf '%s\n' "${FINAL_VERSION_OUTPUT}" | + extract_version +)" + + +[[ -n "${FINAL_VERSION}" ]] || \ + fail "Could not determine final Vaultwarden version" + + +[[ "${FINAL_VERSION}" == "${TARGET_VERSION}" ]] || \ + fail \ + "Final Vaultwarden version mismatch: ${FINAL_VERSION} != ${TARGET_VERSION}" + + +# ============================================================ +# Running image +# ============================================================ RUNNING_IMAGE="$( docker inspect \ @@ -193,14 +1087,49 @@ RUNNING_IMAGE="$( vaultwarden )" -echo "Running image: ${RUNNING_IMAGE}" + +# ============================================================ +# Success +# ============================================================ + +log "Vaultwarden deployment successful" -trap - ERR - +echo "Previous Vaultwarden version:" +echo " ${CURRENT_VERSION}" echo -echo "=================================" -echo " Vaultwarden deployment successful" -echo "=================================" -echo "Backup: ${BACKUP}" +echo "Running Vaultwarden version:" +echo " ${FINAL_VERSION}" + +echo +echo "Running Docker image:" +echo " ${RUNNING_IMAGE}" + +echo +echo "Health:" +echo " ${FINAL_HEALTH}" + + +if [[ "${VERSION_CHANGED}" == "true" ]]; then + + echo + echo "Pre-upgrade backup:" + echo " ${BACKUP_DIR}" + + echo + echo "PostgreSQL dump:" + echo " ${DB_BACKUP_FILE}" + + echo + echo "Backup retention:" + echo " last ${BACKUP_KEEP} backups" + +fi + + +# ============================================================ +# Clear database connection string +# ============================================================ + +unset DATABASE_URL \ No newline at end of file