#!/usr/bin/env bash # shellcheck shell=bash set -Eeuo pipefail umask 077 LIVE="/srv/nfs/media/wishlist" LIVE_COMPOSE="${LIVE}/docker-compose.yml" BACKUP_ROOT="/srv/nfs/media/.deploy-backups/wishlist" DEPLOY_GROUP="infra-deploy" CONTAINER="wishlist" LOCK_FILE="/var/lib/gitea-runner/locks/wishlist.lock" START_TIMEOUT=300 CHECK_INTERVAL=5 log() { echo echo "============================================================" echo "$*" echo "============================================================" } fail() { echo >&2 echo "ERROR: $*" >&2 exit 1 } wait_for_wishlist() { local elapsed=0 local running while (( elapsed < START_TIMEOUT )); do running="$(docker inspect -f '{{.State.Running}}' "${CONTAINER}" 2>/dev/null || echo false)" if [[ "${running}" != "true" ]]; then docker logs --tail 200 "${CONTAINER}" 2>/dev/null || true fail "Wishlist container is not running after rollback" fi if curl --fail --silent --show-error --max-time 5 \ --output /dev/null http://127.0.0.1:3280/; then echo "Wishlist HTTP check: OK" return 0 fi sleep "${CHECK_INTERVAL}" elapsed=$((elapsed + CHECK_INTERVAL)) echo "Waiting after rollback... ${elapsed}s" done docker logs --tail 200 "${CONTAINER}" 2>/dev/null || true fail "Timeout waiting for Wishlist after rollback" } resolve_backup() { local requested="${1:-latest}" local latest if [[ "${requested}" != "latest" ]]; then [[ -d "${requested}" ]] && { printf '%s\n' "${requested}"; return 0; } [[ -d "${BACKUP_ROOT}/${requested}" ]] && { printf '%s\n' "${BACKUP_ROOT}/${requested}"; return 0; } return 1 fi latest="$( find "${BACKUP_ROOT}" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' \ | grep -E '^[0-9]{8}-[0-9]{6}$' \ | sort -r \ | while IFS= read -r name; do if [[ -f "${BACKUP_ROOT}/${name}/.complete" ]]; then echo "${BACKUP_ROOT}/${name}" break fi done )" [[ -n "${latest}" ]] || return 1 printf '%s\n' "${latest}" } for cmd in docker tar gzip sha256sum curl find grep sort flock chgrp chmod; do command -v "${cmd}" >/dev/null 2>&1 || fail "Required command not found: ${cmd}" done docker compose version >/dev/null 2>&1 || fail "Docker Compose plugin is not available" getent group "${DEPLOY_GROUP}" >/dev/null 2>&1 || fail "Group ${DEPLOY_GROUP} does not exist" mkdir -p "$(dirname "${LOCK_FILE}")" if [[ "${WISHLIST_LOCK_HELD:-0}" != "1" ]]; then exec 9>"${LOCK_FILE}" flock -n 9 || fail "Another Wishlist deploy/rollback is already running" fi BACKUP_DIR="$(resolve_backup "${1:-latest}")" || fail "Could not find requested complete backup" log "Rollback Wishlist from ${BACKUP_DIR}" for file in .complete docker-compose.yml metadata.env state.tar.gz old-image.tar.gz SHA256SUMS; do [[ -e "${BACKUP_DIR}/${file}" ]] || fail "Backup file missing: ${BACKUP_DIR}/${file}" done ( cd "${BACKUP_DIR}" sha256sum -c SHA256SUMS ) tar -tzf "${BACKUP_DIR}/state.tar.gz" >/dev/null gzip -t "${BACKUP_DIR}/old-image.tar.gz" # shellcheck disable=SC1090 source "${BACKUP_DIR}/metadata.env" : "${OLD_IMAGE_ID:?OLD_IMAGE_ID is missing from metadata}" : "${OLD_IMAGE_REF:?OLD_IMAGE_REF is missing from metadata}" FAILED_IMAGE_ID="" if docker inspect "${CONTAINER}" >/dev/null 2>&1; then FAILED_IMAGE_ID="$(docker inspect -f '{{.Image}}' "${CONTAINER}" 2>/dev/null || true)" docker logs "${CONTAINER}" > "${BACKUP_DIR}/failed-container.log" 2>&1 || true docker rm -f "${CONTAINER}" >/dev/null || true fi log "Restore previous Compose" cp "${BACKUP_DIR}/docker-compose.yml" "${LIVE_COMPOSE}" log "Restore persistent state" mkdir -p "${LIVE}/data" "${LIVE}/uploads" find "${LIVE}/data" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + find "${LIVE}/uploads" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + tar --no-same-owner -xzf "${BACKUP_DIR}/state.tar.gz" -C "${LIVE}" # Files extracted by the runner become runner-owned. Normalize the group and # group permissions so the container can access them through the shared GID. find "${LIVE}/data" "${LIVE}/uploads" -mindepth 1 -exec chgrp "${DEPLOY_GROUP}" {} + find "${LIVE}/data" "${LIVE}/uploads" -type d -exec chmod g+rwx,g+s {} + find "${LIVE}/data" "${LIVE}/uploads" -type f -exec chmod g+rw {} + log "Restore exact previous Docker image" gzip -dc "${BACKUP_DIR}/old-image.tar.gz" | docker image load docker image inspect "${OLD_IMAGE_ID}" >/dev/null 2>&1 || \ fail "Previous image ${OLD_IMAGE_ID} was not restored" docker image tag "${OLD_IMAGE_ID}" "${OLD_IMAGE_REF}" log "Start previous Wishlist version" cd "${LIVE}" docker compose -p wishlist -f "${LIVE_COMPOSE}" up -d --pull never --remove-orphans wait_for_wishlist RUNNING_IMAGE_ID="$(docker inspect -f '{{.Image}}' "${CONTAINER}")" [[ "${RUNNING_IMAGE_ID}" == "${OLD_IMAGE_ID}" ]] || \ fail "Rollback started unexpected image: ${RUNNING_IMAGE_ID}" if [[ -n "${FAILED_IMAGE_ID}" && "${FAILED_IMAGE_ID}" != "${OLD_IMAGE_ID}" ]]; then docker image rm "${FAILED_IMAGE_ID}" >/dev/null 2>&1 || true fi touch "${BACKUP_DIR}/.rolled-back" log "Wishlist rollback successful" echo "Backup: ${BACKUP_DIR}" echo "Restored ID: ${OLD_IMAGE_ID}" echo "Restored ref: ${OLD_IMAGE_REF}"