#!/usr/bin/env bash # shellcheck shell=bash set -Eeuo pipefail umask 077 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SRC_COMPOSE="${ROOT}/stacks/wishlist/docker-compose.yml" ROLLBACK_SCRIPT="${ROOT}/scripts/rollback-wishlist.sh" LIVE="/srv/nfs/media/wishlist" LIVE_COMPOSE="${LIVE}/docker-compose.yml" BACKUP_ROOT="/srv/nfs/media/.deploy-backups/wishlist" BACKUP_KEEP=3 DEPLOY_GROUP="infra-deploy" CONTAINER="wishlist" TARGET_IMAGE_REF="ghcr.io/cmintey/wishlist:latest" LOCK_FILE="/var/lib/gitea-runner/locks/wishlist.lock" START_TIMEOUT=300 CHECK_INTERVAL=5 STAMP="$(date '+%Y%m%d-%H%M%S')" BACKUP_DIR="${BACKUP_ROOT}/${STAMP}" IMAGE_CHANGED=false CONFIG_CHANGED=false SERVICE_STOPPED=false NEW_CONTAINER_STARTED=false BACKUP_COMPLETE=false log() { echo echo "============================================================" echo "$*" echo "============================================================" } fail() { echo >&2 echo "ERROR: $*" >&2 return 1 } short_id() { printf '%s\n' "$1" | sed 's/^sha256://' | cut -c1-16 } wait_for_wishlist() { local elapsed=0 local running echo echo "Waiting for Wishlist on http://127.0.0.1:3280/ ..." while (( elapsed < START_TIMEOUT )); do running="$(docker inspect -f '{{.State.Running}}' "${CONTAINER}" 2>/dev/null || echo false)" if [[ "${running}" != "true" ]]; then echo echo "--- Wishlist logs ---" docker logs --tail 200 "${CONTAINER}" || true fail "Wishlist container is not running" fi if curl --fail --silent --show-error --max-time 5 \ --output /dev/null http://127.0.0.1:3280/; then echo "Wishlist HTTP check: OK" return 0 fi sleep "${CHECK_INTERVAL}" elapsed=$((elapsed + CHECK_INTERVAL)) echo "Waiting... ${elapsed}s" done echo echo "--- Wishlist logs ---" docker logs --tail 200 "${CONTAINER}" || true fail "Timeout waiting for Wishlist HTTP service" } cleanup_old_backups() { local -a backups=() local name mkdir -p "${BACKUP_ROOT}" while IFS= read -r name; do [[ -f "${BACKUP_ROOT}/${name}/.complete" ]] && backups+=("${name}") done < <( find "${BACKUP_ROOT}" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' \ | grep -E '^[0-9]{8}-[0-9]{6}$' \ | sort -r ) echo echo "Complete backups: ${#backups[@]}" echo "Retention: ${BACKUP_KEEP}" if (( ${#backups[@]} <= BACKUP_KEEP )); then return 0 fi for name in "${backups[@]:${BACKUP_KEEP}}"; do echo "Removing old backup: ${BACKUP_ROOT}/${name}" rm -rf -- "${BACKUP_ROOT:?}/${name}" done } on_error() { local rc=$? local failed_command="${BASH_COMMAND}" local failed_line="${BASH_LINENO[0]:-unknown}" trap - ERR echo echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" echo " Wishlist deployment FAILED" echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" echo "Exit code: ${rc}" echo "Failed at line: ${failed_line}" echo "Failed command: ${failed_command}" if [[ "${NEW_CONTAINER_STARTED}" == "true" && "${BACKUP_COMPLETE}" == "true" ]]; then echo echo "New Wishlist container may already have modified SQLite." echo "Starting automatic rollback from:" echo " ${BACKUP_DIR}" if WISHLIST_LOCK_HELD=1 "${ROLLBACK_SCRIPT}" "${BACKUP_DIR}"; then echo echo "Automatic rollback completed successfully." else echo echo "CRITICAL: automatic rollback FAILED." >&2 echo "Run manually:" >&2 echo " ${ROLLBACK_SCRIPT} ${BACKUP_DIR}" >&2 fi elif [[ "${SERVICE_STOPPED}" == "true" ]]; then echo echo "New container was not started. Starting previous container." docker start "${CONTAINER}" >/dev/null 2>&1 || true else echo echo "Production was not modified." fi if [[ "${BACKUP_COMPLETE}" != "true" && -d "${BACKUP_DIR}" ]]; then echo "Removing incomplete backup: ${BACKUP_DIR}" rm -rf -- "${BACKUP_DIR}" || true fi exit "${rc}" } trap on_error ERR log "Pre-flight checks" for cmd in docker tar gzip sha256sum curl diff mktemp sort find grep sed cut flock; do command -v "${cmd}" >/dev/null 2>&1 || fail "Required command not found: ${cmd}" done docker compose version >/dev/null 2>&1 || fail "Docker Compose plugin is not available" getent group "${DEPLOY_GROUP}" >/dev/null 2>&1 || fail "Group ${DEPLOY_GROUP} does not exist" [[ -x "${ROLLBACK_SCRIPT}" ]] || fail "Rollback script is not executable: ${ROLLBACK_SCRIPT}" [[ -f "${SRC_COMPOSE}" ]] || fail "Source compose not found: ${SRC_COMPOSE}" [[ -f "${LIVE_COMPOSE}" ]] || fail "Live compose not found: ${LIVE_COMPOSE}" [[ -d "${LIVE}/data" ]] || fail "${LIVE}/data not found" [[ -d "${LIVE}/uploads" ]] || fail "${LIVE}/uploads not found" mkdir -p "$(dirname "${LOCK_FILE}")" "${BACKUP_ROOT}" exec 9>"${LOCK_FILE}" flock -n 9 || fail "Another Wishlist deploy/rollback is already running" [[ -w "${LIVE}" ]] || fail "${LIVE} is not writable by $(id -un)" [[ -w "${BACKUP_ROOT}" ]] || fail "${BACKUP_ROOT} is not writable by $(id -un)" log "Validate requested Docker Compose" set +e COMPOSE_OUTPUT="$(docker compose -p wishlist -f "${SRC_COMPOSE}" config 2>&1)" COMPOSE_RC=$? set -e if [[ ${COMPOSE_RC} -ne 0 ]]; then echo "${COMPOSE_OUTPUT}" fail "Requested Docker Compose configuration is invalid" fi echo "Docker Compose configuration is valid." mapfile -t COMPOSE_IMAGES < <(docker compose -p wishlist -f "${SRC_COMPOSE}" config --images | sort -u) [[ ${#COMPOSE_IMAGES[@]} -eq 1 ]] || fail "Expected exactly one image in Wishlist compose" [[ "${COMPOSE_IMAGES[0]}" == "${TARGET_IMAGE_REF}" ]] || \ fail "Wishlist compose must use ${TARGET_IMAGE_REF}; got ${COMPOSE_IMAGES[0]}" log "Detect current Wishlist state" docker inspect "${CONTAINER}" >/dev/null 2>&1 || fail "Current Wishlist container does not exist" CURRENT_RUNNING="$(docker inspect -f '{{.State.Running}}' "${CONTAINER}")" [[ "${CURRENT_RUNNING}" == "true" ]] || fail "Current Wishlist container is not running" CURRENT_IMAGE_ID="$(docker inspect -f '{{.Image}}' "${CONTAINER}")" CURRENT_IMAGE_REF="$(docker inspect -f '{{.Config.Image}}' "${CONTAINER}")" echo "Current image reference: ${CURRENT_IMAGE_REF}" echo "Current image ID: ${CURRENT_IMAGE_ID}" log "Compare semantic Compose configuration" TMP_CURRENT="$(mktemp)" TMP_REQUESTED="$(mktemp)" docker compose -p wishlist -f "${LIVE_COMPOSE}" config > "${TMP_CURRENT}" docker compose -p wishlist -f "${SRC_COMPOSE}" config > "${TMP_REQUESTED}" if ! diff -q "${TMP_CURRENT}" "${TMP_REQUESTED}" >/dev/null; then CONFIG_CHANGED=true echo "Compose configuration changed: true" echo diff -u "${TMP_CURRENT}" "${TMP_REQUESTED}" || true else echo "Compose configuration changed: false" fi rm -f "${TMP_CURRENT}" "${TMP_REQUESTED}" log "Pull and compare latest image" docker pull "${TARGET_IMAGE_REF}" TARGET_IMAGE_ID="$(docker image inspect "${TARGET_IMAGE_REF}" -f '{{.Id}}')" [[ -n "${TARGET_IMAGE_ID}" ]] || fail "Could not determine target image ID" if [[ "${CURRENT_IMAGE_ID}" != "${TARGET_IMAGE_ID}" ]]; then IMAGE_CHANGED=true fi echo "Current image ID: $(short_id "${CURRENT_IMAGE_ID}")" echo "Target image ID: $(short_id "${TARGET_IMAGE_ID}")" echo "Image changed: ${IMAGE_CHANGED}" echo "Compose changed: ${CONFIG_CHANGED}" if [[ "${IMAGE_CHANGED}" != "true" && "${CONFIG_CHANGED}" != "true" ]]; then log "Wishlist is already current" echo "The comment triggered an update check, but the image digest and semantic Compose configuration are unchanged." exit 0 fi log "Prepare pre-deployment backup" mkdir -p "${BACKUP_DIR}" cp "${LIVE_COMPOSE}" "${BACKUP_DIR}/docker-compose.yml" { printf 'BACKUP_TIMESTAMP=%q\n' "${STAMP}" printf 'OLD_IMAGE_ID=%q\n' "${CURRENT_IMAGE_ID}" printf 'OLD_IMAGE_REF=%q\n' "${CURRENT_IMAGE_REF}" printf 'TARGET_IMAGE_ID=%q\n' "${TARGET_IMAGE_ID}" printf 'TARGET_IMAGE_REF=%q\n' "${TARGET_IMAGE_REF}" printf 'IMAGE_CHANGED=%q\n' "${IMAGE_CHANGED}" printf 'CONFIG_CHANGED=%q\n' "${CONFIG_CHANGED}" } > "${BACKUP_DIR}/metadata.env" log "Stop Wishlist for consistent SQLite backup" docker stop "${CONTAINER}" >/dev/null SERVICE_STOPPED=true log "Backup persistent state" tar -C "${LIVE}" -czf "${BACKUP_DIR}/state.tar.gz" data uploads [[ -s "${BACKUP_DIR}/state.tar.gz" ]] || fail "State backup is empty" tar -tzf "${BACKUP_DIR}/state.tar.gz" >/dev/null log "Backup exact previous Docker image" docker image save "${CURRENT_IMAGE_ID}" | gzip -1 > "${BACKUP_DIR}/old-image.tar.gz" [[ -s "${BACKUP_DIR}/old-image.tar.gz" ]] || fail "Docker image backup is empty" gzip -t "${BACKUP_DIR}/old-image.tar.gz" log "Generate and validate checksums" ( cd "${BACKUP_DIR}" sha256sum state.tar.gz old-image.tar.gz docker-compose.yml metadata.env > SHA256SUMS sha256sum -c SHA256SUMS ) touch "${BACKUP_DIR}/.complete" BACKUP_COMPLETE=true cleanup_old_backups log "Deploy requested Compose" cp "${SRC_COMPOSE}" "${LIVE_COMPOSE}" cd "${LIVE}" NEW_CONTAINER_STARTED=true docker compose -p wishlist -f "${LIVE_COMPOSE}" up -d --remove-orphans wait_for_wishlist log "Validate running image" RUNNING_IMAGE_ID="$(docker inspect -f '{{.Image}}' "${CONTAINER}")" echo "Expected: ${TARGET_IMAGE_ID}" echo "Running: ${RUNNING_IMAGE_ID}" [[ "${RUNNING_IMAGE_ID}" == "${TARGET_IMAGE_ID}" ]] || fail "Wishlist is not running the expected image" if [[ "${IMAGE_CHANGED}" == "true" ]]; then log "Remove previous local image" if docker image rm "${CURRENT_IMAGE_ID}"; then echo "Previous local image removed. Exact copy remains in ${BACKUP_DIR}/old-image.tar.gz" else echo "WARNING: previous image could not be removed; it may still be used by another container." fi fi log "Wishlist deployment successful" echo "Backup: ${BACKUP_DIR}" echo "Previous image: $(short_id "${CURRENT_IMAGE_ID}")" echo "Running image: $(short_id "${RUNNING_IMAGE_ID}")" echo "Image changed: ${IMAGE_CHANGED}" echo "Compose changed: ${CONFIG_CHANGED}"