From 6d13cee7411d0b968dbc2da46481ef669fc0ef25 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 5 Oct 2026 17:44:23 +0300 Subject: [PATCH] Add Wishlist CI/CD on media host --- .gitea/workflows/wishlist-rollback.yml | 24 ++ .gitea/workflows/wishlist.yml | 32 +++ scripts/deploy-wishlist.sh | 314 +++++++++++++++++++++++++ scripts/rollback-wishlist.sh | 166 +++++++++++++ stacks/wishlist/docker-compose.yml | 16 ++ 5 files changed, 552 insertions(+) create mode 100644 .gitea/workflows/wishlist-rollback.yml create mode 100644 .gitea/workflows/wishlist.yml create mode 100755 scripts/deploy-wishlist.sh create mode 100755 scripts/rollback-wishlist.sh create mode 100644 stacks/wishlist/docker-compose.yml diff --git a/.gitea/workflows/wishlist-rollback.yml b/.gitea/workflows/wishlist-rollback.yml new file mode 100644 index 0000000..3ec0448 --- /dev/null +++ b/.gitea/workflows/wishlist-rollback.yml @@ -0,0 +1,24 @@ +name: Wishlist Rollback + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + rollback: + name: Rollback Wishlist + runs-on: docker-media-01 + + steps: + - name: Checkout repository + uses: builtin:checkout + with: + fetch-depth: 1 + + - name: Validate rollback script + run: bash -n scripts/rollback-wishlist.sh + + - name: Rollback to latest backup + run: ./scripts/rollback-wishlist.sh latest diff --git a/.gitea/workflows/wishlist.yml b/.gitea/workflows/wishlist.yml new file mode 100644 index 0000000..8391265 --- /dev/null +++ b/.gitea/workflows/wishlist.yml @@ -0,0 +1,32 @@ +name: Wishlist CI/CD + +on: + push: + branches: + - main + paths: + - "stacks/wishlist/**" + - "scripts/deploy-wishlist.sh" + - ".gitea/workflows/wishlist.yml" + + workflow_dispatch: + +permissions: + contents: read + +jobs: + deploy: + name: Deploy Wishlist + runs-on: docker-media-01 + + steps: + - name: Checkout repository + uses: builtin:checkout + with: + fetch-depth: 1 + + - name: Validate deployment script + run: bash -n scripts/deploy-wishlist.sh + + - name: Deploy Wishlist + run: ./scripts/deploy-wishlist.sh diff --git a/scripts/deploy-wishlist.sh b/scripts/deploy-wishlist.sh new file mode 100755 index 0000000..9019d03 --- /dev/null +++ b/scripts/deploy-wishlist.sh @@ -0,0 +1,314 @@ +#!/usr/bin/env bash +# shellcheck shell=bash + +set -Eeuo pipefail +umask 077 + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SRC_COMPOSE="${ROOT}/stacks/wishlist/docker-compose.yml" +ROLLBACK_SCRIPT="${ROOT}/scripts/rollback-wishlist.sh" + +LIVE="/srv/nfs/media/wishlist" +LIVE_COMPOSE="${LIVE}/docker-compose.yml" +BACKUP_ROOT="/srv/nfs/media/.deploy-backups/wishlist" +BACKUP_KEEP=3 +DEPLOY_GROUP="infra-deploy" +CONTAINER="wishlist" +TARGET_IMAGE_REF="ghcr.io/cmintey/wishlist:latest" +LOCK_FILE="/var/lib/gitea-runner/locks/wishlist.lock" +START_TIMEOUT=300 +CHECK_INTERVAL=5 +STAMP="$(date '+%Y%m%d-%H%M%S')" +BACKUP_DIR="${BACKUP_ROOT}/${STAMP}" + +IMAGE_CHANGED=false +CONFIG_CHANGED=false +SERVICE_STOPPED=false +NEW_CONTAINER_STARTED=false +BACKUP_COMPLETE=false + +log() { + echo + echo "============================================================" + echo "$*" + echo "============================================================" +} + +fail() { + echo >&2 + echo "ERROR: $*" >&2 + return 1 +} + +short_id() { + printf '%s\n' "$1" | sed 's/^sha256://' | cut -c1-16 +} + +wait_for_wishlist() { + local elapsed=0 + local running + + echo + echo "Waiting for Wishlist on http://127.0.0.1:3280/ ..." + + while (( elapsed < START_TIMEOUT )); do + running="$(docker inspect -f '{{.State.Running}}' "${CONTAINER}" 2>/dev/null || echo false)" + + if [[ "${running}" != "true" ]]; then + echo + echo "--- Wishlist logs ---" + docker logs --tail 200 "${CONTAINER}" || true + fail "Wishlist container is not running" + fi + + if curl --fail --silent --show-error --max-time 5 \ + --output /dev/null http://127.0.0.1:3280/; then + echo "Wishlist HTTP check: OK" + return 0 + fi + + sleep "${CHECK_INTERVAL}" + elapsed=$((elapsed + CHECK_INTERVAL)) + echo "Waiting... ${elapsed}s" + done + + echo + echo "--- Wishlist logs ---" + docker logs --tail 200 "${CONTAINER}" || true + fail "Timeout waiting for Wishlist HTTP service" +} + +cleanup_old_backups() { + local -a backups=() + local name + + mkdir -p "${BACKUP_ROOT}" + + while IFS= read -r name; do + [[ -f "${BACKUP_ROOT}/${name}/.complete" ]] && backups+=("${name}") + done < <( + find "${BACKUP_ROOT}" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' \ + | grep -E '^[0-9]{8}-[0-9]{6}$' \ + | sort -r + ) + + echo + echo "Complete backups: ${#backups[@]}" + echo "Retention: ${BACKUP_KEEP}" + + if (( ${#backups[@]} <= BACKUP_KEEP )); then + return 0 + fi + + for name in "${backups[@]:${BACKUP_KEEP}}"; do + echo "Removing old backup: ${BACKUP_ROOT}/${name}" + rm -rf -- "${BACKUP_ROOT:?}/${name}" + done +} + +on_error() { + local rc=$? + local failed_command="${BASH_COMMAND}" + local failed_line="${BASH_LINENO[0]:-unknown}" + + trap - ERR + + echo + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo " Wishlist deployment FAILED" + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "Exit code: ${rc}" + echo "Failed at line: ${failed_line}" + echo "Failed command: ${failed_command}" + + if [[ "${NEW_CONTAINER_STARTED}" == "true" && "${BACKUP_COMPLETE}" == "true" ]]; then + echo + echo "New Wishlist container may already have modified SQLite." + echo "Starting automatic rollback from:" + echo " ${BACKUP_DIR}" + + if WISHLIST_LOCK_HELD=1 "${ROLLBACK_SCRIPT}" "${BACKUP_DIR}"; then + echo + echo "Automatic rollback completed successfully." + else + echo + echo "CRITICAL: automatic rollback FAILED." >&2 + echo "Run manually:" >&2 + echo " ${ROLLBACK_SCRIPT} ${BACKUP_DIR}" >&2 + fi + + elif [[ "${SERVICE_STOPPED}" == "true" ]]; then + echo + echo "New container was not started. Starting previous container." + docker start "${CONTAINER}" >/dev/null 2>&1 || true + else + echo + echo "Production was not modified." + fi + + if [[ "${BACKUP_COMPLETE}" != "true" && -d "${BACKUP_DIR}" ]]; then + echo "Removing incomplete backup: ${BACKUP_DIR}" + rm -rf -- "${BACKUP_DIR}" || true + fi + + exit "${rc}" +} + +trap on_error ERR + +log "Pre-flight checks" + +for cmd in docker tar gzip sha256sum curl diff mktemp sort find grep sed cut flock; do + command -v "${cmd}" >/dev/null 2>&1 || fail "Required command not found: ${cmd}" +done + +docker compose version >/dev/null 2>&1 || fail "Docker Compose plugin is not available" +getent group "${DEPLOY_GROUP}" >/dev/null 2>&1 || fail "Group ${DEPLOY_GROUP} does not exist" + +[[ -x "${ROLLBACK_SCRIPT}" ]] || fail "Rollback script is not executable: ${ROLLBACK_SCRIPT}" +[[ -f "${SRC_COMPOSE}" ]] || fail "Source compose not found: ${SRC_COMPOSE}" +[[ -f "${LIVE_COMPOSE}" ]] || fail "Live compose not found: ${LIVE_COMPOSE}" +[[ -d "${LIVE}/data" ]] || fail "${LIVE}/data not found" +[[ -d "${LIVE}/uploads" ]] || fail "${LIVE}/uploads not found" + +mkdir -p "$(dirname "${LOCK_FILE}")" "${BACKUP_ROOT}" +exec 9>"${LOCK_FILE}" +flock -n 9 || fail "Another Wishlist deploy/rollback is already running" + +[[ -w "${LIVE}" ]] || fail "${LIVE} is not writable by $(id -un)" +[[ -w "${BACKUP_ROOT}" ]] || fail "${BACKUP_ROOT} is not writable by $(id -un)" + +log "Validate requested Docker Compose" +set +e +COMPOSE_OUTPUT="$(docker compose -p wishlist -f "${SRC_COMPOSE}" config 2>&1)" +COMPOSE_RC=$? +set -e +if [[ ${COMPOSE_RC} -ne 0 ]]; then + echo "${COMPOSE_OUTPUT}" + fail "Requested Docker Compose configuration is invalid" +fi + +echo "Docker Compose configuration is valid." + +mapfile -t COMPOSE_IMAGES < <(docker compose -p wishlist -f "${SRC_COMPOSE}" config --images | sort -u) +[[ ${#COMPOSE_IMAGES[@]} -eq 1 ]] || fail "Expected exactly one image in Wishlist compose" +[[ "${COMPOSE_IMAGES[0]}" == "${TARGET_IMAGE_REF}" ]] || \ + fail "Wishlist compose must use ${TARGET_IMAGE_REF}; got ${COMPOSE_IMAGES[0]}" + +log "Detect current Wishlist state" +docker inspect "${CONTAINER}" >/dev/null 2>&1 || fail "Current Wishlist container does not exist" +CURRENT_RUNNING="$(docker inspect -f '{{.State.Running}}' "${CONTAINER}")" +[[ "${CURRENT_RUNNING}" == "true" ]] || fail "Current Wishlist container is not running" + +CURRENT_IMAGE_ID="$(docker inspect -f '{{.Image}}' "${CONTAINER}")" +CURRENT_IMAGE_REF="$(docker inspect -f '{{.Config.Image}}' "${CONTAINER}")" + +echo "Current image reference: ${CURRENT_IMAGE_REF}" +echo "Current image ID: ${CURRENT_IMAGE_ID}" + +log "Compare semantic Compose configuration" +TMP_CURRENT="$(mktemp)" +TMP_REQUESTED="$(mktemp)" + +docker compose -p wishlist -f "${LIVE_COMPOSE}" config > "${TMP_CURRENT}" +docker compose -p wishlist -f "${SRC_COMPOSE}" config > "${TMP_REQUESTED}" + +if ! diff -q "${TMP_CURRENT}" "${TMP_REQUESTED}" >/dev/null; then + CONFIG_CHANGED=true + echo "Compose configuration changed: true" + echo + diff -u "${TMP_CURRENT}" "${TMP_REQUESTED}" || true +else + echo "Compose configuration changed: false" +fi + +rm -f "${TMP_CURRENT}" "${TMP_REQUESTED}" + +log "Pull and compare latest image" +docker pull "${TARGET_IMAGE_REF}" +TARGET_IMAGE_ID="$(docker image inspect "${TARGET_IMAGE_REF}" -f '{{.Id}}')" +[[ -n "${TARGET_IMAGE_ID}" ]] || fail "Could not determine target image ID" + +if [[ "${CURRENT_IMAGE_ID}" != "${TARGET_IMAGE_ID}" ]]; then + IMAGE_CHANGED=true +fi + +echo "Current image ID: $(short_id "${CURRENT_IMAGE_ID}")" +echo "Target image ID: $(short_id "${TARGET_IMAGE_ID}")" +echo "Image changed: ${IMAGE_CHANGED}" +echo "Compose changed: ${CONFIG_CHANGED}" + +if [[ "${IMAGE_CHANGED}" != "true" && "${CONFIG_CHANGED}" != "true" ]]; then + log "Wishlist is already current" + echo "The comment triggered an update check, but the image digest and semantic Compose configuration are unchanged." + exit 0 +fi + +log "Prepare pre-deployment backup" +mkdir -p "${BACKUP_DIR}" +cp "${LIVE_COMPOSE}" "${BACKUP_DIR}/docker-compose.yml" + +{ + printf 'BACKUP_TIMESTAMP=%q\n' "${STAMP}" + printf 'OLD_IMAGE_ID=%q\n' "${CURRENT_IMAGE_ID}" + printf 'OLD_IMAGE_REF=%q\n' "${CURRENT_IMAGE_REF}" + printf 'TARGET_IMAGE_ID=%q\n' "${TARGET_IMAGE_ID}" + printf 'TARGET_IMAGE_REF=%q\n' "${TARGET_IMAGE_REF}" + printf 'IMAGE_CHANGED=%q\n' "${IMAGE_CHANGED}" + printf 'CONFIG_CHANGED=%q\n' "${CONFIG_CHANGED}" +} > "${BACKUP_DIR}/metadata.env" + +log "Stop Wishlist for consistent SQLite backup" +docker stop "${CONTAINER}" >/dev/null +SERVICE_STOPPED=true + +log "Backup persistent state" +tar -C "${LIVE}" -czf "${BACKUP_DIR}/state.tar.gz" data uploads +[[ -s "${BACKUP_DIR}/state.tar.gz" ]] || fail "State backup is empty" +tar -tzf "${BACKUP_DIR}/state.tar.gz" >/dev/null + +log "Backup exact previous Docker image" +docker image save "${CURRENT_IMAGE_ID}" | gzip -1 > "${BACKUP_DIR}/old-image.tar.gz" +[[ -s "${BACKUP_DIR}/old-image.tar.gz" ]] || fail "Docker image backup is empty" +gzip -t "${BACKUP_DIR}/old-image.tar.gz" + +log "Generate and validate checksums" +( + cd "${BACKUP_DIR}" + sha256sum state.tar.gz old-image.tar.gz docker-compose.yml metadata.env > SHA256SUMS + sha256sum -c SHA256SUMS +) + +touch "${BACKUP_DIR}/.complete" +BACKUP_COMPLETE=true +cleanup_old_backups + +log "Deploy requested Compose" +cp "${SRC_COMPOSE}" "${LIVE_COMPOSE}" +cd "${LIVE}" +NEW_CONTAINER_STARTED=true + +docker compose -p wishlist -f "${LIVE_COMPOSE}" up -d --remove-orphans +wait_for_wishlist + +log "Validate running image" +RUNNING_IMAGE_ID="$(docker inspect -f '{{.Image}}' "${CONTAINER}")" +echo "Expected: ${TARGET_IMAGE_ID}" +echo "Running: ${RUNNING_IMAGE_ID}" +[[ "${RUNNING_IMAGE_ID}" == "${TARGET_IMAGE_ID}" ]] || fail "Wishlist is not running the expected image" + +if [[ "${IMAGE_CHANGED}" == "true" ]]; then + log "Remove previous local image" + if docker image rm "${CURRENT_IMAGE_ID}"; then + echo "Previous local image removed. Exact copy remains in ${BACKUP_DIR}/old-image.tar.gz" + else + echo "WARNING: previous image could not be removed; it may still be used by another container." + fi +fi + +log "Wishlist deployment successful" +echo "Backup: ${BACKUP_DIR}" +echo "Previous image: $(short_id "${CURRENT_IMAGE_ID}")" +echo "Running image: $(short_id "${RUNNING_IMAGE_ID}")" +echo "Image changed: ${IMAGE_CHANGED}" +echo "Compose changed: ${CONFIG_CHANGED}" diff --git a/scripts/rollback-wishlist.sh b/scripts/rollback-wishlist.sh new file mode 100755 index 0000000..65cf230 --- /dev/null +++ b/scripts/rollback-wishlist.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# shellcheck shell=bash + +set -Eeuo pipefail +umask 077 + +LIVE="/srv/nfs/media/wishlist" +LIVE_COMPOSE="${LIVE}/docker-compose.yml" +BACKUP_ROOT="/srv/nfs/media/.deploy-backups/wishlist" +DEPLOY_GROUP="infra-deploy" +CONTAINER="wishlist" +LOCK_FILE="/var/lib/gitea-runner/locks/wishlist.lock" +START_TIMEOUT=300 +CHECK_INTERVAL=5 + +log() { + echo + echo "============================================================" + echo "$*" + echo "============================================================" +} + +fail() { + echo >&2 + echo "ERROR: $*" >&2 + exit 1 +} + +wait_for_wishlist() { + local elapsed=0 + local running + + while (( elapsed < START_TIMEOUT )); do + running="$(docker inspect -f '{{.State.Running}}' "${CONTAINER}" 2>/dev/null || echo false)" + + if [[ "${running}" != "true" ]]; then + docker logs --tail 200 "${CONTAINER}" 2>/dev/null || true + fail "Wishlist container is not running after rollback" + fi + + if curl --fail --silent --show-error --max-time 5 \ + --output /dev/null http://127.0.0.1:3280/; then + echo "Wishlist HTTP check: OK" + return 0 + fi + + sleep "${CHECK_INTERVAL}" + elapsed=$((elapsed + CHECK_INTERVAL)) + echo "Waiting after rollback... ${elapsed}s" + done + + docker logs --tail 200 "${CONTAINER}" 2>/dev/null || true + fail "Timeout waiting for Wishlist after rollback" +} + +resolve_backup() { + local requested="${1:-latest}" + local latest + + if [[ "${requested}" != "latest" ]]; then + [[ -d "${requested}" ]] && { printf '%s\n' "${requested}"; return 0; } + [[ -d "${BACKUP_ROOT}/${requested}" ]] && { printf '%s\n' "${BACKUP_ROOT}/${requested}"; return 0; } + return 1 + fi + + latest="$( + find "${BACKUP_ROOT}" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' \ + | grep -E '^[0-9]{8}-[0-9]{6}$' \ + | sort -r \ + | while IFS= read -r name; do + if [[ -f "${BACKUP_ROOT}/${name}/.complete" ]]; then + echo "${BACKUP_ROOT}/${name}" + break + fi + done + )" + + [[ -n "${latest}" ]] || return 1 + printf '%s\n' "${latest}" +} + +for cmd in docker tar gzip sha256sum curl find grep sort flock chgrp chmod; do + command -v "${cmd}" >/dev/null 2>&1 || fail "Required command not found: ${cmd}" +done + +docker compose version >/dev/null 2>&1 || fail "Docker Compose plugin is not available" +getent group "${DEPLOY_GROUP}" >/dev/null 2>&1 || fail "Group ${DEPLOY_GROUP} does not exist" + +mkdir -p "$(dirname "${LOCK_FILE}")" +if [[ "${WISHLIST_LOCK_HELD:-0}" != "1" ]]; then + exec 9>"${LOCK_FILE}" + flock -n 9 || fail "Another Wishlist deploy/rollback is already running" +fi + +BACKUP_DIR="$(resolve_backup "${1:-latest}")" || fail "Could not find requested complete backup" + +log "Rollback Wishlist from ${BACKUP_DIR}" + +for file in .complete docker-compose.yml metadata.env state.tar.gz old-image.tar.gz SHA256SUMS; do + [[ -e "${BACKUP_DIR}/${file}" ]] || fail "Backup file missing: ${BACKUP_DIR}/${file}" +done + +( + cd "${BACKUP_DIR}" + sha256sum -c SHA256SUMS +) + +tar -tzf "${BACKUP_DIR}/state.tar.gz" >/dev/null +gzip -t "${BACKUP_DIR}/old-image.tar.gz" + +# shellcheck disable=SC1090 +source "${BACKUP_DIR}/metadata.env" + +: "${OLD_IMAGE_ID:?OLD_IMAGE_ID is missing from metadata}" +: "${OLD_IMAGE_REF:?OLD_IMAGE_REF is missing from metadata}" + +FAILED_IMAGE_ID="" +if docker inspect "${CONTAINER}" >/dev/null 2>&1; then + FAILED_IMAGE_ID="$(docker inspect -f '{{.Image}}' "${CONTAINER}" 2>/dev/null || true)" + docker logs "${CONTAINER}" > "${BACKUP_DIR}/failed-container.log" 2>&1 || true + docker rm -f "${CONTAINER}" >/dev/null || true +fi + +log "Restore previous Compose" +cp "${BACKUP_DIR}/docker-compose.yml" "${LIVE_COMPOSE}" + +log "Restore persistent state" +mkdir -p "${LIVE}/data" "${LIVE}/uploads" +find "${LIVE}/data" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + +find "${LIVE}/uploads" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + + +tar --no-same-owner -xzf "${BACKUP_DIR}/state.tar.gz" -C "${LIVE}" + +# Files extracted by the runner become runner-owned. Normalize the group and +# group permissions so the container can access them through the shared GID. +find "${LIVE}/data" "${LIVE}/uploads" -mindepth 1 -exec chgrp "${DEPLOY_GROUP}" {} + +find "${LIVE}/data" "${LIVE}/uploads" -type d -exec chmod g+rwx,g+s {} + +find "${LIVE}/data" "${LIVE}/uploads" -type f -exec chmod g+rw {} + + +log "Restore exact previous Docker image" +gzip -dc "${BACKUP_DIR}/old-image.tar.gz" | docker image load + +docker image inspect "${OLD_IMAGE_ID}" >/dev/null 2>&1 || \ + fail "Previous image ${OLD_IMAGE_ID} was not restored" + +docker image tag "${OLD_IMAGE_ID}" "${OLD_IMAGE_REF}" + +log "Start previous Wishlist version" +cd "${LIVE}" +docker compose -p wishlist -f "${LIVE_COMPOSE}" up -d --pull never --remove-orphans +wait_for_wishlist + +RUNNING_IMAGE_ID="$(docker inspect -f '{{.Image}}' "${CONTAINER}")" +[[ "${RUNNING_IMAGE_ID}" == "${OLD_IMAGE_ID}" ]] || \ + fail "Rollback started unexpected image: ${RUNNING_IMAGE_ID}" + +if [[ -n "${FAILED_IMAGE_ID}" && "${FAILED_IMAGE_ID}" != "${OLD_IMAGE_ID}" ]]; then + docker image rm "${FAILED_IMAGE_ID}" >/dev/null 2>&1 || true +fi + +touch "${BACKUP_DIR}/.rolled-back" + +log "Wishlist rollback successful" +echo "Backup: ${BACKUP_DIR}" +echo "Restored ID: ${OLD_IMAGE_ID}" +echo "Restored ref: ${OLD_IMAGE_REF}" diff --git a/stacks/wishlist/docker-compose.yml b/stacks/wishlist/docker-compose.yml new file mode 100644 index 0000000..a2e4359 --- /dev/null +++ b/stacks/wishlist/docker-compose.yml @@ -0,0 +1,16 @@ +services: + wishlist: + container_name: wishlist + image: ghcr.io/cmintey/wishlist:latest + restart: unless-stopped + ports: + - "3280:3280" + volumes: + - /srv/nfs/media/wishlist/uploads:/usr/src/app/uploads # This is where user image uploads will be stored + - /srv/nfs/media/wishlist/data:/usr/src/app/data # This is where the sqlite database will be stored + group_add: + - "2000" + environment: + ORIGIN: https://wish.zk-tech.info + TOKEN_TIME: 72 # hours until signup and password reset tokens expire + DEFAULT_CURRENCY: RUB