Merge branch 'main' of https://git.zk-tech.info/a.zubarev/home-infra

.gitea/workflows/wishlist-rollback.yml

@@ -0,0 +1,24 @@
+name: Wishlist Rollback
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  rollback:
+    name: Rollback Wishlist
+    runs-on: docker-media-01
+
+    steps:
+      - name: Checkout repository
+        uses: builtin:checkout
+        with:
+          fetch-depth: 1
+
+      - name: Validate rollback script
+        run: bash -n scripts/rollback-wishlist.sh
+
+      - name: Rollback to latest backup
+        run: ./scripts/rollback-wishlist.sh latest

.gitea/workflows/wishlist.yml

@@ -0,0 +1,32 @@
+name: Wishlist CI/CD
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "stacks/wishlist/**"
+      - "scripts/deploy-wishlist.sh"
+      - ".gitea/workflows/wishlist.yml"
+
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+    name: Deploy Wishlist
+    runs-on: docker-media-01
+
+    steps:
+      - name: Checkout repository
+        uses: builtin:checkout
+        with:
+          fetch-depth: 1
+
+      - name: Validate deployment script
+        run: bash -n scripts/deploy-wishlist.sh
+
+      - name: Deploy Wishlist
+        run: ./scripts/deploy-wishlist.sh

scripts/deploy-wishlist.sh

@@ -0,0 +1,314 @@
+#!/usr/bin/env bash
+# shellcheck shell=bash
+
+set -Eeuo pipefail
+umask 077
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+SRC_COMPOSE="${ROOT}/stacks/wishlist/docker-compose.yml"
+ROLLBACK_SCRIPT="${ROOT}/scripts/rollback-wishlist.sh"
+
+LIVE="/srv/nfs/media/wishlist"
+LIVE_COMPOSE="${LIVE}/docker-compose.yml"
+BACKUP_ROOT="/srv/nfs/media/.deploy-backups/wishlist"
+BACKUP_KEEP=3
+DEPLOY_GROUP="infra-deploy"
+CONTAINER="wishlist"
+TARGET_IMAGE_REF="ghcr.io/cmintey/wishlist:latest"
+LOCK_FILE="/var/lib/gitea-runner/locks/wishlist.lock"
+START_TIMEOUT=300
+CHECK_INTERVAL=5
+STAMP="$(date '+%Y%m%d-%H%M%S')"
+BACKUP_DIR="${BACKUP_ROOT}/${STAMP}"
+
+IMAGE_CHANGED=false
+CONFIG_CHANGED=false
+SERVICE_STOPPED=false
+NEW_CONTAINER_STARTED=false
+BACKUP_COMPLETE=false
+
+log() {
+    echo
+    echo "============================================================"
+    echo "$*"
+    echo "============================================================"
+}
+
+fail() {
+    echo >&2
+    echo "ERROR: $*" >&2
+    return 1
+}
+
+short_id() {
+    printf '%s\n' "$1" | sed 's/^sha256://' | cut -c1-16
+}
+
+wait_for_wishlist() {
+    local elapsed=0
+    local running
+
+    echo
+    echo "Waiting for Wishlist on http://127.0.0.1:3280/ ..."
+
+    while (( elapsed < START_TIMEOUT )); do
+        running="$(docker inspect -f '{{.State.Running}}' "${CONTAINER}" 2>/dev/null || echo false)"
+
+        if [[ "${running}" != "true" ]]; then
+            echo
+            echo "--- Wishlist logs ---"
+            docker logs --tail 200 "${CONTAINER}" || true
+            fail "Wishlist container is not running"
+        fi
+
+        if curl --fail --silent --show-error --max-time 5 \
+            --output /dev/null http://127.0.0.1:3280/; then
+            echo "Wishlist HTTP check: OK"
+            return 0
+        fi
+
+        sleep "${CHECK_INTERVAL}"
+        elapsed=$((elapsed + CHECK_INTERVAL))
+        echo "Waiting... ${elapsed}s"
+    done
+
+    echo
+    echo "--- Wishlist logs ---"
+    docker logs --tail 200 "${CONTAINER}" || true
+    fail "Timeout waiting for Wishlist HTTP service"
+}
+
+cleanup_old_backups() {
+    local -a backups=()
+    local name
+
+    mkdir -p "${BACKUP_ROOT}"
+
+    while IFS= read -r name; do
+        [[ -f "${BACKUP_ROOT}/${name}/.complete" ]] && backups+=("${name}")
+    done < <(
+        find "${BACKUP_ROOT}" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' \
+            | grep -E '^[0-9]{8}-[0-9]{6}$' \
+            | sort -r
+    )
+
+    echo
+    echo "Complete backups: ${#backups[@]}"
+    echo "Retention: ${BACKUP_KEEP}"
+
+    if (( ${#backups[@]} <= BACKUP_KEEP )); then
+        return 0
+    fi
+
+    for name in "${backups[@]:${BACKUP_KEEP}}"; do
+        echo "Removing old backup: ${BACKUP_ROOT}/${name}"
+        rm -rf -- "${BACKUP_ROOT:?}/${name}"
+    done
+}
+
+on_error() {
+    local rc=$?
+    local failed_command="${BASH_COMMAND}"
+    local failed_line="${BASH_LINENO[0]:-unknown}"
+
+    trap - ERR
+
+    echo
+    echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+    echo " Wishlist deployment FAILED"
+    echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
+    echo "Exit code: ${rc}"
+    echo "Failed at line: ${failed_line}"
+    echo "Failed command: ${failed_command}"
+
+    if [[ "${NEW_CONTAINER_STARTED}" == "true" && "${BACKUP_COMPLETE}" == "true" ]]; then
+        echo
+        echo "New Wishlist container may already have modified SQLite."
+        echo "Starting automatic rollback from:"
+        echo "  ${BACKUP_DIR}"
+
+        if WISHLIST_LOCK_HELD=1 "${ROLLBACK_SCRIPT}" "${BACKUP_DIR}"; then
+            echo
+            echo "Automatic rollback completed successfully."
+        else
+            echo
+            echo "CRITICAL: automatic rollback FAILED." >&2
+            echo "Run manually:" >&2
+            echo "  ${ROLLBACK_SCRIPT} ${BACKUP_DIR}" >&2
+        fi
+
+    elif [[ "${SERVICE_STOPPED}" == "true" ]]; then
+        echo
+        echo "New container was not started. Starting previous container."
+        docker start "${CONTAINER}" >/dev/null 2>&1 || true
+    else
+        echo
+        echo "Production was not modified."
+    fi
+
+    if [[ "${BACKUP_COMPLETE}" != "true" && -d "${BACKUP_DIR}" ]]; then
+        echo "Removing incomplete backup: ${BACKUP_DIR}"
+        rm -rf -- "${BACKUP_DIR}" || true
+    fi
+
+    exit "${rc}"
+}
+
+trap on_error ERR
+
+log "Pre-flight checks"
+
+for cmd in docker tar gzip sha256sum curl diff mktemp sort find grep sed cut flock; do
+    command -v "${cmd}" >/dev/null 2>&1 || fail "Required command not found: ${cmd}"
+done
+
+docker compose version >/dev/null 2>&1 || fail "Docker Compose plugin is not available"
+getent group "${DEPLOY_GROUP}" >/dev/null 2>&1 || fail "Group ${DEPLOY_GROUP} does not exist"
+
+[[ -x "${ROLLBACK_SCRIPT}" ]] || fail "Rollback script is not executable: ${ROLLBACK_SCRIPT}"
+[[ -f "${SRC_COMPOSE}" ]] || fail "Source compose not found: ${SRC_COMPOSE}"
+[[ -f "${LIVE_COMPOSE}" ]] || fail "Live compose not found: ${LIVE_COMPOSE}"
+[[ -d "${LIVE}/data" ]] || fail "${LIVE}/data not found"
+[[ -d "${LIVE}/uploads" ]] || fail "${LIVE}/uploads not found"
+
+mkdir -p "$(dirname "${LOCK_FILE}")" "${BACKUP_ROOT}"
+exec 9>"${LOCK_FILE}"
+flock -n 9 || fail "Another Wishlist deploy/rollback is already running"
+
+[[ -w "${LIVE}" ]] || fail "${LIVE} is not writable by $(id -un)"
+[[ -w "${BACKUP_ROOT}" ]] || fail "${BACKUP_ROOT} is not writable by $(id -un)"
+
+log "Validate requested Docker Compose"
+set +e
+COMPOSE_OUTPUT="$(docker compose -p wishlist -f "${SRC_COMPOSE}" config 2>&1)"
+COMPOSE_RC=$?
+set -e
+if [[ ${COMPOSE_RC} -ne 0 ]]; then
+    echo "${COMPOSE_OUTPUT}"
+    fail "Requested Docker Compose configuration is invalid"
+fi
+
+echo "Docker Compose configuration is valid."
+
+mapfile -t COMPOSE_IMAGES < <(docker compose -p wishlist -f "${SRC_COMPOSE}" config --images | sort -u)
+[[ ${#COMPOSE_IMAGES[@]} -eq 1 ]] || fail "Expected exactly one image in Wishlist compose"
+[[ "${COMPOSE_IMAGES[0]}" == "${TARGET_IMAGE_REF}" ]] || \
+    fail "Wishlist compose must use ${TARGET_IMAGE_REF}; got ${COMPOSE_IMAGES[0]}"
+
+log "Detect current Wishlist state"
+docker inspect "${CONTAINER}" >/dev/null 2>&1 || fail "Current Wishlist container does not exist"
+CURRENT_RUNNING="$(docker inspect -f '{{.State.Running}}' "${CONTAINER}")"
+[[ "${CURRENT_RUNNING}" == "true" ]] || fail "Current Wishlist container is not running"
+
+CURRENT_IMAGE_ID="$(docker inspect -f '{{.Image}}' "${CONTAINER}")"
+CURRENT_IMAGE_REF="$(docker inspect -f '{{.Config.Image}}' "${CONTAINER}")"
+
+echo "Current image reference: ${CURRENT_IMAGE_REF}"
+echo "Current image ID:        ${CURRENT_IMAGE_ID}"
+
+log "Compare semantic Compose configuration"
+TMP_CURRENT="$(mktemp)"
+TMP_REQUESTED="$(mktemp)"
+
+docker compose -p wishlist -f "${LIVE_COMPOSE}" config > "${TMP_CURRENT}"
+docker compose -p wishlist -f "${SRC_COMPOSE}" config > "${TMP_REQUESTED}"
+
+if ! diff -q "${TMP_CURRENT}" "${TMP_REQUESTED}" >/dev/null; then
+    CONFIG_CHANGED=true
+    echo "Compose configuration changed: true"
+    echo
+    diff -u "${TMP_CURRENT}" "${TMP_REQUESTED}" || true
+else
+    echo "Compose configuration changed: false"
+fi
+
+rm -f "${TMP_CURRENT}" "${TMP_REQUESTED}"
+
+log "Pull and compare latest image"
+docker pull "${TARGET_IMAGE_REF}"
+TARGET_IMAGE_ID="$(docker image inspect "${TARGET_IMAGE_REF}" -f '{{.Id}}')"
+[[ -n "${TARGET_IMAGE_ID}" ]] || fail "Could not determine target image ID"
+
+if [[ "${CURRENT_IMAGE_ID}" != "${TARGET_IMAGE_ID}" ]]; then
+    IMAGE_CHANGED=true
+fi
+
+echo "Current image ID: $(short_id "${CURRENT_IMAGE_ID}")"
+echo "Target image ID:  $(short_id "${TARGET_IMAGE_ID}")"
+echo "Image changed:    ${IMAGE_CHANGED}"
+echo "Compose changed:  ${CONFIG_CHANGED}"
+
+if [[ "${IMAGE_CHANGED}" != "true" && "${CONFIG_CHANGED}" != "true" ]]; then
+    log "Wishlist is already current"
+    echo "The comment triggered an update check, but the image digest and semantic Compose configuration are unchanged."
+    exit 0
+fi
+
+log "Prepare pre-deployment backup"
+mkdir -p "${BACKUP_DIR}"
+cp "${LIVE_COMPOSE}" "${BACKUP_DIR}/docker-compose.yml"
+
+{
+    printf 'BACKUP_TIMESTAMP=%q\n' "${STAMP}"
+    printf 'OLD_IMAGE_ID=%q\n' "${CURRENT_IMAGE_ID}"
+    printf 'OLD_IMAGE_REF=%q\n' "${CURRENT_IMAGE_REF}"
+    printf 'TARGET_IMAGE_ID=%q\n' "${TARGET_IMAGE_ID}"
+    printf 'TARGET_IMAGE_REF=%q\n' "${TARGET_IMAGE_REF}"
+    printf 'IMAGE_CHANGED=%q\n' "${IMAGE_CHANGED}"
+    printf 'CONFIG_CHANGED=%q\n' "${CONFIG_CHANGED}"
+} > "${BACKUP_DIR}/metadata.env"
+
+log "Stop Wishlist for consistent SQLite backup"
+docker stop "${CONTAINER}" >/dev/null
+SERVICE_STOPPED=true
+
+log "Backup persistent state"
+tar -C "${LIVE}" -czf "${BACKUP_DIR}/state.tar.gz" data uploads
+[[ -s "${BACKUP_DIR}/state.tar.gz" ]] || fail "State backup is empty"
+tar -tzf "${BACKUP_DIR}/state.tar.gz" >/dev/null
+
+log "Backup exact previous Docker image"
+docker image save "${CURRENT_IMAGE_ID}" | gzip -1 > "${BACKUP_DIR}/old-image.tar.gz"
+[[ -s "${BACKUP_DIR}/old-image.tar.gz" ]] || fail "Docker image backup is empty"
+gzip -t "${BACKUP_DIR}/old-image.tar.gz"
+
+log "Generate and validate checksums"
+(
+    cd "${BACKUP_DIR}"
+    sha256sum state.tar.gz old-image.tar.gz docker-compose.yml metadata.env > SHA256SUMS
+    sha256sum -c SHA256SUMS
+)
+
+touch "${BACKUP_DIR}/.complete"
+BACKUP_COMPLETE=true
+cleanup_old_backups
+
+log "Deploy requested Compose"
+cp "${SRC_COMPOSE}" "${LIVE_COMPOSE}"
+cd "${LIVE}"
+NEW_CONTAINER_STARTED=true
+
+docker compose -p wishlist -f "${LIVE_COMPOSE}" up -d --remove-orphans
+wait_for_wishlist
+
+log "Validate running image"
+RUNNING_IMAGE_ID="$(docker inspect -f '{{.Image}}' "${CONTAINER}")"
+echo "Expected: ${TARGET_IMAGE_ID}"
+echo "Running:  ${RUNNING_IMAGE_ID}"
+[[ "${RUNNING_IMAGE_ID}" == "${TARGET_IMAGE_ID}" ]] || fail "Wishlist is not running the expected image"
+
+if [[ "${IMAGE_CHANGED}" == "true" ]]; then
+    log "Remove previous local image"
+    if docker image rm "${CURRENT_IMAGE_ID}"; then
+        echo "Previous local image removed. Exact copy remains in ${BACKUP_DIR}/old-image.tar.gz"
+    else
+        echo "WARNING: previous image could not be removed; it may still be used by another container."
+    fi
+fi
+
+log "Wishlist deployment successful"
+echo "Backup:            ${BACKUP_DIR}"
+echo "Previous image:    $(short_id "${CURRENT_IMAGE_ID}")"
+echo "Running image:     $(short_id "${RUNNING_IMAGE_ID}")"
+echo "Image changed:     ${IMAGE_CHANGED}"
+echo "Compose changed:   ${CONFIG_CHANGED}"

scripts/rollback-wishlist.sh

@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+# shellcheck shell=bash
+
+set -Eeuo pipefail
+umask 077
+
+LIVE="/srv/nfs/media/wishlist"
+LIVE_COMPOSE="${LIVE}/docker-compose.yml"
+BACKUP_ROOT="/srv/nfs/media/.deploy-backups/wishlist"
+DEPLOY_GROUP="infra-deploy"
+CONTAINER="wishlist"
+LOCK_FILE="/var/lib/gitea-runner/locks/wishlist.lock"
+START_TIMEOUT=300
+CHECK_INTERVAL=5
+
+log() {
+    echo
+    echo "============================================================"
+    echo "$*"
+    echo "============================================================"
+}
+
+fail() {
+    echo >&2
+    echo "ERROR: $*" >&2
+    exit 1
+}
+
+wait_for_wishlist() {
+    local elapsed=0
+    local running
+
+    while (( elapsed < START_TIMEOUT )); do
+        running="$(docker inspect -f '{{.State.Running}}' "${CONTAINER}" 2>/dev/null || echo false)"
+
+        if [[ "${running}" != "true" ]]; then
+            docker logs --tail 200 "${CONTAINER}" 2>/dev/null || true
+            fail "Wishlist container is not running after rollback"
+        fi
+
+        if curl --fail --silent --show-error --max-time 5 \
+            --output /dev/null http://127.0.0.1:3280/; then
+            echo "Wishlist HTTP check: OK"
+            return 0
+        fi
+
+        sleep "${CHECK_INTERVAL}"
+        elapsed=$((elapsed + CHECK_INTERVAL))
+        echo "Waiting after rollback... ${elapsed}s"
+    done
+
+    docker logs --tail 200 "${CONTAINER}" 2>/dev/null || true
+    fail "Timeout waiting for Wishlist after rollback"
+}
+
+resolve_backup() {
+    local requested="${1:-latest}"
+    local latest
+
+    if [[ "${requested}" != "latest" ]]; then
+        [[ -d "${requested}" ]] && { printf '%s\n' "${requested}"; return 0; }
+        [[ -d "${BACKUP_ROOT}/${requested}" ]] && { printf '%s\n' "${BACKUP_ROOT}/${requested}"; return 0; }
+        return 1
+    fi
+
+    latest="$(
+        find "${BACKUP_ROOT}" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' \
+            | grep -E '^[0-9]{8}-[0-9]{6}$' \
+            | sort -r \
+            | while IFS= read -r name; do
+                if [[ -f "${BACKUP_ROOT}/${name}/.complete" ]]; then
+                    echo "${BACKUP_ROOT}/${name}"
+                    break
+                fi
+              done
+    )"
+
+    [[ -n "${latest}" ]] || return 1
+    printf '%s\n' "${latest}"
+}
+
+for cmd in docker tar gzip sha256sum curl find grep sort flock chgrp chmod; do
+    command -v "${cmd}" >/dev/null 2>&1 || fail "Required command not found: ${cmd}"
+done
+
+docker compose version >/dev/null 2>&1 || fail "Docker Compose plugin is not available"
+getent group "${DEPLOY_GROUP}" >/dev/null 2>&1 || fail "Group ${DEPLOY_GROUP} does not exist"
+
+mkdir -p "$(dirname "${LOCK_FILE}")"
+if [[ "${WISHLIST_LOCK_HELD:-0}" != "1" ]]; then
+    exec 9>"${LOCK_FILE}"
+    flock -n 9 || fail "Another Wishlist deploy/rollback is already running"
+fi
+
+BACKUP_DIR="$(resolve_backup "${1:-latest}")" || fail "Could not find requested complete backup"
+
+log "Rollback Wishlist from ${BACKUP_DIR}"
+
+for file in .complete docker-compose.yml metadata.env state.tar.gz old-image.tar.gz SHA256SUMS; do
+    [[ -e "${BACKUP_DIR}/${file}" ]] || fail "Backup file missing: ${BACKUP_DIR}/${file}"
+done
+
+(
+    cd "${BACKUP_DIR}"
+    sha256sum -c SHA256SUMS
+)
+
+tar -tzf "${BACKUP_DIR}/state.tar.gz" >/dev/null
+gzip -t "${BACKUP_DIR}/old-image.tar.gz"
+
+# shellcheck disable=SC1090
+source "${BACKUP_DIR}/metadata.env"
+
+: "${OLD_IMAGE_ID:?OLD_IMAGE_ID is missing from metadata}"
+: "${OLD_IMAGE_REF:?OLD_IMAGE_REF is missing from metadata}"
+
+FAILED_IMAGE_ID=""
+if docker inspect "${CONTAINER}" >/dev/null 2>&1; then
+    FAILED_IMAGE_ID="$(docker inspect -f '{{.Image}}' "${CONTAINER}" 2>/dev/null || true)"
+    docker logs "${CONTAINER}" > "${BACKUP_DIR}/failed-container.log" 2>&1 || true
+    docker rm -f "${CONTAINER}" >/dev/null || true
+fi
+
+log "Restore previous Compose"
+cp "${BACKUP_DIR}/docker-compose.yml" "${LIVE_COMPOSE}"
+
+log "Restore persistent state"
+mkdir -p "${LIVE}/data" "${LIVE}/uploads"
+find "${LIVE}/data" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +
+find "${LIVE}/uploads" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +
+
+tar --no-same-owner -xzf "${BACKUP_DIR}/state.tar.gz" -C "${LIVE}"
+
+# Files extracted by the runner become runner-owned. Normalize the group and
+# group permissions so the container can access them through the shared GID.
+find "${LIVE}/data" "${LIVE}/uploads" -mindepth 1 -exec chgrp "${DEPLOY_GROUP}" {} +
+find "${LIVE}/data" "${LIVE}/uploads" -type d -exec chmod g+rwx,g+s {} +
+find "${LIVE}/data" "${LIVE}/uploads" -type f -exec chmod g+rw {} +
+
+log "Restore exact previous Docker image"
+gzip -dc "${BACKUP_DIR}/old-image.tar.gz" | docker image load
+
+docker image inspect "${OLD_IMAGE_ID}" >/dev/null 2>&1 || \
+    fail "Previous image ${OLD_IMAGE_ID} was not restored"
+
+docker image tag "${OLD_IMAGE_ID}" "${OLD_IMAGE_REF}"
+
+log "Start previous Wishlist version"
+cd "${LIVE}"
+docker compose -p wishlist -f "${LIVE_COMPOSE}" up -d --pull never --remove-orphans
+wait_for_wishlist
+
+RUNNING_IMAGE_ID="$(docker inspect -f '{{.Image}}' "${CONTAINER}")"
+[[ "${RUNNING_IMAGE_ID}" == "${OLD_IMAGE_ID}" ]] || \
+    fail "Rollback started unexpected image: ${RUNNING_IMAGE_ID}"
+
+if [[ -n "${FAILED_IMAGE_ID}" && "${FAILED_IMAGE_ID}" != "${OLD_IMAGE_ID}" ]]; then
+    docker image rm "${FAILED_IMAGE_ID}" >/dev/null 2>&1 || true
+fi
+
+touch "${BACKUP_DIR}/.rolled-back"
+
+log "Wishlist rollback successful"
+echo "Backup:        ${BACKUP_DIR}"
+echo "Restored ID:   ${OLD_IMAGE_ID}"
+echo "Restored ref:  ${OLD_IMAGE_REF}"

stacks/wishlist/docker-compose.yml

@@ -0,0 +1,16 @@
+services:
+  wishlist:
+    container_name: wishlist
+    image: ghcr.io/cmintey/wishlist:latest
+    restart: unless-stopped
+    ports:
+      - "3280:3280"
+    volumes:
+      - /srv/nfs/media/wishlist/uploads:/usr/src/app/uploads  # This is where user image uploads will be stored
+      - /srv/nfs/media/wishlist/data:/usr/src/app/data        # This is where the sqlite database will be stored
+    group_add:
+      - "2000"
+    environment:
+      ORIGIN: https://wish.zk-tech.info
+      TOKEN_TIME: 72 # hours until signup and password reset tokens expire
+      DEFAULT_CURRENCY: RUB