Add Wishlist CI/CD on media host
This commit is contained in:
root
Executable
+314
@@ -0,0 +1,314 @@
|
||||
#!/usr/bin/env bash
|
||||
# shellcheck shell=bash
|
||||
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
SRC_COMPOSE="${ROOT}/stacks/wishlist/docker-compose.yml"
|
||||
ROLLBACK_SCRIPT="${ROOT}/scripts/rollback-wishlist.sh"
|
||||
|
||||
LIVE="/srv/nfs/media/wishlist"
|
||||
LIVE_COMPOSE="${LIVE}/docker-compose.yml"
|
||||
BACKUP_ROOT="/srv/nfs/media/.deploy-backups/wishlist"
|
||||
BACKUP_KEEP=3
|
||||
DEPLOY_GROUP="infra-deploy"
|
||||
CONTAINER="wishlist"
|
||||
TARGET_IMAGE_REF="ghcr.io/cmintey/wishlist:latest"
|
||||
LOCK_FILE="/var/lib/gitea-runner/locks/wishlist.lock"
|
||||
START_TIMEOUT=300
|
||||
CHECK_INTERVAL=5
|
||||
STAMP="$(date '+%Y%m%d-%H%M%S')"
|
||||
BACKUP_DIR="${BACKUP_ROOT}/${STAMP}"
|
||||
|
||||
IMAGE_CHANGED=false
|
||||
CONFIG_CHANGED=false
|
||||
SERVICE_STOPPED=false
|
||||
NEW_CONTAINER_STARTED=false
|
||||
BACKUP_COMPLETE=false
|
||||
|
||||
log() {
|
||||
echo
|
||||
echo "============================================================"
|
||||
echo "$*"
|
||||
echo "============================================================"
|
||||
}
|
||||
|
||||
fail() {
|
||||
echo >&2
|
||||
echo "ERROR: $*" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
short_id() {
|
||||
printf '%s\n' "$1" | sed 's/^sha256://' | cut -c1-16
|
||||
}
|
||||
|
||||
wait_for_wishlist() {
|
||||
local elapsed=0
|
||||
local running
|
||||
|
||||
echo
|
||||
echo "Waiting for Wishlist on http://127.0.0.1:3280/ ..."
|
||||
|
||||
while (( elapsed < START_TIMEOUT )); do
|
||||
running="$(docker inspect -f '{{.State.Running}}' "${CONTAINER}" 2>/dev/null || echo false)"
|
||||
|
||||
if [[ "${running}" != "true" ]]; then
|
||||
echo
|
||||
echo "--- Wishlist logs ---"
|
||||
docker logs --tail 200 "${CONTAINER}" || true
|
||||
fail "Wishlist container is not running"
|
||||
fi
|
||||
|
||||
if curl --fail --silent --show-error --max-time 5 \
|
||||
--output /dev/null http://127.0.0.1:3280/; then
|
||||
echo "Wishlist HTTP check: OK"
|
||||
return 0
|
||||
fi
|
||||
|
||||
sleep "${CHECK_INTERVAL}"
|
||||
elapsed=$((elapsed + CHECK_INTERVAL))
|
||||
echo "Waiting... ${elapsed}s"
|
||||
done
|
||||
|
||||
echo
|
||||
echo "--- Wishlist logs ---"
|
||||
docker logs --tail 200 "${CONTAINER}" || true
|
||||
fail "Timeout waiting for Wishlist HTTP service"
|
||||
}
|
||||
|
||||
cleanup_old_backups() {
|
||||
local -a backups=()
|
||||
local name
|
||||
|
||||
mkdir -p "${BACKUP_ROOT}"
|
||||
|
||||
while IFS= read -r name; do
|
||||
[[ -f "${BACKUP_ROOT}/${name}/.complete" ]] && backups+=("${name}")
|
||||
done < <(
|
||||
find "${BACKUP_ROOT}" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' \
|
||||
| grep -E '^[0-9]{8}-[0-9]{6}$' \
|
||||
| sort -r
|
||||
)
|
||||
|
||||
echo
|
||||
echo "Complete backups: ${#backups[@]}"
|
||||
echo "Retention: ${BACKUP_KEEP}"
|
||||
|
||||
if (( ${#backups[@]} <= BACKUP_KEEP )); then
|
||||
return 0
|
||||
fi
|
||||
|
||||
for name in "${backups[@]:${BACKUP_KEEP}}"; do
|
||||
echo "Removing old backup: ${BACKUP_ROOT}/${name}"
|
||||
rm -rf -- "${BACKUP_ROOT:?}/${name}"
|
||||
done
|
||||
}
|
||||
|
||||
on_error() {
|
||||
local rc=$?
|
||||
local failed_command="${BASH_COMMAND}"
|
||||
local failed_line="${BASH_LINENO[0]:-unknown}"
|
||||
|
||||
trap - ERR
|
||||
|
||||
echo
|
||||
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||
echo " Wishlist deployment FAILED"
|
||||
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
||||
echo "Exit code: ${rc}"
|
||||
echo "Failed at line: ${failed_line}"
|
||||
echo "Failed command: ${failed_command}"
|
||||
|
||||
if [[ "${NEW_CONTAINER_STARTED}" == "true" && "${BACKUP_COMPLETE}" == "true" ]]; then
|
||||
echo
|
||||
echo "New Wishlist container may already have modified SQLite."
|
||||
echo "Starting automatic rollback from:"
|
||||
echo " ${BACKUP_DIR}"
|
||||
|
||||
if WISHLIST_LOCK_HELD=1 "${ROLLBACK_SCRIPT}" "${BACKUP_DIR}"; then
|
||||
echo
|
||||
echo "Automatic rollback completed successfully."
|
||||
else
|
||||
echo
|
||||
echo "CRITICAL: automatic rollback FAILED." >&2
|
||||
echo "Run manually:" >&2
|
||||
echo " ${ROLLBACK_SCRIPT} ${BACKUP_DIR}" >&2
|
||||
fi
|
||||
|
||||
elif [[ "${SERVICE_STOPPED}" == "true" ]]; then
|
||||
echo
|
||||
echo "New container was not started. Starting previous container."
|
||||
docker start "${CONTAINER}" >/dev/null 2>&1 || true
|
||||
else
|
||||
echo
|
||||
echo "Production was not modified."
|
||||
fi
|
||||
|
||||
if [[ "${BACKUP_COMPLETE}" != "true" && -d "${BACKUP_DIR}" ]]; then
|
||||
echo "Removing incomplete backup: ${BACKUP_DIR}"
|
||||
rm -rf -- "${BACKUP_DIR}" || true
|
||||
fi
|
||||
|
||||
exit "${rc}"
|
||||
}
|
||||
|
||||
trap on_error ERR
|
||||
|
||||
log "Pre-flight checks"
|
||||
|
||||
for cmd in docker tar gzip sha256sum curl diff mktemp sort find grep sed cut flock; do
|
||||
command -v "${cmd}" >/dev/null 2>&1 || fail "Required command not found: ${cmd}"
|
||||
done
|
||||
|
||||
docker compose version >/dev/null 2>&1 || fail "Docker Compose plugin is not available"
|
||||
getent group "${DEPLOY_GROUP}" >/dev/null 2>&1 || fail "Group ${DEPLOY_GROUP} does not exist"
|
||||
|
||||
[[ -x "${ROLLBACK_SCRIPT}" ]] || fail "Rollback script is not executable: ${ROLLBACK_SCRIPT}"
|
||||
[[ -f "${SRC_COMPOSE}" ]] || fail "Source compose not found: ${SRC_COMPOSE}"
|
||||
[[ -f "${LIVE_COMPOSE}" ]] || fail "Live compose not found: ${LIVE_COMPOSE}"
|
||||
[[ -d "${LIVE}/data" ]] || fail "${LIVE}/data not found"
|
||||
[[ -d "${LIVE}/uploads" ]] || fail "${LIVE}/uploads not found"
|
||||
|
||||
mkdir -p "$(dirname "${LOCK_FILE}")" "${BACKUP_ROOT}"
|
||||
exec 9>"${LOCK_FILE}"
|
||||
flock -n 9 || fail "Another Wishlist deploy/rollback is already running"
|
||||
|
||||
[[ -w "${LIVE}" ]] || fail "${LIVE} is not writable by $(id -un)"
|
||||
[[ -w "${BACKUP_ROOT}" ]] || fail "${BACKUP_ROOT} is not writable by $(id -un)"
|
||||
|
||||
log "Validate requested Docker Compose"
|
||||
set +e
|
||||
COMPOSE_OUTPUT="$(docker compose -p wishlist -f "${SRC_COMPOSE}" config 2>&1)"
|
||||
COMPOSE_RC=$?
|
||||
set -e
|
||||
if [[ ${COMPOSE_RC} -ne 0 ]]; then
|
||||
echo "${COMPOSE_OUTPUT}"
|
||||
fail "Requested Docker Compose configuration is invalid"
|
||||
fi
|
||||
|
||||
echo "Docker Compose configuration is valid."
|
||||
|
||||
mapfile -t COMPOSE_IMAGES < <(docker compose -p wishlist -f "${SRC_COMPOSE}" config --images | sort -u)
|
||||
[[ ${#COMPOSE_IMAGES[@]} -eq 1 ]] || fail "Expected exactly one image in Wishlist compose"
|
||||
[[ "${COMPOSE_IMAGES[0]}" == "${TARGET_IMAGE_REF}" ]] || \
|
||||
fail "Wishlist compose must use ${TARGET_IMAGE_REF}; got ${COMPOSE_IMAGES[0]}"
|
||||
|
||||
log "Detect current Wishlist state"
|
||||
docker inspect "${CONTAINER}" >/dev/null 2>&1 || fail "Current Wishlist container does not exist"
|
||||
CURRENT_RUNNING="$(docker inspect -f '{{.State.Running}}' "${CONTAINER}")"
|
||||
[[ "${CURRENT_RUNNING}" == "true" ]] || fail "Current Wishlist container is not running"
|
||||
|
||||
CURRENT_IMAGE_ID="$(docker inspect -f '{{.Image}}' "${CONTAINER}")"
|
||||
CURRENT_IMAGE_REF="$(docker inspect -f '{{.Config.Image}}' "${CONTAINER}")"
|
||||
|
||||
echo "Current image reference: ${CURRENT_IMAGE_REF}"
|
||||
echo "Current image ID: ${CURRENT_IMAGE_ID}"
|
||||
|
||||
log "Compare semantic Compose configuration"
|
||||
TMP_CURRENT="$(mktemp)"
|
||||
TMP_REQUESTED="$(mktemp)"
|
||||
|
||||
docker compose -p wishlist -f "${LIVE_COMPOSE}" config > "${TMP_CURRENT}"
|
||||
docker compose -p wishlist -f "${SRC_COMPOSE}" config > "${TMP_REQUESTED}"
|
||||
|
||||
if ! diff -q "${TMP_CURRENT}" "${TMP_REQUESTED}" >/dev/null; then
|
||||
CONFIG_CHANGED=true
|
||||
echo "Compose configuration changed: true"
|
||||
echo
|
||||
diff -u "${TMP_CURRENT}" "${TMP_REQUESTED}" || true
|
||||
else
|
||||
echo "Compose configuration changed: false"
|
||||
fi
|
||||
|
||||
rm -f "${TMP_CURRENT}" "${TMP_REQUESTED}"
|
||||
|
||||
log "Pull and compare latest image"
|
||||
docker pull "${TARGET_IMAGE_REF}"
|
||||
TARGET_IMAGE_ID="$(docker image inspect "${TARGET_IMAGE_REF}" -f '{{.Id}}')"
|
||||
[[ -n "${TARGET_IMAGE_ID}" ]] || fail "Could not determine target image ID"
|
||||
|
||||
if [[ "${CURRENT_IMAGE_ID}" != "${TARGET_IMAGE_ID}" ]]; then
|
||||
IMAGE_CHANGED=true
|
||||
fi
|
||||
|
||||
echo "Current image ID: $(short_id "${CURRENT_IMAGE_ID}")"
|
||||
echo "Target image ID: $(short_id "${TARGET_IMAGE_ID}")"
|
||||
echo "Image changed: ${IMAGE_CHANGED}"
|
||||
echo "Compose changed: ${CONFIG_CHANGED}"
|
||||
|
||||
if [[ "${IMAGE_CHANGED}" != "true" && "${CONFIG_CHANGED}" != "true" ]]; then
|
||||
log "Wishlist is already current"
|
||||
echo "The comment triggered an update check, but the image digest and semantic Compose configuration are unchanged."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "Prepare pre-deployment backup"
|
||||
mkdir -p "${BACKUP_DIR}"
|
||||
cp "${LIVE_COMPOSE}" "${BACKUP_DIR}/docker-compose.yml"
|
||||
|
||||
{
|
||||
printf 'BACKUP_TIMESTAMP=%q\n' "${STAMP}"
|
||||
printf 'OLD_IMAGE_ID=%q\n' "${CURRENT_IMAGE_ID}"
|
||||
printf 'OLD_IMAGE_REF=%q\n' "${CURRENT_IMAGE_REF}"
|
||||
printf 'TARGET_IMAGE_ID=%q\n' "${TARGET_IMAGE_ID}"
|
||||
printf 'TARGET_IMAGE_REF=%q\n' "${TARGET_IMAGE_REF}"
|
||||
printf 'IMAGE_CHANGED=%q\n' "${IMAGE_CHANGED}"
|
||||
printf 'CONFIG_CHANGED=%q\n' "${CONFIG_CHANGED}"
|
||||
} > "${BACKUP_DIR}/metadata.env"
|
||||
|
||||
log "Stop Wishlist for consistent SQLite backup"
|
||||
docker stop "${CONTAINER}" >/dev/null
|
||||
SERVICE_STOPPED=true
|
||||
|
||||
log "Backup persistent state"
|
||||
tar -C "${LIVE}" -czf "${BACKUP_DIR}/state.tar.gz" data uploads
|
||||
[[ -s "${BACKUP_DIR}/state.tar.gz" ]] || fail "State backup is empty"
|
||||
tar -tzf "${BACKUP_DIR}/state.tar.gz" >/dev/null
|
||||
|
||||
log "Backup exact previous Docker image"
|
||||
docker image save "${CURRENT_IMAGE_ID}" | gzip -1 > "${BACKUP_DIR}/old-image.tar.gz"
|
||||
[[ -s "${BACKUP_DIR}/old-image.tar.gz" ]] || fail "Docker image backup is empty"
|
||||
gzip -t "${BACKUP_DIR}/old-image.tar.gz"
|
||||
|
||||
log "Generate and validate checksums"
|
||||
(
|
||||
cd "${BACKUP_DIR}"
|
||||
sha256sum state.tar.gz old-image.tar.gz docker-compose.yml metadata.env > SHA256SUMS
|
||||
sha256sum -c SHA256SUMS
|
||||
)
|
||||
|
||||
touch "${BACKUP_DIR}/.complete"
|
||||
BACKUP_COMPLETE=true
|
||||
cleanup_old_backups
|
||||
|
||||
log "Deploy requested Compose"
|
||||
cp "${SRC_COMPOSE}" "${LIVE_COMPOSE}"
|
||||
cd "${LIVE}"
|
||||
NEW_CONTAINER_STARTED=true
|
||||
|
||||
docker compose -p wishlist -f "${LIVE_COMPOSE}" up -d --remove-orphans
|
||||
wait_for_wishlist
|
||||
|
||||
log "Validate running image"
|
||||
RUNNING_IMAGE_ID="$(docker inspect -f '{{.Image}}' "${CONTAINER}")"
|
||||
echo "Expected: ${TARGET_IMAGE_ID}"
|
||||
echo "Running: ${RUNNING_IMAGE_ID}"
|
||||
[[ "${RUNNING_IMAGE_ID}" == "${TARGET_IMAGE_ID}" ]] || fail "Wishlist is not running the expected image"
|
||||
|
||||
if [[ "${IMAGE_CHANGED}" == "true" ]]; then
|
||||
log "Remove previous local image"
|
||||
if docker image rm "${CURRENT_IMAGE_ID}"; then
|
||||
echo "Previous local image removed. Exact copy remains in ${BACKUP_DIR}/old-image.tar.gz"
|
||||
else
|
||||
echo "WARNING: previous image could not be removed; it may still be used by another container."
|
||||
fi
|
||||
fi
|
||||
|
||||
log "Wishlist deployment successful"
|
||||
echo "Backup: ${BACKUP_DIR}"
|
||||
echo "Previous image: $(short_id "${CURRENT_IMAGE_ID}")"
|
||||
echo "Running image: $(short_id "${RUNNING_IMAGE_ID}")"
|
||||
echo "Image changed: ${IMAGE_CHANGED}"
|
||||
echo "Compose changed: ${CONFIG_CHANGED}"
|
||||
Reference in New Issue
Block a user