#!/usr/bin/env bash set -Eeuo pipefail umask 077 # ============================================================ # Configuration # ============================================================ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SRC="${ROOT}/stacks/zabbix" LIVE="/srv/nfs/infra/zabbix" ENV_FILE="${LIVE}/.env" BACKUP_ROOT="/srv/nfs/infra/.deploy-backups/zabbix" BACKUP_KEEP=3 STAMP="$(date '+%Y%m%d-%H%M%S')" BACKUP_DIR="${BACKUP_ROOT}/${STAMP}" DB_BACKUP_DIR="${BACKUP_DIR}/database" DB_BACKUP_FILE="${DB_BACKUP_DIR}/zabbix.dump" SERVER_START_TIMEOUT=1800 SERVER_CHECK_INTERVAL=5 WEB_START_TIMEOUT=300 WEB_CHECK_INTERVAL=5 # ============================================================ # Runtime state # ============================================================ SERVICES_STOPPED=false MIGRATION_STARTED=false VERSION_CHANGED=false # ============================================================ # Helpers # ============================================================ log() { echo echo "============================================================" echo "$*" echo "============================================================" } fail() { echo echo "ERROR: $*" >&2 exit 1 } extract_version() { grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n1 } image_version() { local IMAGE="$1" printf '%s\n' "${IMAGE}" | sed -nE 's/.*:alpine-([0-9]+\.[0-9]+\.[0-9]+)$/\1/p' } version_less_than() { local A="$1" local B="$2" [[ "${A}" == "${B}" ]] && return 1 [[ "$( printf '%s\n%s\n' "${A}" "${B}" | sort -V | head -n1 )" == "${A}" ]] } cleanup_old_backups() { echo echo "--- Backup retention ---" echo "Keeping last ${BACKUP_KEEP} pre-upgrade backups." mkdir -p "${BACKUP_ROOT}" mapfile -t BACKUPS < <( find "${BACKUP_ROOT}" \ -mindepth 1 \ -maxdepth 1 \ -type d \ -printf '%f\n' | grep -E '^[0-9]{8}-[0-9]{6}$' | sort -r ) if (( ${#BACKUPS[@]} <= BACKUP_KEEP )); then echo "Current backup count: ${#BACKUPS[@]}" echo "Nothing to delete." return 0 fi for BACKUP_NAME in "${BACKUPS[@]:${BACKUP_KEEP}}" do OLD_BACKUP="${BACKUP_ROOT}/${BACKUP_NAME}" echo echo "Removing old backup:" echo " ${OLD_BACKUP}" rm -rf -- "${OLD_BACKUP}" done } restore_old_services() { echo echo "--- Restore previous Zabbix services ---" if [[ -f "${BACKUP_DIR}/docker-compose.yml" ]]; then cp \ "${BACKUP_DIR}/docker-compose.yml" \ "${LIVE}/docker-compose.yml" fi ( cd "${LIVE}" docker compose \ --env-file "${ENV_FILE}" \ up -d \ zabbix-server \ zabbix-web ) || true } on_error() { rc=$? trap - ERR echo echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" echo " Zabbix deployment FAILED" echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" if [[ -d "${BACKUP_DIR}" ]]; then echo echo "Pre-upgrade backup:" echo " ${BACKUP_DIR}" fi # # До старта нового zabbix-server схема БД точно # не могла измениться. # # Поэтому старую версию можно безопасно вернуть. # if [[ "${SERVICES_STOPPED}" == "true" ]] && [[ "${MIGRATION_STARTED}" != "true" ]] then echo echo "Database migration has NOT started." echo "Restoring previous containers." restore_old_services else echo echo "Automatic rollback is NOT performed." if [[ "${MIGRATION_STARTED}" == "true" ]]; then echo echo "Reason:" echo " new Zabbix server has already been started;" echo " database schema migration may have begun." echo echo "Rollback requires restoration of the" echo "pre-upgrade PostgreSQL dump." fi fi exit "${rc}" } wait_for_zabbix_server() { local elapsed=0 echo echo "Waiting for Zabbix server startup..." while (( elapsed < SERVER_START_TIMEOUT )) do RUNNING="$( docker inspect \ -f '{{.State.Running}}' \ zabbix-server-pgsql 2>/dev/null || echo false )" if [[ "${RUNNING}" != "true" ]]; then echo echo "--- Zabbix server logs ---" docker logs \ --since "${UPGRADE_STARTED}" \ zabbix-server-pgsql || true fail "Zabbix server container stopped during startup" fi SERVER_LOG="$( docker logs \ --since "${UPGRADE_STARTED}" \ zabbix-server-pgsql 2>&1 || true )" # # Successful server start means mandatory DB schema # is compatible. Zabbix does not start if required # patches failed. # if echo "${SERVER_LOG}" | grep -q 'server #0 started \[main process\]' then echo echo "Zabbix server main process started." return 0 fi if echo "${SERVER_LOG}" | grep -Eqi \ 'database upgrade.*(failed|failure)|cannot upgrade database|database version.*newer' then echo "${SERVER_LOG}" fail "Zabbix database upgrade failed" fi sleep "${SERVER_CHECK_INTERVAL}" elapsed=$((elapsed + SERVER_CHECK_INTERVAL)) echo "Waiting... ${elapsed}s" done echo echo "--- Zabbix server logs ---" docker logs \ --since "${UPGRADE_STARTED}" \ zabbix-server-pgsql || true fail \ "Timeout waiting for Zabbix server after ${SERVER_START_TIMEOUT}s" } wait_for_zabbix_web() { local elapsed=0 echo echo "Waiting for Zabbix web health=healthy..." while (( elapsed < WEB_START_TIMEOUT )) do RUNNING="$( docker inspect \ -f '{{.State.Running}}' \ zabbix-web-nginx-pgsql 2>/dev/null || echo false )" if [[ "${RUNNING}" != "true" ]]; then echo echo "--- Zabbix web logs ---" docker logs \ zabbix-web-nginx-pgsql || true fail "Zabbix web container stopped" fi HEALTH="$( docker inspect \ -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ zabbix-web-nginx-pgsql )" echo "Zabbix web health: ${HEALTH} (${elapsed}s)" if [[ "${HEALTH}" == "healthy" ]]; then return 0 fi sleep "${WEB_CHECK_INTERVAL}" elapsed=$((elapsed + WEB_CHECK_INTERVAL)) done docker logs \ zabbix-web-nginx-pgsql || true fail \ "Timeout waiting for Zabbix web health=healthy" } trap on_error ERR # ============================================================ # Required commands # ============================================================ log "Pre-flight checks" for CMD in \ docker \ psql \ pg_dump \ pg_restore \ sha256sum \ grep \ sed \ sort \ find \ awk do command -v "${CMD}" >/dev/null 2>&1 || \ fail "Required command not found: ${CMD}" done docker compose version >/dev/null 2>&1 || \ fail "Docker Compose plugin is not available" # ============================================================ # Required files # ============================================================ [[ -f "${SRC}/docker-compose.yml" ]] || \ fail "${SRC}/docker-compose.yml not found" [[ -f "${LIVE}/docker-compose.yml" ]] || \ fail "${LIVE}/docker-compose.yml not found" [[ -f "${ENV_FILE}" ]] || \ fail "${ENV_FILE} not found" # ============================================================ # Backup directory # ============================================================ mkdir -p "${BACKUP_ROOT}" [[ -w "${BACKUP_ROOT}" ]] || \ fail "${BACKUP_ROOT} is not writable" # ============================================================ # Load production environment # # .env must be shell compatible. # ============================================================ set -a # shellcheck disable=SC1090 source "${ENV_FILE}" set +a DB_HOST="${DB_SERVER_HOST:-${POSTGRES_HOST:-}}" DB_PORT="${DB_SERVER_PORT:-${POSTGRES_PORT:-5432}}" DB_NAME="${POSTGRES_DB:-}" DB_USER="${POSTGRES_USER:-}" DB_PASSWORD="${POSTGRES_PASSWORD:-}" : "${DB_HOST:?DB_SERVER_HOST not defined}" : "${DB_PORT:?DB_SERVER_PORT not defined}" : "${DB_NAME:?POSTGRES_DB not defined}" : "${DB_USER:?POSTGRES_USER not defined}" : "${DB_PASSWORD:?POSTGRES_PASSWORD not defined}" export PGPASSWORD="${POSTGRES_PASSWORD}" export PGCONNECT_TIMEOUT=10 # ============================================================ # Validate Compose # ============================================================ log "Validate Docker Compose" docker compose \ --env-file "${ENV_FILE}" \ -f "${SRC}/docker-compose.yml" \ config >/dev/null echo "Docker Compose configuration is valid." # ============================================================ # Detect target images # ============================================================ log "Detect target Zabbix images" NEW_SERVER_IMAGE="$( docker compose \ --env-file "${ENV_FILE}" \ -f "${SRC}/docker-compose.yml" \ config --images | grep -m1 'zabbix/zabbix-server-pgsql:' )" NEW_WEB_IMAGE="$( docker compose \ --env-file "${ENV_FILE}" \ -f "${SRC}/docker-compose.yml" \ config --images | grep -m1 'zabbix/zabbix-web-nginx-pgsql:' )" [[ -n "${NEW_SERVER_IMAGE}" ]] || \ fail "Could not detect target Zabbix server image" [[ -n "${NEW_WEB_IMAGE}" ]] || \ fail "Could not detect target Zabbix web image" TARGET_SERVER_VERSION="$( image_version "${NEW_SERVER_IMAGE}" )" TARGET_WEB_VERSION="$( image_version "${NEW_WEB_IMAGE}" )" [[ -n "${TARGET_SERVER_VERSION}" ]] || \ fail \ "Zabbix server must use a pinned alpine-X.Y.Z tag" [[ -n "${TARGET_WEB_VERSION}" ]] || \ fail \ "Zabbix web must use a pinned alpine-X.Y.Z tag" [[ "${TARGET_SERVER_VERSION}" == "${TARGET_WEB_VERSION}" ]] || \ fail \ "Zabbix server/web versions differ: ${TARGET_SERVER_VERSION} != ${TARGET_WEB_VERSION}" TARGET_VERSION="${TARGET_SERVER_VERSION}" echo "Target server image:" echo " ${NEW_SERVER_IMAGE}" echo echo "Target web image:" echo " ${NEW_WEB_IMAGE}" echo echo "Target Zabbix version:" echo " ${TARGET_VERSION}" # ============================================================ # Current containers # ============================================================ docker inspect zabbix-server-pgsql >/dev/null 2>&1 || \ fail "Current Zabbix server container does not exist" docker inspect zabbix-web-nginx-pgsql >/dev/null 2>&1 || \ fail "Current Zabbix web container does not exist" SERVER_RUNNING="$( docker inspect \ -f '{{.State.Running}}' \ zabbix-server-pgsql )" WEB_RUNNING="$( docker inspect \ -f '{{.State.Running}}' \ zabbix-web-nginx-pgsql )" [[ "${SERVER_RUNNING}" == "true" ]] || \ fail "Current Zabbix server is not running" [[ "${WEB_RUNNING}" == "true" ]] || \ fail "Current Zabbix web is not running" # ============================================================ # Current actual Zabbix version # ============================================================ log "Detect current Zabbix version" CURRENT_VERSION_OUTPUT="$( docker exec \ zabbix-server-pgsql \ zabbix_server \ -V )" CURRENT_VERSION="$( printf '%s\n' "${CURRENT_VERSION_OUTPUT}" | extract_version )" [[ -n "${CURRENT_VERSION}" ]] || \ fail "Could not determine current Zabbix version" OLD_SERVER_IMAGE="$( docker inspect \ -f '{{.Config.Image}}' \ zabbix-server-pgsql )" OLD_WEB_IMAGE="$( docker inspect \ -f '{{.Config.Image}}' \ zabbix-web-nginx-pgsql )" echo "Current Zabbix version:" echo " ${CURRENT_VERSION}" echo echo "Current server image:" echo " ${OLD_SERVER_IMAGE}" echo echo "Current web image:" echo " ${OLD_WEB_IMAGE}" # ============================================================ # Version validation # ============================================================ log "Validate version transition" if version_less_than \ "${TARGET_VERSION}" \ "${CURRENT_VERSION}" then fail \ "Zabbix downgrade is not allowed: ${CURRENT_VERSION} -> ${TARGET_VERSION}" fi if [[ "${CURRENT_VERSION}" != "${TARGET_VERSION}" ]] then VERSION_CHANGED=true fi echo "Current version: ${CURRENT_VERSION}" echo "Target version: ${TARGET_VERSION}" echo "Version changed: ${VERSION_CHANGED}" # ============================================================ # Pull target images before downtime # ============================================================ log "Pull target Docker images" docker pull "${NEW_SERVER_IMAGE}" docker pull "${NEW_WEB_IMAGE}" # ============================================================ # VERSION UPDATE # ============================================================ if [[ "${VERSION_CHANGED}" == "true" ]] then # ======================================================== # Prepare backup directory before shutdown # ======================================================== log "Prepare Zabbix pre-upgrade backup" mkdir -p "${DB_BACKUP_DIR}" cp \ "${LIVE}/docker-compose.yml" \ "${BACKUP_DIR}/docker-compose.yml" cat > "${BACKUP_DIR}/metadata.txt" < "${DB_BACKUP_DIR}/postgresql-version.txt" SERVER_MAJOR="${SERVER_VERSION%%.*}" PG_DUMP_VERSION="$( pg_dump --version | sed -E 's/.* ([0-9]+(\.[0-9]+)?).*/\1/' )" PG_DUMP_MAJOR="${PG_DUMP_VERSION%%.*}" echo "PostgreSQL server:" echo " ${SERVER_VERSION}" echo "pg_dump client:" echo " ${PG_DUMP_VERSION}" if (( PG_DUMP_MAJOR < SERVER_MAJOR )) then fail \ "pg_dump ${PG_DUMP_VERSION} is older than PostgreSQL server ${SERVER_VERSION}" fi # ======================================================== # DB size # ======================================================== DB_SIZE="$( psql \ -h "${DB_HOST}" \ -p "${DB_PORT}" \ -U "${POSTGRES_USER}" \ -d "${POSTGRES_DB}" \ -Atqc \ 'SELECT pg_size_pretty(pg_database_size(current_database()));' )" echo "Database size:" echo " ${DB_SIZE}" # ======================================================== # pg_dump # ======================================================== log "Create PostgreSQL backup" pg_dump \ -h "${DB_HOST}" \ -p "${DB_PORT}" \ -U "${POSTGRES_USER}" \ -d "${POSTGRES_DB}" \ --format=custom \ --no-owner \ --no-privileges \ --file="${DB_BACKUP_FILE}" [[ -s "${DB_BACKUP_FILE}" ]] || \ fail "PostgreSQL dump does not exist or is empty" # ======================================================== # Validate dump # ======================================================== log "Validate PostgreSQL backup" pg_restore \ --list \ "${DB_BACKUP_FILE}" \ > "${DB_BACKUP_DIR}/zabbix.dump.list" [[ -s "${DB_BACKUP_DIR}/zabbix.dump.list" ]] || \ fail "pg_restore could not validate dump" # ======================================================== # SHA256 # ======================================================== ( cd "${DB_BACKUP_DIR}" sha256sum \ zabbix.dump \ > zabbix.dump.sha256 sha256sum \ -c zabbix.dump.sha256 ) DB_BACKUP_SIZE="$( du -h "${DB_BACKUP_FILE}" | awk '{print $1}' )" echo echo "PostgreSQL backup successful." echo echo "Backup:" echo " ${DB_BACKUP_FILE}" echo echo "Dump size:" echo " ${DB_BACKUP_SIZE}" # ======================================================== # Retention # ======================================================== cleanup_old_backups # ======================================================== # Deploy new Compose # ======================================================== log "Deploy new docker-compose.yml" cp \ "${SRC}/docker-compose.yml" \ "${LIVE}/docker-compose.yml" cd "${LIVE}" # ======================================================== # Start NEW server only # # From this point DB migration may start. # No automatic rollback after this flag. # ======================================================== log "Start new Zabbix server" UPGRADE_STARTED="$( date -u '+%Y-%m-%dT%H:%M:%SZ' )" MIGRATION_STARTED=true docker compose \ --env-file "${ENV_FILE}" \ up -d \ --no-deps \ zabbix-server # ======================================================== # Wait for automatic DB schema upgrade + server startup # ======================================================== log "Wait for Zabbix database upgrade and server startup" wait_for_zabbix_server # ======================================================== # Upgrade log # ======================================================== SERVER_LOG="$( docker logs \ --since "${UPGRADE_STARTED}" \ zabbix-server-pgsql 2>&1 )" echo echo "--- Zabbix server startup log ---" echo "${SERVER_LOG}" if echo "${SERVER_LOG}" | grep -q 'database upgrade fully completed' then echo echo "Database schema upgrade:" echo " fully completed" else echo echo "Database schema upgrade marker:" echo " not present" echo echo "This is normal when the target release" echo "does not require a schema change." fi # ======================================================== # Verify new server version # ======================================================== log "Validate Zabbix server version" RUNNING_VERSION_OUTPUT="$( docker exec \ zabbix-server-pgsql \ zabbix_server \ -V )" RUNNING_VERSION="$( printf '%s\n' "${RUNNING_VERSION_OUTPUT}" | extract_version )" [[ "${RUNNING_VERSION}" == "${TARGET_VERSION}" ]] || \ fail \ "Running Zabbix version ${RUNNING_VERSION} does not match ${TARGET_VERSION}" # ======================================================== # Start web after server/DB upgrade is complete # ======================================================== log "Start Zabbix web" docker compose \ --env-file "${ENV_FILE}" \ up -d \ --no-deps \ zabbix-web wait_for_zabbix_web # ============================================================ # NO VERSION CHANGE # ============================================================ else log "No Zabbix version change" cp \ "${SRC}/docker-compose.yml" \ "${LIVE}/docker-compose.yml" cd "${LIVE}" docker compose \ --env-file "${ENV_FILE}" \ pull docker compose \ --env-file "${ENV_FILE}" \ up -d \ --remove-orphans UPGRADE_STARTED="$( date -u -d '10 minutes ago' '+%Y-%m-%dT%H:%M:%SZ' )" wait_for_zabbix_server wait_for_zabbix_web fi # ============================================================ # Final server state # ============================================================ log "Final Zabbix server checks" SERVER_RUNNING="$( docker inspect \ -f '{{.State.Running}}' \ zabbix-server-pgsql )" [[ "${SERVER_RUNNING}" == "true" ]] || \ fail "Zabbix server is not running" FINAL_VERSION_OUTPUT="$( docker exec \ zabbix-server-pgsql \ zabbix_server \ -V )" FINAL_VERSION="$( printf '%s\n' "${FINAL_VERSION_OUTPUT}" | extract_version )" [[ "${FINAL_VERSION}" == "${TARGET_VERSION}" ]] || \ fail \ "Final Zabbix version mismatch: ${FINAL_VERSION} != ${TARGET_VERSION}" # ============================================================ # Final web state # ============================================================ WEB_RUNNING="$( docker inspect \ -f '{{.State.Running}}' \ zabbix-web-nginx-pgsql )" WEB_HEALTH="$( docker inspect \ -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ zabbix-web-nginx-pgsql )" [[ "${WEB_RUNNING}" == "true" ]] || \ fail "Zabbix web is not running" [[ "${WEB_HEALTH}" == "healthy" ]] || \ fail "Zabbix web is not healthy" # ============================================================ # Running images # ============================================================ RUNNING_SERVER_IMAGE="$( docker inspect \ -f '{{.Config.Image}}' \ zabbix-server-pgsql )" RUNNING_WEB_IMAGE="$( docker inspect \ -f '{{.Config.Image}}' \ zabbix-web-nginx-pgsql )" # ============================================================ # Success # ============================================================ log "Zabbix deployment successful" echo "Previous Zabbix version:" echo " ${CURRENT_VERSION}" echo echo "Running Zabbix version:" echo " ${FINAL_VERSION}" echo echo "Server image:" echo " ${RUNNING_SERVER_IMAGE}" echo echo "Web image:" echo " ${RUNNING_WEB_IMAGE}" echo echo "Web health:" echo " ${WEB_HEALTH}" if [[ "${VERSION_CHANGED}" == "true" ]] then echo echo "Pre-upgrade backup:" echo " ${BACKUP_DIR}" echo echo "PostgreSQL dump:" echo " ${DB_BACKUP_FILE}" echo echo "Backup retention:" echo " last ${BACKUP_KEEP} backups" fi # ============================================================ # Clear credentials # ============================================================ unset PGPASSWORD unset POSTGRES_PASSWORD