Files
home-infra/scripts/deploy-zabbix.sh
T
root bcd8bf7e3f
Zabbix CI/CD / Deploy Zabbix (push) Failing after 2s
Add Zabbix CI/CD with PostgreSQL backup
2026-09-30 11:11:35 +03:00

1213 lines
24 KiB
Bash
Executable File

#!/usr/bin/env bash
set -Eeuo pipefail
umask 077
# ============================================================
# Configuration
# ============================================================
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
SRC="${ROOT}/stacks/zabbix"
LIVE="/srv/nfs/infra/zabbix"
ENV_FILE="${LIVE}/.env"
BACKUP_ROOT="/srv/nfs/infra/.deploy-backups/zabbix"
BACKUP_KEEP=3
STAMP="$(date '+%Y%m%d-%H%M%S')"
BACKUP_DIR="${BACKUP_ROOT}/${STAMP}"
DB_BACKUP_DIR="${BACKUP_DIR}/database"
DB_BACKUP_FILE="${DB_BACKUP_DIR}/zabbix.dump"
SERVER_START_TIMEOUT=1800
SERVER_CHECK_INTERVAL=5
WEB_START_TIMEOUT=300
WEB_CHECK_INTERVAL=5
# ============================================================
# Runtime state
# ============================================================
SERVICES_STOPPED=false
MIGRATION_STARTED=false
VERSION_CHANGED=false
# ============================================================
# Helpers
# ============================================================
log()
{
echo
echo "============================================================"
echo "$*"
echo "============================================================"
}
fail()
{
echo
echo "ERROR: $*" >&2
exit 1
}
extract_version()
{
grep -oE '[0-9]+\.[0-9]+\.[0-9]+' |
head -n1
}
image_version()
{
local IMAGE="$1"
printf '%s\n' "${IMAGE}" |
sed -nE 's/.*:alpine-([0-9]+\.[0-9]+\.[0-9]+)$/\1/p'
}
version_less_than()
{
local A="$1"
local B="$2"
[[ "${A}" == "${B}" ]] && return 1
[[ "$(
printf '%s\n%s\n' "${A}" "${B}" |
sort -V |
head -n1
)" == "${A}" ]]
}
cleanup_old_backups()
{
echo
echo "--- Backup retention ---"
echo "Keeping last ${BACKUP_KEEP} pre-upgrade backups."
mkdir -p "${BACKUP_ROOT}"
mapfile -t BACKUPS < <(
find "${BACKUP_ROOT}" \
-mindepth 1 \
-maxdepth 1 \
-type d \
-printf '%f\n' |
grep -E '^[0-9]{8}-[0-9]{6}$' |
sort -r
)
if (( ${#BACKUPS[@]} <= BACKUP_KEEP )); then
echo "Current backup count: ${#BACKUPS[@]}"
echo "Nothing to delete."
return 0
fi
for BACKUP_NAME in "${BACKUPS[@]:${BACKUP_KEEP}}"
do
OLD_BACKUP="${BACKUP_ROOT}/${BACKUP_NAME}"
echo
echo "Removing old backup:"
echo " ${OLD_BACKUP}"
rm -rf -- "${OLD_BACKUP}"
done
}
restore_old_services()
{
echo
echo "--- Restore previous Zabbix services ---"
if [[ -f "${BACKUP_DIR}/docker-compose.yml" ]]; then
cp \
"${BACKUP_DIR}/docker-compose.yml" \
"${LIVE}/docker-compose.yml"
fi
(
cd "${LIVE}"
docker compose \
--env-file "${ENV_FILE}" \
up -d \
zabbix-server \
zabbix-web
) || true
}
on_error()
{
rc=$?
trap - ERR
echo
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
echo " Zabbix deployment FAILED"
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
if [[ -d "${BACKUP_DIR}" ]]; then
echo
echo "Pre-upgrade backup:"
echo " ${BACKUP_DIR}"
fi
#
# До старта нового zabbix-server схема БД точно
# не могла измениться.
#
# Поэтому старую версию можно безопасно вернуть.
#
if [[ "${SERVICES_STOPPED}" == "true" ]] &&
[[ "${MIGRATION_STARTED}" != "true" ]]
then
echo
echo "Database migration has NOT started."
echo "Restoring previous containers."
restore_old_services
else
echo
echo "Automatic rollback is NOT performed."
if [[ "${MIGRATION_STARTED}" == "true" ]]; then
echo
echo "Reason:"
echo " new Zabbix server has already been started;"
echo " database schema migration may have begun."
echo
echo "Rollback requires restoration of the"
echo "pre-upgrade PostgreSQL dump."
fi
fi
exit "${rc}"
}
wait_for_zabbix_server()
{
local elapsed=0
echo
echo "Waiting for Zabbix server startup..."
while (( elapsed < SERVER_START_TIMEOUT ))
do
RUNNING="$(
docker inspect \
-f '{{.State.Running}}' \
zabbix-server-pgsql 2>/dev/null ||
echo false
)"
if [[ "${RUNNING}" != "true" ]]; then
echo
echo "--- Zabbix server logs ---"
docker logs \
--since "${UPGRADE_STARTED}" \
zabbix-server-pgsql || true
fail "Zabbix server container stopped during startup"
fi
SERVER_LOG="$(
docker logs \
--since "${UPGRADE_STARTED}" \
zabbix-server-pgsql 2>&1 || true
)"
#
# Successful server start means mandatory DB schema
# is compatible. Zabbix does not start if required
# patches failed.
#
if echo "${SERVER_LOG}" |
grep -q 'server #0 started \[main process\]'
then
echo
echo "Zabbix server main process started."
return 0
fi
if echo "${SERVER_LOG}" |
grep -Eqi \
'database upgrade.*(failed|failure)|cannot upgrade database|database version.*newer'
then
echo "${SERVER_LOG}"
fail "Zabbix database upgrade failed"
fi
sleep "${SERVER_CHECK_INTERVAL}"
elapsed=$((elapsed + SERVER_CHECK_INTERVAL))
echo "Waiting... ${elapsed}s"
done
echo
echo "--- Zabbix server logs ---"
docker logs \
--since "${UPGRADE_STARTED}" \
zabbix-server-pgsql || true
fail \
"Timeout waiting for Zabbix server after ${SERVER_START_TIMEOUT}s"
}
wait_for_zabbix_web()
{
local elapsed=0
echo
echo "Waiting for Zabbix web health=healthy..."
while (( elapsed < WEB_START_TIMEOUT ))
do
RUNNING="$(
docker inspect \
-f '{{.State.Running}}' \
zabbix-web-nginx-pgsql 2>/dev/null ||
echo false
)"
if [[ "${RUNNING}" != "true" ]]; then
echo
echo "--- Zabbix web logs ---"
docker logs \
zabbix-web-nginx-pgsql || true
fail "Zabbix web container stopped"
fi
HEALTH="$(
docker inspect \
-f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
zabbix-web-nginx-pgsql
)"
echo "Zabbix web health: ${HEALTH} (${elapsed}s)"
if [[ "${HEALTH}" == "healthy" ]]; then
return 0
fi
sleep "${WEB_CHECK_INTERVAL}"
elapsed=$((elapsed + WEB_CHECK_INTERVAL))
done
docker logs \
zabbix-web-nginx-pgsql || true
fail \
"Timeout waiting for Zabbix web health=healthy"
}
trap on_error ERR
# ============================================================
# Required commands
# ============================================================
log "Pre-flight checks"
for CMD in \
docker \
psql \
pg_dump \
pg_restore \
sha256sum \
grep \
sed \
sort \
find \
awk
do
command -v "${CMD}" >/dev/null 2>&1 || \
fail "Required command not found: ${CMD}"
done
docker compose version >/dev/null 2>&1 || \
fail "Docker Compose plugin is not available"
# ============================================================
# Required files
# ============================================================
[[ -f "${SRC}/docker-compose.yml" ]] || \
fail "${SRC}/docker-compose.yml not found"
[[ -f "${LIVE}/docker-compose.yml" ]] || \
fail "${LIVE}/docker-compose.yml not found"
[[ -f "${ENV_FILE}" ]] || \
fail "${ENV_FILE} not found"
# ============================================================
# Backup directory
# ============================================================
mkdir -p "${BACKUP_ROOT}"
[[ -w "${BACKUP_ROOT}" ]] || \
fail "${BACKUP_ROOT} is not writable"
# ============================================================
# Load production environment
#
# .env must be shell compatible.
# ============================================================
set -a
# shellcheck disable=SC1090
source "${ENV_FILE}"
set +a
: "${POSTGRES_HOST:?POSTGRES_HOST not defined}"
: "${POSTGRES_PORT:?POSTGRES_PORT not defined}"
: "${POSTGRES_DB:?POSTGRES_DB not defined}"
: "${POSTGRES_USER:?POSTGRES_USER not defined}"
: "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD not defined}"
export PGPASSWORD="${POSTGRES_PASSWORD}"
export PGCONNECT_TIMEOUT=10
# ============================================================
# Validate Compose
# ============================================================
log "Validate Docker Compose"
docker compose \
--env-file "${ENV_FILE}" \
-f "${SRC}/docker-compose.yml" \
config >/dev/null
echo "Docker Compose configuration is valid."
# ============================================================
# Detect target images
# ============================================================
log "Detect target Zabbix images"
NEW_SERVER_IMAGE="$(
docker compose \
--env-file "${ENV_FILE}" \
-f "${SRC}/docker-compose.yml" \
config --images |
grep -m1 'zabbix/zabbix-server-pgsql:'
)"
NEW_WEB_IMAGE="$(
docker compose \
--env-file "${ENV_FILE}" \
-f "${SRC}/docker-compose.yml" \
config --images |
grep -m1 'zabbix/zabbix-web-nginx-pgsql:'
)"
[[ -n "${NEW_SERVER_IMAGE}" ]] || \
fail "Could not detect target Zabbix server image"
[[ -n "${NEW_WEB_IMAGE}" ]] || \
fail "Could not detect target Zabbix web image"
TARGET_SERVER_VERSION="$(
image_version "${NEW_SERVER_IMAGE}"
)"
TARGET_WEB_VERSION="$(
image_version "${NEW_WEB_IMAGE}"
)"
[[ -n "${TARGET_SERVER_VERSION}" ]] || \
fail \
"Zabbix server must use a pinned alpine-X.Y.Z tag"
[[ -n "${TARGET_WEB_VERSION}" ]] || \
fail \
"Zabbix web must use a pinned alpine-X.Y.Z tag"
[[ "${TARGET_SERVER_VERSION}" == "${TARGET_WEB_VERSION}" ]] || \
fail \
"Zabbix server/web versions differ: ${TARGET_SERVER_VERSION} != ${TARGET_WEB_VERSION}"
TARGET_VERSION="${TARGET_SERVER_VERSION}"
echo "Target server image:"
echo " ${NEW_SERVER_IMAGE}"
echo
echo "Target web image:"
echo " ${NEW_WEB_IMAGE}"
echo
echo "Target Zabbix version:"
echo " ${TARGET_VERSION}"
# ============================================================
# Current containers
# ============================================================
docker inspect zabbix-server-pgsql >/dev/null 2>&1 || \
fail "Current Zabbix server container does not exist"
docker inspect zabbix-web-nginx-pgsql >/dev/null 2>&1 || \
fail "Current Zabbix web container does not exist"
SERVER_RUNNING="$(
docker inspect \
-f '{{.State.Running}}' \
zabbix-server-pgsql
)"
WEB_RUNNING="$(
docker inspect \
-f '{{.State.Running}}' \
zabbix-web-nginx-pgsql
)"
[[ "${SERVER_RUNNING}" == "true" ]] || \
fail "Current Zabbix server is not running"
[[ "${WEB_RUNNING}" == "true" ]] || \
fail "Current Zabbix web is not running"
# ============================================================
# Current actual Zabbix version
# ============================================================
log "Detect current Zabbix version"
CURRENT_VERSION_OUTPUT="$(
docker exec \
zabbix-server-pgsql \
zabbix_server \
-V
)"
CURRENT_VERSION="$(
printf '%s\n' "${CURRENT_VERSION_OUTPUT}" |
extract_version
)"
[[ -n "${CURRENT_VERSION}" ]] || \
fail "Could not determine current Zabbix version"
OLD_SERVER_IMAGE="$(
docker inspect \
-f '{{.Config.Image}}' \
zabbix-server-pgsql
)"
OLD_WEB_IMAGE="$(
docker inspect \
-f '{{.Config.Image}}' \
zabbix-web-nginx-pgsql
)"
echo "Current Zabbix version:"
echo " ${CURRENT_VERSION}"
echo
echo "Current server image:"
echo " ${OLD_SERVER_IMAGE}"
echo
echo "Current web image:"
echo " ${OLD_WEB_IMAGE}"
# ============================================================
# Version validation
# ============================================================
log "Validate version transition"
if version_less_than \
"${TARGET_VERSION}" \
"${CURRENT_VERSION}"
then
fail \
"Zabbix downgrade is not allowed: ${CURRENT_VERSION} -> ${TARGET_VERSION}"
fi
if [[ "${CURRENT_VERSION}" != "${TARGET_VERSION}" ]]
then
VERSION_CHANGED=true
fi
echo "Current version: ${CURRENT_VERSION}"
echo "Target version: ${TARGET_VERSION}"
echo "Version changed: ${VERSION_CHANGED}"
# ============================================================
# Pull target images before downtime
# ============================================================
log "Pull target Docker images"
docker pull "${NEW_SERVER_IMAGE}"
docker pull "${NEW_WEB_IMAGE}"
# ============================================================
# VERSION UPDATE
# ============================================================
if [[ "${VERSION_CHANGED}" == "true" ]]
then
# ========================================================
# Prepare backup directory before shutdown
# ========================================================
log "Prepare Zabbix pre-upgrade backup"
mkdir -p "${DB_BACKUP_DIR}"
cp \
"${LIVE}/docker-compose.yml" \
"${BACKUP_DIR}/docker-compose.yml"
cat > "${BACKUP_DIR}/metadata.txt" <<EOF
timestamp=${STAMP}
current_version=${CURRENT_VERSION}
target_version=${TARGET_VERSION}
current_server_image=${OLD_SERVER_IMAGE}
target_server_image=${NEW_SERVER_IMAGE}
current_web_image=${OLD_WEB_IMAGE}
target_web_image=${NEW_WEB_IMAGE}
postgres_host=${POSTGRES_HOST}
postgres_port=${POSTGRES_PORT}
postgres_database=${POSTGRES_DB}
postgres_user=${POSTGRES_USER}
EOF
# ========================================================
# Stop application writes before DB backup
#
# Official Zabbix upgrade procedure requires stopping
# server before database backup.
# Web is also stopped to prevent configuration writes.
# ========================================================
log "Stop Zabbix before database backup"
cd "${LIVE}"
docker compose \
--env-file "${ENV_FILE}" \
stop \
zabbix-web \
zabbix-server
SERVICES_STOPPED=true
# ========================================================
# DB connection test
# ========================================================
log "Test PostgreSQL connection"
DB_TEST="$(
psql \
-h "${POSTGRES_HOST}" \
-p "${POSTGRES_PORT}" \
-U "${POSTGRES_USER}" \
-d "${POSTGRES_DB}" \
-Atqc 'SELECT 1;'
)"
[[ "${DB_TEST}" == "1" ]] || \
fail "Cannot connect to PostgreSQL"
# ========================================================
# PostgreSQL versions
# ========================================================
SERVER_VERSION="$(
psql \
-h "${POSTGRES_HOST}" \
-p "${POSTGRES_PORT}" \
-U "${POSTGRES_USER}" \
-d "${POSTGRES_DB}" \
-Atqc 'SHOW server_version;'
)"
echo "${SERVER_VERSION}" \
> "${DB_BACKUP_DIR}/postgresql-version.txt"
SERVER_MAJOR="${SERVER_VERSION%%.*}"
PG_DUMP_VERSION="$(
pg_dump --version |
sed -E 's/.* ([0-9]+(\.[0-9]+)?).*/\1/'
)"
PG_DUMP_MAJOR="${PG_DUMP_VERSION%%.*}"
echo "PostgreSQL server:"
echo " ${SERVER_VERSION}"
echo "pg_dump client:"
echo " ${PG_DUMP_VERSION}"
if (( PG_DUMP_MAJOR < SERVER_MAJOR ))
then
fail \
"pg_dump ${PG_DUMP_VERSION} is older than PostgreSQL server ${SERVER_VERSION}"
fi
# ========================================================
# DB size
# ========================================================
DB_SIZE="$(
psql \
-h "${POSTGRES_HOST}" \
-p "${POSTGRES_PORT}" \
-U "${POSTGRES_USER}" \
-d "${POSTGRES_DB}" \
-Atqc \
'SELECT pg_size_pretty(pg_database_size(current_database()));'
)"
echo "Database size:"
echo " ${DB_SIZE}"
# ========================================================
# pg_dump
# ========================================================
log "Create PostgreSQL backup"
pg_dump \
-h "${POSTGRES_HOST}" \
-p "${POSTGRES_PORT}" \
-U "${POSTGRES_USER}" \
-d "${POSTGRES_DB}" \
--format=custom \
--no-owner \
--no-privileges \
--file="${DB_BACKUP_FILE}"
[[ -s "${DB_BACKUP_FILE}" ]] || \
fail "PostgreSQL dump does not exist or is empty"
# ========================================================
# Validate dump
# ========================================================
log "Validate PostgreSQL backup"
pg_restore \
--list \
"${DB_BACKUP_FILE}" \
> "${DB_BACKUP_DIR}/zabbix.dump.list"
[[ -s "${DB_BACKUP_DIR}/zabbix.dump.list" ]] || \
fail "pg_restore could not validate dump"
# ========================================================
# SHA256
# ========================================================
(
cd "${DB_BACKUP_DIR}"
sha256sum \
zabbix.dump \
> zabbix.dump.sha256
sha256sum \
-c zabbix.dump.sha256
)
DB_BACKUP_SIZE="$(
du -h "${DB_BACKUP_FILE}" |
awk '{print $1}'
)"
echo
echo "PostgreSQL backup successful."
echo
echo "Backup:"
echo " ${DB_BACKUP_FILE}"
echo
echo "Dump size:"
echo " ${DB_BACKUP_SIZE}"
# ========================================================
# Retention
# ========================================================
cleanup_old_backups
# ========================================================
# Deploy new Compose
# ========================================================
log "Deploy new docker-compose.yml"
cp \
"${SRC}/docker-compose.yml" \
"${LIVE}/docker-compose.yml"
cd "${LIVE}"
# ========================================================
# Start NEW server only
#
# From this point DB migration may start.
# No automatic rollback after this flag.
# ========================================================
log "Start new Zabbix server"
UPGRADE_STARTED="$(
date -u '+%Y-%m-%dT%H:%M:%SZ'
)"
MIGRATION_STARTED=true
docker compose \
--env-file "${ENV_FILE}" \
up -d \
--no-deps \
zabbix-server
# ========================================================
# Wait for automatic DB schema upgrade + server startup
# ========================================================
log "Wait for Zabbix database upgrade and server startup"
wait_for_zabbix_server
# ========================================================
# Upgrade log
# ========================================================
SERVER_LOG="$(
docker logs \
--since "${UPGRADE_STARTED}" \
zabbix-server-pgsql 2>&1
)"
echo
echo "--- Zabbix server startup log ---"
echo "${SERVER_LOG}"
if echo "${SERVER_LOG}" |
grep -q 'database upgrade fully completed'
then
echo
echo "Database schema upgrade:"
echo " fully completed"
else
echo
echo "Database schema upgrade marker:"
echo " not present"
echo
echo "This is normal when the target release"
echo "does not require a schema change."
fi
# ========================================================
# Verify new server version
# ========================================================
log "Validate Zabbix server version"
RUNNING_VERSION_OUTPUT="$(
docker exec \
zabbix-server-pgsql \
zabbix_server \
-V
)"
RUNNING_VERSION="$(
printf '%s\n' "${RUNNING_VERSION_OUTPUT}" |
extract_version
)"
[[ "${RUNNING_VERSION}" == "${TARGET_VERSION}" ]] || \
fail \
"Running Zabbix version ${RUNNING_VERSION} does not match ${TARGET_VERSION}"
# ========================================================
# Start web after server/DB upgrade is complete
# ========================================================
log "Start Zabbix web"
docker compose \
--env-file "${ENV_FILE}" \
up -d \
--no-deps \
zabbix-web
wait_for_zabbix_web
# ============================================================
# NO VERSION CHANGE
# ============================================================
else
log "No Zabbix version change"
cp \
"${SRC}/docker-compose.yml" \
"${LIVE}/docker-compose.yml"
cd "${LIVE}"
docker compose \
--env-file "${ENV_FILE}" \
pull
docker compose \
--env-file "${ENV_FILE}" \
up -d \
--remove-orphans
UPGRADE_STARTED="$(
date -u -d '10 minutes ago' '+%Y-%m-%dT%H:%M:%SZ'
)
wait_for_zabbix_server
wait_for_zabbix_web
fi
# ============================================================
# Final server state
# ============================================================
log "Final Zabbix server checks"
SERVER_RUNNING="$(
docker inspect \
-f '{{.State.Running}}' \
zabbix-server-pgsql
)"
[[ "${SERVER_RUNNING}" == "true" ]] || \
fail "Zabbix server is not running"
FINAL_VERSION_OUTPUT="$(
docker exec \
zabbix-server-pgsql \
zabbix_server \
-V
)"
FINAL_VERSION="$(
printf '%s\n' "${FINAL_VERSION_OUTPUT}" |
extract_version
)"
[[ "${FINAL_VERSION}" == "${TARGET_VERSION}" ]] || \
fail \
"Final Zabbix version mismatch: ${FINAL_VERSION} != ${TARGET_VERSION}"
# ============================================================
# Final web state
# ============================================================
WEB_RUNNING="$(
docker inspect \
-f '{{.State.Running}}' \
zabbix-web-nginx-pgsql
)"
WEB_HEALTH="$(
docker inspect \
-f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
zabbix-web-nginx-pgsql
)"
[[ "${WEB_RUNNING}" == "true" ]] || \
fail "Zabbix web is not running"
[[ "${WEB_HEALTH}" == "healthy" ]] || \
fail "Zabbix web is not healthy"
# ============================================================
# Running images
# ============================================================
RUNNING_SERVER_IMAGE="$(
docker inspect \
-f '{{.Config.Image}}' \
zabbix-server-pgsql
)"
RUNNING_WEB_IMAGE="$(
docker inspect \
-f '{{.Config.Image}}' \
zabbix-web-nginx-pgsql
)"
# ============================================================
# Success
# ============================================================
log "Zabbix deployment successful"
echo "Previous Zabbix version:"
echo " ${CURRENT_VERSION}"
echo
echo "Running Zabbix version:"
echo " ${FINAL_VERSION}"
echo
echo "Server image:"
echo " ${RUNNING_SERVER_IMAGE}"
echo
echo "Web image:"
echo " ${RUNNING_WEB_IMAGE}"
echo
echo "Web health:"
echo " ${WEB_HEALTH}"
if [[ "${VERSION_CHANGED}" == "true" ]]
then
echo
echo "Pre-upgrade backup:"
echo " ${BACKUP_DIR}"
echo
echo "PostgreSQL dump:"
echo " ${DB_BACKUP_FILE}"
echo
echo "Backup retention:"
echo " last ${BACKUP_KEEP} backups"
fi
# ============================================================
# Clear credentials
# ============================================================
unset PGPASSWORD
unset POSTGRES_PASSWORD