1481 lines
30 KiB
Bash
Executable File
1481 lines
30 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -Eeuo pipefail
|
|
|
|
umask 077
|
|
|
|
|
|
# ============================================================
|
|
# Configuration
|
|
# ============================================================
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
|
|
SRC="${ROOT}/stacks/zabbix"
|
|
|
|
LIVE="/srv/nfs/infra/zabbix"
|
|
ENV_FILE="${LIVE}/.env"
|
|
|
|
BACKUP_ROOT="/srv/nfs/infra/.deploy-backups/zabbix"
|
|
BACKUP_KEEP=3
|
|
|
|
STAMP="$(date '+%Y%m%d-%H%M%S')"
|
|
|
|
BACKUP_DIR="${BACKUP_ROOT}/${STAMP}"
|
|
DB_BACKUP_DIR="${BACKUP_DIR}/database"
|
|
DB_BACKUP_FILE="${DB_BACKUP_DIR}/zabbix.dump"
|
|
|
|
SERVER_START_TIMEOUT=1800
|
|
SERVER_CHECK_INTERVAL=5
|
|
|
|
WEB_START_TIMEOUT=300
|
|
WEB_CHECK_INTERVAL=5
|
|
|
|
|
|
# ============================================================
|
|
# Runtime state
|
|
# ============================================================
|
|
|
|
SERVICES_STOPPED=false
|
|
MIGRATION_STARTED=false
|
|
VERSION_CHANGED=false
|
|
|
|
|
|
# ============================================================
|
|
# Helpers
|
|
# ============================================================
|
|
|
|
log()
|
|
{
|
|
echo
|
|
echo "============================================================"
|
|
echo "$*"
|
|
echo "============================================================"
|
|
}
|
|
|
|
|
|
fail()
|
|
{
|
|
echo
|
|
echo "ERROR: $*" >&2
|
|
exit 1
|
|
}
|
|
|
|
|
|
extract_version()
|
|
{
|
|
grep -oE '[0-9]+\.[0-9]+\.[0-9]+' |
|
|
head -n1
|
|
}
|
|
|
|
|
|
image_version()
|
|
{
|
|
local IMAGE="$1"
|
|
|
|
printf '%s\n' "${IMAGE}" |
|
|
sed -nE 's/.*:alpine-([0-9]+\.[0-9]+\.[0-9]+)$/\1/p'
|
|
}
|
|
|
|
|
|
version_less_than()
|
|
{
|
|
local A="$1"
|
|
local B="$2"
|
|
|
|
[[ "${A}" == "${B}" ]] && return 1
|
|
|
|
[[ "$(
|
|
printf '%s\n%s\n' "${A}" "${B}" |
|
|
sort -V |
|
|
head -n1
|
|
)" == "${A}" ]]
|
|
}
|
|
|
|
|
|
cleanup_old_backups()
|
|
{
|
|
echo
|
|
echo "--- Backup retention ---"
|
|
echo "Keeping last ${BACKUP_KEEP} pre-upgrade backups."
|
|
|
|
mkdir -p "${BACKUP_ROOT}"
|
|
|
|
mapfile -t BACKUPS < <(
|
|
find "${BACKUP_ROOT}" \
|
|
-mindepth 1 \
|
|
-maxdepth 1 \
|
|
-type d \
|
|
-printf '%f\n' |
|
|
grep -E '^[0-9]{8}-[0-9]{6}$' |
|
|
sort -r
|
|
)
|
|
|
|
|
|
if (( ${#BACKUPS[@]} <= BACKUP_KEEP )); then
|
|
|
|
echo "Current backup count: ${#BACKUPS[@]}"
|
|
echo "Nothing to delete."
|
|
|
|
return 0
|
|
|
|
fi
|
|
|
|
|
|
for BACKUP_NAME in "${BACKUPS[@]:${BACKUP_KEEP}}"
|
|
do
|
|
|
|
OLD_BACKUP="${BACKUP_ROOT}/${BACKUP_NAME}"
|
|
|
|
echo
|
|
echo "Removing old backup:"
|
|
echo " ${OLD_BACKUP}"
|
|
|
|
rm -rf -- "${OLD_BACKUP}"
|
|
|
|
done
|
|
}
|
|
|
|
|
|
restore_old_services()
|
|
{
|
|
echo
|
|
echo "--- Restore previous Zabbix services ---"
|
|
|
|
if [[ -f "${BACKUP_DIR}/docker-compose.yml" ]]; then
|
|
|
|
echo "Restoring previous docker-compose.yml"
|
|
|
|
cp \
|
|
"${BACKUP_DIR}/docker-compose.yml" \
|
|
"${LIVE}/docker-compose.yml"
|
|
|
|
fi
|
|
|
|
|
|
(
|
|
cd "${LIVE}"
|
|
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
up -d \
|
|
zabbix-server \
|
|
zabbix-web
|
|
|
|
) || true
|
|
}
|
|
|
|
|
|
check_zabbix_server_tcp()
|
|
{
|
|
local HOST="${1:-127.0.0.1}"
|
|
local PORT="${2:-10051}"
|
|
local ATTEMPTS=30
|
|
local DELAY=2
|
|
|
|
echo
|
|
echo "Checking Zabbix server TCP ${HOST}:${PORT}..."
|
|
|
|
|
|
for ((i=1; i<=ATTEMPTS; i++))
|
|
do
|
|
|
|
if timeout 2 \
|
|
bash -c \
|
|
"exec 3<>/dev/tcp/${HOST}/${PORT}" \
|
|
2>/dev/null
|
|
then
|
|
|
|
echo "Zabbix server TCP ${HOST}:${PORT}: available"
|
|
|
|
return 0
|
|
|
|
fi
|
|
|
|
|
|
echo \
|
|
"Waiting for TCP ${HOST}:${PORT}... (${i}/${ATTEMPTS})"
|
|
|
|
sleep "${DELAY}"
|
|
|
|
done
|
|
|
|
|
|
fail \
|
|
"Zabbix server does not accept TCP connections on ${HOST}:${PORT}"
|
|
}
|
|
|
|
|
|
wait_for_zabbix_server()
|
|
{
|
|
local ELAPSED=0
|
|
|
|
echo
|
|
echo "Waiting for Zabbix server startup..."
|
|
|
|
|
|
while (( ELAPSED < SERVER_START_TIMEOUT ))
|
|
do
|
|
|
|
RUNNING="$(
|
|
docker inspect \
|
|
-f '{{.State.Running}}' \
|
|
zabbix-server-pgsql \
|
|
2>/dev/null ||
|
|
echo false
|
|
)"
|
|
|
|
|
|
if [[ "${RUNNING}" != "true" ]]; then
|
|
|
|
echo
|
|
echo "--- Zabbix server logs ---"
|
|
|
|
docker logs \
|
|
--since "${UPGRADE_STARTED}" \
|
|
zabbix-server-pgsql \
|
|
|| true
|
|
|
|
fail "Zabbix server container stopped during startup"
|
|
|
|
fi
|
|
|
|
|
|
SERVER_LOG="$(
|
|
docker logs \
|
|
--since "${UPGRADE_STARTED}" \
|
|
zabbix-server-pgsql \
|
|
2>&1 ||
|
|
true
|
|
)"
|
|
|
|
|
|
# ----------------------------------------------------
|
|
# Explicit database upgrade failures
|
|
# ----------------------------------------------------
|
|
|
|
if echo "${SERVER_LOG}" |
|
|
grep -Eqi \
|
|
'database upgrade.*(failed|failure)|cannot upgrade database|database version.*newer'
|
|
then
|
|
|
|
echo "${SERVER_LOG}"
|
|
|
|
fail "Zabbix database upgrade failed"
|
|
|
|
fi
|
|
|
|
|
|
# ----------------------------------------------------
|
|
# Standard Zabbix startup marker
|
|
# ----------------------------------------------------
|
|
|
|
if echo "${SERVER_LOG}" |
|
|
grep -q 'server #0 started'
|
|
then
|
|
|
|
echo
|
|
echo "Zabbix server main process started."
|
|
|
|
return 0
|
|
|
|
fi
|
|
|
|
|
|
# ----------------------------------------------------
|
|
# Functional fallback:
|
|
# TCP/10051 is already accepting connections
|
|
# ----------------------------------------------------
|
|
|
|
if timeout 2 \
|
|
bash -c \
|
|
'exec 3<>/dev/tcp/127.0.0.1/10051' \
|
|
2>/dev/null
|
|
then
|
|
|
|
echo
|
|
echo "Zabbix server TCP/10051 is available."
|
|
|
|
return 0
|
|
|
|
fi
|
|
|
|
|
|
sleep "${SERVER_CHECK_INTERVAL}"
|
|
|
|
ELAPSED=$((ELAPSED + SERVER_CHECK_INTERVAL))
|
|
|
|
echo "Waiting... ${ELAPSED}s"
|
|
|
|
done
|
|
|
|
|
|
echo
|
|
echo "--- Zabbix server logs ---"
|
|
|
|
docker logs \
|
|
--since "${UPGRADE_STARTED}" \
|
|
zabbix-server-pgsql \
|
|
|| true
|
|
|
|
|
|
fail \
|
|
"Timeout waiting for Zabbix server after ${SERVER_START_TIMEOUT}s"
|
|
}
|
|
|
|
|
|
wait_for_zabbix_web()
|
|
{
|
|
local ELAPSED=0
|
|
|
|
echo
|
|
echo "Waiting for Zabbix web health=healthy..."
|
|
|
|
|
|
while (( ELAPSED < WEB_START_TIMEOUT ))
|
|
do
|
|
|
|
RUNNING="$(
|
|
docker inspect \
|
|
-f '{{.State.Running}}' \
|
|
zabbix-web-nginx-pgsql \
|
|
2>/dev/null ||
|
|
echo false
|
|
)"
|
|
|
|
|
|
if [[ "${RUNNING}" != "true" ]]; then
|
|
|
|
echo
|
|
echo "--- Zabbix web logs ---"
|
|
|
|
docker logs \
|
|
--tail 200 \
|
|
zabbix-web-nginx-pgsql \
|
|
|| true
|
|
|
|
fail "Zabbix web container stopped"
|
|
|
|
fi
|
|
|
|
|
|
HEALTH="$(
|
|
docker inspect \
|
|
-f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
|
|
zabbix-web-nginx-pgsql
|
|
)"
|
|
|
|
|
|
echo "Zabbix web health: ${HEALTH} (${ELAPSED}s)"
|
|
|
|
|
|
if [[ "${HEALTH}" == "healthy" ]]; then
|
|
|
|
echo
|
|
echo "Zabbix web is healthy."
|
|
|
|
return 0
|
|
|
|
fi
|
|
|
|
|
|
sleep "${WEB_CHECK_INTERVAL}"
|
|
|
|
ELAPSED=$((ELAPSED + WEB_CHECK_INTERVAL))
|
|
|
|
done
|
|
|
|
|
|
echo
|
|
echo "--- Zabbix web logs ---"
|
|
|
|
docker logs \
|
|
--tail 200 \
|
|
zabbix-web-nginx-pgsql \
|
|
|| true
|
|
|
|
|
|
fail \
|
|
"Timeout waiting for Zabbix web health=healthy after ${WEB_START_TIMEOUT}s"
|
|
}
|
|
|
|
|
|
on_error()
|
|
{
|
|
rc=$?
|
|
|
|
trap - ERR
|
|
|
|
echo
|
|
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
|
echo " Zabbix deployment FAILED"
|
|
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
|
|
|
|
|
if [[ -d "${BACKUP_DIR}" ]]; then
|
|
|
|
echo
|
|
echo "Pre-upgrade backup:"
|
|
echo " ${BACKUP_DIR}"
|
|
|
|
fi
|
|
|
|
|
|
# --------------------------------------------------------
|
|
# If new Zabbix server has NOT been started yet,
|
|
# DB migration cannot have started.
|
|
#
|
|
# In this case restoring the old Compose/services is safe.
|
|
# --------------------------------------------------------
|
|
|
|
if [[ "${SERVICES_STOPPED}" == "true" ]] &&
|
|
[[ "${MIGRATION_STARTED}" != "true" ]]
|
|
then
|
|
|
|
echo
|
|
echo "Database migration has NOT started."
|
|
echo "Restoring previous Zabbix containers."
|
|
|
|
restore_old_services
|
|
|
|
else
|
|
|
|
echo
|
|
echo "Automatic rollback is NOT performed."
|
|
|
|
if [[ "${MIGRATION_STARTED}" == "true" ]]; then
|
|
|
|
echo
|
|
echo "Reason:"
|
|
echo " New Zabbix server has already been started."
|
|
echo " Database schema migration may have begun."
|
|
echo
|
|
echo "Rollback requires restoration of the"
|
|
echo "pre-upgrade PostgreSQL dump."
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
exit "${rc}"
|
|
}
|
|
|
|
|
|
trap on_error ERR
|
|
|
|
|
|
# ============================================================
|
|
# Required commands
|
|
# ============================================================
|
|
|
|
log "Pre-flight checks"
|
|
|
|
for CMD in \
|
|
docker \
|
|
psql \
|
|
pg_dump \
|
|
pg_restore \
|
|
sha256sum \
|
|
grep \
|
|
sed \
|
|
sort \
|
|
find \
|
|
awk \
|
|
timeout
|
|
do
|
|
|
|
command -v "${CMD}" >/dev/null 2>&1 || \
|
|
fail "Required command not found: ${CMD}"
|
|
|
|
done
|
|
|
|
|
|
docker compose version >/dev/null 2>&1 || \
|
|
fail "Docker Compose plugin is not available"
|
|
|
|
|
|
# ============================================================
|
|
# Required files
|
|
# ============================================================
|
|
|
|
[[ -f "${SRC}/docker-compose.yml" ]] || \
|
|
fail "${SRC}/docker-compose.yml not found"
|
|
|
|
[[ -f "${LIVE}/docker-compose.yml" ]] || \
|
|
fail "${LIVE}/docker-compose.yml not found"
|
|
|
|
[[ -f "${ENV_FILE}" ]] || \
|
|
fail "${ENV_FILE} not found"
|
|
|
|
|
|
# ============================================================
|
|
# Backup directory
|
|
# ============================================================
|
|
|
|
mkdir -p "${BACKUP_ROOT}"
|
|
|
|
[[ -w "${BACKUP_ROOT}" ]] || \
|
|
fail "${BACKUP_ROOT} is not writable"
|
|
|
|
|
|
# ============================================================
|
|
# Load production environment
|
|
# ============================================================
|
|
|
|
set -a
|
|
|
|
# shellcheck disable=SC1090
|
|
source "${ENV_FILE}"
|
|
|
|
set +a
|
|
|
|
|
|
# ============================================================
|
|
# Normalize PostgreSQL connection settings
|
|
#
|
|
# Preferred Zabbix variables:
|
|
#
|
|
# DB_SERVER_HOST
|
|
# DB_SERVER_PORT
|
|
#
|
|
# Legacy fallback:
|
|
#
|
|
# POSTGRES_HOST
|
|
# POSTGRES_PORT
|
|
# ============================================================
|
|
|
|
DB_HOST="${DB_SERVER_HOST:-${POSTGRES_HOST:-}}"
|
|
DB_PORT="${DB_SERVER_PORT:-${POSTGRES_PORT:-5432}}"
|
|
DB_NAME="${POSTGRES_DB:-}"
|
|
DB_USER="${POSTGRES_USER:-}"
|
|
DB_PASSWORD="${POSTGRES_PASSWORD:-}"
|
|
|
|
|
|
: "${DB_HOST:?DB_SERVER_HOST not defined}"
|
|
: "${DB_PORT:?DB_SERVER_PORT not defined}"
|
|
: "${DB_NAME:?POSTGRES_DB not defined}"
|
|
: "${DB_USER:?POSTGRES_USER not defined}"
|
|
: "${DB_PASSWORD:?POSTGRES_PASSWORD not defined}"
|
|
|
|
|
|
export PGPASSWORD="${DB_PASSWORD}"
|
|
export PGCONNECT_TIMEOUT=10
|
|
|
|
|
|
# ============================================================
|
|
# Validate Compose
|
|
# ============================================================
|
|
|
|
log "Validate Docker Compose"
|
|
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
-f "${SRC}/docker-compose.yml" \
|
|
config >/dev/null
|
|
|
|
|
|
echo "Docker Compose configuration is valid."
|
|
|
|
|
|
# ============================================================
|
|
# Detect target Zabbix images
|
|
# ============================================================
|
|
|
|
log "Detect target Zabbix images"
|
|
|
|
|
|
NEW_SERVER_IMAGE="$(
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
-f "${SRC}/docker-compose.yml" \
|
|
config --images |
|
|
grep -m1 'zabbix/zabbix-server-pgsql:'
|
|
)"
|
|
|
|
|
|
NEW_WEB_IMAGE="$(
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
-f "${SRC}/docker-compose.yml" \
|
|
config --images |
|
|
grep -m1 'zabbix/zabbix-web-nginx-pgsql:'
|
|
)"
|
|
|
|
|
|
[[ -n "${NEW_SERVER_IMAGE}" ]] || \
|
|
fail "Could not detect target Zabbix server image"
|
|
|
|
[[ -n "${NEW_WEB_IMAGE}" ]] || \
|
|
fail "Could not detect target Zabbix web image"
|
|
|
|
|
|
TARGET_SERVER_VERSION="$(
|
|
image_version "${NEW_SERVER_IMAGE}"
|
|
)"
|
|
|
|
|
|
TARGET_WEB_VERSION="$(
|
|
image_version "${NEW_WEB_IMAGE}"
|
|
)"
|
|
|
|
|
|
[[ -n "${TARGET_SERVER_VERSION}" ]] || \
|
|
fail "Zabbix server must use pinned tag alpine-X.Y.Z"
|
|
|
|
[[ -n "${TARGET_WEB_VERSION}" ]] || \
|
|
fail "Zabbix web must use pinned tag alpine-X.Y.Z"
|
|
|
|
|
|
[[ "${TARGET_SERVER_VERSION}" == "${TARGET_WEB_VERSION}" ]] || \
|
|
fail \
|
|
"Zabbix server/web versions differ: ${TARGET_SERVER_VERSION} != ${TARGET_WEB_VERSION}"
|
|
|
|
|
|
TARGET_VERSION="${TARGET_SERVER_VERSION}"
|
|
|
|
|
|
echo "Target server image:"
|
|
echo " ${NEW_SERVER_IMAGE}"
|
|
|
|
echo
|
|
echo "Target web image:"
|
|
echo " ${NEW_WEB_IMAGE}"
|
|
|
|
echo
|
|
echo "Target Zabbix version:"
|
|
echo " ${TARGET_VERSION}"
|
|
|
|
|
|
# ============================================================
|
|
# Current containers
|
|
# ============================================================
|
|
|
|
docker inspect zabbix-server-pgsql >/dev/null 2>&1 || \
|
|
fail "Current Zabbix server container does not exist"
|
|
|
|
docker inspect zabbix-web-nginx-pgsql >/dev/null 2>&1 || \
|
|
fail "Current Zabbix web container does not exist"
|
|
|
|
|
|
SERVER_RUNNING="$(
|
|
docker inspect \
|
|
-f '{{.State.Running}}' \
|
|
zabbix-server-pgsql
|
|
)"
|
|
|
|
|
|
WEB_RUNNING="$(
|
|
docker inspect \
|
|
-f '{{.State.Running}}' \
|
|
zabbix-web-nginx-pgsql
|
|
)"
|
|
|
|
|
|
[[ "${SERVER_RUNNING}" == "true" ]] || \
|
|
fail "Current Zabbix server is not running"
|
|
|
|
[[ "${WEB_RUNNING}" == "true" ]] || \
|
|
fail "Current Zabbix web is not running"
|
|
|
|
|
|
# ============================================================
|
|
# Detect current actual Zabbix version
|
|
# ============================================================
|
|
|
|
log "Detect current Zabbix version"
|
|
|
|
|
|
CURRENT_VERSION_OUTPUT="$(
|
|
docker exec \
|
|
zabbix-server-pgsql \
|
|
zabbix_server \
|
|
-V
|
|
)"
|
|
|
|
|
|
CURRENT_VERSION="$(
|
|
printf '%s\n' "${CURRENT_VERSION_OUTPUT}" |
|
|
extract_version
|
|
)"
|
|
|
|
|
|
[[ -n "${CURRENT_VERSION}" ]] || \
|
|
fail "Could not determine current Zabbix version"
|
|
|
|
|
|
OLD_SERVER_IMAGE="$(
|
|
docker inspect \
|
|
-f '{{.Config.Image}}' \
|
|
zabbix-server-pgsql
|
|
)"
|
|
|
|
|
|
OLD_WEB_IMAGE="$(
|
|
docker inspect \
|
|
-f '{{.Config.Image}}' \
|
|
zabbix-web-nginx-pgsql
|
|
)"
|
|
|
|
|
|
echo "Current Zabbix version:"
|
|
echo " ${CURRENT_VERSION}"
|
|
|
|
echo
|
|
echo "Current server image:"
|
|
echo " ${OLD_SERVER_IMAGE}"
|
|
|
|
echo
|
|
echo "Current web image:"
|
|
echo " ${OLD_WEB_IMAGE}"
|
|
|
|
|
|
# ============================================================
|
|
# Validate version transition
|
|
# ============================================================
|
|
|
|
log "Validate version transition"
|
|
|
|
|
|
if version_less_than \
|
|
"${TARGET_VERSION}" \
|
|
"${CURRENT_VERSION}"
|
|
then
|
|
|
|
fail \
|
|
"Zabbix downgrade is not allowed: ${CURRENT_VERSION} -> ${TARGET_VERSION}"
|
|
|
|
fi
|
|
|
|
|
|
if [[ "${CURRENT_VERSION}" != "${TARGET_VERSION}" ]]; then
|
|
|
|
VERSION_CHANGED=true
|
|
|
|
fi
|
|
|
|
|
|
echo "Current version: ${CURRENT_VERSION}"
|
|
echo "Target version: ${TARGET_VERSION}"
|
|
echo "Version changed: ${VERSION_CHANGED}"
|
|
|
|
|
|
# ============================================================
|
|
# Pull target images before any downtime
|
|
# ============================================================
|
|
|
|
log "Pull target Docker images"
|
|
|
|
|
|
docker pull "${NEW_SERVER_IMAGE}"
|
|
docker pull "${NEW_WEB_IMAGE}"
|
|
|
|
|
|
# ============================================================
|
|
# ZABBIX VERSION UPDATE
|
|
# ============================================================
|
|
|
|
if [[ "${VERSION_CHANGED}" == "true" ]]
|
|
then
|
|
|
|
# ========================================================
|
|
# Prepare backup directory
|
|
# ========================================================
|
|
|
|
log "Prepare Zabbix pre-upgrade backup"
|
|
|
|
|
|
mkdir -p "${DB_BACKUP_DIR}"
|
|
|
|
|
|
# --------------------------------------------------------
|
|
# Backup current compose
|
|
#
|
|
# Do NOT use cp -a because current NFS does not support
|
|
# preserving all POSIX metadata.
|
|
# --------------------------------------------------------
|
|
|
|
cp \
|
|
"${LIVE}/docker-compose.yml" \
|
|
"${BACKUP_DIR}/docker-compose.yml"
|
|
|
|
|
|
# --------------------------------------------------------
|
|
# Metadata
|
|
# --------------------------------------------------------
|
|
|
|
cat > "${BACKUP_DIR}/metadata.txt" <<EOF
|
|
timestamp=${STAMP}
|
|
|
|
current_version=${CURRENT_VERSION}
|
|
target_version=${TARGET_VERSION}
|
|
|
|
current_server_image=${OLD_SERVER_IMAGE}
|
|
target_server_image=${NEW_SERVER_IMAGE}
|
|
|
|
current_web_image=${OLD_WEB_IMAGE}
|
|
target_web_image=${NEW_WEB_IMAGE}
|
|
|
|
postgres_host=${DB_HOST}
|
|
postgres_port=${DB_PORT}
|
|
postgres_database=${DB_NAME}
|
|
postgres_user=${DB_USER}
|
|
EOF
|
|
|
|
|
|
# ========================================================
|
|
# Stop Zabbix before database backup
|
|
#
|
|
# Server is stopped to freeze DB writes.
|
|
# Web is also stopped to prevent configuration changes.
|
|
# ========================================================
|
|
|
|
log "Stop Zabbix before database backup"
|
|
|
|
|
|
cd "${LIVE}"
|
|
|
|
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
stop \
|
|
zabbix-web \
|
|
zabbix-server
|
|
|
|
|
|
SERVICES_STOPPED=true
|
|
|
|
|
|
# ========================================================
|
|
# PostgreSQL connectivity
|
|
# ========================================================
|
|
|
|
log "Test PostgreSQL connection"
|
|
|
|
|
|
DB_TEST="$(
|
|
psql \
|
|
-h "${DB_HOST}" \
|
|
-p "${DB_PORT}" \
|
|
-U "${DB_USER}" \
|
|
-d "${DB_NAME}" \
|
|
-Atqc 'SELECT 1;'
|
|
)"
|
|
|
|
|
|
[[ "${DB_TEST}" == "1" ]] || \
|
|
fail "Cannot connect to PostgreSQL"
|
|
|
|
|
|
echo "PostgreSQL connection successful."
|
|
|
|
|
|
# ========================================================
|
|
# PostgreSQL version compatibility
|
|
# ========================================================
|
|
|
|
log "Check PostgreSQL versions"
|
|
|
|
|
|
SERVER_VERSION="$(
|
|
psql \
|
|
-h "${DB_HOST}" \
|
|
-p "${DB_PORT}" \
|
|
-U "${DB_USER}" \
|
|
-d "${DB_NAME}" \
|
|
-Atqc 'SHOW server_version;'
|
|
)"
|
|
|
|
|
|
[[ -n "${SERVER_VERSION}" ]] || \
|
|
fail "Could not determine PostgreSQL server version"
|
|
|
|
|
|
echo "${SERVER_VERSION}" \
|
|
> "${DB_BACKUP_DIR}/postgresql-version.txt"
|
|
|
|
|
|
SERVER_MAJOR="${SERVER_VERSION%%.*}"
|
|
|
|
|
|
PG_DUMP_VERSION="$(
|
|
pg_dump --version |
|
|
sed -E 's/.* ([0-9]+(\.[0-9]+)?).*/\1/'
|
|
)"
|
|
|
|
|
|
PG_DUMP_MAJOR="${PG_DUMP_VERSION%%.*}"
|
|
|
|
|
|
echo "PostgreSQL server:"
|
|
echo " ${SERVER_VERSION}"
|
|
|
|
echo "pg_dump client:"
|
|
echo " ${PG_DUMP_VERSION}"
|
|
|
|
|
|
if (( PG_DUMP_MAJOR < SERVER_MAJOR )); then
|
|
|
|
fail \
|
|
"pg_dump ${PG_DUMP_VERSION} is older than PostgreSQL server ${SERVER_VERSION}"
|
|
|
|
fi
|
|
|
|
|
|
# ========================================================
|
|
# Database size
|
|
# ========================================================
|
|
|
|
DB_SIZE="$(
|
|
psql \
|
|
-h "${DB_HOST}" \
|
|
-p "${DB_PORT}" \
|
|
-U "${DB_USER}" \
|
|
-d "${DB_NAME}" \
|
|
-Atqc \
|
|
'SELECT pg_size_pretty(pg_database_size(current_database()));'
|
|
)"
|
|
|
|
|
|
echo
|
|
echo "Database:"
|
|
echo " ${DB_NAME}"
|
|
|
|
echo "Database size:"
|
|
echo " ${DB_SIZE}"
|
|
|
|
|
|
# ========================================================
|
|
# PostgreSQL dump
|
|
# ========================================================
|
|
|
|
log "Create PostgreSQL backup"
|
|
|
|
|
|
pg_dump \
|
|
-h "${DB_HOST}" \
|
|
-p "${DB_PORT}" \
|
|
-U "${DB_USER}" \
|
|
-d "${DB_NAME}" \
|
|
--format=custom \
|
|
--no-owner \
|
|
--no-privileges \
|
|
--file="${DB_BACKUP_FILE}"
|
|
|
|
|
|
[[ -s "${DB_BACKUP_FILE}" ]] || \
|
|
fail "PostgreSQL dump does not exist or is empty"
|
|
|
|
|
|
# ========================================================
|
|
# Validate dump
|
|
# ========================================================
|
|
|
|
log "Validate PostgreSQL backup"
|
|
|
|
|
|
pg_restore \
|
|
--list \
|
|
"${DB_BACKUP_FILE}" \
|
|
> "${DB_BACKUP_DIR}/zabbix.dump.list"
|
|
|
|
|
|
[[ -s "${DB_BACKUP_DIR}/zabbix.dump.list" ]] || \
|
|
fail "pg_restore could not validate PostgreSQL dump"
|
|
|
|
|
|
# ========================================================
|
|
# SHA256
|
|
# ========================================================
|
|
|
|
(
|
|
cd "${DB_BACKUP_DIR}"
|
|
|
|
sha256sum \
|
|
zabbix.dump \
|
|
> zabbix.dump.sha256
|
|
|
|
sha256sum \
|
|
-c zabbix.dump.sha256
|
|
)
|
|
|
|
|
|
DB_BACKUP_SIZE="$(
|
|
du -h "${DB_BACKUP_FILE}" |
|
|
awk '{print $1}'
|
|
)"
|
|
|
|
|
|
echo
|
|
echo "PostgreSQL backup successful."
|
|
|
|
echo
|
|
echo "Backup directory:"
|
|
echo " ${BACKUP_DIR}"
|
|
|
|
echo
|
|
echo "Database dump:"
|
|
echo " ${DB_BACKUP_FILE}"
|
|
|
|
echo
|
|
echo "Dump size:"
|
|
echo " ${DB_BACKUP_SIZE}"
|
|
|
|
echo
|
|
echo "SHA256:"
|
|
cat "${DB_BACKUP_DIR}/zabbix.dump.sha256"
|
|
|
|
|
|
# ========================================================
|
|
# Retention
|
|
#
|
|
# Old backups are deleted only after the new dump
|
|
# has been created and validated successfully.
|
|
# ========================================================
|
|
|
|
cleanup_old_backups
|
|
|
|
|
|
# ========================================================
|
|
# Deploy new Compose
|
|
# ========================================================
|
|
|
|
log "Deploy new docker-compose.yml"
|
|
|
|
|
|
cp \
|
|
"${SRC}/docker-compose.yml" \
|
|
"${LIVE}/docker-compose.yml"
|
|
|
|
|
|
cd "${LIVE}"
|
|
|
|
|
|
# ========================================================
|
|
# Start new Zabbix server only
|
|
#
|
|
# After this point DB migration may begin.
|
|
# Automatic rollback is therefore disabled.
|
|
# ========================================================
|
|
|
|
log "Start new Zabbix server"
|
|
|
|
|
|
UPGRADE_STARTED="$(
|
|
date -u '+%Y-%m-%dT%H:%M:%SZ'
|
|
)"
|
|
|
|
|
|
MIGRATION_STARTED=true
|
|
|
|
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
up -d \
|
|
--no-deps \
|
|
zabbix-server
|
|
|
|
|
|
# ========================================================
|
|
# Wait for DB upgrade / server startup
|
|
# ========================================================
|
|
|
|
log "Wait for Zabbix database upgrade and server startup"
|
|
|
|
|
|
wait_for_zabbix_server
|
|
|
|
|
|
# ========================================================
|
|
# Capture startup log
|
|
# ========================================================
|
|
|
|
SERVER_LOG="$(
|
|
docker logs \
|
|
--since "${UPGRADE_STARTED}" \
|
|
zabbix-server-pgsql \
|
|
2>&1
|
|
)"
|
|
|
|
|
|
echo
|
|
echo "--- Zabbix server startup log ---"
|
|
echo "${SERVER_LOG}"
|
|
|
|
|
|
if echo "${SERVER_LOG}" |
|
|
grep -q 'database upgrade fully completed'
|
|
then
|
|
|
|
echo
|
|
echo "Database schema upgrade:"
|
|
echo " fully completed"
|
|
|
|
else
|
|
|
|
echo
|
|
echo "Database schema upgrade marker:"
|
|
echo " not present"
|
|
|
|
echo
|
|
echo "This can be normal if the target Zabbix release"
|
|
echo "does not require a database schema change."
|
|
|
|
fi
|
|
|
|
|
|
# ========================================================
|
|
# Verify running server version
|
|
# ========================================================
|
|
|
|
log "Validate Zabbix server version"
|
|
|
|
|
|
RUNNING_VERSION_OUTPUT="$(
|
|
docker exec \
|
|
zabbix-server-pgsql \
|
|
zabbix_server \
|
|
-V
|
|
)"
|
|
|
|
|
|
RUNNING_VERSION="$(
|
|
printf '%s\n' "${RUNNING_VERSION_OUTPUT}" |
|
|
extract_version
|
|
)"
|
|
|
|
|
|
[[ -n "${RUNNING_VERSION}" ]] || \
|
|
fail "Could not determine running Zabbix version"
|
|
|
|
|
|
echo "Expected version:"
|
|
echo " ${TARGET_VERSION}"
|
|
|
|
echo "Running version:"
|
|
echo " ${RUNNING_VERSION}"
|
|
|
|
|
|
[[ "${RUNNING_VERSION}" == "${TARGET_VERSION}" ]] || \
|
|
fail \
|
|
"Running Zabbix version ${RUNNING_VERSION} does not match ${TARGET_VERSION}"
|
|
|
|
|
|
# ========================================================
|
|
# Start Web only after server/DB migration succeeds
|
|
# ========================================================
|
|
|
|
log "Start Zabbix web"
|
|
|
|
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
up -d \
|
|
--no-deps \
|
|
zabbix-web
|
|
|
|
|
|
wait_for_zabbix_web
|
|
|
|
|
|
# ============================================================
|
|
# NO ZABBIX VERSION CHANGE
|
|
# ============================================================
|
|
|
|
else
|
|
|
|
log "No Zabbix version change"
|
|
|
|
|
|
echo "Current version:"
|
|
echo " ${CURRENT_VERSION}"
|
|
|
|
|
|
# --------------------------------------------------------
|
|
# Deploy changed Compose.
|
|
#
|
|
# Even without a version change Docker Compose may need
|
|
# to recreate containers because environment, networks,
|
|
# volumes, healthchecks, resources, etc. changed.
|
|
# --------------------------------------------------------
|
|
|
|
cp \
|
|
"${SRC}/docker-compose.yml" \
|
|
"${LIVE}/docker-compose.yml"
|
|
|
|
|
|
cd "${LIVE}"
|
|
|
|
|
|
echo
|
|
echo "--- Pull images ---"
|
|
|
|
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
pull
|
|
|
|
|
|
echo
|
|
echo "--- Apply Docker Compose ---"
|
|
|
|
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
up -d \
|
|
--remove-orphans
|
|
|
|
|
|
# ========================================================
|
|
# Check server container
|
|
# ========================================================
|
|
|
|
echo
|
|
echo "--- Check Zabbix server container ---"
|
|
|
|
|
|
SERVER_RUNNING="$(
|
|
docker inspect \
|
|
-f '{{.State.Running}}' \
|
|
zabbix-server-pgsql \
|
|
2>/dev/null ||
|
|
echo false
|
|
)"
|
|
|
|
|
|
if [[ "${SERVER_RUNNING}" != "true" ]]; then
|
|
|
|
echo
|
|
echo "--- Zabbix server logs ---"
|
|
|
|
docker logs \
|
|
--tail 200 \
|
|
zabbix-server-pgsql \
|
|
|| true
|
|
|
|
fail "Zabbix server container is not running"
|
|
|
|
fi
|
|
|
|
|
|
echo "Zabbix server container: running"
|
|
|
|
|
|
# ========================================================
|
|
# Functional server check
|
|
#
|
|
# Do NOT search for old startup messages here:
|
|
# container may have been running before this pipeline.
|
|
# ========================================================
|
|
|
|
check_zabbix_server_tcp \
|
|
127.0.0.1 \
|
|
10051
|
|
|
|
|
|
# ========================================================
|
|
# Verify running version
|
|
# ========================================================
|
|
|
|
echo
|
|
echo "--- Validate running Zabbix version ---"
|
|
|
|
|
|
RUNNING_VERSION_OUTPUT="$(
|
|
docker exec \
|
|
zabbix-server-pgsql \
|
|
zabbix_server \
|
|
-V
|
|
)"
|
|
|
|
|
|
RUNNING_VERSION="$(
|
|
printf '%s\n' "${RUNNING_VERSION_OUTPUT}" |
|
|
extract_version
|
|
)"
|
|
|
|
|
|
[[ -n "${RUNNING_VERSION}" ]] || \
|
|
fail "Could not determine running Zabbix version"
|
|
|
|
|
|
echo "Expected version:"
|
|
echo " ${TARGET_VERSION}"
|
|
|
|
echo "Running version:"
|
|
echo " ${RUNNING_VERSION}"
|
|
|
|
|
|
[[ "${RUNNING_VERSION}" == "${TARGET_VERSION}" ]] || \
|
|
fail \
|
|
"Running Zabbix version ${RUNNING_VERSION} does not match ${TARGET_VERSION}"
|
|
|
|
|
|
# ========================================================
|
|
# Verify Web
|
|
# ========================================================
|
|
|
|
wait_for_zabbix_web
|
|
|
|
fi
|
|
|
|
|
|
# ============================================================
|
|
# Final Zabbix server checks
|
|
# ============================================================
|
|
|
|
log "Final Zabbix server checks"
|
|
|
|
|
|
SERVER_RUNNING="$(
|
|
docker inspect \
|
|
-f '{{.State.Running}}' \
|
|
zabbix-server-pgsql
|
|
)"
|
|
|
|
|
|
[[ "${SERVER_RUNNING}" == "true" ]] || \
|
|
fail "Zabbix server is not running"
|
|
|
|
|
|
# ------------------------------------------------------------
|
|
# TCP readiness
|
|
# ------------------------------------------------------------
|
|
|
|
check_zabbix_server_tcp \
|
|
127.0.0.1 \
|
|
10051
|
|
|
|
|
|
# ------------------------------------------------------------
|
|
# Version
|
|
# ------------------------------------------------------------
|
|
|
|
FINAL_VERSION_OUTPUT="$(
|
|
docker exec \
|
|
zabbix-server-pgsql \
|
|
zabbix_server \
|
|
-V
|
|
)"
|
|
|
|
|
|
FINAL_VERSION="$(
|
|
printf '%s\n' "${FINAL_VERSION_OUTPUT}" |
|
|
extract_version
|
|
)"
|
|
|
|
|
|
[[ -n "${FINAL_VERSION}" ]] || \
|
|
fail "Could not determine final Zabbix version"
|
|
|
|
|
|
[[ "${FINAL_VERSION}" == "${TARGET_VERSION}" ]] || \
|
|
fail \
|
|
"Final Zabbix version mismatch: ${FINAL_VERSION} != ${TARGET_VERSION}"
|
|
|
|
|
|
# ============================================================
|
|
# Final Web checks
|
|
# ============================================================
|
|
|
|
log "Final Zabbix web checks"
|
|
|
|
|
|
WEB_RUNNING="$(
|
|
docker inspect \
|
|
-f '{{.State.Running}}' \
|
|
zabbix-web-nginx-pgsql
|
|
)"
|
|
|
|
|
|
WEB_HEALTH="$(
|
|
docker inspect \
|
|
-f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
|
|
zabbix-web-nginx-pgsql
|
|
)"
|
|
|
|
|
|
[[ "${WEB_RUNNING}" == "true" ]] || \
|
|
fail "Zabbix web is not running"
|
|
|
|
|
|
[[ "${WEB_HEALTH}" == "healthy" ]] || \
|
|
fail "Zabbix web is not healthy"
|
|
|
|
|
|
echo "Zabbix web container:"
|
|
echo " running"
|
|
|
|
echo "Zabbix web health:"
|
|
echo " ${WEB_HEALTH}"
|
|
|
|
|
|
# ============================================================
|
|
# Running images
|
|
# ============================================================
|
|
|
|
RUNNING_SERVER_IMAGE="$(
|
|
docker inspect \
|
|
-f '{{.Config.Image}}' \
|
|
zabbix-server-pgsql
|
|
)"
|
|
|
|
|
|
RUNNING_WEB_IMAGE="$(
|
|
docker inspect \
|
|
-f '{{.Config.Image}}' \
|
|
zabbix-web-nginx-pgsql
|
|
)"
|
|
|
|
|
|
# ============================================================
|
|
# Success
|
|
# ============================================================
|
|
|
|
log "Zabbix deployment successful"
|
|
|
|
|
|
echo "Previous Zabbix version:"
|
|
echo " ${CURRENT_VERSION}"
|
|
|
|
echo
|
|
echo "Running Zabbix version:"
|
|
echo " ${FINAL_VERSION}"
|
|
|
|
echo
|
|
echo "Server image:"
|
|
echo " ${RUNNING_SERVER_IMAGE}"
|
|
|
|
echo
|
|
echo "Web image:"
|
|
echo " ${RUNNING_WEB_IMAGE}"
|
|
|
|
echo
|
|
echo "Web health:"
|
|
echo " ${WEB_HEALTH}"
|
|
|
|
|
|
if [[ "${VERSION_CHANGED}" == "true" ]]; then
|
|
|
|
echo
|
|
echo "Pre-upgrade backup:"
|
|
echo " ${BACKUP_DIR}"
|
|
|
|
echo
|
|
echo "PostgreSQL dump:"
|
|
echo " ${DB_BACKUP_FILE}"
|
|
|
|
echo
|
|
echo "Backup retention:"
|
|
echo " last ${BACKUP_KEEP} backups"
|
|
|
|
fi
|
|
|
|
|
|
# ============================================================
|
|
# Clear credentials
|
|
# ============================================================
|
|
|
|
unset PGPASSWORD
|
|
unset DB_PASSWORD
|
|
unset POSTGRES_PASSWORD |