root
1220 lines
24 KiB
Bash
Executable File
1220 lines
24 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -Eeuo pipefail
|
|
|
|
umask 077
|
|
|
|
|
|
# ============================================================
|
|
# Configuration
|
|
# ============================================================
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
|
|
SRC="${ROOT}/stacks/zabbix"
|
|
|
|
LIVE="/srv/nfs/infra/zabbix"
|
|
ENV_FILE="${LIVE}/.env"
|
|
|
|
BACKUP_ROOT="/srv/nfs/infra/.deploy-backups/zabbix"
|
|
BACKUP_KEEP=3
|
|
|
|
STAMP="$(date '+%Y%m%d-%H%M%S')"
|
|
|
|
BACKUP_DIR="${BACKUP_ROOT}/${STAMP}"
|
|
DB_BACKUP_DIR="${BACKUP_DIR}/database"
|
|
|
|
DB_BACKUP_FILE="${DB_BACKUP_DIR}/zabbix.dump"
|
|
|
|
SERVER_START_TIMEOUT=1800
|
|
SERVER_CHECK_INTERVAL=5
|
|
|
|
WEB_START_TIMEOUT=300
|
|
WEB_CHECK_INTERVAL=5
|
|
|
|
|
|
# ============================================================
|
|
# Runtime state
|
|
# ============================================================
|
|
|
|
SERVICES_STOPPED=false
|
|
MIGRATION_STARTED=false
|
|
VERSION_CHANGED=false
|
|
|
|
|
|
# ============================================================
|
|
# Helpers
|
|
# ============================================================
|
|
|
|
log()
|
|
{
|
|
echo
|
|
echo "============================================================"
|
|
echo "$*"
|
|
echo "============================================================"
|
|
}
|
|
|
|
|
|
fail()
|
|
{
|
|
echo
|
|
echo "ERROR: $*" >&2
|
|
exit 1
|
|
}
|
|
|
|
|
|
extract_version()
|
|
{
|
|
grep -oE '[0-9]+\.[0-9]+\.[0-9]+' |
|
|
head -n1
|
|
}
|
|
|
|
|
|
image_version()
|
|
{
|
|
local IMAGE="$1"
|
|
|
|
printf '%s\n' "${IMAGE}" |
|
|
sed -nE 's/.*:alpine-([0-9]+\.[0-9]+\.[0-9]+)$/\1/p'
|
|
}
|
|
|
|
|
|
version_less_than()
|
|
{
|
|
local A="$1"
|
|
local B="$2"
|
|
|
|
[[ "${A}" == "${B}" ]] && return 1
|
|
|
|
[[ "$(
|
|
printf '%s\n%s\n' "${A}" "${B}" |
|
|
sort -V |
|
|
head -n1
|
|
)" == "${A}" ]]
|
|
}
|
|
|
|
|
|
cleanup_old_backups()
|
|
{
|
|
echo
|
|
echo "--- Backup retention ---"
|
|
echo "Keeping last ${BACKUP_KEEP} pre-upgrade backups."
|
|
|
|
mkdir -p "${BACKUP_ROOT}"
|
|
|
|
mapfile -t BACKUPS < <(
|
|
find "${BACKUP_ROOT}" \
|
|
-mindepth 1 \
|
|
-maxdepth 1 \
|
|
-type d \
|
|
-printf '%f\n' |
|
|
grep -E '^[0-9]{8}-[0-9]{6}$' |
|
|
sort -r
|
|
)
|
|
|
|
|
|
if (( ${#BACKUPS[@]} <= BACKUP_KEEP )); then
|
|
|
|
echo "Current backup count: ${#BACKUPS[@]}"
|
|
echo "Nothing to delete."
|
|
|
|
return 0
|
|
|
|
fi
|
|
|
|
|
|
for BACKUP_NAME in "${BACKUPS[@]:${BACKUP_KEEP}}"
|
|
do
|
|
|
|
OLD_BACKUP="${BACKUP_ROOT}/${BACKUP_NAME}"
|
|
|
|
echo
|
|
echo "Removing old backup:"
|
|
echo " ${OLD_BACKUP}"
|
|
|
|
rm -rf -- "${OLD_BACKUP}"
|
|
|
|
done
|
|
}
|
|
|
|
|
|
restore_old_services()
|
|
{
|
|
echo
|
|
echo "--- Restore previous Zabbix services ---"
|
|
|
|
if [[ -f "${BACKUP_DIR}/docker-compose.yml" ]]; then
|
|
|
|
cp \
|
|
"${BACKUP_DIR}/docker-compose.yml" \
|
|
"${LIVE}/docker-compose.yml"
|
|
|
|
fi
|
|
|
|
|
|
(
|
|
cd "${LIVE}"
|
|
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
up -d \
|
|
zabbix-server \
|
|
zabbix-web
|
|
) || true
|
|
}
|
|
|
|
|
|
on_error()
|
|
{
|
|
rc=$?
|
|
|
|
trap - ERR
|
|
|
|
echo
|
|
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
|
echo " Zabbix deployment FAILED"
|
|
echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
|
|
|
|
|
|
if [[ -d "${BACKUP_DIR}" ]]; then
|
|
|
|
echo
|
|
echo "Pre-upgrade backup:"
|
|
echo " ${BACKUP_DIR}"
|
|
|
|
fi
|
|
|
|
|
|
#
|
|
# До старта нового zabbix-server схема БД точно
|
|
# не могла измениться.
|
|
#
|
|
# Поэтому старую версию можно безопасно вернуть.
|
|
#
|
|
|
|
if [[ "${SERVICES_STOPPED}" == "true" ]] &&
|
|
[[ "${MIGRATION_STARTED}" != "true" ]]
|
|
then
|
|
|
|
echo
|
|
echo "Database migration has NOT started."
|
|
echo "Restoring previous containers."
|
|
|
|
restore_old_services
|
|
|
|
else
|
|
|
|
echo
|
|
echo "Automatic rollback is NOT performed."
|
|
|
|
if [[ "${MIGRATION_STARTED}" == "true" ]]; then
|
|
|
|
echo
|
|
echo "Reason:"
|
|
echo " new Zabbix server has already been started;"
|
|
echo " database schema migration may have begun."
|
|
echo
|
|
echo "Rollback requires restoration of the"
|
|
echo "pre-upgrade PostgreSQL dump."
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
exit "${rc}"
|
|
}
|
|
|
|
|
|
wait_for_zabbix_server()
|
|
{
|
|
local elapsed=0
|
|
|
|
echo
|
|
echo "Waiting for Zabbix server startup..."
|
|
|
|
|
|
while (( elapsed < SERVER_START_TIMEOUT ))
|
|
do
|
|
|
|
RUNNING="$(
|
|
docker inspect \
|
|
-f '{{.State.Running}}' \
|
|
zabbix-server-pgsql 2>/dev/null ||
|
|
echo false
|
|
)"
|
|
|
|
|
|
if [[ "${RUNNING}" != "true" ]]; then
|
|
|
|
echo
|
|
echo "--- Zabbix server logs ---"
|
|
|
|
docker logs \
|
|
--since "${UPGRADE_STARTED}" \
|
|
zabbix-server-pgsql || true
|
|
|
|
fail "Zabbix server container stopped during startup"
|
|
|
|
fi
|
|
|
|
|
|
SERVER_LOG="$(
|
|
docker logs \
|
|
--since "${UPGRADE_STARTED}" \
|
|
zabbix-server-pgsql 2>&1 || true
|
|
)"
|
|
|
|
|
|
#
|
|
# Successful server start means mandatory DB schema
|
|
# is compatible. Zabbix does not start if required
|
|
# patches failed.
|
|
#
|
|
|
|
if echo "${SERVER_LOG}" |
|
|
grep -q 'server #0 started \[main process\]'
|
|
then
|
|
|
|
echo
|
|
echo "Zabbix server main process started."
|
|
|
|
return 0
|
|
|
|
fi
|
|
|
|
|
|
if echo "${SERVER_LOG}" |
|
|
grep -Eqi \
|
|
'database upgrade.*(failed|failure)|cannot upgrade database|database version.*newer'
|
|
then
|
|
|
|
echo "${SERVER_LOG}"
|
|
|
|
fail "Zabbix database upgrade failed"
|
|
|
|
fi
|
|
|
|
|
|
sleep "${SERVER_CHECK_INTERVAL}"
|
|
|
|
elapsed=$((elapsed + SERVER_CHECK_INTERVAL))
|
|
|
|
echo "Waiting... ${elapsed}s"
|
|
|
|
done
|
|
|
|
|
|
echo
|
|
echo "--- Zabbix server logs ---"
|
|
|
|
docker logs \
|
|
--since "${UPGRADE_STARTED}" \
|
|
zabbix-server-pgsql || true
|
|
|
|
|
|
fail \
|
|
"Timeout waiting for Zabbix server after ${SERVER_START_TIMEOUT}s"
|
|
}
|
|
|
|
|
|
wait_for_zabbix_web()
|
|
{
|
|
local elapsed=0
|
|
|
|
echo
|
|
echo "Waiting for Zabbix web health=healthy..."
|
|
|
|
|
|
while (( elapsed < WEB_START_TIMEOUT ))
|
|
do
|
|
|
|
RUNNING="$(
|
|
docker inspect \
|
|
-f '{{.State.Running}}' \
|
|
zabbix-web-nginx-pgsql 2>/dev/null ||
|
|
echo false
|
|
)"
|
|
|
|
|
|
if [[ "${RUNNING}" != "true" ]]; then
|
|
|
|
echo
|
|
echo "--- Zabbix web logs ---"
|
|
|
|
docker logs \
|
|
zabbix-web-nginx-pgsql || true
|
|
|
|
fail "Zabbix web container stopped"
|
|
|
|
fi
|
|
|
|
|
|
HEALTH="$(
|
|
docker inspect \
|
|
-f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
|
|
zabbix-web-nginx-pgsql
|
|
)"
|
|
|
|
|
|
echo "Zabbix web health: ${HEALTH} (${elapsed}s)"
|
|
|
|
|
|
if [[ "${HEALTH}" == "healthy" ]]; then
|
|
|
|
return 0
|
|
|
|
fi
|
|
|
|
|
|
sleep "${WEB_CHECK_INTERVAL}"
|
|
|
|
elapsed=$((elapsed + WEB_CHECK_INTERVAL))
|
|
|
|
done
|
|
|
|
|
|
docker logs \
|
|
zabbix-web-nginx-pgsql || true
|
|
|
|
|
|
fail \
|
|
"Timeout waiting for Zabbix web health=healthy"
|
|
}
|
|
|
|
|
|
trap on_error ERR
|
|
|
|
|
|
# ============================================================
|
|
# Required commands
|
|
# ============================================================
|
|
|
|
log "Pre-flight checks"
|
|
|
|
for CMD in \
|
|
docker \
|
|
psql \
|
|
pg_dump \
|
|
pg_restore \
|
|
sha256sum \
|
|
grep \
|
|
sed \
|
|
sort \
|
|
find \
|
|
awk
|
|
do
|
|
|
|
command -v "${CMD}" >/dev/null 2>&1 || \
|
|
fail "Required command not found: ${CMD}"
|
|
|
|
done
|
|
|
|
|
|
docker compose version >/dev/null 2>&1 || \
|
|
fail "Docker Compose plugin is not available"
|
|
|
|
|
|
# ============================================================
|
|
# Required files
|
|
# ============================================================
|
|
|
|
[[ -f "${SRC}/docker-compose.yml" ]] || \
|
|
fail "${SRC}/docker-compose.yml not found"
|
|
|
|
[[ -f "${LIVE}/docker-compose.yml" ]] || \
|
|
fail "${LIVE}/docker-compose.yml not found"
|
|
|
|
[[ -f "${ENV_FILE}" ]] || \
|
|
fail "${ENV_FILE} not found"
|
|
|
|
|
|
# ============================================================
|
|
# Backup directory
|
|
# ============================================================
|
|
|
|
mkdir -p "${BACKUP_ROOT}"
|
|
|
|
[[ -w "${BACKUP_ROOT}" ]] || \
|
|
fail "${BACKUP_ROOT} is not writable"
|
|
|
|
|
|
# ============================================================
|
|
# Load production environment
|
|
#
|
|
# .env must be shell compatible.
|
|
# ============================================================
|
|
|
|
set -a
|
|
|
|
# shellcheck disable=SC1090
|
|
source "${ENV_FILE}"
|
|
|
|
set +a
|
|
|
|
|
|
|
|
DB_HOST="${DB_SERVER_HOST:-${POSTGRES_HOST:-}}"
|
|
DB_PORT="${DB_SERVER_PORT:-${POSTGRES_PORT:-5432}}"
|
|
DB_NAME="${POSTGRES_DB:-}"
|
|
DB_USER="${POSTGRES_USER:-}"
|
|
DB_PASSWORD="${POSTGRES_PASSWORD:-}"
|
|
|
|
|
|
: "${DB_HOST:?DB_SERVER_HOST not defined}"
|
|
: "${DB_PORT:?DB_SERVER_PORT not defined}"
|
|
: "${DB_NAME:?POSTGRES_DB not defined}"
|
|
: "${DB_USER:?POSTGRES_USER not defined}"
|
|
: "${DB_PASSWORD:?POSTGRES_PASSWORD not defined}"
|
|
|
|
export PGPASSWORD="${POSTGRES_PASSWORD}"
|
|
export PGCONNECT_TIMEOUT=10
|
|
|
|
|
|
# ============================================================
|
|
# Validate Compose
|
|
# ============================================================
|
|
|
|
log "Validate Docker Compose"
|
|
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
-f "${SRC}/docker-compose.yml" \
|
|
config >/dev/null
|
|
|
|
|
|
echo "Docker Compose configuration is valid."
|
|
|
|
|
|
# ============================================================
|
|
# Detect target images
|
|
# ============================================================
|
|
|
|
log "Detect target Zabbix images"
|
|
|
|
|
|
NEW_SERVER_IMAGE="$(
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
-f "${SRC}/docker-compose.yml" \
|
|
config --images |
|
|
grep -m1 'zabbix/zabbix-server-pgsql:'
|
|
)"
|
|
|
|
|
|
NEW_WEB_IMAGE="$(
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
-f "${SRC}/docker-compose.yml" \
|
|
config --images |
|
|
grep -m1 'zabbix/zabbix-web-nginx-pgsql:'
|
|
)"
|
|
|
|
|
|
[[ -n "${NEW_SERVER_IMAGE}" ]] || \
|
|
fail "Could not detect target Zabbix server image"
|
|
|
|
|
|
[[ -n "${NEW_WEB_IMAGE}" ]] || \
|
|
fail "Could not detect target Zabbix web image"
|
|
|
|
|
|
TARGET_SERVER_VERSION="$(
|
|
image_version "${NEW_SERVER_IMAGE}"
|
|
)"
|
|
|
|
|
|
TARGET_WEB_VERSION="$(
|
|
image_version "${NEW_WEB_IMAGE}"
|
|
)"
|
|
|
|
|
|
[[ -n "${TARGET_SERVER_VERSION}" ]] || \
|
|
fail \
|
|
"Zabbix server must use a pinned alpine-X.Y.Z tag"
|
|
|
|
|
|
[[ -n "${TARGET_WEB_VERSION}" ]] || \
|
|
fail \
|
|
"Zabbix web must use a pinned alpine-X.Y.Z tag"
|
|
|
|
|
|
[[ "${TARGET_SERVER_VERSION}" == "${TARGET_WEB_VERSION}" ]] || \
|
|
fail \
|
|
"Zabbix server/web versions differ: ${TARGET_SERVER_VERSION} != ${TARGET_WEB_VERSION}"
|
|
|
|
|
|
TARGET_VERSION="${TARGET_SERVER_VERSION}"
|
|
|
|
|
|
echo "Target server image:"
|
|
echo " ${NEW_SERVER_IMAGE}"
|
|
|
|
echo
|
|
echo "Target web image:"
|
|
echo " ${NEW_WEB_IMAGE}"
|
|
|
|
echo
|
|
echo "Target Zabbix version:"
|
|
echo " ${TARGET_VERSION}"
|
|
|
|
|
|
# ============================================================
|
|
# Current containers
|
|
# ============================================================
|
|
|
|
docker inspect zabbix-server-pgsql >/dev/null 2>&1 || \
|
|
fail "Current Zabbix server container does not exist"
|
|
|
|
|
|
docker inspect zabbix-web-nginx-pgsql >/dev/null 2>&1 || \
|
|
fail "Current Zabbix web container does not exist"
|
|
|
|
|
|
SERVER_RUNNING="$(
|
|
docker inspect \
|
|
-f '{{.State.Running}}' \
|
|
zabbix-server-pgsql
|
|
)"
|
|
|
|
|
|
WEB_RUNNING="$(
|
|
docker inspect \
|
|
-f '{{.State.Running}}' \
|
|
zabbix-web-nginx-pgsql
|
|
)"
|
|
|
|
|
|
[[ "${SERVER_RUNNING}" == "true" ]] || \
|
|
fail "Current Zabbix server is not running"
|
|
|
|
|
|
[[ "${WEB_RUNNING}" == "true" ]] || \
|
|
fail "Current Zabbix web is not running"
|
|
|
|
|
|
# ============================================================
|
|
# Current actual Zabbix version
|
|
# ============================================================
|
|
|
|
log "Detect current Zabbix version"
|
|
|
|
|
|
CURRENT_VERSION_OUTPUT="$(
|
|
docker exec \
|
|
zabbix-server-pgsql \
|
|
zabbix_server \
|
|
-V
|
|
)"
|
|
|
|
|
|
CURRENT_VERSION="$(
|
|
printf '%s\n' "${CURRENT_VERSION_OUTPUT}" |
|
|
extract_version
|
|
)"
|
|
|
|
|
|
[[ -n "${CURRENT_VERSION}" ]] || \
|
|
fail "Could not determine current Zabbix version"
|
|
|
|
|
|
OLD_SERVER_IMAGE="$(
|
|
docker inspect \
|
|
-f '{{.Config.Image}}' \
|
|
zabbix-server-pgsql
|
|
)"
|
|
|
|
|
|
OLD_WEB_IMAGE="$(
|
|
docker inspect \
|
|
-f '{{.Config.Image}}' \
|
|
zabbix-web-nginx-pgsql
|
|
)"
|
|
|
|
|
|
echo "Current Zabbix version:"
|
|
echo " ${CURRENT_VERSION}"
|
|
|
|
echo
|
|
echo "Current server image:"
|
|
echo " ${OLD_SERVER_IMAGE}"
|
|
|
|
echo
|
|
echo "Current web image:"
|
|
echo " ${OLD_WEB_IMAGE}"
|
|
|
|
|
|
# ============================================================
|
|
# Version validation
|
|
# ============================================================
|
|
|
|
log "Validate version transition"
|
|
|
|
|
|
if version_less_than \
|
|
"${TARGET_VERSION}" \
|
|
"${CURRENT_VERSION}"
|
|
then
|
|
|
|
fail \
|
|
"Zabbix downgrade is not allowed: ${CURRENT_VERSION} -> ${TARGET_VERSION}"
|
|
|
|
fi
|
|
|
|
|
|
if [[ "${CURRENT_VERSION}" != "${TARGET_VERSION}" ]]
|
|
then
|
|
|
|
VERSION_CHANGED=true
|
|
|
|
fi
|
|
|
|
|
|
echo "Current version: ${CURRENT_VERSION}"
|
|
echo "Target version: ${TARGET_VERSION}"
|
|
echo "Version changed: ${VERSION_CHANGED}"
|
|
|
|
|
|
# ============================================================
|
|
# Pull target images before downtime
|
|
# ============================================================
|
|
|
|
log "Pull target Docker images"
|
|
|
|
|
|
docker pull "${NEW_SERVER_IMAGE}"
|
|
docker pull "${NEW_WEB_IMAGE}"
|
|
|
|
|
|
# ============================================================
|
|
# VERSION UPDATE
|
|
# ============================================================
|
|
|
|
if [[ "${VERSION_CHANGED}" == "true" ]]
|
|
then
|
|
|
|
# ========================================================
|
|
# Prepare backup directory before shutdown
|
|
# ========================================================
|
|
|
|
log "Prepare Zabbix pre-upgrade backup"
|
|
|
|
|
|
mkdir -p "${DB_BACKUP_DIR}"
|
|
|
|
|
|
cp \
|
|
"${LIVE}/docker-compose.yml" \
|
|
"${BACKUP_DIR}/docker-compose.yml"
|
|
|
|
|
|
cat > "${BACKUP_DIR}/metadata.txt" <<EOF
|
|
timestamp=${STAMP}
|
|
|
|
current_version=${CURRENT_VERSION}
|
|
target_version=${TARGET_VERSION}
|
|
|
|
current_server_image=${OLD_SERVER_IMAGE}
|
|
target_server_image=${NEW_SERVER_IMAGE}
|
|
|
|
current_web_image=${OLD_WEB_IMAGE}
|
|
target_web_image=${NEW_WEB_IMAGE}
|
|
|
|
postgres_host=${POSTGRES_HOST}
|
|
postgres_port=${POSTGRES_PORT}
|
|
postgres_database=${POSTGRES_DB}
|
|
postgres_user=${POSTGRES_USER}
|
|
EOF
|
|
|
|
|
|
# ========================================================
|
|
# Stop application writes before DB backup
|
|
#
|
|
# Official Zabbix upgrade procedure requires stopping
|
|
# server before database backup.
|
|
# Web is also stopped to prevent configuration writes.
|
|
# ========================================================
|
|
|
|
log "Stop Zabbix before database backup"
|
|
|
|
|
|
cd "${LIVE}"
|
|
|
|
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
stop \
|
|
zabbix-web \
|
|
zabbix-server
|
|
|
|
|
|
SERVICES_STOPPED=true
|
|
|
|
|
|
# ========================================================
|
|
# DB connection test
|
|
# ========================================================
|
|
|
|
log "Test PostgreSQL connection"
|
|
|
|
|
|
DB_TEST="$(
|
|
psql \
|
|
-h "${DB_HOST}" \
|
|
-p "${DB_PORT}" \
|
|
-U "${POSTGRES_USER}" \
|
|
-d "${POSTGRES_DB}" \
|
|
-Atqc 'SELECT 1;'
|
|
)"
|
|
|
|
|
|
[[ "${DB_TEST}" == "1" ]] || \
|
|
fail "Cannot connect to PostgreSQL"
|
|
|
|
|
|
# ========================================================
|
|
# PostgreSQL versions
|
|
# ========================================================
|
|
|
|
SERVER_VERSION="$(
|
|
psql \
|
|
-h "${DB_HOST}" \
|
|
-p "${DB_PORT}" \
|
|
-U "${POSTGRES_USER}" \
|
|
-d "${POSTGRES_DB}" \
|
|
-Atqc 'SHOW server_version;'
|
|
)"
|
|
|
|
|
|
echo "${SERVER_VERSION}" \
|
|
> "${DB_BACKUP_DIR}/postgresql-version.txt"
|
|
|
|
|
|
SERVER_MAJOR="${SERVER_VERSION%%.*}"
|
|
|
|
|
|
PG_DUMP_VERSION="$(
|
|
pg_dump --version |
|
|
sed -E 's/.* ([0-9]+(\.[0-9]+)?).*/\1/'
|
|
)"
|
|
|
|
|
|
PG_DUMP_MAJOR="${PG_DUMP_VERSION%%.*}"
|
|
|
|
|
|
echo "PostgreSQL server:"
|
|
echo " ${SERVER_VERSION}"
|
|
|
|
echo "pg_dump client:"
|
|
echo " ${PG_DUMP_VERSION}"
|
|
|
|
|
|
if (( PG_DUMP_MAJOR < SERVER_MAJOR ))
|
|
then
|
|
|
|
fail \
|
|
"pg_dump ${PG_DUMP_VERSION} is older than PostgreSQL server ${SERVER_VERSION}"
|
|
|
|
fi
|
|
|
|
|
|
# ========================================================
|
|
# DB size
|
|
# ========================================================
|
|
|
|
DB_SIZE="$(
|
|
psql \
|
|
-h "${DB_HOST}" \
|
|
-p "${DB_PORT}" \
|
|
-U "${POSTGRES_USER}" \
|
|
-d "${POSTGRES_DB}" \
|
|
-Atqc \
|
|
'SELECT pg_size_pretty(pg_database_size(current_database()));'
|
|
)"
|
|
|
|
|
|
echo "Database size:"
|
|
echo " ${DB_SIZE}"
|
|
|
|
|
|
# ========================================================
|
|
# pg_dump
|
|
# ========================================================
|
|
|
|
log "Create PostgreSQL backup"
|
|
|
|
|
|
pg_dump \
|
|
-h "${DB_HOST}" \
|
|
-p "${DB_PORT}" \
|
|
-U "${POSTGRES_USER}" \
|
|
-d "${POSTGRES_DB}" \
|
|
--format=custom \
|
|
--no-owner \
|
|
--no-privileges \
|
|
--file="${DB_BACKUP_FILE}"
|
|
|
|
|
|
[[ -s "${DB_BACKUP_FILE}" ]] || \
|
|
fail "PostgreSQL dump does not exist or is empty"
|
|
|
|
|
|
# ========================================================
|
|
# Validate dump
|
|
# ========================================================
|
|
|
|
log "Validate PostgreSQL backup"
|
|
|
|
|
|
pg_restore \
|
|
--list \
|
|
"${DB_BACKUP_FILE}" \
|
|
> "${DB_BACKUP_DIR}/zabbix.dump.list"
|
|
|
|
|
|
[[ -s "${DB_BACKUP_DIR}/zabbix.dump.list" ]] || \
|
|
fail "pg_restore could not validate dump"
|
|
|
|
|
|
# ========================================================
|
|
# SHA256
|
|
# ========================================================
|
|
|
|
(
|
|
cd "${DB_BACKUP_DIR}"
|
|
|
|
sha256sum \
|
|
zabbix.dump \
|
|
> zabbix.dump.sha256
|
|
|
|
sha256sum \
|
|
-c zabbix.dump.sha256
|
|
)
|
|
|
|
|
|
DB_BACKUP_SIZE="$(
|
|
du -h "${DB_BACKUP_FILE}" |
|
|
awk '{print $1}'
|
|
)"
|
|
|
|
|
|
echo
|
|
echo "PostgreSQL backup successful."
|
|
|
|
echo
|
|
echo "Backup:"
|
|
echo " ${DB_BACKUP_FILE}"
|
|
|
|
echo
|
|
echo "Dump size:"
|
|
echo " ${DB_BACKUP_SIZE}"
|
|
|
|
|
|
# ========================================================
|
|
# Retention
|
|
# ========================================================
|
|
|
|
cleanup_old_backups
|
|
|
|
|
|
# ========================================================
|
|
# Deploy new Compose
|
|
# ========================================================
|
|
|
|
log "Deploy new docker-compose.yml"
|
|
|
|
|
|
cp \
|
|
"${SRC}/docker-compose.yml" \
|
|
"${LIVE}/docker-compose.yml"
|
|
|
|
|
|
cd "${LIVE}"
|
|
|
|
|
|
# ========================================================
|
|
# Start NEW server only
|
|
#
|
|
# From this point DB migration may start.
|
|
# No automatic rollback after this flag.
|
|
# ========================================================
|
|
|
|
log "Start new Zabbix server"
|
|
|
|
|
|
UPGRADE_STARTED="$(
|
|
date -u '+%Y-%m-%dT%H:%M:%SZ'
|
|
)"
|
|
|
|
|
|
MIGRATION_STARTED=true
|
|
|
|
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
up -d \
|
|
--no-deps \
|
|
zabbix-server
|
|
|
|
|
|
# ========================================================
|
|
# Wait for automatic DB schema upgrade + server startup
|
|
# ========================================================
|
|
|
|
log "Wait for Zabbix database upgrade and server startup"
|
|
|
|
|
|
wait_for_zabbix_server
|
|
|
|
|
|
# ========================================================
|
|
# Upgrade log
|
|
# ========================================================
|
|
|
|
SERVER_LOG="$(
|
|
docker logs \
|
|
--since "${UPGRADE_STARTED}" \
|
|
zabbix-server-pgsql 2>&1
|
|
)"
|
|
|
|
|
|
echo
|
|
echo "--- Zabbix server startup log ---"
|
|
echo "${SERVER_LOG}"
|
|
|
|
|
|
if echo "${SERVER_LOG}" |
|
|
grep -q 'database upgrade fully completed'
|
|
then
|
|
|
|
echo
|
|
echo "Database schema upgrade:"
|
|
echo " fully completed"
|
|
|
|
else
|
|
|
|
echo
|
|
echo "Database schema upgrade marker:"
|
|
echo " not present"
|
|
echo
|
|
echo "This is normal when the target release"
|
|
echo "does not require a schema change."
|
|
|
|
fi
|
|
|
|
|
|
# ========================================================
|
|
# Verify new server version
|
|
# ========================================================
|
|
|
|
log "Validate Zabbix server version"
|
|
|
|
|
|
RUNNING_VERSION_OUTPUT="$(
|
|
docker exec \
|
|
zabbix-server-pgsql \
|
|
zabbix_server \
|
|
-V
|
|
)"
|
|
|
|
|
|
RUNNING_VERSION="$(
|
|
printf '%s\n' "${RUNNING_VERSION_OUTPUT}" |
|
|
extract_version
|
|
)"
|
|
|
|
|
|
[[ "${RUNNING_VERSION}" == "${TARGET_VERSION}" ]] || \
|
|
fail \
|
|
"Running Zabbix version ${RUNNING_VERSION} does not match ${TARGET_VERSION}"
|
|
|
|
|
|
# ========================================================
|
|
# Start web after server/DB upgrade is complete
|
|
# ========================================================
|
|
|
|
log "Start Zabbix web"
|
|
|
|
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
up -d \
|
|
--no-deps \
|
|
zabbix-web
|
|
|
|
|
|
wait_for_zabbix_web
|
|
|
|
|
|
# ============================================================
|
|
# NO VERSION CHANGE
|
|
# ============================================================
|
|
|
|
else
|
|
|
|
log "No Zabbix version change"
|
|
|
|
|
|
cp \
|
|
"${SRC}/docker-compose.yml" \
|
|
"${LIVE}/docker-compose.yml"
|
|
|
|
|
|
cd "${LIVE}"
|
|
|
|
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
pull
|
|
|
|
|
|
docker compose \
|
|
--env-file "${ENV_FILE}" \
|
|
up -d \
|
|
--remove-orphans
|
|
|
|
|
|
UPGRADE_STARTED="$(
|
|
date -u -d '10 minutes ago' '+%Y-%m-%dT%H:%M:%SZ'
|
|
)
|
|
|
|
|
|
wait_for_zabbix_server
|
|
wait_for_zabbix_web
|
|
|
|
fi
|
|
|
|
|
|
# ============================================================
|
|
# Final server state
|
|
# ============================================================
|
|
|
|
log "Final Zabbix server checks"
|
|
|
|
|
|
SERVER_RUNNING="$(
|
|
docker inspect \
|
|
-f '{{.State.Running}}' \
|
|
zabbix-server-pgsql
|
|
)"
|
|
|
|
|
|
[[ "${SERVER_RUNNING}" == "true" ]] || \
|
|
fail "Zabbix server is not running"
|
|
|
|
|
|
FINAL_VERSION_OUTPUT="$(
|
|
docker exec \
|
|
zabbix-server-pgsql \
|
|
zabbix_server \
|
|
-V
|
|
)"
|
|
|
|
|
|
FINAL_VERSION="$(
|
|
printf '%s\n' "${FINAL_VERSION_OUTPUT}" |
|
|
extract_version
|
|
)"
|
|
|
|
|
|
[[ "${FINAL_VERSION}" == "${TARGET_VERSION}" ]] || \
|
|
fail \
|
|
"Final Zabbix version mismatch: ${FINAL_VERSION} != ${TARGET_VERSION}"
|
|
|
|
|
|
# ============================================================
|
|
# Final web state
|
|
# ============================================================
|
|
|
|
WEB_RUNNING="$(
|
|
docker inspect \
|
|
-f '{{.State.Running}}' \
|
|
zabbix-web-nginx-pgsql
|
|
)"
|
|
|
|
|
|
WEB_HEALTH="$(
|
|
docker inspect \
|
|
-f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
|
|
zabbix-web-nginx-pgsql
|
|
)"
|
|
|
|
|
|
[[ "${WEB_RUNNING}" == "true" ]] || \
|
|
fail "Zabbix web is not running"
|
|
|
|
|
|
[[ "${WEB_HEALTH}" == "healthy" ]] || \
|
|
fail "Zabbix web is not healthy"
|
|
|
|
|
|
# ============================================================
|
|
# Running images
|
|
# ============================================================
|
|
|
|
RUNNING_SERVER_IMAGE="$(
|
|
docker inspect \
|
|
-f '{{.Config.Image}}' \
|
|
zabbix-server-pgsql
|
|
)"
|
|
|
|
|
|
RUNNING_WEB_IMAGE="$(
|
|
docker inspect \
|
|
-f '{{.Config.Image}}' \
|
|
zabbix-web-nginx-pgsql
|
|
)"
|
|
|
|
|
|
# ============================================================
|
|
# Success
|
|
# ============================================================
|
|
|
|
log "Zabbix deployment successful"
|
|
|
|
|
|
echo "Previous Zabbix version:"
|
|
echo " ${CURRENT_VERSION}"
|
|
|
|
echo
|
|
echo "Running Zabbix version:"
|
|
echo " ${FINAL_VERSION}"
|
|
|
|
echo
|
|
echo "Server image:"
|
|
echo " ${RUNNING_SERVER_IMAGE}"
|
|
|
|
echo
|
|
echo "Web image:"
|
|
echo " ${RUNNING_WEB_IMAGE}"
|
|
|
|
echo
|
|
echo "Web health:"
|
|
echo " ${WEB_HEALTH}"
|
|
|
|
|
|
if [[ "${VERSION_CHANGED}" == "true" ]]
|
|
then
|
|
|
|
echo
|
|
echo "Pre-upgrade backup:"
|
|
echo " ${BACKUP_DIR}"
|
|
|
|
echo
|
|
echo "PostgreSQL dump:"
|
|
echo " ${DB_BACKUP_FILE}"
|
|
|
|
echo
|
|
echo "Backup retention:"
|
|
echo " last ${BACKUP_KEEP} backups"
|
|
|
|
fi
|
|
|
|
|
|
# ============================================================
|
|
# Clear credentials
|
|
# ============================================================
|
|
|
|
unset PGPASSWORD
|
|
unset POSTGRES_PASSWORD
|