#!/usr/bin/env bash set -Eeuo pipefail umask 077 # ============================================================ # Configuration # ============================================================ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" SRC="${ROOT}/stacks/keycloak" LIVE="/srv/nfs/infra/keycloak" ENV_FILE="${LIVE}/env/kc.env" BACKUP_ROOT="/srv/nfs/infra/.deploy-backups/keycloak" BACKUP_KEEP=3 STAMP="$(date '+%Y%m%d-%H%M%S')" BACKUP_DIR="${BACKUP_ROOT}/${STAMP}" DB_BACKUP_DIR="${BACKUP_DIR}/database" DB_BACKUP_FILE="${DB_BACKUP_DIR}/keycloak.dump" # Максимальное ожидание старта Keycloak. # # Keycloak по умолчанию допускает довольно долгую DB migration. # KEYCLOAK_START_TIMEOUT=1800 KEYCLOAK_CHECK_INTERVAL=10 # ============================================================ # Helpers # ============================================================ log() { echo echo "============================================================" echo "$*" echo "============================================================" } fail() { echo echo "ERROR: $*" >&2 exit 1 } on_error() { rc=$? trap - ERR echo echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" echo " Keycloak deployment FAILED" echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" if [[ -d "${BACKUP_DIR}" ]]; then echo echo "Pre-upgrade backup:" echo " ${BACKUP_DIR}" fi echo echo "Automatic rollback is intentionally NOT performed." echo echo "Reason:" echo " Keycloak may already have changed the PostgreSQL schema" echo " during Liquibase migration." echo echo "For rollback restore the pre-upgrade database backup first." exit "${rc}" } cleanup_old_backups() { echo echo "--- Backup retention ---" echo "Keeping last ${BACKUP_KEEP} pre-upgrade backups." mkdir -p "${BACKUP_ROOT}" mapfile -t BACKUPS < <( find "${BACKUP_ROOT}" \ -mindepth 1 \ -maxdepth 1 \ -type d \ -printf '%f\n' | grep -E '^[0-9]{8}-[0-9]{6}$' | sort -r ) if (( ${#BACKUPS[@]} <= BACKUP_KEEP )); then echo "Current backup count: ${#BACKUPS[@]}" echo "Nothing to delete." return 0 fi for BACKUP_NAME in "${BACKUPS[@]:${BACKUP_KEEP}}" do OLD_BACKUP="${BACKUP_ROOT}/${BACKUP_NAME}" echo echo "Removing old backup:" echo " ${OLD_BACKUP}" rm -rf -- "${OLD_BACKUP}" done } get_container_env() { local VARIABLE="$1" docker inspect \ --format '{{range .Config.Env}}{{println .}}{{end}}' \ keycloak | grep -m1 "^${VARIABLE}=" | sed "s/^${VARIABLE}=//" || true } extract_version() { grep -oE \ '[0-9]+(\.[0-9]+){1,3}' | head -n1 } version_less_than() { local A="$1" local B="$2" [[ "${A}" == "${B}" ]] && return 1 [[ "$( printf '%s\n%s\n' "${A}" "${B}" | sort -V | head -n1 )" == "${A}" ]] } wait_for_keycloak() { local elapsed=0 echo echo "Waiting for Keycloak health=healthy..." while (( elapsed < KEYCLOAK_START_TIMEOUT )) do RUNNING="$( docker inspect \ -f '{{.State.Running}}' \ keycloak 2>/dev/null || echo false )" if [[ "${RUNNING}" != "true" ]]; then echo echo "--- Keycloak logs ---" docker logs \ --since "${UPGRADE_STARTED:-10m}" \ keycloak || true fail "Keycloak container stopped during startup" fi HEALTH="$( docker inspect \ -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ keycloak )" echo "Keycloak health: ${HEALTH} (${elapsed}s)" if [[ "${HEALTH}" == "healthy" ]]; then echo echo "Keycloak is healthy." return 0 fi sleep "${KEYCLOAK_CHECK_INTERVAL}" elapsed=$((elapsed + KEYCLOAK_CHECK_INTERVAL)) done echo echo "--- Keycloak logs ---" docker logs \ --since "${UPGRADE_STARTED:-30m}" \ keycloak || true fail \ "Timeout waiting for Keycloak health=healthy after ${KEYCLOAK_START_TIMEOUT}s" } trap on_error ERR # ============================================================ # Required commands # ============================================================ log "Pre-flight checks" for CMD in \ docker \ psql \ pg_dump \ pg_restore \ sha256sum \ sort \ grep \ find \ sed \ awk do command -v "${CMD}" >/dev/null 2>&1 || \ fail "Required command not found: ${CMD}" done docker compose version >/dev/null 2>&1 || \ fail "Docker Compose plugin is not available" # ============================================================ # Required files # ============================================================ [[ -f "${SRC}/docker-compose.yml" ]] || \ fail "${SRC}/docker-compose.yml not found" [[ -f "${LIVE}/docker-compose.yml" ]] || \ fail "${LIVE}/docker-compose.yml not found" [[ -f "${ENV_FILE}" ]] || \ fail "${ENV_FILE} not found" # ============================================================ # Backup directory # ============================================================ mkdir -p "${BACKUP_ROOT}" [[ -w "${BACKUP_ROOT}" ]] || \ fail "${BACKUP_ROOT} is not writable" # ============================================================ # Validate Compose # ============================================================ log "Validate Docker Compose" docker compose \ -f "${SRC}/docker-compose.yml" \ config >/dev/null echo "Docker Compose configuration is valid." # ============================================================ # Detect target Keycloak image # ============================================================ log "Detect requested Keycloak image" mapfile -t KEYCLOAK_IMAGES < <( docker compose \ -f "${SRC}/docker-compose.yml" \ config --images | sort -u ) if [[ "${#KEYCLOAK_IMAGES[@]}" -ne 1 ]]; then echo "Detected images:" if (( ${#KEYCLOAK_IMAGES[@]} > 0 )); then printf ' %s\n' "${KEYCLOAK_IMAGES[@]}" else echo " none" fi fail "Expected exactly one Keycloak image" fi NEW_IMAGE="${KEYCLOAK_IMAGES[0]}" echo "Requested image:" echo " ${NEW_IMAGE}" # ============================================================ # Current Keycloak container # ============================================================ docker inspect keycloak >/dev/null 2>&1 || \ fail "Current Keycloak container does not exist" CURRENT_RUNNING="$( docker inspect \ -f '{{.State.Running}}' \ keycloak )" [[ "${CURRENT_RUNNING}" == "true" ]] || \ fail "Current Keycloak container is not running" OLD_IMAGE="$( docker inspect \ -f '{{.Config.Image}}' \ keycloak )" # ============================================================ # Current health # ============================================================ CURRENT_HEALTH="$( docker inspect \ -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ keycloak )" echo echo "Current Keycloak health:" echo " ${CURRENT_HEALTH}" [[ "${CURRENT_HEALTH}" == "healthy" ]] || \ fail "Current Keycloak is not healthy before deployment" # ============================================================ # Detect actual current Keycloak version # ============================================================ log "Detect current Keycloak version" CURRENT_VERSION_OUTPUT="$( docker exec \ keycloak \ /opt/keycloak/bin/kc.sh \ --version )" CURRENT_KC_VERSION="$( printf '%s\n' "${CURRENT_VERSION_OUTPUT}" | extract_version )" [[ -n "${CURRENT_KC_VERSION}" ]] || \ fail "Could not determine current Keycloak version" echo "Current Docker image:" echo " ${OLD_IMAGE}" echo "Current Keycloak version:" echo " ${CURRENT_KC_VERSION}" # ============================================================ # Pull target image # # This does not modify the running container. # ============================================================ log "Pull target Keycloak image" docker pull "${NEW_IMAGE}" # ============================================================ # Detect actual target version # ============================================================ log "Detect target Keycloak version" TARGET_VERSION_OUTPUT="$( docker run \ --rm \ --entrypoint /opt/keycloak/bin/kc.sh \ "${NEW_IMAGE}" \ --version )" TARGET_KC_VERSION="$( printf '%s\n' "${TARGET_VERSION_OUTPUT}" | extract_version )" [[ -n "${TARGET_KC_VERSION}" ]] || \ fail "Could not determine target Keycloak version" echo "Target Docker image:" echo " ${NEW_IMAGE}" echo "Target Keycloak version:" echo " ${TARGET_KC_VERSION}" # ============================================================ # Version safety # ============================================================ log "Validate version transition" if version_less_than \ "${TARGET_KC_VERSION}" \ "${CURRENT_KC_VERSION}" then fail \ "Keycloak downgrade is not allowed: ${CURRENT_KC_VERSION} -> ${TARGET_KC_VERSION}" fi VERSION_CHANGED=false if [[ "${CURRENT_KC_VERSION}" != "${TARGET_KC_VERSION}" ]] then VERSION_CHANGED=true fi echo "Current version: ${CURRENT_KC_VERSION}" echo "Target version: ${TARGET_KC_VERSION}" echo "Version changed: ${VERSION_CHANGED}" # ============================================================ # Extract effective DB configuration from CURRENT container # ============================================================ log "Detect Keycloak database configuration" KC_DB="$( get_container_env KC_DB )" KC_DB_URL="$( get_container_env KC_DB_URL )" KC_DB_URL_HOST="$( get_container_env KC_DB_URL_HOST )" KC_DB_URL_PORT="$( get_container_env KC_DB_URL_PORT )" KC_DB_URL_DATABASE="$( get_container_env KC_DB_URL_DATABASE )" KC_DB_USERNAME="$( get_container_env KC_DB_USERNAME )" KC_DB_PASSWORD="$( get_container_env KC_DB_PASSWORD )" KCRAW_DB_PASSWORD="$( get_container_env KCRAW_DB_PASSWORD )" # ============================================================ # Database vendor # ============================================================ if [[ -z "${KC_DB}" ]] && [[ "${KC_DB_URL}" == jdbc:postgresql://* ]] then KC_DB="postgres" fi [[ "${KC_DB}" == "postgres" ]] || \ fail "This deployment script currently supports PostgreSQL only. KC_DB=${KC_DB:-undefined}" # ============================================================ # DB username/password # ============================================================ [[ -n "${KC_DB_USERNAME}" ]] || \ fail "KC_DB_USERNAME is not defined" if [[ -n "${KCRAW_DB_PASSWORD}" ]]; then DB_PASSWORD="${KCRAW_DB_PASSWORD}" else DB_PASSWORD="${KC_DB_PASSWORD}" fi [[ -n "${DB_PASSWORD}" ]] || \ fail "Neither KC_DB_PASSWORD nor KCRAW_DB_PASSWORD is defined" # ============================================================ # Parse PostgreSQL connection settings # # Priority: # # 1. KC_DB_URL # # 2. KC_DB_URL_HOST # KC_DB_URL_PORT # KC_DB_URL_DATABASE # ============================================================ if [[ -n "${KC_DB_URL}" ]] then [[ "${KC_DB_URL}" == jdbc:postgresql://* ]] || \ fail "Unsupported KC_DB_URL: expected jdbc:postgresql://..." DB_URL_NO_PREFIX="${KC_DB_URL#jdbc:postgresql://}" # Remove JDBC query parameters. DB_URL_NO_PARAMS="${DB_URL_NO_PREFIX%%\?*}" DB_HOSTPORT="${DB_URL_NO_PARAMS%%/*}" DB_NAME="${DB_URL_NO_PARAMS#*/}" [[ "${DB_NAME}" != "${DB_URL_NO_PARAMS}" ]] || \ fail "Could not determine database name from KC_DB_URL" # -------------------------------------------------------- # IPv6 [address]:port # -------------------------------------------------------- if [[ "${DB_HOSTPORT}" =~ ^\[([^]]+)\](:([0-9]+))?$ ]] then DB_HOST="${BASH_REMATCH[1]}" DB_PORT="${BASH_REMATCH[3]:-5432}" # -------------------------------------------------------- # Standard hostname:port # -------------------------------------------------------- elif [[ "${DB_HOSTPORT}" == *:* ]] then DB_HOST="${DB_HOSTPORT%%:*}" DB_PORT="${DB_HOSTPORT##*:}" else DB_HOST="${DB_HOSTPORT}" DB_PORT="5432" fi else [[ -n "${KC_DB_URL_HOST}" ]] || \ fail "KC_DB_URL_HOST is not defined" [[ -n "${KC_DB_URL_DATABASE}" ]] || \ fail "KC_DB_URL_DATABASE is not defined" DB_HOST="${KC_DB_URL_HOST}" DB_PORT="${KC_DB_URL_PORT:-5432}" DB_NAME="${KC_DB_URL_DATABASE}" fi DB_USER="${KC_DB_USERNAME}" echo "Database vendor:" echo " postgres" echo "Database host:" echo " ${DB_HOST}" echo "Database port:" echo " ${DB_PORT}" echo "Database name:" echo " ${DB_NAME}" echo "Database user:" echo " ${DB_USER}" # ============================================================ # PostgreSQL client environment # ============================================================ export PGPASSWORD="${DB_PASSWORD}" export PGCONNECT_TIMEOUT=10 # ============================================================ # PRE-UPGRADE BACKUP # # Created ONLY when Keycloak version changes. # ============================================================ if [[ "${VERSION_CHANGED}" == "true" ]] then log "Prepare Keycloak pre-upgrade backup" # ======================================================== # Backup directories # ======================================================== mkdir -p "${DB_BACKUP_DIR}" # ======================================================== # Save current Compose # # No cp -a because NFS does not support preserving # all POSIX attributes in this setup. # ======================================================== echo echo "--- Backup current docker-compose.yml ---" cp \ "${LIVE}/docker-compose.yml" \ "${BACKUP_DIR}/docker-compose.yml" # ======================================================== # Metadata # # No passwords are written here. # ======================================================== cat > "${BACKUP_DIR}/metadata.txt" < "${DB_BACKUP_DIR}/postgresql-version.txt" SERVER_MAJOR="${SERVER_VERSION%%.*}" PG_DUMP_VERSION="$( pg_dump --version | sed -E 's/.* ([0-9]+(\.[0-9]+)?).*/\1/' )" PG_DUMP_MAJOR="${PG_DUMP_VERSION%%.*}" echo "PostgreSQL server:" echo " ${SERVER_VERSION}" echo "pg_dump client:" echo " ${PG_DUMP_VERSION}" if (( PG_DUMP_MAJOR < SERVER_MAJOR )) then fail \ "pg_dump ${PG_DUMP_VERSION} is older than PostgreSQL server ${SERVER_VERSION}" fi # ======================================================== # DB size # ======================================================== echo echo "--- Database size ---" DB_SIZE="$( psql \ -h "${DB_HOST}" \ -p "${DB_PORT}" \ -U "${DB_USER}" \ -d "${DB_NAME}" \ -Atqc \ 'SELECT pg_size_pretty(pg_database_size(current_database()));' )" echo "Database size:" echo " ${DB_SIZE}" # ======================================================== # pg_dump # ======================================================== log "Create PostgreSQL backup" pg_dump \ -h "${DB_HOST}" \ -p "${DB_PORT}" \ -U "${DB_USER}" \ -d "${DB_NAME}" \ --format=custom \ --no-owner \ --no-privileges \ --file="${DB_BACKUP_FILE}" [[ -s "${DB_BACKUP_FILE}" ]] || \ fail "PostgreSQL dump does not exist or is empty" # ======================================================== # Validate archive # ======================================================== echo echo "--- Validate PostgreSQL dump ---" pg_restore \ --list \ "${DB_BACKUP_FILE}" \ > "${DB_BACKUP_DIR}/keycloak.dump.list" [[ -s "${DB_BACKUP_DIR}/keycloak.dump.list" ]] || \ fail "pg_restore could not validate PostgreSQL dump" # ======================================================== # SHA256 # ======================================================== echo echo "--- Generate SHA256 ---" ( cd "${DB_BACKUP_DIR}" sha256sum \ keycloak.dump \ > keycloak.dump.sha256 sha256sum \ -c keycloak.dump.sha256 ) DB_BACKUP_SIZE="$( du -h "${DB_BACKUP_FILE}" | awk '{print $1}' )" echo echo "PostgreSQL backup successful." echo echo "Backup directory:" echo " ${BACKUP_DIR}" echo echo "Database dump:" echo " ${DB_BACKUP_FILE}" echo echo "Dump size:" echo " ${DB_BACKUP_SIZE}" echo echo "SHA256:" cat "${DB_BACKUP_DIR}/keycloak.dump.sha256" # ======================================================== # Retention # # Delete old copies only after new dump has been # successfully created and validated. # ======================================================== cleanup_old_backups fi # ============================================================ # Deploy new Compose # # Nothing in production is changed before successful DB backup. # ============================================================ log "Deploy docker-compose.yml" cp \ "${SRC}/docker-compose.yml" \ "${LIVE}/docker-compose.yml" cd "${LIVE}" # ============================================================ # Pull Compose images # ============================================================ log "Pull Docker images" docker compose pull # ============================================================ # KEYCLOAK VERSION UPGRADE # ============================================================ if [[ "${VERSION_CHANGED}" == "true" ]] then log "Keycloak upgrade ${CURRENT_KC_VERSION} -> ${TARGET_KC_VERSION}" # ======================================================== # Stop old Keycloak before DB migration # ======================================================== echo echo "--- Stop old Keycloak ---" docker compose stop keycloak # ======================================================== # Timestamp for new container logs # ======================================================== UPGRADE_STARTED="$( date -u '+%Y-%m-%dT%H:%M:%SZ' )" # ======================================================== # Start new Keycloak # # Keycloak performs required Liquibase schema migration # automatically during startup. # ======================================================== echo echo "--- Start new Keycloak ---" docker compose \ up -d \ --no-deps \ keycloak # ======================================================== # Wait until migration/startup is complete # ======================================================== log "Wait for Keycloak startup and DB migration" wait_for_keycloak # ======================================================== # Startup logs # ======================================================== log "Keycloak startup log" STARTUP_LOG="$( docker logs \ --since "${UPGRADE_STARTED}" \ keycloak 2>&1 )" echo "${STARTUP_LOG}" # ======================================================== # Obvious migration failure detection # # health=healthy remains the authoritative success check. # ======================================================== if echo "${STARTUP_LOG}" | grep -Eqi \ 'liquibase[^[:cntrl:]]*(failed|failure|exception)|database migration[^[:cntrl:]]*(failed|failure|exception)' then fail "Keycloak startup log contains a database migration failure" fi # ======================================================== # Verify actual running version # ======================================================== log "Validate running Keycloak version" RUNNING_VERSION_OUTPUT="$( docker exec \ keycloak \ /opt/keycloak/bin/kc.sh \ --version )" RUNNING_KC_VERSION="$( printf '%s\n' "${RUNNING_VERSION_OUTPUT}" | extract_version )" [[ -n "${RUNNING_KC_VERSION}" ]] || \ fail "Could not determine running Keycloak version" echo "Expected version:" echo " ${TARGET_KC_VERSION}" echo "Running version:" echo " ${RUNNING_KC_VERSION}" [[ "${RUNNING_KC_VERSION}" == "${TARGET_KC_VERSION}" ]] || \ fail \ "Running Keycloak version does not match target: ${RUNNING_KC_VERSION} != ${TARGET_KC_VERSION}" # ============================================================ # NO KEYCLOAK VERSION CHANGE # ============================================================ else log "No Keycloak version change" echo "Current version:" echo " ${CURRENT_KC_VERSION}" docker compose \ up -d \ --remove-orphans UPGRADE_STARTED="$( date -u -d '10 minutes ago' '+%Y-%m-%dT%H:%M:%SZ' )" wait_for_keycloak fi # ============================================================ # Final container status # ============================================================ log "Final container status" FINAL_RUNNING="$( docker inspect \ -f '{{.State.Running}}' \ keycloak )" FINAL_HEALTH="$( docker inspect \ -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ keycloak )" [[ "${FINAL_RUNNING}" == "true" ]] || \ fail "Keycloak container is not running" [[ "${FINAL_HEALTH}" == "healthy" ]] || \ fail "Keycloak container is not healthy" echo "Container:" echo " running" echo "Health:" echo " ${FINAL_HEALTH}" # ============================================================ # Final version # ============================================================ FINAL_VERSION_OUTPUT="$( docker exec \ keycloak \ /opt/keycloak/bin/kc.sh \ --version )" FINAL_KC_VERSION="$( printf '%s\n' "${FINAL_VERSION_OUTPUT}" | extract_version )" [[ -n "${FINAL_KC_VERSION}" ]] || \ fail "Could not determine final Keycloak version" [[ "${FINAL_KC_VERSION}" == "${TARGET_KC_VERSION}" ]] || \ fail \ "Final Keycloak version mismatch: ${FINAL_KC_VERSION} != ${TARGET_KC_VERSION}" # ============================================================ # Final image # ============================================================ RUNNING_IMAGE="$( docker inspect \ -f '{{.Config.Image}}' \ keycloak )" # ============================================================ # Success # ============================================================ log "Keycloak deployment successful" echo "Previous Keycloak version:" echo " ${CURRENT_KC_VERSION}" echo echo "Running Keycloak version:" echo " ${FINAL_KC_VERSION}" echo echo "Previous Docker image:" echo " ${OLD_IMAGE}" echo echo "Running Docker image:" echo " ${RUNNING_IMAGE}" if [[ "${VERSION_CHANGED}" == "true" ]] then echo echo "Pre-upgrade backup:" echo " ${BACKUP_DIR}" echo echo "PostgreSQL dump:" echo " ${DB_BACKUP_FILE}" echo echo "Backup retention:" echo " last ${BACKUP_KEEP} backups" fi # ============================================================ # Clear DB credentials # ============================================================ unset PGPASSWORD unset DB_PASSWORD unset KC_DB_PASSWORD unset KCRAW_DB_PASSWORD