Загрузить файлы в «alert-processor/app»

alert-processor/app/__init__.py

@@ -0,0 +1 @@
+# empty

alert-processor/app/audit_logger.py

@@ -0,0 +1,330 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import Any
+
+from redis.asyncio import Redis
+
+from app.models import NotificationDecision, ProcessorForwardEnvelope
+
+
+def _utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+class AuditLogger:
+    def __init__(
+        self,
+        client: Redis,
+        key_prefix: str,
+        ttl_seconds: int = 604800,
+        max_stage_records: int = 200,
+    ) -> None:
+        self.client = client
+        self.key_prefix = key_prefix
+        self.ttl_seconds = ttl_seconds
+        self.max_stage_records = max_stage_records
+
+    def _journal_key(self, correlation_id: str) -> str:
+        return f"{self.key_prefix}:journal:{correlation_id}"
+
+    def _stages_key(self, correlation_id: str) -> str:
+        return f"{self.key_prefix}:stages:{correlation_id}"
+
+    def _event_index_key(self, event_id: str) -> str:
+        return f"{self.key_prefix}:event:{event_id}"
+
+    def _recent_key(self) -> str:
+        return f"{self.key_prefix}:recent"
+
+    async def log_ingest_queued(
+        self,
+        envelope: ProcessorForwardEnvelope,
+        job_id: str,
+        queue_name: str,
+    ) -> None:
+        event = envelope.event
+        correlation_id = event.correlation_id
+        event_id = event.event_id or ""
+
+        await self._upsert_journal(
+            correlation_id=correlation_id,
+            event_id=event_id or None,
+            fields={
+                "state": "queued",
+                "job_id": job_id,
+                "queue_name": queue_name,
+                "correlation_id": correlation_id,
+                "event_id": event_id,
+                "host": event.host or "",
+                "service": event.service or "",
+                "trigger_name": event.trigger_name or "",
+                "severity": event.severity or "",
+                "queued_at": _utc_now(),
+                "updated_at": _utc_now(),
+            },
+        )
+
+        await self.log_stage(
+            correlation_id=correlation_id,
+            event_id=event.event_id,
+            stage="ingest_queued",
+            status="ok",
+            details={
+                "job_id": job_id,
+                "queue_name": queue_name,
+            },
+        )
+
+    async def log_worker_started(
+        self,
+        envelope: ProcessorForwardEnvelope,
+        job_id: str,
+        attempt: int,
+        identity: str,
+    ) -> None:
+        event = envelope.event
+        await self._upsert_journal(
+            correlation_id=event.correlation_id,
+            event_id=event.event_id,
+            fields={
+                "state": "processing",
+                "worker_started_at": _utc_now(),
+                "worker_attempt": attempt,
+                "processing_identity": identity,
+                "updated_at": _utc_now(),
+            },
+        )
+
+        await self.log_stage(
+            correlation_id=event.correlation_id,
+            event_id=event.event_id,
+            stage="worker_started",
+            status="ok",
+            details={
+                "job_id": job_id,
+                "attempt": attempt,
+                "identity": identity,
+            },
+        )
+
+    async def log_stage(
+        self,
+        correlation_id: str,
+        event_id: str | None,
+        stage: str,
+        status: str = "ok",
+        details: dict[str, Any] | None = None,
+    ) -> None:
+        payload = {
+            "ts": _utc_now(),
+            "stage": stage,
+            "status": status,
+            "details": details or {},
+        }
+        raw = json.dumps(payload, ensure_ascii=False)
+
+        stages_key = self._stages_key(correlation_id)
+        await self.client.rpush(stages_key, raw)
+        await self.client.ltrim(stages_key, -self.max_stage_records, -1)
+        await self.client.expire(stages_key, self.ttl_seconds)
+
+        await self._upsert_journal(
+            correlation_id=correlation_id,
+            event_id=event_id,
+            fields={
+                "last_stage": stage,
+                "last_stage_status": status,
+                "updated_at": _utc_now(),
+            },
+        )
+
+    async def log_decision(
+        self,
+        envelope: ProcessorForwardEnvelope,
+        decision: NotificationDecision,
+    ) -> None:
+        event = envelope.event
+        await self._upsert_journal(
+            correlation_id=event.correlation_id,
+            event_id=event.event_id,
+            fields={
+                "state": "decision_made",
+                "decision_at": _utc_now(),
+                "notify": str(decision.notify).lower(),
+                "suppressed": str(decision.suppressed).lower(),
+                "routing_class": decision.routing_class or "",
+                "decision_reason": decision.reason or "",
+                "decision_json": json.dumps(
+                    decision.model_dump(mode="json"),
+                    ensure_ascii=False,
+                ),
+                "updated_at": _utc_now(),
+            },
+        )
+
+        await self.log_stage(
+            correlation_id=event.correlation_id,
+            event_id=event.event_id,
+            stage="decision_made",
+            status="ok",
+            details={
+                "notify": decision.notify,
+                "suppressed": decision.suppressed,
+                "routing_class": decision.routing_class,
+                "reason": decision.reason,
+            },
+        )
+
+    async def log_delivery(
+        self,
+        envelope: ProcessorForwardEnvelope,
+        delivery_payload: dict[str, Any],
+    ) -> None:
+        event = envelope.event
+
+        await self._upsert_journal(
+            correlation_id=event.correlation_id,
+            event_id=event.event_id,
+            fields={
+                "state": "delivered",
+                "delivery_at": _utc_now(),
+                "delivery_json": json.dumps(delivery_payload, ensure_ascii=False),
+                "updated_at": _utc_now(),
+            },
+        )
+
+        await self.log_stage(
+            correlation_id=event.correlation_id,
+            event_id=event.event_id,
+            stage="delivery_completed",
+            status="ok",
+            details=delivery_payload,
+        )
+
+    async def log_worker_outcome(
+        self,
+        envelope: ProcessorForwardEnvelope,
+        state: str,
+        error: str | None = None,
+        attempt: int | None = None,
+    ) -> None:
+        event = envelope.event
+
+        fields = {
+            "state": state,
+            "updated_at": _utc_now(),
+        }
+        if error:
+            fields["last_error"] = error
+            fields["last_error_at"] = _utc_now()
+        if attempt is not None:
+            fields["worker_attempt"] = attempt
+
+        await self._upsert_journal(
+            correlation_id=event.correlation_id,
+            event_id=event.event_id,
+            fields=fields,
+        )
+
+        await self.log_stage(
+            correlation_id=event.correlation_id,
+            event_id=event.event_id,
+            stage=f"worker_{state}",
+            status="error" if error else "ok",
+            details={
+                "attempt": attempt,
+                "error": error,
+            },
+        )
+
+    async def get_event_audit(
+        self,
+        correlation_id: str | None = None,
+        event_id: str | None = None,
+    ) -> dict[str, Any] | None:
+        resolved_correlation = correlation_id
+
+        if not resolved_correlation and event_id:
+            value = await self.client.get(self._event_index_key(event_id))
+            if value is None:
+                return None
+            resolved_correlation = value.decode() if isinstance(value, bytes) else str(value)
+
+        if not resolved_correlation:
+            return None
+
+        journal_raw = await self.client.hgetall(self._journal_key(resolved_correlation))
+        if not journal_raw:
+            return None
+
+        stages_raw = await self.client.lrange(self._stages_key(resolved_correlation), 0, -1)
+
+        journal = self._decode_hash(journal_raw)
+        stages = [json.loads(item.decode() if isinstance(item, bytes) else str(item)) for item in stages_raw]
+
+        if journal.get("decision_json"):
+            try:
+                journal["decision"] = json.loads(journal["decision_json"])
+            except Exception:
+                journal["decision"] = None
+
+        if journal.get("delivery_json"):
+            try:
+                journal["delivery"] = json.loads(journal["delivery_json"])
+            except Exception:
+                journal["delivery"] = None
+
+        return {
+            "journal": journal,
+            "stages": stages,
+        }
+
+    async def list_recent(self, limit: int = 20) -> list[dict[str, Any]]:
+        correlation_ids_raw = await self.client.zrevrange(self._recent_key(), 0, max(limit - 1, 0))
+        results: list[dict[str, Any]] = []
+
+        for raw in correlation_ids_raw:
+            correlation_id = raw.decode() if isinstance(raw, bytes) else str(raw)
+            audit = await self.get_event_audit(correlation_id=correlation_id)
+            if audit is not None:
+                results.append(audit)
+
+        return results
+
+    async def _upsert_journal(
+        self,
+        correlation_id: str,
+        event_id: str | None,
+        fields: dict[str, Any],
+    ) -> None:
+        journal_key = self._journal_key(correlation_id)
+
+        mapping: dict[str, str] = {}
+        for key, value in fields.items():
+            if value is None:
+                continue
+            mapping[key] = str(value)
+
+        await self.client.hset(journal_key, mapping=mapping)
+        await self.client.expire(journal_key, self.ttl_seconds)
+
+        if event_id:
+            await self.client.set(
+                self._event_index_key(event_id),
+                correlation_id,
+                ex=self.ttl_seconds,
+            )
+
+        await self.client.zadd(self._recent_key(), {correlation_id: datetime.now(timezone.utc).timestamp()})
+        await self.client.expire(self._recent_key(), self.ttl_seconds)
+
+    @staticmethod
+    def _decode_hash(raw: dict[bytes, bytes]) -> dict[str, str]:
+        result: dict[str, str] = {}
+        for k, v in raw.items():
+            key = k.decode() if isinstance(k, bytes) else str(k)
+            value = v.decode() if isinstance(v, bytes) else str(v)
+            result[key] = value
+        return result

alert-processor/app/config.py

@@ -0,0 +1,257 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+from dotenv import load_dotenv
+
+
+load_dotenv()
+
+
+def _parse_bool(value: str | None, default: bool = False) -> bool:
+    if value is None:
+        return default
+    return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+@dataclass(frozen=True)
+class Settings:
+    app_name: str = os.getenv("APP_NAME", "alert-processor")
+    app_host: str = os.getenv("APP_HOST", "0.0.0.0")
+    app_port: int = int(os.getenv("APP_PORT", "8081"))
+
+    internal_api_token: str = os.getenv("INTERNAL_API_TOKEN", "").strip()
+    require_internal_api_token: bool = _parse_bool(
+        os.getenv("REQUIRE_INTERNAL_API_TOKEN", "true"),
+        default=True,
+    )
+
+    redis_enabled: bool = _parse_bool(
+        os.getenv("REDIS_ENABLED", "true"),
+        default=True,
+    )
+    redis_url: str = os.getenv("REDIS_URL", "redis://127.0.0.1:6379/0").strip()
+    redis_key_prefix: str = os.getenv("REDIS_KEY_PREFIX", "alert").strip()
+    redis_fingerprint_ttl_seconds: int = int(
+        os.getenv("REDIS_FINGERPRINT_TTL_SECONDS", "86400")
+    )
+    redis_event_ttl_seconds: int = int(
+        os.getenv("REDIS_EVENT_TTL_SECONDS", "604800")
+    )
+
+    suppress_enabled: bool = _parse_bool(
+        os.getenv("SUPPRESS_ENABLED", "true"),
+        default=True,
+    )
+    suppress_window_seconds: int = int(
+        os.getenv("SUPPRESS_WINDOW_SECONDS", "900")
+    )
+    suppress_apply_to_average: bool = _parse_bool(
+        os.getenv("SUPPRESS_APPLY_TO_AVERAGE", "true"),
+        default=True,
+    )
+
+    flap_enabled: bool = _parse_bool(
+        os.getenv("FLAP_ENABLED", "true"),
+        default=True,
+    )
+    flap_window_seconds: int = int(
+        os.getenv("FLAP_WINDOW_SECONDS", "120")
+    )
+    flap_threshold: int = int(
+        os.getenv("FLAP_THRESHOLD", "4")
+    )
+    flap_apply_to_average: bool = _parse_bool(
+        os.getenv("FLAP_APPLY_TO_AVERAGE", "true"),
+        default=True,
+    )
+
+    correlation_enabled: bool = _parse_bool(
+        os.getenv("CORRELATION_ENABLED", "true"),
+        default=True,
+    )
+    correlation_window_seconds: int = int(
+        os.getenv("CORRELATION_WINDOW_SECONDS", "180")
+    )
+    correlation_suppress_children: bool = _parse_bool(
+        os.getenv("CORRELATION_SUPPRESS_CHILDREN", "true"),
+        default=True,
+    )
+
+    matrix_enabled: bool = _parse_bool(
+        os.getenv("MATRIX_ENABLED", "false"),
+        default=False,
+    )
+    matrix_homeserver_url: str = os.getenv("MATRIX_HOMESERVER_URL", "").strip().rstrip("/")
+    matrix_room_id: str = os.getenv("MATRIX_ROOM_ID", "").strip()
+
+    matrix_access_token: str = os.getenv("MATRIX_ACCESS_TOKEN", "").strip()
+    matrix_refresh_token: str = os.getenv("MATRIX_REFRESH_TOKEN", "").strip()
+
+    matrix_oauth_token_endpoint: str = os.getenv("MATRIX_OAUTH_TOKEN_ENDPOINT", "").strip()
+    matrix_oauth_client_id: str = os.getenv("MATRIX_OAUTH_CLIENT_ID", "").strip()
+    matrix_oauth_client_secret: str = os.getenv("MATRIX_OAUTH_CLIENT_SECRET", "").strip()
+
+    matrix_access_token_expires_in_seconds: int = int(
+        os.getenv("MATRIX_ACCESS_TOKEN_EXPIRES_IN_SECONDS", "300")
+    )
+    matrix_refresh_margin_seconds: int = int(
+        os.getenv("MATRIX_REFRESH_MARGIN_SECONDS", "60")
+    )
+    matrix_token_state_file: str = os.getenv(
+        "MATRIX_TOKEN_STATE_FILE",
+        ".matrix_token_state.json",
+    ).strip()
+
+    matrix_message_type: str = os.getenv("MATRIX_MESSAGE_TYPE", "m.notice").strip()
+    matrix_request_timeout_seconds: float = float(
+        os.getenv("MATRIX_REQUEST_TIMEOUT_SECONDS", "10")
+    )
+    matrix_verify_tls: bool = _parse_bool(
+        os.getenv("MATRIX_VERIFY_TLS", "true"),
+        default=True,
+    )
+
+    mail_enabled: bool = _parse_bool(
+        os.getenv("MAIL_ENABLED", "false"),
+        default=False,
+    )
+    mail_smtp_host: str = os.getenv("MAIL_SMTP_HOST", "").strip()
+    mail_smtp_port: int = int(os.getenv("MAIL_SMTP_PORT", "587"))
+    mail_smtp_username: str = os.getenv("MAIL_SMTP_USERNAME", "").strip()
+    mail_smtp_password: str = os.getenv("MAIL_SMTP_PASSWORD", "").strip()
+    mail_from: str = os.getenv("MAIL_FROM", "").strip()
+    mail_to: str = os.getenv("MAIL_TO", "").strip()
+    mail_use_starttls: bool = _parse_bool(
+        os.getenv("MAIL_USE_STARTTLS", "true"),
+        default=True,
+    )
+    mail_use_tls: bool = _parse_bool(
+        os.getenv("MAIL_USE_TLS", "false"),
+        default=False,
+    )
+    mail_timeout_seconds: float = float(
+        os.getenv("MAIL_TIMEOUT_SECONDS", "15")
+    )
+    mail_subject_prefix: str = os.getenv(
+        "MAIL_SUBJECT_PREFIX",
+        "[LLM-Zabbix]",
+    ).strip()
+
+    zabbix_api_enabled: bool = _parse_bool(
+        os.getenv("ZABBIX_API_ENABLED", "false"),
+        default=False,
+    )
+    zabbix_api_url: str = os.getenv("ZABBIX_API_URL", "").strip()
+    zabbix_api_token: str = os.getenv("ZABBIX_API_TOKEN", "").strip()
+    zabbix_web_url: str = os.getenv("ZABBIX_WEB_URL", "").strip().rstrip("/")
+    zabbix_api_timeout_seconds: float = float(
+        os.getenv("ZABBIX_API_TIMEOUT_SECONDS", "10")
+    )
+    zabbix_api_verify_tls: bool = _parse_bool(
+        os.getenv("ZABBIX_API_VERIFY_TLS", "true"),
+        default=True,
+    )
+    zabbix_graph_period_hours: int = int(
+        os.getenv("ZABBIX_GRAPH_PERIOD_HOURS", "1")
+    )
+    zabbix_graph_timezone: str = os.getenv(
+        "ZABBIX_GRAPH_TIMEZONE",
+        "UTC",
+    ).strip()
+    zabbix_enrich_only_notify: bool = _parse_bool(
+        os.getenv("ZABBIX_ENRICH_ONLY_NOTIFY", "true"),
+        default=True,
+    )
+
+    llm_enabled: bool = _parse_bool(
+        os.getenv("LLM_ENABLED", "false"),
+        default=False,
+    )
+    llm_base_url: str = os.getenv("LLM_BASE_URL", "").strip().rstrip("/")
+    llm_model: str = os.getenv("LLM_MODEL", "").strip()
+    llm_timeout_seconds: float = float(
+        os.getenv("LLM_TIMEOUT_SECONDS", "45")
+    )
+    llm_verify_tls: bool = _parse_bool(
+        os.getenv("LLM_VERIFY_TLS", "true"),
+        default=True,
+    )
+    llm_temperature: float = float(
+        os.getenv("LLM_TEMPERATURE", "0.1")
+    )
+    llm_max_steps: int = int(
+        os.getenv("LLM_MAX_STEPS", "4")
+    )
+    llm_max_commands: int = int(
+        os.getenv("LLM_MAX_COMMANDS", "4")
+    )
+
+    llm_triage_enabled: bool = _parse_bool(
+        os.getenv("LLM_TRIAGE_ENABLED", "false"),
+        default=False,
+    )
+    llm_triage_cache_ttl_seconds: int = int(
+        os.getenv("LLM_TRIAGE_CACHE_TTL_SECONDS", "3600")
+    )
+
+    queue_enabled: bool = _parse_bool(
+        os.getenv("QUEUE_ENABLED", "true"),
+        default=True,
+    )
+    queue_name: str = os.getenv("QUEUE_NAME", "alert:queue:events").strip()
+    queue_processing_name: str = os.getenv(
+        "QUEUE_PROCESSING_NAME",
+        "alert:queue:processing",
+    ).strip()
+    queue_deadletter_name: str = os.getenv(
+        "QUEUE_DEADLETTER_NAME",
+        "alert:queue:deadletter",
+    ).strip()
+    queue_block_timeout_seconds: int = int(
+        os.getenv("QUEUE_BLOCK_TIMEOUT_SECONDS", "5")
+    )
+    queue_max_attempts: int = int(
+        os.getenv("QUEUE_MAX_ATTEMPTS", "3")
+    )
+    queue_dedup_ttl_seconds: int = int(
+        os.getenv("QUEUE_DEDUP_TTL_SECONDS", "86400")
+    )
+    queue_requeue_processing_on_startup: bool = _parse_bool(
+        os.getenv("QUEUE_REQUEUE_PROCESSING_ON_STARTUP", "true"),
+        default=True,
+    )
+    correlation_kind_rules_path: str = os.getenv(
+        "CORRELATION_KIND_RULES_PATH",
+        "./config/event_kind_rules.yaml",
+    ).strip()
+    correlation_root_cause_path: str = os.getenv(
+        "CORRELATION_ROOT_CAUSE_PATH",
+        "./config/root_cause_map.yaml",
+    ).strip()
+    audit_enabled: bool = _parse_bool(
+        os.getenv("AUDIT_ENABLED", "true"),
+        default=True,
+    )
+    audit_key_prefix: str = os.getenv(
+        "AUDIT_KEY_PREFIX",
+        "alert:audit",
+    ).strip()
+    audit_ttl_seconds: int = int(
+        os.getenv("AUDIT_TTL_SECONDS", "604800")
+    )
+    audit_max_stage_records: int = int(
+        os.getenv("AUDIT_MAX_STAGE_RECORDS", "200")
+    )
+    llm_correlation_enabled: bool = _parse_bool(
+        os.getenv("LLM_CORRELATION_ENABLED", "false"),
+        default=False,
+    )
+    llm_correlation_min_confidence: str = os.getenv(
+        "LLM_CORRELATION_MIN_CONFIDENCE",
+        "medium",
+    ).strip().lower()
+
+
+settings = Settings()

alert-processor/app/correlation.py

@@ -0,0 +1,530 @@
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from app.config import settings
+from app.models import NotificationDecision, ProcessorForwardEnvelope
+
+logger = logging.getLogger(__name__)
+
+
+def _norm(value: str | None) -> str:
+    return (value or "").strip().lower()
+
+
+def severity_rank(severity: str | None) -> int:
+    value = _norm(severity)
+    if value in {"disaster"}:
+        return 4
+    if value in {"high"}:
+        return 3
+    if value in {"average"}:
+        return 2
+    if value in {"warning", "information", "not classified", "not_classified"}:
+        return 1
+    return 0
+
+
+@dataclass(frozen=True)
+class EventKindRule:
+    kind: str
+    trigger_patterns: tuple[str, ...] = ()
+    item_key_patterns: tuple[str, ...] = ()
+    service_patterns: tuple[str, ...] = ()
+    tag_patterns: tuple[str, ...] = ()
+    scope_in: tuple[str, ...] = ()
+    severity_in: tuple[str, ...] = ()
+
+    def matches(
+        self,
+        envelope: ProcessorForwardEnvelope,
+        decision: NotificationDecision,
+    ) -> bool:
+        event = envelope.event
+        zbx_context = event.zabbix_context or {}
+
+        trigger_name = _norm(event.trigger_name)
+        item_key = _norm(zbx_context.get("item_key"))
+        service = _norm(zbx_context.get("service") or event.service)
+        alert_scope = _norm(zbx_context.get("alert_scope"))
+        severity = _norm(decision.severity)
+
+        tags_joined = " ".join(
+            f"{_norm(str(k))}={_norm(str(v))}" for k, v in (event.tags or {}).items()
+        )
+
+        if self.scope_in and alert_scope not in self.scope_in:
+            return False
+
+        if self.severity_in and severity not in self.severity_in:
+            return False
+
+        if self.trigger_patterns and not any(re.search(pattern, trigger_name) for pattern in self.trigger_patterns):
+            return False
+
+        if self.item_key_patterns and not any(re.search(pattern, item_key) for pattern in self.item_key_patterns):
+            return False
+
+        if self.service_patterns and not any(re.search(pattern, service) for pattern in self.service_patterns):
+            return False
+
+        if self.tag_patterns and not all(re.search(pattern, tags_joined) for pattern in self.tag_patterns):
+            return False
+
+        return True
+
+
+@dataclass
+class CorrelationEventRecord:
+    host: str
+    event_id: str | None
+    correlation_id: str
+    kind: str
+    severity: str | None
+    routing_class: str | None
+    fingerprint: str | None
+    root_candidate: bool
+    role: str
+    group_id: str | None
+    parent_event_id: str | None
+    parent_correlation_id: str | None
+    timestamp: float
+    tags: dict[str, str] = field(default_factory=dict)
+    service: str | None = None
+    scope: str | None = None
+    component: str | None = None
+    domain: str | None = None
+
+
+@dataclass
+class CorrelationAssessment:
+    applied: bool
+    role: str
+    kind: str
+    group_id: str | None
+    reason: str | None
+    parent_event_id: str | None = None
+    parent_correlation_id: str | None = None
+    root_cause_candidate: bool = False
+    correlated_event_count: int = 0
+    suppress_child: bool = False
+    source: str = "deterministic"
+    confidence: str | None = "high"
+
+
+@dataclass
+class CorrelationRegistry:
+    event_kind_rules: list[EventKindRule] = field(default_factory=list)
+    root_cause_map: dict[str, set[str]] = field(default_factory=dict)
+
+    def classify_event_kind(
+        self,
+        envelope: ProcessorForwardEnvelope,
+        decision: NotificationDecision,
+    ) -> str:
+        for rule in self.event_kind_rules:
+            if rule.matches(envelope, decision):
+                return rule.kind
+        return "unknown"
+
+    def is_root_candidate(self, kind: str) -> bool:
+        return kind in self.root_cause_map
+
+    def explains(self, parent_kind: str, child_kind: str) -> bool:
+        if parent_kind == child_kind:
+            return False
+        return child_kind in self.root_cause_map.get(parent_kind, set())
+
+    @classmethod
+    def default(cls) -> "CorrelationRegistry":
+        return cls(
+            event_kind_rules=[
+                EventKindRule(
+                    kind="postgresql_unavailable",
+                    trigger_patterns=(r"postgresql.+unavailable", r"postgres.+unavailable"),
+                ),
+                EventKindRule(
+                    kind="host_down",
+                    trigger_patterns=(r"host unavailable", r"icmp.+unreachable", r"unreachable"),
+                    scope_in=("host_os",),
+                ),
+                EventKindRule(
+                    kind="network_port_down",
+                    trigger_patterns=(r"port down", r"interface down"),
+                ),
+                EventKindRule(
+                    kind="ap_down",
+                    trigger_patterns=(r"ap down",),
+                ),
+                EventKindRule(
+                    kind="disk_full",
+                    item_key_patterns=(r"^vfs\.fs\.",),
+                    trigger_patterns=(r"space", r"full", r"disk"),
+                    scope_in=("host_os",),
+                ),
+                EventKindRule(
+                    kind="container_unavailable",
+                    trigger_patterns=(
+                        r"container",
+                        r"docker",
+                        r"health state",
+                        r"not running",
+                        r"stopped",
+                        r"unavailable",
+                        r"down",
+                        r"failed",
+                    ),
+                    scope_in=("container",),
+                ),
+                EventKindRule(
+                    kind="cpu_high",
+                    item_key_patterns=(r"^system\.cpu\.",),
+                    trigger_patterns=(r"cpu utilization",),
+                    scope_in=("host_os",),
+                ),
+                EventKindRule(
+                    kind="memory_high",
+                    item_key_patterns=(r"^vm\.memory\.", r"^system\.swap\."),
+                    trigger_patterns=(r"memory utilization",),
+                    scope_in=("host_os",),
+                ),
+                EventKindRule(
+                    kind="ssl_expiry",
+                    trigger_patterns=(r"ssl", r"certificate", r"expire"),
+                ),
+                EventKindRule(
+                    kind="service_unavailable",
+                    trigger_patterns=(r"service unavailable", r"unavailable"),
+                ),
+                EventKindRule(
+                    kind="site_unavailable",
+                    trigger_patterns=(r"site.+is down", r"failed step of scenario"),
+                ),
+                EventKindRule(
+                    kind="db_connection_error",
+                    trigger_patterns=(r"connection", r"db"),
+                ),
+            ],
+            root_cause_map={
+                "host_down": {
+                    "service_unavailable",
+                    "container_unavailable",
+                    "postgresql_unavailable",
+                    "cpu_high",
+                    "memory_high",
+                    "disk_full",
+                    "ssl_expiry",
+                    "db_connection_error",
+                    "site_unavailable",
+                },
+                "network_port_down": {
+                    "host_down",
+                    "service_unavailable",
+                    "ap_down",
+                    "site_unavailable",
+                },
+                "postgresql_unavailable": {
+                    "service_unavailable",
+                    "db_connection_error",
+                    "site_unavailable",
+                },
+                "disk_full": {
+                    "postgresql_unavailable",
+                    "container_unavailable",
+                    "service_unavailable",
+                    "db_connection_error",
+                    "site_unavailable",
+                },
+                "container_unavailable": {
+                    "service_unavailable",
+                    "site_unavailable",
+                },
+            },
+        )
+
+    @classmethod
+    def from_yaml_files(
+        cls,
+        event_kind_rules_path: str,
+        root_cause_map_path: str,
+    ) -> "CorrelationRegistry":
+        base = cls.default()
+
+        custom_kind_rules = _load_event_kind_rules(event_kind_rules_path)
+        custom_root_cause_map = _load_root_cause_map(root_cause_map_path)
+
+        merged_kind_rules = custom_kind_rules + base.event_kind_rules
+
+        merged_root_cause_map: dict[str, set[str]] = {
+            root_kind: set(children)
+            for root_kind, children in base.root_cause_map.items()
+        }
+
+        for root_kind, explained in custom_root_cause_map.items():
+            if root_kind not in merged_root_cause_map:
+                merged_root_cause_map[root_kind] = set()
+            merged_root_cause_map[root_kind].update(explained)
+
+        logger.info(
+            "Correlation registry loaded: custom_kind_rules=%s custom_root_kinds=%s total_kind_rules=%s total_root_kinds=%s",
+            len(custom_kind_rules),
+            len(custom_root_cause_map),
+            len(merged_kind_rules),
+            len(merged_root_cause_map),
+        )
+
+        return cls(
+            event_kind_rules=merged_kind_rules,
+            root_cause_map=merged_root_cause_map,
+        )
+
+
+def _load_yaml(path: str) -> dict[str, Any]:
+    file_path = Path(path)
+    if not file_path.exists():
+        logger.warning("Correlation YAML file not found: %s. Using defaults/merge fallback.", path)
+        return {}
+
+    try:
+        with file_path.open("r", encoding="utf-8") as f:
+            data = yaml.safe_load(f) or {}
+        if not isinstance(data, dict):
+            logger.warning("Correlation YAML root must be a mapping: %s", path)
+            return {}
+        return data
+    except Exception as exc:
+        logger.exception("Failed to load correlation YAML %s: %s", path, exc)
+        return {}
+
+
+def _load_event_kind_rules(path: str) -> list[EventKindRule]:
+    data = _load_yaml(path)
+    raw_rules = data.get("event_kind_rules", [])
+    if not isinstance(raw_rules, list):
+        logger.warning("event_kind_rules must be a list in %s", path)
+        return []
+
+    rules: list[EventKindRule] = []
+
+    for idx, item in enumerate(raw_rules, start=1):
+        if not isinstance(item, dict):
+            logger.warning("Skipping invalid event_kind_rules[%s] in %s", idx, path)
+            continue
+
+        kind = _norm(item.get("kind"))
+        if not kind:
+            logger.warning("Skipping event_kind_rules[%s] without kind in %s", idx, path)
+            continue
+
+        rules.append(
+            EventKindRule(
+                kind=kind,
+                trigger_patterns=tuple(item.get("trigger_patterns", []) or []),
+                item_key_patterns=tuple(item.get("item_key_patterns", []) or []),
+                service_patterns=tuple(item.get("service_patterns", []) or []),
+                tag_patterns=tuple(item.get("tag_patterns", []) or []),
+                scope_in=tuple(_norm(v) for v in (item.get("scope_in", []) or [])),
+                severity_in=tuple(_norm(v) for v in (item.get("severity_in", []) or [])),
+            )
+        )
+
+    return rules
+
+
+def _load_root_cause_map(path: str) -> dict[str, set[str]]:
+    data = _load_yaml(path)
+    raw_map = data.get("root_cause_map", {})
+    if not isinstance(raw_map, dict):
+        logger.warning("root_cause_map must be a mapping in %s", path)
+        return {}
+
+    result: dict[str, set[str]] = {}
+    for root_kind_raw, payload in raw_map.items():
+        root_kind = _norm(root_kind_raw)
+        if not root_kind:
+            continue
+
+        explains: list[str] = []
+        if isinstance(payload, dict):
+            explains_raw = payload.get("explains", [])
+            if isinstance(explains_raw, list):
+                explains = [_norm(v) for v in explains_raw if _norm(v)]
+        elif isinstance(payload, list):
+            explains = [_norm(v) for v in payload if _norm(v)]
+
+        if explains:
+            result[root_kind] = set(explains)
+
+    return result
+
+
+def _shared_service_scope_or_domain(
+    envelope: ProcessorForwardEnvelope,
+    candidate: CorrelationEventRecord,
+) -> bool:
+    event = envelope.event
+    zbx_context = event.zabbix_context or {}
+
+    current_tags = {str(k).lower(): str(v).strip() for k, v in (event.tags or {}).items()}
+    current_service = _norm(current_tags.get("service") or zbx_context.get("service") or event.service)
+    current_scope = _norm(current_tags.get("scope") or zbx_context.get("scope"))
+    current_domain = _norm(current_tags.get("domain") or zbx_context.get("domain"))
+
+    candidate_service = _norm(candidate.service or candidate.tags.get("service"))
+    candidate_scope = _norm(candidate.scope or candidate.tags.get("scope"))
+    candidate_domain = _norm(candidate.domain or candidate.tags.get("domain"))
+
+    if current_scope and candidate_scope and current_scope == candidate_scope:
+        return True
+    if current_service and candidate_service and current_service == candidate_service:
+        return True
+    if current_domain and candidate_domain and current_domain == candidate_domain:
+        return True
+    if event.host and candidate.host and _norm(event.host) == _norm(candidate.host):
+        return True
+    return False
+
+
+def assess_correlation(
+    envelope: ProcessorForwardEnvelope,
+    decision: NotificationDecision,
+    recent_events: list[CorrelationEventRecord],
+    registry: CorrelationRegistry,
+) -> CorrelationAssessment:
+    kind = registry.classify_event_kind(envelope, decision)
+    root_candidate = registry.is_root_candidate(kind)
+
+    if kind == "unknown":
+        return CorrelationAssessment(
+            applied=False,
+            role="standalone",
+            kind=kind,
+            group_id=None,
+            reason=None,
+            root_cause_candidate=False,
+            correlated_event_count=0,
+        )
+
+    parent_candidates = [
+        item
+        for item in recent_events
+        if item.root_candidate
+        and registry.explains(item.kind, kind)
+        and _shared_service_scope_or_domain(envelope, item)
+    ]
+    parent_candidates.sort(
+        key=lambda item: (item.timestamp, severity_rank(item.severity)),
+        reverse=True,
+    )
+
+    if parent_candidates:
+        parent = parent_candidates[0]
+        suppress_child = (
+            settings.correlation_suppress_children
+            and decision.routing_class in {
+                "average_priority",
+                "low_priority",
+                "triage_low_priority_notify",
+                "triage_low_priority_hold",
+                "triage_low_priority_suppressed",
+            }
+        )
+
+        return CorrelationAssessment(
+            applied=True,
+            role="child",
+            kind=kind,
+            group_id=parent.group_id or parent.event_id or parent.correlation_id,
+            reason=(
+                f"Likely downstream of {parent.kind}; matched recent event "
+                f"in the same service/scope/domain window within {settings.correlation_window_seconds}s"
+            ),
+            parent_event_id=parent.event_id,
+            parent_correlation_id=parent.correlation_id,
+            root_cause_candidate=False,
+            correlated_event_count=0,
+            suppress_child=suppress_child,
+        )
+
+    if root_candidate:
+        correlated_count = sum(
+            1
+            for item in recent_events
+            if registry.explains(kind, item.kind) and _shared_service_scope_or_domain(envelope, item)
+        )
+
+        return CorrelationAssessment(
+            applied=True,
+            role="root",
+            kind=kind,
+            group_id=envelope.event.event_id or envelope.event.correlation_id,
+            reason=(
+                f"Potential root cause candidate; {correlated_count} related alerts seen "
+                f"in matching service/scope/domain window within {settings.correlation_window_seconds}s"
+            ),
+            root_cause_candidate=True,
+            correlated_event_count=correlated_count,
+            suppress_child=False,
+        )
+
+    return CorrelationAssessment(
+        applied=True,
+        role="standalone",
+        kind=kind,
+        group_id=envelope.event.event_id or envelope.event.correlation_id,
+        reason="No root cause candidate found in matching service/scope/domain correlation window",
+        root_cause_candidate=False,
+        correlated_event_count=0,
+        suppress_child=False,
+    )
+
+
+def apply_correlation_to_decision(
+    decision: NotificationDecision,
+    assessment: CorrelationAssessment,
+) -> NotificationDecision:
+    if not assessment.applied:
+        return decision
+
+    update = {
+        "correlation_applied": True,
+        "correlation_role": assessment.role,
+        "correlation_group_id": assessment.group_id,
+        "correlation_kind": assessment.kind,
+        "correlation_reason": assessment.reason,
+        "correlation_source": assessment.source,
+        "correlation_confidence": assessment.confidence,
+        "parent_event_id": assessment.parent_event_id,
+        "parent_correlation_id": assessment.parent_correlation_id,
+        "root_cause_candidate": assessment.root_cause_candidate,
+        "correlated_event_count": assessment.correlated_event_count,
+    }
+
+    if assessment.role == "child" and assessment.suppress_child:
+        update.update(
+            {
+                "notify": False,
+                "suppressed": True,
+                "channels": [],
+                "routing_class": "correlated_child_suppressed",
+                "reason": "Correlated child alert suppressed by RCA baseline",
+                "suppress_reason": assessment.reason,
+            }
+        )
+        return decision.model_copy(update=update)
+
+    if assessment.role == "child":
+        update["reason"] = f"{decision.reason}. RCA baseline matched a parent event in the same service/scope."
+        return decision.model_copy(update=update)
+
+    if assessment.role == "root":
+        update["reason"] = f"{decision.reason}. RCA baseline marked this event as root cause candidate"
+        return decision.model_copy(update=update)
+
+    return decision.model_copy(update=update)

alert-processor/app/fingerprint.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+import re
+
+from app.models import NormalizedEvent
+
+
+def _norm(value: str | None) -> str:
+    if not value:
+        return "unknown"
+    value = value.strip().lower()
+    value = re.sub(r"\s+", "_", value)
+    value = re.sub(r"[^a-z0-9_\-\.]+", "_", value)
+    return value.strip("_") or "unknown"
+
+
+def build_fingerprint(event: NormalizedEvent) -> str:
+    host = _norm(event.host)
+    service = _norm(event.service)
+    trigger = _norm(event.trigger_name)
+
+    return f"{host}|{service}|{trigger}"