269 lines
8.5 KiB
Python
269 lines
8.5 KiB
Python
from __future__ import annotations
|
|
|
|
from app.models import NormalizedEvent, NotificationDecision
|
|
|
|
|
|
HIGH_SEVERITIES = {"high", "disaster"}
|
|
AVERAGE_SEVERITIES = {"average"}
|
|
LOW_SEVERITIES = {"warning", "information", "not classified", "not_classified", "info"}
|
|
|
|
RECOVERY_EVENT_TYPES = {"recovery", "resolved", "resolve", "ok", "clear", "closed"}
|
|
|
|
|
|
def normalize_severity(value: str | None) -> str | None:
|
|
if value is None:
|
|
return None
|
|
return value.strip().lower()
|
|
|
|
|
|
def normalize_event_phase(event_type: str | None, value: str | None = None) -> str:
|
|
raw = (event_type or "").strip().lower()
|
|
if raw in RECOVERY_EVENT_TYPES:
|
|
return "recovery"
|
|
|
|
if not raw and str(value or "").strip() == "0":
|
|
return "recovery"
|
|
|
|
return "problem"
|
|
|
|
|
|
def evaluate_event(
|
|
event: NormalizedEvent,
|
|
fingerprint: str,
|
|
repeat_count: int,
|
|
) -> NotificationDecision:
|
|
severity = normalize_severity(event.severity)
|
|
|
|
if severity in HIGH_SEVERITIES:
|
|
return NotificationDecision(
|
|
notify=True,
|
|
severity=event.severity,
|
|
channels=["matrix", "mail"],
|
|
reason="Severity is High/Disaster: mandatory notification",
|
|
routing_class="high_priority",
|
|
fingerprint=fingerprint,
|
|
repeat_count=repeat_count,
|
|
event_phase="problem",
|
|
)
|
|
|
|
if severity in AVERAGE_SEVERITIES:
|
|
return NotificationDecision(
|
|
notify=True,
|
|
severity=event.severity,
|
|
channels=["matrix"],
|
|
reason="Severity is Average: notify by deterministic baseline policy",
|
|
routing_class="average_priority",
|
|
fingerprint=fingerprint,
|
|
repeat_count=repeat_count,
|
|
event_phase="problem",
|
|
)
|
|
|
|
if severity in LOW_SEVERITIES:
|
|
return NotificationDecision(
|
|
notify=False,
|
|
severity=event.severity,
|
|
channels=[],
|
|
reason="Severity is Warning or lower: held for triage/suppress pipeline",
|
|
routing_class="low_priority",
|
|
fingerprint=fingerprint,
|
|
repeat_count=repeat_count,
|
|
event_phase="problem",
|
|
)
|
|
|
|
return NotificationDecision(
|
|
notify=False,
|
|
severity=event.severity,
|
|
channels=[],
|
|
reason="Unknown severity: conservative hold until policy is expanded",
|
|
routing_class="unknown_priority",
|
|
fingerprint=fingerprint,
|
|
repeat_count=repeat_count,
|
|
event_phase="problem",
|
|
)
|
|
|
|
|
|
def build_recovery_decision(
|
|
fingerprint: str,
|
|
repeat_count: int,
|
|
open_incident_found: bool,
|
|
previous_severity: str | None,
|
|
previous_channels: list[str] | None,
|
|
previous_routing_class: str | None,
|
|
) -> NotificationDecision:
|
|
severity_norm = normalize_severity(previous_severity)
|
|
|
|
if not open_incident_found:
|
|
return NotificationDecision(
|
|
notify=False,
|
|
severity=previous_severity,
|
|
channels=[],
|
|
reason="Recovery received but no matching open incident was found",
|
|
routing_class="recovery_ignored",
|
|
fingerprint=fingerprint,
|
|
repeat_count=repeat_count,
|
|
event_phase="recovery",
|
|
open_incident_found=False,
|
|
recovered_from_severity=previous_severity,
|
|
)
|
|
|
|
if previous_routing_class == "triage_low_priority_notify":
|
|
return NotificationDecision(
|
|
notify=True,
|
|
severity=previous_severity,
|
|
channels=previous_channels or ["matrix"],
|
|
reason="Recovery matched a triaged low-severity incident",
|
|
routing_class="recovery_triage_low_priority",
|
|
fingerprint=fingerprint,
|
|
repeat_count=repeat_count,
|
|
event_phase="recovery",
|
|
open_incident_found=True,
|
|
recovered_from_severity=previous_severity,
|
|
)
|
|
|
|
if severity_norm in HIGH_SEVERITIES:
|
|
return NotificationDecision(
|
|
notify=True,
|
|
severity=previous_severity,
|
|
channels=previous_channels or ["matrix", "mail"],
|
|
reason="Recovery matched an open High/Disaster incident",
|
|
routing_class="recovery_high_priority",
|
|
fingerprint=fingerprint,
|
|
repeat_count=repeat_count,
|
|
event_phase="recovery",
|
|
open_incident_found=True,
|
|
recovered_from_severity=previous_severity,
|
|
)
|
|
|
|
if severity_norm in AVERAGE_SEVERITIES:
|
|
return NotificationDecision(
|
|
notify=True,
|
|
severity=previous_severity,
|
|
channels=previous_channels or ["matrix"],
|
|
reason="Recovery matched an open Average incident",
|
|
routing_class="recovery_average_priority",
|
|
fingerprint=fingerprint,
|
|
repeat_count=repeat_count,
|
|
event_phase="recovery",
|
|
open_incident_found=True,
|
|
recovered_from_severity=previous_severity,
|
|
)
|
|
|
|
return NotificationDecision(
|
|
notify=False,
|
|
severity=previous_severity,
|
|
channels=[],
|
|
reason="Recovery matched a low-priority incident: no notification by baseline policy",
|
|
routing_class="recovery_low_priority",
|
|
fingerprint=fingerprint,
|
|
repeat_count=repeat_count,
|
|
event_phase="recovery",
|
|
open_incident_found=True,
|
|
recovered_from_severity=previous_severity,
|
|
)
|
|
|
|
|
|
def decision_supports_suppress(decision: NotificationDecision) -> bool:
|
|
return decision.routing_class == "average_priority" and decision.notify
|
|
|
|
|
|
def decision_supports_flap_suppress(decision: NotificationDecision) -> bool:
|
|
return decision.routing_class in {
|
|
"average_priority",
|
|
"recovery_average_priority",
|
|
} and decision.notify
|
|
|
|
|
|
def is_low_severity_problem_candidate(
|
|
decision: NotificationDecision,
|
|
) -> bool:
|
|
return decision.event_phase == "problem" and decision.routing_class == "low_priority"
|
|
|
|
|
|
def apply_suppress_window(
|
|
decision: NotificationDecision,
|
|
ttl_seconds: int,
|
|
) -> NotificationDecision:
|
|
return decision.model_copy(
|
|
update={
|
|
"notify": False,
|
|
"channels": [],
|
|
"suppressed": True,
|
|
"suppress_reason": f"Suppress window active, {ttl_seconds}s remaining",
|
|
"reason": "Duplicate Average event suppressed by re-notify window",
|
|
}
|
|
)
|
|
|
|
|
|
def apply_flap_suppress(
|
|
decision: NotificationDecision,
|
|
event_count: int,
|
|
window_seconds: int,
|
|
) -> NotificationDecision:
|
|
return decision.model_copy(
|
|
update={
|
|
"notify": False,
|
|
"channels": [],
|
|
"suppressed": True,
|
|
"flap_detected": True,
|
|
"flap_event_count": event_count,
|
|
"flap_reason": f"Flapping detected: {event_count} phase changes within {window_seconds}s",
|
|
"reason": "Average event suppressed because the series is flapping",
|
|
}
|
|
)
|
|
|
|
|
|
def apply_low_severity_triage(
|
|
decision: NotificationDecision,
|
|
verdict: str,
|
|
classification: str | None,
|
|
reason: str | None,
|
|
source: str,
|
|
) -> NotificationDecision:
|
|
verdict_norm = (verdict or "hold").strip().lower()
|
|
classification_norm = (classification or "unknown").strip().lower()
|
|
reason_text = (reason or "").strip() or "LLM triage applied"
|
|
|
|
common = {
|
|
"triage_applied": True,
|
|
"triage_source": source,
|
|
"triage_verdict": verdict_norm,
|
|
"triage_reason": reason_text,
|
|
"triage_classification": classification_norm,
|
|
}
|
|
|
|
if verdict_norm == "notify":
|
|
return decision.model_copy(
|
|
update={
|
|
**common,
|
|
"notify": True,
|
|
"suppressed": False,
|
|
"suppress_reason": None,
|
|
"channels": ["matrix"],
|
|
"routing_class": "triage_low_priority_notify",
|
|
"reason": f"LLM triage marked low-severity event as actionable: {reason_text}",
|
|
}
|
|
)
|
|
|
|
if verdict_norm == "suppress":
|
|
return decision.model_copy(
|
|
update={
|
|
**common,
|
|
"notify": False,
|
|
"suppressed": True,
|
|
"channels": [],
|
|
"routing_class": "triage_low_priority_suppressed",
|
|
"reason": "LLM triage suppressed low-severity event",
|
|
"suppress_reason": reason_text,
|
|
}
|
|
)
|
|
|
|
return decision.model_copy(
|
|
update={
|
|
**common,
|
|
"notify": False,
|
|
"suppressed": False,
|
|
"channels": [],
|
|
"routing_class": "triage_low_priority_hold",
|
|
"reason": f"LLM triage kept low-severity event on hold: {reason_text}",
|
|
}
|
|
) |