from __future__ import annotations import logging from app.audit_logger import AuditLogger from app.config import settings from app.correlation import ( CorrelationAssessment, CorrelationRegistry, apply_correlation_to_decision, assess_correlation, ) from app.fingerprint import build_fingerprint from app.llm_correlation import LLMCorrelationAdapter, LLMCorrelationResult from app.llm_remediation import LLMRemediationAdapter from app.llm_triage import LLMTriageAdapter from app.models import NotificationDecision, ProcessorForwardEnvelope from app.notifications.dispatcher import NotificationDispatcher from app.policy import ( apply_flap_suppress, apply_low_severity_triage, apply_suppress_window, build_recovery_decision, decision_supports_flap_suppress, decision_supports_suppress, evaluate_event, is_low_severity_problem_candidate, normalize_event_phase, ) from app.redis_repo import FlapState, OpenIncidentState, RedisStateRepository from app.zabbix_enricher import ZabbixEnricher logger = logging.getLogger(__name__) def _confidence_rank(value: str | None) -> int: normalized = (value or "").strip().lower() if normalized == "high": return 3 if normalized == "medium": return 2 if normalized == "low": return 1 return 0 def _is_high_or_disaster(severity: str | None) -> bool: value = (severity or "").strip().lower() return value in {"high", "disaster"} class ProcessorService: def __init__( self, redis_repo: RedisStateRepository, notification_dispatcher: NotificationDispatcher, zabbix_enricher: ZabbixEnricher | None = None, llm_remediation_adapter: LLMRemediationAdapter | None = None, llm_triage_adapter: LLMTriageAdapter | None = None, correlation_registry: CorrelationRegistry | None = None, audit_logger: AuditLogger | None = None, llm_correlation_adapter: LLMCorrelationAdapter | None = None, ) -> None: self.redis_repo = redis_repo self.notification_dispatcher = notification_dispatcher self.zabbix_enricher = zabbix_enricher self.llm_remediation_adapter = llm_remediation_adapter self.llm_triage_adapter = llm_triage_adapter self.correlation_registry = correlation_registry self.audit_logger = audit_logger self.llm_correlation_adapter = llm_correlation_adapter async def process(self, envelope: ProcessorForwardEnvelope) -> NotificationDecision: event = envelope.event fingerprint = build_fingerprint(event) repeat_count = 1 event_phase = normalize_event_phase(event.event_type, event.value) async def audit(stage: str, status: str = "ok", details: dict | None = None) -> None: if self.audit_logger is not None: await self.audit_logger.log_stage( correlation_id=event.correlation_id, event_id=event.event_id, stage=stage, status=status, details=details, ) flap_state = FlapState( active=False, event_count=0, phases=[], window_seconds=settings.flap_window_seconds, ) fp_state = await self.redis_repo.update_fingerprint_state( fingerprint=fingerprint, event=event, ) repeat_count = fp_state.count await self.redis_repo.save_event_snapshot( event=event, fingerprint=fingerprint, repeat_count=repeat_count, ) await audit( "state_updated", details={ "fingerprint": fingerprint, "repeat_count": repeat_count, "event_phase": event_phase, }, ) if settings.flap_enabled: flap_state = await self.redis_repo.record_phase_transition( fingerprint=fingerprint, event_phase=event_phase, correlation_id=event.correlation_id, event_id=event.event_id, ) await audit( "flap_evaluated", details={ "flap_active": flap_state.active, "flap_event_count": flap_state.event_count, "window_seconds": flap_state.window_seconds, }, ) baseline_low_severity_candidate = False if event_phase == "recovery": open_incident: OpenIncidentState | None = await self.redis_repo.get_open_incident( fingerprint ) decision = build_recovery_decision( fingerprint=fingerprint, repeat_count=repeat_count, open_incident_found=open_incident is not None, previous_severity=open_incident.severity if open_incident else None, previous_channels=open_incident.channels if open_incident else None, previous_routing_class=open_incident.routing_class if open_incident else None, ) if ( flap_state.active and settings.flap_enabled and settings.flap_apply_to_average and decision_supports_flap_suppress(decision) ): decision = apply_flap_suppress( decision=decision, event_count=flap_state.event_count, window_seconds=flap_state.window_seconds, ) await self.redis_repo.clear_suppress_window(fingerprint) await self.redis_repo.clear_open_incident(fingerprint) else: decision = evaluate_event( event=event, fingerprint=fingerprint, repeat_count=repeat_count, ) if ( flap_state.active and settings.flap_enabled and settings.flap_apply_to_average and decision_supports_flap_suppress(decision) ): decision = apply_flap_suppress( decision=decision, event_count=flap_state.event_count, window_seconds=flap_state.window_seconds, ) elif ( settings.suppress_enabled and settings.suppress_apply_to_average and decision_supports_suppress(decision) ): suppress_state = await self.redis_repo.get_suppress_state(fingerprint) if suppress_state.active: decision = apply_suppress_window( decision=decision, ttl_seconds=suppress_state.ttl_seconds, ) elif decision.notify: await self.redis_repo.activate_suppress_window(fingerprint) baseline_low_severity_candidate = ( settings.llm_triage_enabled and self.llm_triage_adapter is not None and is_low_severity_problem_candidate(decision) and not decision.suppressed ) await audit( "baseline_policy_applied", details={ "notify": decision.notify, "suppressed": decision.suppressed, "routing_class": decision.routing_class, "reason": decision.reason, }, ) should_enrich = self.zabbix_enricher is not None if settings.zabbix_enrich_only_notify: should_enrich = should_enrich and ( (decision.notify and not decision.suppressed) or baseline_low_severity_candidate ) if should_enrich and self.zabbix_enricher is not None: try: await self.zabbix_enricher.enrich_event(event) await audit( "zabbix_enrichment", details={ "event_url": event.event_url, "graph_url": event.graph_url, "graph_image_path": event.graph_image_path, "zabbix_context_keys": sorted(list((event.zabbix_context or {}).keys())), }, ) except Exception as exc: await audit( "zabbix_enrichment", status="error", details={"error": str(exc)}, ) logger.exception( "Zabbix enrichment failed: correlation_id=%s event_id=%s error=%s", event.correlation_id, event.event_id, exc, ) if baseline_low_severity_candidate and self.llm_triage_adapter is not None: triage_applied = False cached = await self.redis_repo.get_triage_cache( fingerprint=fingerprint, severity=decision.severity, ) if cached is not None: decision = apply_low_severity_triage( decision=decision, verdict=cached.verdict, classification=cached.classification, reason=cached.reason, source="cache", ) triage_applied = True await audit( "llm_triage", details={ "source": "cache", "verdict": cached.verdict, "classification": cached.classification, "reason": cached.reason, }, ) logger.info( "LLM triage reused from cache: correlation_id=%s event_id=%s verdict=%s classification=%s ttl=%s", event.correlation_id, event.event_id, cached.verdict, cached.classification, cached.ttl_seconds, ) if not triage_applied: try: triage = await self.llm_triage_adapter.generate(envelope, decision) if triage.ok: decision = apply_low_severity_triage( decision=decision, verdict=triage.verdict, classification=triage.classification, reason=triage.reason, source="llm", ) await self.redis_repo.save_triage_cache( fingerprint=fingerprint, severity=decision.severity, verdict=triage.verdict, classification=triage.classification, reason=triage.reason, source="llm", ) await audit( "llm_triage", details={ "source": "llm", "verdict": triage.verdict, "classification": triage.classification, "reason": triage.reason, }, ) else: await audit( "llm_triage", status="error", details={"error": triage.error}, ) logger.warning( "LLM triage returned no usable content: correlation_id=%s event_id=%s error=%s", event.correlation_id, event.event_id, triage.error, ) except Exception as exc: await audit( "llm_triage", status="error", details={"error": str(exc)}, ) logger.exception( "LLM triage failed: correlation_id=%s event_id=%s error=%s", event.correlation_id, event.event_id, exc, ) if ( settings.correlation_enabled and self.correlation_registry is not None and decision.event_phase == "problem" ): try: zbx_context = event.zabbix_context or {} scope_keys = list(zbx_context.get("correlation_scopes") or []) recent = await self.redis_repo.get_recent_correlation_events( host=event.host, scope_keys=scope_keys, window_seconds=settings.correlation_window_seconds, ) deterministic_assessment = assess_correlation( envelope=envelope, decision=decision, recent_events=recent, registry=self.correlation_registry, ) chosen_assessment: CorrelationAssessment | None = ( deterministic_assessment if deterministic_assessment.applied else None ) strong_deterministic = ( deterministic_assessment.applied and deterministic_assessment.role in {"root", "child"} ) if ( self.llm_correlation_adapter is not None and settings.llm_correlation_enabled and not strong_deterministic ): try: llm_result = await self.llm_correlation_adapter.generate( envelope=envelope, decision=decision, recent_events=recent, ) llm_assessment = self._build_llm_correlation_assessment( llm_result=llm_result, decision=decision, event=envelope.event, ) if llm_assessment is not None: chosen_assessment = llm_assessment await audit( "llm_correlation", details={ "role": llm_assessment.role, "kind": llm_assessment.kind, "reason": llm_assessment.reason, "confidence": llm_assessment.confidence, "parent_event_id": llm_assessment.parent_event_id, "parent_correlation_id": llm_assessment.parent_correlation_id, "suppress_child": llm_assessment.suppress_child, "scope_keys": scope_keys, }, ) else: await audit( "llm_correlation", status="skipped", details={ "reason": llm_result.reason, "confidence": llm_result.confidence, "error": llm_result.error, "scope_keys": scope_keys, }, ) except Exception as exc: await audit( "llm_correlation", status="error", details={"error": str(exc), "scope_keys": scope_keys}, ) logger.exception( "LLM correlation fallback failed: correlation_id=%s event_id=%s error=%s", event.correlation_id, event.event_id, exc, ) if chosen_assessment is not None: decision = apply_correlation_to_decision( decision=decision, assessment=chosen_assessment, ) await self.redis_repo.save_correlation_event( host=event.host or "", event_id=event.event_id, correlation_id=event.correlation_id, kind=chosen_assessment.kind, severity=decision.severity, routing_class=decision.routing_class, fingerprint=decision.fingerprint, root_candidate=chosen_assessment.root_cause_candidate, role=chosen_assessment.role, group_id=chosen_assessment.group_id, parent_event_id=chosen_assessment.parent_event_id, parent_correlation_id=chosen_assessment.parent_correlation_id, scope_keys=scope_keys, tags=event.tags, service=zbx_context.get("service") or event.service, scope=zbx_context.get("scope"), component=zbx_context.get("component"), domain=zbx_context.get("domain"), ) await audit( "correlation", details={ "applied": chosen_assessment.applied, "role": chosen_assessment.role, "kind": chosen_assessment.kind, "group_id": chosen_assessment.group_id, "parent_event_id": chosen_assessment.parent_event_id, "parent_correlation_id": chosen_assessment.parent_correlation_id, "root_cause_candidate": chosen_assessment.root_cause_candidate, "correlated_event_count": chosen_assessment.correlated_event_count, "reason": chosen_assessment.reason, "suppress_child": chosen_assessment.suppress_child, "source": chosen_assessment.source, "confidence": chosen_assessment.confidence, "scope_keys": scope_keys, }, ) except Exception as exc: await audit( "correlation", status="error", details={"error": str(exc)}, ) logger.exception( "Correlation baseline failed: correlation_id=%s event_id=%s error=%s", event.correlation_id, event.event_id, exc, ) if ( decision.event_phase == "problem" and decision.notify and not decision.suppressed and decision.routing_class in { "high_priority", "average_priority", "triage_low_priority_notify", } ): await self.redis_repo.upsert_open_incident( fingerprint=fingerprint, event=event, decision=decision, ) if ( self.llm_remediation_adapter is not None and decision.event_phase == "problem" and decision.notify and not decision.suppressed ): try: remediation = await self.llm_remediation_adapter.generate(envelope, decision) if remediation.ok: decision = decision.model_copy( update={ "llm_enriched": True, "remediation_summary": remediation.summary, "remediation_steps": remediation.steps, "remediation_commands": remediation.commands, } ) await audit( "llm_remediation", details={ "summary": remediation.summary, "steps_count": len(remediation.steps), "commands_count": len(remediation.commands), }, ) else: await audit( "llm_remediation", status="error", details={"error": remediation.error}, ) logger.warning( "LLM remediation returned no usable content: correlation_id=%s event_id=%s error=%s", event.correlation_id, event.event_id, remediation.error, ) except Exception as exc: await audit( "llm_remediation", status="error", details={"error": str(exc)}, ) logger.exception( "LLM remediation failed: correlation_id=%s event_id=%s error=%s", event.correlation_id, event.event_id, exc, ) if self.audit_logger is not None: await self.audit_logger.log_decision( envelope=envelope, decision=decision, ) logger.info( "Processed event: correlation_id=%s event_id=%s event_phase=%s severity=%s " "host=%s trigger=%s fingerprint=%s repeat_count=%s " "open_incident_found=%s flap_detected=%s flap_event_count=%s " "notify=%s suppressed=%s channels=%s reason=%s " "triage_applied=%s triage_source=%s triage_verdict=%s triage_classification=%s triage_reason=%s " "correlation_applied=%s correlation_role=%s correlation_kind=%s correlation_group_id=%s parent_event_id=%s root_cause_candidate=%s correlated_event_count=%s correlation_reason=%s correlation_source=%s correlation_confidence=%s " "suppress_reason=%s flap_reason=%s recovered_from_severity=%s " "event_url=%s graph_url=%s llm_enriched=%s", event.correlation_id, event.event_id, decision.event_phase, event.severity, event.host, event.trigger_name, fingerprint, repeat_count, decision.open_incident_found, decision.flap_detected, decision.flap_event_count, decision.notify, decision.suppressed, ",".join(decision.channels), decision.reason, decision.triage_applied, decision.triage_source, decision.triage_verdict, decision.triage_classification, decision.triage_reason, decision.correlation_applied, decision.correlation_role, decision.correlation_kind, decision.correlation_group_id, decision.parent_event_id, decision.root_cause_candidate, decision.correlated_event_count, decision.correlation_reason, decision.correlation_source, decision.correlation_confidence, decision.suppress_reason, decision.flap_reason, decision.recovered_from_severity, event.event_url, event.graph_url, decision.llm_enriched, ) dispatch_report = await self.notification_dispatcher.dispatch( envelope=envelope, decision=decision, ) delivery_payload = { "attempted": dispatch_report.attempted, "matrix_attempted": dispatch_report.matrix_attempted, "matrix_sent": dispatch_report.matrix_sent, "matrix_event_id": dispatch_report.matrix_event_id, "matrix_error": dispatch_report.matrix_error, "matrix_image_attempted": dispatch_report.matrix_image_attempted, "matrix_image_sent": dispatch_report.matrix_image_sent, "matrix_image_event_id": dispatch_report.matrix_image_event_id, "matrix_image_mxc_uri": dispatch_report.matrix_image_mxc_uri, "matrix_image_error": dispatch_report.matrix_image_error, "mail_attempted": dispatch_report.mail_attempted, "mail_sent": dispatch_report.mail_sent, "mail_error": dispatch_report.mail_error, "errors": dispatch_report.errors, } if self.audit_logger is not None: await self.audit_logger.log_delivery( envelope=envelope, delivery_payload=delivery_payload, ) if dispatch_report.matrix_attempted: if dispatch_report.matrix_sent: logger.info( "Matrix notification sent: correlation_id=%s event_id=%s matrix_event_id=%s", event.correlation_id, event.event_id, dispatch_report.matrix_event_id, ) else: logger.error( "Matrix notification failed: correlation_id=%s event_id=%s error=%s", event.correlation_id, event.event_id, dispatch_report.matrix_error, ) if dispatch_report.matrix_image_attempted: if dispatch_report.matrix_image_sent: logger.info( "Matrix graph image sent: correlation_id=%s event_id=%s matrix_event_id=%s content_uri=%s", event.correlation_id, event.event_id, dispatch_report.matrix_image_event_id, dispatch_report.matrix_image_mxc_uri, ) else: logger.error( "Matrix graph image failed: correlation_id=%s event_id=%s error=%s", event.correlation_id, event.event_id, dispatch_report.matrix_image_error, ) if dispatch_report.mail_attempted: if dispatch_report.mail_sent: logger.info( "Mail notification sent: correlation_id=%s event_id=%s recipient=%s", event.correlation_id, event.event_id, settings.mail_to, ) else: logger.error( "Mail notification failed: correlation_id=%s event_id=%s error=%s", event.correlation_id, event.event_id, dispatch_report.mail_error, ) return decision def _build_llm_correlation_assessment( self, llm_result: LLMCorrelationResult, decision: NotificationDecision, event, ) -> CorrelationAssessment | None: if not llm_result.ok: return None min_confidence_rank = _confidence_rank(settings.llm_correlation_min_confidence) actual_confidence_rank = _confidence_rank(llm_result.confidence) role = llm_result.role confidence = llm_result.confidence or "low" if role in {"root", "child"} and actual_confidence_rank < min_confidence_rank: return None if role == "child" and not (llm_result.parent_event_id or llm_result.parent_correlation_id): return None suppress_child = False if ( role == "child" and llm_result.suppress_child and actual_confidence_rank >= _confidence_rank("high") and not _is_high_or_disaster(decision.severity) ): suppress_child = True if role == "root": group_id = event.event_id or event.correlation_id return CorrelationAssessment( applied=True, role="root", kind=llm_result.kind, group_id=group_id, reason=llm_result.reason, parent_event_id=None, parent_correlation_id=None, root_cause_candidate=True, correlated_event_count=0, suppress_child=False, source="llm", confidence=confidence, ) if role == "child": group_id = llm_result.parent_event_id or llm_result.parent_correlation_id return CorrelationAssessment( applied=True, role="child", kind=llm_result.kind, group_id=group_id, reason=llm_result.reason, parent_event_id=llm_result.parent_event_id, parent_correlation_id=llm_result.parent_correlation_id, root_cause_candidate=False, correlated_event_count=0, suppress_child=suppress_child, source="llm", confidence=confidence, ) return None