Загрузить файлы в «alert-processor/app»
This commit is contained in:
@@ -0,0 +1,744 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from app.audit_logger import AuditLogger
|
||||
from app.config import settings
|
||||
from app.correlation import (
|
||||
CorrelationAssessment,
|
||||
CorrelationRegistry,
|
||||
apply_correlation_to_decision,
|
||||
assess_correlation,
|
||||
)
|
||||
from app.fingerprint import build_fingerprint
|
||||
from app.llm_correlation import LLMCorrelationAdapter, LLMCorrelationResult
|
||||
from app.llm_remediation import LLMRemediationAdapter
|
||||
from app.llm_triage import LLMTriageAdapter
|
||||
from app.models import NotificationDecision, ProcessorForwardEnvelope
|
||||
from app.notifications.dispatcher import NotificationDispatcher
|
||||
from app.policy import (
|
||||
apply_flap_suppress,
|
||||
apply_low_severity_triage,
|
||||
apply_suppress_window,
|
||||
build_recovery_decision,
|
||||
decision_supports_flap_suppress,
|
||||
decision_supports_suppress,
|
||||
evaluate_event,
|
||||
is_low_severity_problem_candidate,
|
||||
normalize_event_phase,
|
||||
)
|
||||
from app.redis_repo import FlapState, OpenIncidentState, RedisStateRepository
|
||||
from app.zabbix_enricher import ZabbixEnricher
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _confidence_rank(value: str | None) -> int:
|
||||
normalized = (value or "").strip().lower()
|
||||
if normalized == "high":
|
||||
return 3
|
||||
if normalized == "medium":
|
||||
return 2
|
||||
if normalized == "low":
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
def _is_high_or_disaster(severity: str | None) -> bool:
|
||||
value = (severity or "").strip().lower()
|
||||
return value in {"high", "disaster"}
|
||||
|
||||
|
||||
class ProcessorService:
|
||||
def __init__(
|
||||
self,
|
||||
redis_repo: RedisStateRepository,
|
||||
notification_dispatcher: NotificationDispatcher,
|
||||
zabbix_enricher: ZabbixEnricher | None = None,
|
||||
llm_remediation_adapter: LLMRemediationAdapter | None = None,
|
||||
llm_triage_adapter: LLMTriageAdapter | None = None,
|
||||
correlation_registry: CorrelationRegistry | None = None,
|
||||
audit_logger: AuditLogger | None = None,
|
||||
llm_correlation_adapter: LLMCorrelationAdapter | None = None,
|
||||
) -> None:
|
||||
self.redis_repo = redis_repo
|
||||
self.notification_dispatcher = notification_dispatcher
|
||||
self.zabbix_enricher = zabbix_enricher
|
||||
self.llm_remediation_adapter = llm_remediation_adapter
|
||||
self.llm_triage_adapter = llm_triage_adapter
|
||||
self.correlation_registry = correlation_registry
|
||||
self.audit_logger = audit_logger
|
||||
self.llm_correlation_adapter = llm_correlation_adapter
|
||||
|
||||
async def process(self, envelope: ProcessorForwardEnvelope) -> NotificationDecision:
|
||||
event = envelope.event
|
||||
fingerprint = build_fingerprint(event)
|
||||
repeat_count = 1
|
||||
event_phase = normalize_event_phase(event.event_type, event.value)
|
||||
|
||||
async def audit(stage: str, status: str = "ok", details: dict | None = None) -> None:
|
||||
if self.audit_logger is not None:
|
||||
await self.audit_logger.log_stage(
|
||||
correlation_id=event.correlation_id,
|
||||
event_id=event.event_id,
|
||||
stage=stage,
|
||||
status=status,
|
||||
details=details,
|
||||
)
|
||||
|
||||
flap_state = FlapState(
|
||||
active=False,
|
||||
event_count=0,
|
||||
phases=[],
|
||||
window_seconds=settings.flap_window_seconds,
|
||||
)
|
||||
|
||||
fp_state = await self.redis_repo.update_fingerprint_state(
|
||||
fingerprint=fingerprint,
|
||||
event=event,
|
||||
)
|
||||
repeat_count = fp_state.count
|
||||
|
||||
await self.redis_repo.save_event_snapshot(
|
||||
event=event,
|
||||
fingerprint=fingerprint,
|
||||
repeat_count=repeat_count,
|
||||
)
|
||||
|
||||
await audit(
|
||||
"state_updated",
|
||||
details={
|
||||
"fingerprint": fingerprint,
|
||||
"repeat_count": repeat_count,
|
||||
"event_phase": event_phase,
|
||||
},
|
||||
)
|
||||
|
||||
if settings.flap_enabled:
|
||||
flap_state = await self.redis_repo.record_phase_transition(
|
||||
fingerprint=fingerprint,
|
||||
event_phase=event_phase,
|
||||
correlation_id=event.correlation_id,
|
||||
event_id=event.event_id,
|
||||
)
|
||||
await audit(
|
||||
"flap_evaluated",
|
||||
details={
|
||||
"flap_active": flap_state.active,
|
||||
"flap_event_count": flap_state.event_count,
|
||||
"window_seconds": flap_state.window_seconds,
|
||||
},
|
||||
)
|
||||
|
||||
baseline_low_severity_candidate = False
|
||||
|
||||
if event_phase == "recovery":
|
||||
open_incident: OpenIncidentState | None = await self.redis_repo.get_open_incident(
|
||||
fingerprint
|
||||
)
|
||||
|
||||
decision = build_recovery_decision(
|
||||
fingerprint=fingerprint,
|
||||
repeat_count=repeat_count,
|
||||
open_incident_found=open_incident is not None,
|
||||
previous_severity=open_incident.severity if open_incident else None,
|
||||
previous_channels=open_incident.channels if open_incident else None,
|
||||
previous_routing_class=open_incident.routing_class if open_incident else None,
|
||||
)
|
||||
|
||||
if (
|
||||
flap_state.active
|
||||
and settings.flap_enabled
|
||||
and settings.flap_apply_to_average
|
||||
and decision_supports_flap_suppress(decision)
|
||||
):
|
||||
decision = apply_flap_suppress(
|
||||
decision=decision,
|
||||
event_count=flap_state.event_count,
|
||||
window_seconds=flap_state.window_seconds,
|
||||
)
|
||||
|
||||
await self.redis_repo.clear_suppress_window(fingerprint)
|
||||
await self.redis_repo.clear_open_incident(fingerprint)
|
||||
|
||||
else:
|
||||
decision = evaluate_event(
|
||||
event=event,
|
||||
fingerprint=fingerprint,
|
||||
repeat_count=repeat_count,
|
||||
)
|
||||
|
||||
if (
|
||||
flap_state.active
|
||||
and settings.flap_enabled
|
||||
and settings.flap_apply_to_average
|
||||
and decision_supports_flap_suppress(decision)
|
||||
):
|
||||
decision = apply_flap_suppress(
|
||||
decision=decision,
|
||||
event_count=flap_state.event_count,
|
||||
window_seconds=flap_state.window_seconds,
|
||||
)
|
||||
elif (
|
||||
settings.suppress_enabled
|
||||
and settings.suppress_apply_to_average
|
||||
and decision_supports_suppress(decision)
|
||||
):
|
||||
suppress_state = await self.redis_repo.get_suppress_state(fingerprint)
|
||||
|
||||
if suppress_state.active:
|
||||
decision = apply_suppress_window(
|
||||
decision=decision,
|
||||
ttl_seconds=suppress_state.ttl_seconds,
|
||||
)
|
||||
elif decision.notify:
|
||||
await self.redis_repo.activate_suppress_window(fingerprint)
|
||||
|
||||
baseline_low_severity_candidate = (
|
||||
settings.llm_triage_enabled
|
||||
and self.llm_triage_adapter is not None
|
||||
and is_low_severity_problem_candidate(decision)
|
||||
and not decision.suppressed
|
||||
)
|
||||
|
||||
await audit(
|
||||
"baseline_policy_applied",
|
||||
details={
|
||||
"notify": decision.notify,
|
||||
"suppressed": decision.suppressed,
|
||||
"routing_class": decision.routing_class,
|
||||
"reason": decision.reason,
|
||||
},
|
||||
)
|
||||
|
||||
should_enrich = self.zabbix_enricher is not None
|
||||
if settings.zabbix_enrich_only_notify:
|
||||
should_enrich = should_enrich and (
|
||||
(decision.notify and not decision.suppressed) or baseline_low_severity_candidate
|
||||
)
|
||||
|
||||
if should_enrich and self.zabbix_enricher is not None:
|
||||
try:
|
||||
await self.zabbix_enricher.enrich_event(event)
|
||||
await audit(
|
||||
"zabbix_enrichment",
|
||||
details={
|
||||
"event_url": event.event_url,
|
||||
"graph_url": event.graph_url,
|
||||
"graph_image_path": event.graph_image_path,
|
||||
"zabbix_context_keys": sorted(list((event.zabbix_context or {}).keys())),
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
await audit(
|
||||
"zabbix_enrichment",
|
||||
status="error",
|
||||
details={"error": str(exc)},
|
||||
)
|
||||
logger.exception(
|
||||
"Zabbix enrichment failed: correlation_id=%s event_id=%s error=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
exc,
|
||||
)
|
||||
|
||||
if baseline_low_severity_candidate and self.llm_triage_adapter is not None:
|
||||
triage_applied = False
|
||||
|
||||
cached = await self.redis_repo.get_triage_cache(
|
||||
fingerprint=fingerprint,
|
||||
severity=decision.severity,
|
||||
)
|
||||
if cached is not None:
|
||||
decision = apply_low_severity_triage(
|
||||
decision=decision,
|
||||
verdict=cached.verdict,
|
||||
classification=cached.classification,
|
||||
reason=cached.reason,
|
||||
source="cache",
|
||||
)
|
||||
triage_applied = True
|
||||
|
||||
await audit(
|
||||
"llm_triage",
|
||||
details={
|
||||
"source": "cache",
|
||||
"verdict": cached.verdict,
|
||||
"classification": cached.classification,
|
||||
"reason": cached.reason,
|
||||
},
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"LLM triage reused from cache: correlation_id=%s event_id=%s verdict=%s classification=%s ttl=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
cached.verdict,
|
||||
cached.classification,
|
||||
cached.ttl_seconds,
|
||||
)
|
||||
|
||||
if not triage_applied:
|
||||
try:
|
||||
triage = await self.llm_triage_adapter.generate(envelope, decision)
|
||||
if triage.ok:
|
||||
decision = apply_low_severity_triage(
|
||||
decision=decision,
|
||||
verdict=triage.verdict,
|
||||
classification=triage.classification,
|
||||
reason=triage.reason,
|
||||
source="llm",
|
||||
)
|
||||
await self.redis_repo.save_triage_cache(
|
||||
fingerprint=fingerprint,
|
||||
severity=decision.severity,
|
||||
verdict=triage.verdict,
|
||||
classification=triage.classification,
|
||||
reason=triage.reason,
|
||||
source="llm",
|
||||
)
|
||||
|
||||
await audit(
|
||||
"llm_triage",
|
||||
details={
|
||||
"source": "llm",
|
||||
"verdict": triage.verdict,
|
||||
"classification": triage.classification,
|
||||
"reason": triage.reason,
|
||||
},
|
||||
)
|
||||
else:
|
||||
await audit(
|
||||
"llm_triage",
|
||||
status="error",
|
||||
details={"error": triage.error},
|
||||
)
|
||||
logger.warning(
|
||||
"LLM triage returned no usable content: correlation_id=%s event_id=%s error=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
triage.error,
|
||||
)
|
||||
except Exception as exc:
|
||||
await audit(
|
||||
"llm_triage",
|
||||
status="error",
|
||||
details={"error": str(exc)},
|
||||
)
|
||||
logger.exception(
|
||||
"LLM triage failed: correlation_id=%s event_id=%s error=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
exc,
|
||||
)
|
||||
|
||||
if (
|
||||
settings.correlation_enabled
|
||||
and self.correlation_registry is not None
|
||||
and decision.event_phase == "problem"
|
||||
):
|
||||
try:
|
||||
zbx_context = event.zabbix_context or {}
|
||||
scope_keys = list(zbx_context.get("correlation_scopes") or [])
|
||||
|
||||
recent = await self.redis_repo.get_recent_correlation_events(
|
||||
host=event.host,
|
||||
scope_keys=scope_keys,
|
||||
window_seconds=settings.correlation_window_seconds,
|
||||
)
|
||||
|
||||
deterministic_assessment = assess_correlation(
|
||||
envelope=envelope,
|
||||
decision=decision,
|
||||
recent_events=recent,
|
||||
registry=self.correlation_registry,
|
||||
)
|
||||
|
||||
chosen_assessment: CorrelationAssessment | None = (
|
||||
deterministic_assessment if deterministic_assessment.applied else None
|
||||
)
|
||||
|
||||
strong_deterministic = (
|
||||
deterministic_assessment.applied
|
||||
and deterministic_assessment.role in {"root", "child"}
|
||||
)
|
||||
|
||||
if (
|
||||
self.llm_correlation_adapter is not None
|
||||
and settings.llm_correlation_enabled
|
||||
and not strong_deterministic
|
||||
):
|
||||
try:
|
||||
llm_result = await self.llm_correlation_adapter.generate(
|
||||
envelope=envelope,
|
||||
decision=decision,
|
||||
recent_events=recent,
|
||||
)
|
||||
|
||||
llm_assessment = self._build_llm_correlation_assessment(
|
||||
llm_result=llm_result,
|
||||
decision=decision,
|
||||
event=envelope.event,
|
||||
)
|
||||
|
||||
if llm_assessment is not None:
|
||||
chosen_assessment = llm_assessment
|
||||
await audit(
|
||||
"llm_correlation",
|
||||
details={
|
||||
"role": llm_assessment.role,
|
||||
"kind": llm_assessment.kind,
|
||||
"reason": llm_assessment.reason,
|
||||
"confidence": llm_assessment.confidence,
|
||||
"parent_event_id": llm_assessment.parent_event_id,
|
||||
"parent_correlation_id": llm_assessment.parent_correlation_id,
|
||||
"suppress_child": llm_assessment.suppress_child,
|
||||
"scope_keys": scope_keys,
|
||||
},
|
||||
)
|
||||
else:
|
||||
await audit(
|
||||
"llm_correlation",
|
||||
status="skipped",
|
||||
details={
|
||||
"reason": llm_result.reason,
|
||||
"confidence": llm_result.confidence,
|
||||
"error": llm_result.error,
|
||||
"scope_keys": scope_keys,
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
await audit(
|
||||
"llm_correlation",
|
||||
status="error",
|
||||
details={"error": str(exc), "scope_keys": scope_keys},
|
||||
)
|
||||
logger.exception(
|
||||
"LLM correlation fallback failed: correlation_id=%s event_id=%s error=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
exc,
|
||||
)
|
||||
|
||||
if chosen_assessment is not None:
|
||||
decision = apply_correlation_to_decision(
|
||||
decision=decision,
|
||||
assessment=chosen_assessment,
|
||||
)
|
||||
|
||||
await self.redis_repo.save_correlation_event(
|
||||
host=event.host or "",
|
||||
event_id=event.event_id,
|
||||
correlation_id=event.correlation_id,
|
||||
kind=chosen_assessment.kind,
|
||||
severity=decision.severity,
|
||||
routing_class=decision.routing_class,
|
||||
fingerprint=decision.fingerprint,
|
||||
root_candidate=chosen_assessment.root_cause_candidate,
|
||||
role=chosen_assessment.role,
|
||||
group_id=chosen_assessment.group_id,
|
||||
parent_event_id=chosen_assessment.parent_event_id,
|
||||
parent_correlation_id=chosen_assessment.parent_correlation_id,
|
||||
scope_keys=scope_keys,
|
||||
tags=event.tags,
|
||||
service=zbx_context.get("service") or event.service,
|
||||
scope=zbx_context.get("scope"),
|
||||
component=zbx_context.get("component"),
|
||||
domain=zbx_context.get("domain"),
|
||||
)
|
||||
|
||||
await audit(
|
||||
"correlation",
|
||||
details={
|
||||
"applied": chosen_assessment.applied,
|
||||
"role": chosen_assessment.role,
|
||||
"kind": chosen_assessment.kind,
|
||||
"group_id": chosen_assessment.group_id,
|
||||
"parent_event_id": chosen_assessment.parent_event_id,
|
||||
"parent_correlation_id": chosen_assessment.parent_correlation_id,
|
||||
"root_cause_candidate": chosen_assessment.root_cause_candidate,
|
||||
"correlated_event_count": chosen_assessment.correlated_event_count,
|
||||
"reason": chosen_assessment.reason,
|
||||
"suppress_child": chosen_assessment.suppress_child,
|
||||
"source": chosen_assessment.source,
|
||||
"confidence": chosen_assessment.confidence,
|
||||
"scope_keys": scope_keys,
|
||||
},
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
await audit(
|
||||
"correlation",
|
||||
status="error",
|
||||
details={"error": str(exc)},
|
||||
)
|
||||
logger.exception(
|
||||
"Correlation baseline failed: correlation_id=%s event_id=%s error=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
exc,
|
||||
)
|
||||
|
||||
if (
|
||||
decision.event_phase == "problem"
|
||||
and decision.notify
|
||||
and not decision.suppressed
|
||||
and decision.routing_class in {
|
||||
"high_priority",
|
||||
"average_priority",
|
||||
"triage_low_priority_notify",
|
||||
}
|
||||
):
|
||||
await self.redis_repo.upsert_open_incident(
|
||||
fingerprint=fingerprint,
|
||||
event=event,
|
||||
decision=decision,
|
||||
)
|
||||
|
||||
if (
|
||||
self.llm_remediation_adapter is not None
|
||||
and decision.event_phase == "problem"
|
||||
and decision.notify
|
||||
and not decision.suppressed
|
||||
):
|
||||
try:
|
||||
remediation = await self.llm_remediation_adapter.generate(envelope, decision)
|
||||
if remediation.ok:
|
||||
decision = decision.model_copy(
|
||||
update={
|
||||
"llm_enriched": True,
|
||||
"remediation_summary": remediation.summary,
|
||||
"remediation_steps": remediation.steps,
|
||||
"remediation_commands": remediation.commands,
|
||||
}
|
||||
)
|
||||
await audit(
|
||||
"llm_remediation",
|
||||
details={
|
||||
"summary": remediation.summary,
|
||||
"steps_count": len(remediation.steps),
|
||||
"commands_count": len(remediation.commands),
|
||||
},
|
||||
)
|
||||
else:
|
||||
await audit(
|
||||
"llm_remediation",
|
||||
status="error",
|
||||
details={"error": remediation.error},
|
||||
)
|
||||
logger.warning(
|
||||
"LLM remediation returned no usable content: correlation_id=%s event_id=%s error=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
remediation.error,
|
||||
)
|
||||
except Exception as exc:
|
||||
await audit(
|
||||
"llm_remediation",
|
||||
status="error",
|
||||
details={"error": str(exc)},
|
||||
)
|
||||
logger.exception(
|
||||
"LLM remediation failed: correlation_id=%s event_id=%s error=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
exc,
|
||||
)
|
||||
|
||||
if self.audit_logger is not None:
|
||||
await self.audit_logger.log_decision(
|
||||
envelope=envelope,
|
||||
decision=decision,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Processed event: correlation_id=%s event_id=%s event_phase=%s severity=%s "
|
||||
"host=%s trigger=%s fingerprint=%s repeat_count=%s "
|
||||
"open_incident_found=%s flap_detected=%s flap_event_count=%s "
|
||||
"notify=%s suppressed=%s channels=%s reason=%s "
|
||||
"triage_applied=%s triage_source=%s triage_verdict=%s triage_classification=%s triage_reason=%s "
|
||||
"correlation_applied=%s correlation_role=%s correlation_kind=%s correlation_group_id=%s parent_event_id=%s root_cause_candidate=%s correlated_event_count=%s correlation_reason=%s correlation_source=%s correlation_confidence=%s "
|
||||
"suppress_reason=%s flap_reason=%s recovered_from_severity=%s "
|
||||
"event_url=%s graph_url=%s llm_enriched=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
decision.event_phase,
|
||||
event.severity,
|
||||
event.host,
|
||||
event.trigger_name,
|
||||
fingerprint,
|
||||
repeat_count,
|
||||
decision.open_incident_found,
|
||||
decision.flap_detected,
|
||||
decision.flap_event_count,
|
||||
decision.notify,
|
||||
decision.suppressed,
|
||||
",".join(decision.channels),
|
||||
decision.reason,
|
||||
decision.triage_applied,
|
||||
decision.triage_source,
|
||||
decision.triage_verdict,
|
||||
decision.triage_classification,
|
||||
decision.triage_reason,
|
||||
decision.correlation_applied,
|
||||
decision.correlation_role,
|
||||
decision.correlation_kind,
|
||||
decision.correlation_group_id,
|
||||
decision.parent_event_id,
|
||||
decision.root_cause_candidate,
|
||||
decision.correlated_event_count,
|
||||
decision.correlation_reason,
|
||||
decision.correlation_source,
|
||||
decision.correlation_confidence,
|
||||
decision.suppress_reason,
|
||||
decision.flap_reason,
|
||||
decision.recovered_from_severity,
|
||||
event.event_url,
|
||||
event.graph_url,
|
||||
decision.llm_enriched,
|
||||
)
|
||||
|
||||
dispatch_report = await self.notification_dispatcher.dispatch(
|
||||
envelope=envelope,
|
||||
decision=decision,
|
||||
)
|
||||
|
||||
delivery_payload = {
|
||||
"attempted": dispatch_report.attempted,
|
||||
"matrix_attempted": dispatch_report.matrix_attempted,
|
||||
"matrix_sent": dispatch_report.matrix_sent,
|
||||
"matrix_event_id": dispatch_report.matrix_event_id,
|
||||
"matrix_error": dispatch_report.matrix_error,
|
||||
"matrix_image_attempted": dispatch_report.matrix_image_attempted,
|
||||
"matrix_image_sent": dispatch_report.matrix_image_sent,
|
||||
"matrix_image_event_id": dispatch_report.matrix_image_event_id,
|
||||
"matrix_image_mxc_uri": dispatch_report.matrix_image_mxc_uri,
|
||||
"matrix_image_error": dispatch_report.matrix_image_error,
|
||||
"mail_attempted": dispatch_report.mail_attempted,
|
||||
"mail_sent": dispatch_report.mail_sent,
|
||||
"mail_error": dispatch_report.mail_error,
|
||||
"errors": dispatch_report.errors,
|
||||
}
|
||||
|
||||
if self.audit_logger is not None:
|
||||
await self.audit_logger.log_delivery(
|
||||
envelope=envelope,
|
||||
delivery_payload=delivery_payload,
|
||||
)
|
||||
|
||||
if dispatch_report.matrix_attempted:
|
||||
if dispatch_report.matrix_sent:
|
||||
logger.info(
|
||||
"Matrix notification sent: correlation_id=%s event_id=%s matrix_event_id=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
dispatch_report.matrix_event_id,
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"Matrix notification failed: correlation_id=%s event_id=%s error=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
dispatch_report.matrix_error,
|
||||
)
|
||||
|
||||
if dispatch_report.matrix_image_attempted:
|
||||
if dispatch_report.matrix_image_sent:
|
||||
logger.info(
|
||||
"Matrix graph image sent: correlation_id=%s event_id=%s matrix_event_id=%s content_uri=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
dispatch_report.matrix_image_event_id,
|
||||
dispatch_report.matrix_image_mxc_uri,
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"Matrix graph image failed: correlation_id=%s event_id=%s error=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
dispatch_report.matrix_image_error,
|
||||
)
|
||||
|
||||
if dispatch_report.mail_attempted:
|
||||
if dispatch_report.mail_sent:
|
||||
logger.info(
|
||||
"Mail notification sent: correlation_id=%s event_id=%s recipient=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
settings.mail_to,
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"Mail notification failed: correlation_id=%s event_id=%s error=%s",
|
||||
event.correlation_id,
|
||||
event.event_id,
|
||||
dispatch_report.mail_error,
|
||||
)
|
||||
|
||||
return decision
|
||||
|
||||
def _build_llm_correlation_assessment(
|
||||
self,
|
||||
llm_result: LLMCorrelationResult,
|
||||
decision: NotificationDecision,
|
||||
event,
|
||||
) -> CorrelationAssessment | None:
|
||||
if not llm_result.ok:
|
||||
return None
|
||||
|
||||
min_confidence_rank = _confidence_rank(settings.llm_correlation_min_confidence)
|
||||
actual_confidence_rank = _confidence_rank(llm_result.confidence)
|
||||
|
||||
role = llm_result.role
|
||||
confidence = llm_result.confidence or "low"
|
||||
|
||||
if role in {"root", "child"} and actual_confidence_rank < min_confidence_rank:
|
||||
return None
|
||||
|
||||
if role == "child" and not (llm_result.parent_event_id or llm_result.parent_correlation_id):
|
||||
return None
|
||||
|
||||
suppress_child = False
|
||||
if (
|
||||
role == "child"
|
||||
and llm_result.suppress_child
|
||||
and actual_confidence_rank >= _confidence_rank("high")
|
||||
and not _is_high_or_disaster(decision.severity)
|
||||
):
|
||||
suppress_child = True
|
||||
|
||||
if role == "root":
|
||||
group_id = event.event_id or event.correlation_id
|
||||
return CorrelationAssessment(
|
||||
applied=True,
|
||||
role="root",
|
||||
kind=llm_result.kind,
|
||||
group_id=group_id,
|
||||
reason=llm_result.reason,
|
||||
parent_event_id=None,
|
||||
parent_correlation_id=None,
|
||||
root_cause_candidate=True,
|
||||
correlated_event_count=0,
|
||||
suppress_child=False,
|
||||
source="llm",
|
||||
confidence=confidence,
|
||||
)
|
||||
|
||||
if role == "child":
|
||||
group_id = llm_result.parent_event_id or llm_result.parent_correlation_id
|
||||
return CorrelationAssessment(
|
||||
applied=True,
|
||||
role="child",
|
||||
kind=llm_result.kind,
|
||||
group_id=group_id,
|
||||
reason=llm_result.reason,
|
||||
parent_event_id=llm_result.parent_event_id,
|
||||
parent_correlation_id=llm_result.parent_correlation_id,
|
||||
root_cause_candidate=False,
|
||||
correlated_event_count=0,
|
||||
suppress_child=suppress_child,
|
||||
source="llm",
|
||||
confidence=confidence,
|
||||
)
|
||||
|
||||
return None
|
||||
Reference in New Issue
Block a user