v6ole
143 lines
5.1 KiB
Python
143 lines
5.1 KiB
Python
"""Alert evaluation and notification service."""
|
|
|
|
import logging
|
|
from datetime import datetime, timezone, timedelta
|
|
from typing import Optional
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.config import settings
|
|
from app.models.alert import AlertRule, AlertHistory, AlertLevel, AlertOperator
|
|
from app.models.log import LogEntry
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class AlertService:
|
|
"""Evaluates alert rules against recent log data and triggers notifications."""
|
|
|
|
def __init__(self, db: AsyncSession):
|
|
self.db = db
|
|
|
|
async def evaluate_rules(self) -> list[AlertHistory]:
|
|
"""Check all enabled alert rules and trigger any that match."""
|
|
result = await self.db.execute(
|
|
select(AlertRule).where(AlertRule.is_enabled == True)
|
|
)
|
|
rules: list[AlertRule] = result.scalars().all()
|
|
triggered: list[AlertHistory] = []
|
|
|
|
for rule in rules:
|
|
try:
|
|
hit = await self._evaluate_single(rule)
|
|
if hit:
|
|
triggered.append(hit)
|
|
except Exception as e:
|
|
logger.error("Error evaluating rule %s: %s", rule.id, e)
|
|
|
|
return triggered
|
|
|
|
async def _evaluate_single(self, rule: AlertRule) -> Optional[AlertHistory]:
|
|
"""Evaluate a single rule. Returns AlertHistory if triggered."""
|
|
now = datetime.now(timezone.utc)
|
|
since = now - timedelta(minutes=rule.window_minutes)
|
|
|
|
stmt = (
|
|
select(func.count(LogEntry.id))
|
|
.where(LogEntry.project_id == rule.project_id)
|
|
.where(LogEntry.timestamp >= since)
|
|
.where(LogEntry.level == rule.level.value)
|
|
)
|
|
count = (await self.db.execute(stmt)).scalar() or 0
|
|
|
|
operator_map = {
|
|
AlertOperator.GT: lambda v, t: v > t,
|
|
AlertOperator.GTE: lambda v, t: v >= t,
|
|
AlertOperator.LT: lambda v, t: v < t,
|
|
AlertOperator.LTE: lambda v, t: v <= t,
|
|
AlertOperator.EQ: lambda v, t: v == t,
|
|
}
|
|
|
|
op_fn = operator_map.get(rule.operator)
|
|
if op_fn and op_fn(float(count), rule.threshold):
|
|
history = AlertHistory(
|
|
rule_id=rule.id,
|
|
project_id=rule.project_id,
|
|
level=rule.level,
|
|
message=(
|
|
f"Alert '{rule.name}': {rule.level.value} logs count ({count}) "
|
|
f"{rule.operator.value} threshold ({rule.threshold}) "
|
|
f"in last {rule.window_minutes} min"
|
|
),
|
|
triggered_value=float(count),
|
|
)
|
|
self.db.add(history)
|
|
await self.db.flush()
|
|
logger.info("Alert triggered: %s", history.message)
|
|
|
|
if rule.notify_channels:
|
|
await self._notify(rule, history)
|
|
|
|
return history
|
|
|
|
return None
|
|
|
|
async def _notify(self, rule: AlertRule, history: AlertHistory) -> None:
|
|
"""Send notification via configured channels."""
|
|
channels = [c.strip() for c in rule.notify_channels.split(",") if c.strip()]
|
|
for channel in channels:
|
|
if channel == "feishu" and settings.FEISHU_WEBHOOK_URL:
|
|
await self._send_feishu(rule, history)
|
|
elif channel == "dingtalk" and settings.DINGTALK_WEBHOOK_URL:
|
|
await self._send_dingtalk(rule, history)
|
|
|
|
async def _send_feishu(self, rule: AlertRule, history: AlertHistory) -> None:
|
|
"""Send a notification to Feishu webhook."""
|
|
import httpx
|
|
|
|
payload = {
|
|
"msg_type": "post",
|
|
"content": {
|
|
"post": {
|
|
"zh_cn": {
|
|
"title": f"[LogHive] Alert: {rule.name}",
|
|
"content": [
|
|
[{"tag": "text", "text": history.message}],
|
|
[
|
|
{
|
|
"tag": "text",
|
|
"text": f"Project: {rule.project_id} | Level: {rule.level.value}",
|
|
}
|
|
],
|
|
],
|
|
}
|
|
}
|
|
},
|
|
}
|
|
try:
|
|
async with httpx.AsyncClient() as client:
|
|
await client.post(settings.FEISHU_WEBHOOK_URL, json=payload, timeout=10)
|
|
except Exception as e:
|
|
logger.error("Feishu notification failed: %s", e)
|
|
|
|
async def _send_dingtalk(self, rule: AlertRule, history: AlertHistory) -> None:
|
|
"""Send a notification to DingTalk webhook."""
|
|
import httpx
|
|
|
|
payload = {
|
|
"msgtype": "text",
|
|
"text": {
|
|
"content": (
|
|
f"[LogHive] Alert: {rule.name}\n"
|
|
f"{history.message}\n"
|
|
f"Project: {rule.project_id} | Level: {rule.level.value}"
|
|
)
|
|
},
|
|
}
|
|
try:
|
|
async with httpx.AsyncClient() as client:
|
|
await client.post(settings.DINGTALK_WEBHOOK_URL, json=payload, timeout=10)
|
|
except Exception as e:
|
|
logger.error("DingTalk notification failed: %s", e)
|