Initial commit: LogHive centralized log management system
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,180 @@
|
||||
"""Alert management API routes."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import List, Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models.alert import AlertRule, AlertHistory, AlertLevel, AlertOperator
|
||||
from app.schemas.alert import (
|
||||
AlertRuleCreate,
|
||||
AlertRuleUpdate,
|
||||
AlertRuleResponse,
|
||||
AlertHistoryResponse,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/alerts", tags=["alerts"])
|
||||
|
||||
|
||||
# ── Rule CRUD ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.post("/rules", response_model=AlertRuleResponse, status_code=201)
|
||||
async def create_alert_rule(
|
||||
data: AlertRuleCreate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Create a new alert rule."""
|
||||
rule = AlertRule(
|
||||
project_id=data.project_id,
|
||||
name=data.name,
|
||||
level=data.level,
|
||||
field=data.field,
|
||||
operator=data.operator,
|
||||
threshold=data.threshold,
|
||||
window_minutes=data.window_minutes,
|
||||
notify_channels=",".join(data.notify_channels) if data.notify_channels else "",
|
||||
)
|
||||
db.add(rule)
|
||||
await db.flush()
|
||||
await db.refresh(rule)
|
||||
return _rule_to_response(rule)
|
||||
|
||||
|
||||
@router.get("/rules", response_model=List[AlertRuleResponse])
|
||||
async def list_alert_rules(
|
||||
project_id: Optional[str] = Query(None),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""List alert rules, optionally filtered by project."""
|
||||
stmt = select(AlertRule)
|
||||
if project_id:
|
||||
stmt = stmt.where(AlertRule.project_id == project_id)
|
||||
stmt = stmt.order_by(AlertRule.created_at.desc())
|
||||
result = await db.execute(stmt)
|
||||
return [_rule_to_response(r) for r in result.scalars().all()]
|
||||
|
||||
|
||||
@router.get("/rules/{rule_id}", response_model=AlertRuleResponse)
|
||||
async def get_alert_rule(rule_id: str, db: AsyncSession = Depends(get_db)):
|
||||
"""Get a single alert rule."""
|
||||
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
||||
rule = result.scalar_one_or_none()
|
||||
if not rule:
|
||||
raise HTTPException(status_code=404, detail="Alert rule not found")
|
||||
return _rule_to_response(rule)
|
||||
|
||||
|
||||
@router.patch("/rules/{rule_id}", response_model=AlertRuleResponse)
|
||||
async def update_alert_rule(
|
||||
rule_id: str,
|
||||
data: AlertRuleUpdate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Update an alert rule."""
|
||||
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
||||
rule = result.scalar_one_or_none()
|
||||
if not rule:
|
||||
raise HTTPException(status_code=404, detail="Alert rule not found")
|
||||
|
||||
if data.name is not None:
|
||||
rule.name = data.name
|
||||
if data.level is not None:
|
||||
rule.level = data.level
|
||||
if data.field is not None:
|
||||
rule.field = data.field
|
||||
if data.operator is not None:
|
||||
rule.operator = data.operator
|
||||
if data.threshold is not None:
|
||||
rule.threshold = data.threshold
|
||||
if data.window_minutes is not None:
|
||||
rule.window_minutes = data.window_minutes
|
||||
if data.is_enabled is not None:
|
||||
rule.is_enabled = data.is_enabled
|
||||
if data.notify_channels is not None:
|
||||
rule.notify_channels = ",".join(data.notify_channels)
|
||||
|
||||
await db.flush()
|
||||
await db.refresh(rule)
|
||||
return _rule_to_response(rule)
|
||||
|
||||
|
||||
@router.delete("/rules/{rule_id}", status_code=204)
|
||||
async def delete_alert_rule(rule_id: str, db: AsyncSession = Depends(get_db)):
|
||||
"""Delete an alert rule."""
|
||||
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
|
||||
rule = result.scalar_one_or_none()
|
||||
if not rule:
|
||||
raise HTTPException(status_code=404, detail="Alert rule not found")
|
||||
await db.delete(rule)
|
||||
|
||||
|
||||
# ── Alert History ──────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("/history", response_model=List[AlertHistoryResponse])
|
||||
async def list_alert_history(
|
||||
project_id: Optional[str] = Query(None),
|
||||
rule_id: Optional[str] = Query(None),
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""List alert trigger history."""
|
||||
stmt = select(AlertHistory).order_by(AlertHistory.triggered_at.desc()).limit(limit)
|
||||
if project_id:
|
||||
stmt = stmt.where(AlertHistory.project_id == project_id)
|
||||
if rule_id:
|
||||
stmt = stmt.where(AlertHistory.rule_id == rule_id)
|
||||
result = await db.execute(stmt)
|
||||
return [_history_to_response(h) for h in result.scalars().all()]
|
||||
|
||||
|
||||
@router.post("/history/{history_id}/acknowledge", response_model=AlertHistoryResponse)
|
||||
async def acknowledge_alert(
|
||||
history_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Mark an alert as acknowledged."""
|
||||
result = await db.execute(select(AlertHistory).where(AlertHistory.id == history_id))
|
||||
history = result.scalar_one_or_none()
|
||||
if not history:
|
||||
raise HTTPException(status_code=404, detail="Alert history not found")
|
||||
history.is_acknowledged = True
|
||||
await db.flush()
|
||||
await db.refresh(history)
|
||||
return _history_to_response(history)
|
||||
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _rule_to_response(rule: AlertRule) -> AlertRuleResponse:
|
||||
return AlertRuleResponse(
|
||||
id=rule.id,
|
||||
project_id=rule.project_id,
|
||||
name=rule.name,
|
||||
level=rule.level.value if hasattr(rule.level, "value") else rule.level,
|
||||
field=rule.field,
|
||||
operator=rule.operator.value if hasattr(rule.operator, "value") else rule.operator,
|
||||
threshold=rule.threshold,
|
||||
window_minutes=rule.window_minutes,
|
||||
is_enabled=rule.is_enabled,
|
||||
notify_channels=rule.notify_channels.split(",") if rule.notify_channels else [],
|
||||
created_at=rule.created_at,
|
||||
)
|
||||
|
||||
|
||||
def _history_to_response(history: AlertHistory) -> AlertHistoryResponse:
|
||||
return AlertHistoryResponse(
|
||||
id=history.id,
|
||||
rule_id=history.rule_id,
|
||||
project_id=history.project_id,
|
||||
level=history.level.value if hasattr(history.level, "value") else history.level,
|
||||
message=history.message,
|
||||
triggered_value=history.triggered_value,
|
||||
is_acknowledged=history.is_acknowledged,
|
||||
triggered_at=history.triggered_at,
|
||||
)
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Dashboard API routes."""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.services.analytics_service import AnalyticsService
|
||||
|
||||
router = APIRouter(prefix="/api/dashboard", tags=["dashboard"])
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
async def dashboard_stats(
|
||||
project: Optional[str] = Query(None),
|
||||
hours: int = Query(24, ge=1, le=720),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Get overview statistics for the dashboard."""
|
||||
service = AnalyticsService(db)
|
||||
return await service.get_dashboard_stats(project=project, hours=hours)
|
||||
|
||||
|
||||
@router.get("/timeseries")
|
||||
async def dashboard_timeseries(
|
||||
project: Optional[str] = Query(None),
|
||||
hours: int = Query(24, ge=1, le=720),
|
||||
interval: str = Query("5m", max_length=16),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Get time-series data for chart rendering."""
|
||||
service = AnalyticsService(db)
|
||||
return await service.get_time_series(project=project, hours=hours, interval=interval)
|
||||
@@ -0,0 +1,253 @@
|
||||
"""Log ingestion and query API routes."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Query, status
|
||||
from sqlalchemy import func, select, delete, and_
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings, to_display_time, from_query_time
|
||||
from app.database import get_db
|
||||
from app.schemas.log import (
|
||||
LogBatchRequest,
|
||||
LogEntry as LogEntrySchema,
|
||||
LogQueryParams,
|
||||
LogSearchResult,
|
||||
LogStatsResult,
|
||||
)
|
||||
from app.models.project import Project
|
||||
from app.models.log import LogEntry
|
||||
from app.core.auth import verify_api_key, resolve_project_by_api_key
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/logs", tags=["logs"])
|
||||
|
||||
|
||||
# ── Ingestion ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.post("/ingest", status_code=201)
|
||||
async def ingest_logs(
|
||||
batch: LogBatchRequest,
|
||||
authorization: Optional[str] = Header(None),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Receive a batch of log entries from a project."""
|
||||
api_key = await verify_api_key(authorization)
|
||||
project = await resolve_project_by_api_key(db, api_key)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
rows = []
|
||||
for entry in batch.entries:
|
||||
row = LogEntry(
|
||||
project_id=project.id,
|
||||
project_name=project.name,
|
||||
timestamp=entry.timestamp or now,
|
||||
level=entry.level,
|
||||
logger=entry.logger,
|
||||
message=entry.message,
|
||||
module=entry.module,
|
||||
function=entry.function,
|
||||
line_no=entry.line_no,
|
||||
trace_id=entry.trace_id,
|
||||
exception=entry.exception,
|
||||
extra=entry.extra,
|
||||
)
|
||||
rows.append(row)
|
||||
|
||||
db.add_all(rows)
|
||||
await db.flush()
|
||||
return {"accepted": len(rows), "project": project.name}
|
||||
|
||||
|
||||
@router.post("/ingest/single", status_code=201)
|
||||
async def ingest_single_log(
|
||||
entry: LogEntrySchema,
|
||||
project_name: str = Query(..., description="Project name"),
|
||||
authorization: Optional[str] = Header(None),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Receive a single log entry (useful for testing / simple clients)."""
|
||||
api_key = await verify_api_key(authorization)
|
||||
project = await resolve_project_by_api_key(db, api_key)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
row = LogEntry(
|
||||
project_id=project.id,
|
||||
project_name=project.name,
|
||||
timestamp=entry.timestamp or now,
|
||||
level=entry.level,
|
||||
logger=entry.logger,
|
||||
message=entry.message,
|
||||
module=entry.module,
|
||||
function=entry.function,
|
||||
line_no=entry.line_no,
|
||||
trace_id=entry.trace_id,
|
||||
exception=entry.exception,
|
||||
extra=entry.extra,
|
||||
)
|
||||
db.add(row)
|
||||
await db.flush()
|
||||
return {"accepted": 1, "project": project.name}
|
||||
|
||||
|
||||
# ── Query ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("/search", response_model=LogSearchResult)
|
||||
async def search_logs(
|
||||
project: Optional[str] = Query(None, description="Filter by project name"),
|
||||
level: Optional[str] = Query(None, pattern="^(debug|info|warning|error|critical)$"),
|
||||
query: Optional[str] = Query(None, max_length=1024),
|
||||
trace_id: Optional[str] = Query(None, max_length=64),
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(50, ge=1, le=1000),
|
||||
sort_by: str = Query("timestamp", max_length=32),
|
||||
sort_order: str = Query("desc", pattern="^(asc|desc)$"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Search logs with full-text query and filters."""
|
||||
conditions = []
|
||||
|
||||
if project:
|
||||
conditions.append(LogEntry.project_name == project)
|
||||
if level:
|
||||
conditions.append(LogEntry.level == level)
|
||||
if trace_id:
|
||||
conditions.append(LogEntry.trace_id == trace_id)
|
||||
if start_time:
|
||||
conditions.append(LogEntry.timestamp >= from_query_time(start_time))
|
||||
if end_time:
|
||||
conditions.append(LogEntry.timestamp <= from_query_time(end_time))
|
||||
if query:
|
||||
conditions.append(
|
||||
func.to_tsvector(
|
||||
"english",
|
||||
func.concat(
|
||||
LogEntry.message, " ",
|
||||
func.coalesce(LogEntry.logger, ""), " ",
|
||||
func.coalesce(LogEntry.module, ""),
|
||||
),
|
||||
).bool_op("@@")(
|
||||
func.plainto_tsquery("english", query)
|
||||
)
|
||||
)
|
||||
|
||||
# Sort
|
||||
sort_col = getattr(LogEntry, sort_by, LogEntry.timestamp)
|
||||
order = sort_col.desc() if sort_order == "desc" else sort_col.asc()
|
||||
|
||||
# Count total
|
||||
count_stmt = select(func.count(LogEntry.id))
|
||||
if conditions:
|
||||
count_stmt = count_stmt.where(and_(*conditions))
|
||||
total = (await db.execute(count_stmt)).scalar() or 0
|
||||
|
||||
# Fetch page
|
||||
stmt = select(LogEntry).order_by(order).offset((page - 1) * page_size).limit(page_size)
|
||||
if conditions:
|
||||
stmt = stmt.where(and_(*conditions))
|
||||
rows = (await db.execute(stmt)).scalars().all()
|
||||
|
||||
hits = []
|
||||
for r in rows:
|
||||
hits.append({
|
||||
"project_id": r.project_id,
|
||||
"project_name": r.project_name,
|
||||
"timestamp": to_display_time(r.timestamp),
|
||||
"level": r.level,
|
||||
"logger": r.logger,
|
||||
"message": r.message,
|
||||
"module": r.module,
|
||||
"function": r.function,
|
||||
"line_no": r.line_no,
|
||||
"trace_id": r.trace_id,
|
||||
"exception": r.exception,
|
||||
"extra": r.extra,
|
||||
})
|
||||
|
||||
return LogSearchResult(total=total, page=page, page_size=page_size, hits=hits)
|
||||
|
||||
|
||||
@router.get("/stats", response_model=LogStatsResult)
|
||||
async def log_stats(
|
||||
project: Optional[str] = Query(None),
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
interval: str = Query("5m", max_length=16),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Get aggregated log statistics over a time range."""
|
||||
conditions = []
|
||||
if project:
|
||||
conditions.append(LogEntry.project_name == project)
|
||||
if start_time:
|
||||
conditions.append(LogEntry.timestamp >= from_query_time(start_time))
|
||||
if end_time:
|
||||
conditions.append(LogEntry.timestamp <= from_query_time(end_time))
|
||||
|
||||
def apply(stmt):
|
||||
if conditions:
|
||||
stmt = stmt.where(and_(*conditions))
|
||||
return stmt
|
||||
|
||||
# Time buckets (hourly)
|
||||
stmt = apply(
|
||||
select(
|
||||
func.date_trunc("hour", LogEntry.timestamp).label("bucket"),
|
||||
func.count(LogEntry.id).label("cnt"),
|
||||
).group_by("bucket").order_by("bucket")
|
||||
)
|
||||
time_rows = (await db.execute(stmt)).all()
|
||||
time_buckets = [
|
||||
{"key": to_display_time(r.bucket), "doc_count": r.cnt}
|
||||
for r in time_rows
|
||||
]
|
||||
|
||||
# Level counts
|
||||
stmt = apply(
|
||||
select(LogEntry.level, func.count(LogEntry.id).label("cnt"))
|
||||
.group_by(LogEntry.level)
|
||||
)
|
||||
level_rows = (await db.execute(stmt)).all()
|
||||
level_counts = {r.level: r.cnt for r in level_rows}
|
||||
|
||||
# Top loggers
|
||||
stmt = apply(
|
||||
select(LogEntry.logger, func.count(LogEntry.id).label("cnt"))
|
||||
.group_by(LogEntry.logger)
|
||||
.order_by(func.count(LogEntry.id).desc())
|
||||
.limit(20)
|
||||
)
|
||||
logger_rows = (await db.execute(stmt)).all()
|
||||
top_loggers = [{"logger": r.logger, "count": r.cnt} for r in logger_rows]
|
||||
|
||||
# Total count
|
||||
total = (await db.execute(apply(select(func.count(LogEntry.id))))).scalar() or 0
|
||||
|
||||
return LogStatsResult(
|
||||
project=project,
|
||||
time_buckets=time_buckets,
|
||||
level_counts=level_counts,
|
||||
top_loggers=top_loggers,
|
||||
total=total,
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/cleanup")
|
||||
async def cleanup_old_logs(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Delete log entries older than the retention period."""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=settings.LOG_RETENTION_DAYS)
|
||||
|
||||
result = await db.execute(
|
||||
delete(LogEntry).where(LogEntry.timestamp < cutoff)
|
||||
)
|
||||
await db.flush()
|
||||
|
||||
return {"deleted_count": result.rowcount}
|
||||
@@ -0,0 +1,123 @@
|
||||
"""Project management API routes."""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from typing import List
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models.project import Project
|
||||
from app.schemas.project import ProjectCreate, ProjectUpdate, ProjectResponse, ProjectWithToken
|
||||
|
||||
router = APIRouter(prefix="/api/projects", tags=["projects"])
|
||||
|
||||
|
||||
@router.post("", response_model=ProjectWithToken, status_code=201)
|
||||
async def create_project(
|
||||
data: ProjectCreate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Register a new project for log collection."""
|
||||
existing = await db.execute(select(Project).where(Project.name == data.name))
|
||||
if existing.scalar_one_or_none():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"Project '{data.name}' already exists",
|
||||
)
|
||||
|
||||
project = Project(
|
||||
name=data.name,
|
||||
description=data.description,
|
||||
api_key=uuid.uuid4().hex,
|
||||
)
|
||||
db.add(project)
|
||||
await db.flush()
|
||||
await db.refresh(project)
|
||||
return project
|
||||
|
||||
|
||||
@router.get("", response_model=List[ProjectResponse])
|
||||
async def list_projects(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""List all registered projects."""
|
||||
result = await db.execute(select(Project).order_by(Project.created_at.desc()))
|
||||
return result.scalars().all()
|
||||
|
||||
|
||||
@router.get("/{project_id}", response_model=ProjectResponse)
|
||||
async def get_project(
|
||||
project_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Get a single project by ID."""
|
||||
result = await db.execute(select(Project).where(Project.id == project_id))
|
||||
project = result.scalar_one_or_none()
|
||||
if not project:
|
||||
raise HTTPException(status_code=404, detail="Project not found")
|
||||
return project
|
||||
|
||||
|
||||
@router.patch("/{project_id}", response_model=ProjectResponse)
|
||||
async def update_project(
|
||||
project_id: str,
|
||||
data: ProjectUpdate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Update a project."""
|
||||
result = await db.execute(select(Project).where(Project.id == project_id))
|
||||
project = result.scalar_one_or_none()
|
||||
if not project:
|
||||
raise HTTPException(status_code=404, detail="Project not found")
|
||||
|
||||
if data.name is not None:
|
||||
# Check uniqueness
|
||||
existing = await db.execute(
|
||||
select(Project).where(Project.name == data.name, Project.id != project_id)
|
||||
)
|
||||
if existing.scalar_one_or_none():
|
||||
raise HTTPException(status_code=409, detail="Name already taken")
|
||||
project.name = data.name
|
||||
if data.description is not None:
|
||||
project.description = data.description
|
||||
if data.is_active is not None:
|
||||
project.is_active = data.is_active
|
||||
|
||||
project.updated_at = datetime.now(timezone.utc)
|
||||
await db.flush()
|
||||
await db.refresh(project)
|
||||
return project
|
||||
|
||||
|
||||
@router.delete("/{project_id}", status_code=204)
|
||||
async def delete_project(
|
||||
project_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Delete a project."""
|
||||
result = await db.execute(select(Project).where(Project.id == project_id))
|
||||
project = result.scalar_one_or_none()
|
||||
if not project:
|
||||
raise HTTPException(status_code=404, detail="Project not found")
|
||||
await db.delete(project)
|
||||
|
||||
|
||||
@router.post("/{project_id}/rotate-key", response_model=ProjectWithToken)
|
||||
async def rotate_api_key(
|
||||
project_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Rotate a project's API key."""
|
||||
result = await db.execute(select(Project).where(Project.id == project_id))
|
||||
project = result.scalar_one_or_none()
|
||||
if not project:
|
||||
raise HTTPException(status_code=404, detail="Project not found")
|
||||
|
||||
project.api_key = uuid.uuid4().hex
|
||||
project.updated_at = datetime.now(timezone.utc)
|
||||
await db.flush()
|
||||
await db.refresh(project)
|
||||
return project
|
||||
@@ -0,0 +1,79 @@
|
||||
"""LogHive configuration."""
|
||||
|
||||
import os
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional
|
||||
|
||||
# Beijing timezone offset
|
||||
_BEIJING_TZ = timezone(timedelta(hours=8))
|
||||
|
||||
|
||||
def to_display_time(dt: datetime) -> str:
|
||||
"""Convert a UTC datetime to Beijing time for display.
|
||||
Returns ISO format without timezone suffix (e.g. '2026-05-08T14:30:00')."""
|
||||
if dt is None:
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
local = dt.astimezone(_BEIJING_TZ)
|
||||
return local.strftime("%Y-%m-%dT%H:%M:%S")
|
||||
|
||||
|
||||
def from_query_time(dt: Optional[datetime]) -> Optional[datetime]:
|
||||
"""Interpret a naive datetime query param as Beijing time, return UTC datetime."""
|
||||
if dt is None:
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=_BEIJING_TZ)
|
||||
return dt.astimezone(timezone.utc)
|
||||
|
||||
|
||||
class Settings:
|
||||
"""Application settings loaded from environment variables."""
|
||||
|
||||
# Service name
|
||||
SERVICE_NAME: str = "LogHive"
|
||||
|
||||
# API
|
||||
API_HOST: str = os.getenv("API_HOST", "0.0.0.0")
|
||||
API_PORT: int = int(os.getenv("API_PORT", "8000"))
|
||||
DEBUG: bool = os.getenv("DEBUG", "false").lower() == "true"
|
||||
CORS_ORIGINS: list[str] = os.getenv("CORS_ORIGINS", "*").split(",")
|
||||
|
||||
# Security
|
||||
SECRET_KEY: str = os.getenv("SECRET_KEY", "change-me-in-production")
|
||||
TOKEN_EXPIRE_HOURS: int = int(os.getenv("TOKEN_EXPIRE_HOURS", "720"))
|
||||
|
||||
# PostgreSQL
|
||||
DATABASE_URL: str = os.getenv(
|
||||
"DATABASE_URL",
|
||||
"postgresql+asyncpg://LogHive:LogHive@localhost:5432/LogHive",
|
||||
)
|
||||
|
||||
# Redis
|
||||
REDIS_URL: str = os.getenv("REDIS_URL", "redis://localhost:6379/0")
|
||||
|
||||
# Celery
|
||||
CELERY_BROKER_URL: str = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/1")
|
||||
CELERY_RESULT_BACKEND: str = os.getenv(
|
||||
"CELERY_RESULT_BACKEND", "redis://localhost:6379/2"
|
||||
)
|
||||
|
||||
# Log retention (days)
|
||||
LOG_RETENTION_DAYS: int = int(os.getenv("LOG_RETENTION_DAYS", "30"))
|
||||
LOG_COLD_STORAGE_DAYS: int = int(os.getenv("LOG_COLD_STORAGE_DAYS", "7"))
|
||||
|
||||
# Rate limit
|
||||
RATE_LIMIT_PER_SECOND: int = int(os.getenv("RATE_LIMIT_PER_SECOND", "1000"))
|
||||
|
||||
# Alert
|
||||
ALERT_CHECK_INTERVAL_SECONDS: int = int(
|
||||
os.getenv("ALERT_CHECK_INTERVAL_SECONDS", "60")
|
||||
)
|
||||
|
||||
# Notification
|
||||
FEISHU_WEBHOOK_URL: Optional[str] = os.getenv("FEISHU_WEBHOOK_URL", None)
|
||||
DINGTALK_WEBHOOK_URL: Optional[str] = os.getenv("DINGTALK_WEBHOOK_URL", None)
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1 @@
|
||||
from .auth import verify_api_key, resolve_project_by_api_key
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Auth utilities — API key verification."""
|
||||
|
||||
import hmac
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Header, HTTPException, status
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.database import get_db
|
||||
from app.models.project import Project
|
||||
|
||||
|
||||
async def verify_api_key(
|
||||
authorization: Optional[str] = Header(None),
|
||||
) -> str:
|
||||
"""Extract and verify the API key from the Authorization header.
|
||||
|
||||
Returns the project_id on success.
|
||||
"""
|
||||
if not authorization:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Missing Authorization header",
|
||||
)
|
||||
|
||||
scheme, _, key = authorization.partition(" ")
|
||||
if scheme.lower() != "bearer" or not key.strip():
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid Authorization format. Use: Bearer <api_key>",
|
||||
)
|
||||
|
||||
return key.strip()
|
||||
|
||||
|
||||
async def resolve_project_by_api_key(
|
||||
db: AsyncSession, api_key: str
|
||||
) -> Project:
|
||||
"""Look up a project by its API key."""
|
||||
result = await db.execute(
|
||||
select(Project).where(Project.api_key == api_key, Project.is_active == True)
|
||||
)
|
||||
project = result.scalar_one_or_none()
|
||||
if not project:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid or inactive API key",
|
||||
)
|
||||
return project
|
||||
@@ -0,0 +1 @@
|
||||
"""Core auth logic."""
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Database connections and session management."""
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
from app.config import settings
|
||||
|
||||
# ── PostgreSQL ────────────────────────────────────────────────────
|
||||
engine = create_async_engine(settings.DATABASE_URL, echo=settings.DEBUG)
|
||||
async_session_factory = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
async def get_db() -> AsyncSession:
|
||||
"""Yield a DB session per request."""
|
||||
async with async_session_factory() as session:
|
||||
try:
|
||||
yield session
|
||||
await session.commit()
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
raise
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def init_db() -> None:
|
||||
"""Create all tables (use Alembic in production)."""
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
|
||||
async def close_db() -> None:
|
||||
await engine.dispose()
|
||||
@@ -0,0 +1,65 @@
|
||||
"""LogHive — Centralized Log Management System.
|
||||
|
||||
FastAPI application that receives, stores, searches and analyzes logs
|
||||
from multiple Python projects.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app.config import settings
|
||||
from app.database import init_db, close_db
|
||||
from app.api import logs, projects, alerts, dashboard
|
||||
|
||||
# ── Logging ────────────────────────────────────────────────────
|
||||
logging.basicConfig(
|
||||
level=logging.INFO if not settings.DEBUG else logging.DEBUG,
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── Lifecycle ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""Startup and shutdown lifecycle."""
|
||||
logger.info("🚀 LogHive starting up...")
|
||||
await init_db()
|
||||
yield
|
||||
logger.info("🛑 LogHive shutting down...")
|
||||
await close_db()
|
||||
|
||||
|
||||
# ── App ────────────────────────────────────────────────────────
|
||||
|
||||
app = FastAPI(
|
||||
title="LogHive API",
|
||||
description="Centralized log management system — ingest, search, monitor, and alert on logs from your Python services.",
|
||||
version="0.1.0",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.CORS_ORIGINS,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# ── Routes ─────────────────────────────────────────────────────
|
||||
|
||||
app.include_router(logs.router)
|
||||
app.include_router(projects.router)
|
||||
app.include_router(alerts.router)
|
||||
app.include_router(dashboard.router)
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
async def health():
|
||||
return {"status": "ok", "service": settings.SERVICE_NAME, "version": "0.1.0"}
|
||||
@@ -0,0 +1,5 @@
|
||||
from app.models.project import Project
|
||||
from app.models.alert import AlertRule, AlertHistory
|
||||
from app.models.log import LogEntry
|
||||
|
||||
__all__ = ["Project", "AlertRule", "AlertHistory", "LogEntry"]
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Alert models — rules and history."""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import String, Integer, Float, Boolean, DateTime, Text, Enum as SAEnum
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
import enum
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class AlertLevel(str, enum.Enum):
|
||||
INFO = "info"
|
||||
WARNING = "warning"
|
||||
CRITICAL = "critical"
|
||||
|
||||
|
||||
class AlertOperator(str, enum.Enum):
|
||||
GT = "gt"
|
||||
GTE = "gte"
|
||||
LT = "lt"
|
||||
LTE = "lte"
|
||||
EQ = "eq"
|
||||
|
||||
|
||||
class AlertRule(Base):
|
||||
__tablename__ = "alert_rules"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
|
||||
)
|
||||
project_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
level: Mapped[AlertLevel] = mapped_column(
|
||||
SAEnum(AlertLevel), default=AlertLevel.WARNING
|
||||
)
|
||||
field: Mapped[str] = mapped_column(
|
||||
String(64), nullable=False, default="level"
|
||||
) # log field to evaluate
|
||||
operator: Mapped[AlertOperator] = mapped_column(
|
||||
SAEnum(AlertOperator), default=AlertOperator.GTE
|
||||
)
|
||||
threshold: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
window_minutes: Mapped[int] = mapped_column(Integer, default=5)
|
||||
is_enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
notify_channels: Mapped[str] = mapped_column(
|
||||
String(256), default=""
|
||||
) # comma-separated
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
|
||||
class AlertHistory(Base):
|
||||
__tablename__ = "alert_history"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
|
||||
)
|
||||
rule_id: Mapped[str] = mapped_column(
|
||||
String(36), nullable=False, index=True
|
||||
)
|
||||
project_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
level: Mapped[AlertLevel] = mapped_column(SAEnum(AlertLevel))
|
||||
message: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
triggered_value: Mapped[float] = mapped_column(Float, nullable=True)
|
||||
is_acknowledged: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
triggered_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Log entry model — stored in PostgreSQL."""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import DateTime, Index, Integer, String, Text
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class LogEntry(Base):
|
||||
__tablename__ = "log_entries"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
|
||||
)
|
||||
project_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||
project_name: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||
timestamp: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, index=True,
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
level: Mapped[str] = mapped_column(String(16), nullable=False, index=True)
|
||||
logger: Mapped[str] = mapped_column(String(128), default="root")
|
||||
message: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
module: Mapped[str | None] = mapped_column(String(256), nullable=True)
|
||||
function: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
line_no: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
trace_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
exception: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
extra: Mapped[dict] = mapped_column(JSONB, default=dict)
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_log_entries_project_time", "project_id", timestamp.desc()),
|
||||
)
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Project model — represents an external service that pushes logs."""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import String, Boolean, DateTime, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class Project(Base):
|
||||
__tablename__ = "projects"
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
|
||||
)
|
||||
name: Mapped[str] = mapped_column(String(128), unique=True, nullable=False)
|
||||
description: Mapped[str] = mapped_column(Text, default="")
|
||||
api_key: Mapped[str] = mapped_column(
|
||||
String(64), unique=True, nullable=False, default=lambda: uuid.uuid4().hex
|
||||
)
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True),
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
onupdate=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<Project {self.name}>"
|
||||
@@ -0,0 +1,35 @@
|
||||
from app.schemas.log import (
|
||||
LogEntry,
|
||||
LogBatchRequest,
|
||||
LogQueryParams,
|
||||
LogSearchResult,
|
||||
LogStatsResult,
|
||||
)
|
||||
from app.schemas.project import (
|
||||
ProjectCreate,
|
||||
ProjectUpdate,
|
||||
ProjectResponse,
|
||||
ProjectWithToken,
|
||||
)
|
||||
from app.schemas.alert import (
|
||||
AlertRuleCreate,
|
||||
AlertRuleUpdate,
|
||||
AlertRuleResponse,
|
||||
AlertHistoryResponse,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"LogEntry",
|
||||
"LogBatchRequest",
|
||||
"LogQueryParams",
|
||||
"LogSearchResult",
|
||||
"LogStatsResult",
|
||||
"ProjectCreate",
|
||||
"ProjectUpdate",
|
||||
"ProjectResponse",
|
||||
"ProjectWithToken",
|
||||
"AlertRuleCreate",
|
||||
"AlertRuleUpdate",
|
||||
"AlertRuleResponse",
|
||||
"AlertHistoryResponse",
|
||||
]
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Alert schemas."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
# ── Alert Rule ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
class AlertRuleCreate(BaseModel):
|
||||
project_id: str = Field(..., min_length=1, max_length=36)
|
||||
name: str = Field(..., min_length=1, max_length=128)
|
||||
level: str = Field(default="warning", pattern="^(info|warning|critical)$")
|
||||
field: str = Field(default="level", max_length=64)
|
||||
operator: str = Field(default="gte", pattern="^(gt|gte|lt|lte|eq)$")
|
||||
threshold: float = Field(..., ge=0)
|
||||
window_minutes: int = Field(default=5, ge=1, le=1440)
|
||||
notify_channels: List[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class AlertRuleUpdate(BaseModel):
|
||||
name: Optional[str] = Field(default=None, max_length=128)
|
||||
level: Optional[str] = Field(default=None, pattern="^(info|warning|critical)$")
|
||||
field: Optional[str] = Field(default=None, max_length=64)
|
||||
operator: Optional[str] = Field(default=None, pattern="^(gt|gte|lt|lte|eq)$")
|
||||
threshold: Optional[float] = Field(default=None, ge=0)
|
||||
window_minutes: Optional[int] = Field(default=None, ge=1, le=1440)
|
||||
is_enabled: Optional[bool] = None
|
||||
notify_channels: Optional[List[str]] = None
|
||||
|
||||
|
||||
class AlertRuleResponse(BaseModel):
|
||||
id: str
|
||||
project_id: str
|
||||
name: str
|
||||
level: str
|
||||
field: str
|
||||
operator: str
|
||||
threshold: float
|
||||
window_minutes: int
|
||||
is_enabled: bool
|
||||
notify_channels: List[str]
|
||||
created_at: datetime
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
# ── Alert History ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class AlertHistoryResponse(BaseModel):
|
||||
id: str
|
||||
rule_id: str
|
||||
project_id: str
|
||||
level: str
|
||||
message: str
|
||||
triggered_value: Optional[float] = None
|
||||
is_acknowledged: bool
|
||||
triggered_at: datetime
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Log entry schemas."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class LogEntry(BaseModel):
|
||||
"""A single log entry pushed by a project."""
|
||||
|
||||
timestamp: Optional[datetime] = None
|
||||
level: str = Field(default="info", pattern="^(debug|info|warning|error|critical)$")
|
||||
logger: str = Field(default="root", max_length=128)
|
||||
message: str = Field(..., min_length=1, max_length=65536)
|
||||
module: Optional[str] = Field(default=None, max_length=256)
|
||||
function: Optional[str] = Field(default=None, max_length=128)
|
||||
line_no: Optional[int] = None
|
||||
trace_id: Optional[str] = Field(default=None, max_length=64)
|
||||
exception: Optional[str] = None
|
||||
extra: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class LogBatchRequest(BaseModel):
|
||||
"""Batch of log entries pushed in a single request."""
|
||||
|
||||
project: str = Field(..., min_length=1, max_length=128)
|
||||
entries: List[LogEntry] = Field(..., min_length=1, max_length=5000)
|
||||
|
||||
|
||||
class LogQueryParams(BaseModel):
|
||||
"""Parameters for log search / query."""
|
||||
|
||||
project: Optional[str] = None
|
||||
level: Optional[str] = Field(
|
||||
default=None, pattern="^(debug|info|warning|error|critical)$"
|
||||
)
|
||||
query: Optional[str] = Field(default=None, max_length=1024)
|
||||
trace_id: Optional[str] = Field(default=None, max_length=64)
|
||||
start_time: Optional[datetime] = None
|
||||
end_time: Optional[datetime] = None
|
||||
page: int = Field(default=1, ge=1)
|
||||
page_size: int = Field(default=50, ge=1, le=1000)
|
||||
sort_by: str = Field(default="timestamp", max_length=32)
|
||||
sort_order: str = Field(default="desc", pattern="^(asc|desc)$")
|
||||
|
||||
|
||||
class LogSearchResult(BaseModel):
|
||||
"""Search result wrapper."""
|
||||
|
||||
total: int
|
||||
page: int
|
||||
page_size: int
|
||||
hits: List[Dict[str, Any]]
|
||||
|
||||
|
||||
class LogStatsResult(BaseModel):
|
||||
"""Aggregated log statistics."""
|
||||
|
||||
project: Optional[str] = None
|
||||
time_buckets: List[Dict[str, Any]]
|
||||
level_counts: Dict[str, int]
|
||||
top_loggers: List[Dict[str, Any]]
|
||||
total: int
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Project schemas."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ProjectCreate(BaseModel):
|
||||
name: str = Field(..., min_length=1, max_length=128, description="Project name")
|
||||
description: str = Field(default="", max_length=1024)
|
||||
|
||||
|
||||
class ProjectUpdate(BaseModel):
|
||||
name: Optional[str] = Field(default=None, max_length=128)
|
||||
description: Optional[str] = Field(default=None, max_length=1024)
|
||||
is_active: Optional[bool] = None
|
||||
|
||||
|
||||
class ProjectResponse(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
description: str
|
||||
is_active: bool
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class ProjectWithToken(ProjectResponse):
|
||||
api_key: str
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
@@ -0,0 +1,142 @@
|
||||
"""Alert evaluation and notification service."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.models.alert import AlertRule, AlertHistory, AlertLevel, AlertOperator
|
||||
from app.models.log import LogEntry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AlertService:
|
||||
"""Evaluates alert rules against recent log data and triggers notifications."""
|
||||
|
||||
def __init__(self, db: AsyncSession):
|
||||
self.db = db
|
||||
|
||||
async def evaluate_rules(self) -> list[AlertHistory]:
|
||||
"""Check all enabled alert rules and trigger any that match."""
|
||||
result = await self.db.execute(
|
||||
select(AlertRule).where(AlertRule.is_enabled == True)
|
||||
)
|
||||
rules: list[AlertRule] = result.scalars().all()
|
||||
triggered: list[AlertHistory] = []
|
||||
|
||||
for rule in rules:
|
||||
try:
|
||||
hit = await self._evaluate_single(rule)
|
||||
if hit:
|
||||
triggered.append(hit)
|
||||
except Exception as e:
|
||||
logger.error("Error evaluating rule %s: %s", rule.id, e)
|
||||
|
||||
return triggered
|
||||
|
||||
async def _evaluate_single(self, rule: AlertRule) -> Optional[AlertHistory]:
|
||||
"""Evaluate a single rule. Returns AlertHistory if triggered."""
|
||||
now = datetime.now(timezone.utc)
|
||||
since = now - timedelta(minutes=rule.window_minutes)
|
||||
|
||||
stmt = (
|
||||
select(func.count(LogEntry.id))
|
||||
.where(LogEntry.project_id == rule.project_id)
|
||||
.where(LogEntry.timestamp >= since)
|
||||
.where(LogEntry.level == rule.level.value)
|
||||
)
|
||||
count = (await self.db.execute(stmt)).scalar() or 0
|
||||
|
||||
operator_map = {
|
||||
AlertOperator.GT: lambda v, t: v > t,
|
||||
AlertOperator.GTE: lambda v, t: v >= t,
|
||||
AlertOperator.LT: lambda v, t: v < t,
|
||||
AlertOperator.LTE: lambda v, t: v <= t,
|
||||
AlertOperator.EQ: lambda v, t: v == t,
|
||||
}
|
||||
|
||||
op_fn = operator_map.get(rule.operator)
|
||||
if op_fn and op_fn(float(count), rule.threshold):
|
||||
history = AlertHistory(
|
||||
rule_id=rule.id,
|
||||
project_id=rule.project_id,
|
||||
level=rule.level,
|
||||
message=(
|
||||
f"Alert '{rule.name}': {rule.level.value} logs count ({count}) "
|
||||
f"{rule.operator.value} threshold ({rule.threshold}) "
|
||||
f"in last {rule.window_minutes} min"
|
||||
),
|
||||
triggered_value=float(count),
|
||||
)
|
||||
self.db.add(history)
|
||||
await self.db.flush()
|
||||
logger.info("Alert triggered: %s", history.message)
|
||||
|
||||
if rule.notify_channels:
|
||||
await self._notify(rule, history)
|
||||
|
||||
return history
|
||||
|
||||
return None
|
||||
|
||||
async def _notify(self, rule: AlertRule, history: AlertHistory) -> None:
|
||||
"""Send notification via configured channels."""
|
||||
channels = [c.strip() for c in rule.notify_channels.split(",") if c.strip()]
|
||||
for channel in channels:
|
||||
if channel == "feishu" and settings.FEISHU_WEBHOOK_URL:
|
||||
await self._send_feishu(rule, history)
|
||||
elif channel == "dingtalk" and settings.DINGTALK_WEBHOOK_URL:
|
||||
await self._send_dingtalk(rule, history)
|
||||
|
||||
async def _send_feishu(self, rule: AlertRule, history: AlertHistory) -> None:
|
||||
"""Send a notification to Feishu webhook."""
|
||||
import httpx
|
||||
|
||||
payload = {
|
||||
"msg_type": "post",
|
||||
"content": {
|
||||
"post": {
|
||||
"zh_cn": {
|
||||
"title": f"[LogHive] Alert: {rule.name}",
|
||||
"content": [
|
||||
[{"tag": "text", "text": history.message}],
|
||||
[
|
||||
{
|
||||
"tag": "text",
|
||||
"text": f"Project: {rule.project_id} | Level: {rule.level.value}",
|
||||
}
|
||||
],
|
||||
],
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
await client.post(settings.FEISHU_WEBHOOK_URL, json=payload, timeout=10)
|
||||
except Exception as e:
|
||||
logger.error("Feishu notification failed: %s", e)
|
||||
|
||||
async def _send_dingtalk(self, rule: AlertRule, history: AlertHistory) -> None:
|
||||
"""Send a notification to DingTalk webhook."""
|
||||
import httpx
|
||||
|
||||
payload = {
|
||||
"msgtype": "text",
|
||||
"text": {
|
||||
"content": (
|
||||
f"[LogHive] Alert: {rule.name}\n"
|
||||
f"{history.message}\n"
|
||||
f"Project: {rule.project_id} | Level: {rule.level.value}"
|
||||
)
|
||||
},
|
||||
}
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
await client.post(settings.DINGTALK_WEBHOOK_URL, json=payload, timeout=10)
|
||||
except Exception as e:
|
||||
logger.error("DingTalk notification failed: %s", e)
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Log analytics service — provides aggregated statistics and insights."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import func, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import to_display_time
|
||||
from app.models.log import LogEntry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AnalyticsService:
|
||||
"""Time-series analytics over log data."""
|
||||
|
||||
def __init__(self, db: AsyncSession):
|
||||
self.db = db
|
||||
|
||||
def _build_filters(self, project: Optional[str], since: datetime):
|
||||
"""Build SQLAlchemy filter list."""
|
||||
filters = [LogEntry.timestamp >= since]
|
||||
if project:
|
||||
filters.append(LogEntry.project_name == project)
|
||||
return filters
|
||||
|
||||
async def get_dashboard_stats(
|
||||
self, project: Optional[str] = None, hours: int = 24
|
||||
) -> dict:
|
||||
"""Get overview statistics for the dashboard."""
|
||||
now = datetime.now(timezone.utc)
|
||||
since = now - timedelta(hours=hours)
|
||||
filters = self._build_filters(project, since)
|
||||
|
||||
# Total log count
|
||||
total = (await self.db.execute(
|
||||
select(func.count(LogEntry.id)).where(*filters)
|
||||
)).scalar() or 0
|
||||
|
||||
# Error count
|
||||
error_count = (await self.db.execute(
|
||||
select(func.count(LogEntry.id)).where(
|
||||
*filters, LogEntry.level.in_(["error", "critical"])
|
||||
)
|
||||
)).scalar() or 0
|
||||
|
||||
# Level breakdown
|
||||
level_rows = (await self.db.execute(
|
||||
select(LogEntry.level, func.count(LogEntry.id))
|
||||
.where(*filters)
|
||||
.group_by(LogEntry.level)
|
||||
)).all()
|
||||
level_breakdown = {row[0]: row[1] for row in level_rows}
|
||||
|
||||
# Error rate
|
||||
error_rate = round((error_count / total * 100), 2) if total > 0 else 0
|
||||
|
||||
# Top error messages
|
||||
top_rows = (await self.db.execute(
|
||||
select(LogEntry.message, func.count(LogEntry.id).label("cnt"))
|
||||
.where(*filters, LogEntry.level.in_(["error", "critical"]))
|
||||
.group_by(LogEntry.message)
|
||||
.order_by(func.count(LogEntry.id).desc())
|
||||
.limit(10)
|
||||
)).all()
|
||||
top_errors = [{"message": row[0], "count": row[1]} for row in top_rows]
|
||||
|
||||
return {
|
||||
"total_logs": total,
|
||||
"error_count": error_count,
|
||||
"error_rate": error_rate,
|
||||
"level_breakdown": level_breakdown,
|
||||
"time_range_hours": hours,
|
||||
"top_errors": top_errors,
|
||||
}
|
||||
|
||||
async def get_time_series(
|
||||
self,
|
||||
project: Optional[str] = None,
|
||||
hours: int = 24,
|
||||
interval: str = "5m",
|
||||
) -> list[dict]:
|
||||
"""Get time-series data for log volume charts."""
|
||||
now = datetime.now(timezone.utc)
|
||||
since = now - timedelta(hours=hours)
|
||||
|
||||
n = int("".join(c for c in interval if c.isdigit()) or "5")
|
||||
unit = "".join(c for c in interval if c.isalpha()) or "m"
|
||||
if unit == "h":
|
||||
n = n * 60
|
||||
n = max(n, 1)
|
||||
|
||||
# Build WHERE clauses
|
||||
where_clauses = [f"timestamp >= '{since.isoformat()}'"]
|
||||
params: dict = {}
|
||||
if project:
|
||||
where_clauses.append("project_name = :project")
|
||||
params["project"] = project
|
||||
where_sql = " AND ".join(where_clauses)
|
||||
|
||||
sql = text(
|
||||
f"SELECT "
|
||||
f" date_trunc('hour', timestamp) + "
|
||||
f" ((floor(extract(minute from timestamp) / {n})::int * {n}) || ' min')::interval AS bucket, "
|
||||
f" level, "
|
||||
f" count(*) AS cnt "
|
||||
f"FROM log_entries "
|
||||
f"WHERE {where_sql} "
|
||||
f"GROUP BY bucket, level "
|
||||
f"ORDER BY bucket"
|
||||
)
|
||||
|
||||
rows = (await self.db.execute(sql, params)).all()
|
||||
|
||||
buckets: dict[str, dict] = {}
|
||||
for row in rows:
|
||||
ts = to_display_time(row.bucket) if isinstance(row.bucket, datetime) else str(row.bucket)
|
||||
if ts not in buckets:
|
||||
buckets[ts] = {"timestamp": ts, "total": 0, "levels": {}}
|
||||
buckets[ts]["total"] += row.cnt
|
||||
buckets[ts]["levels"][row.level] = row.cnt
|
||||
|
||||
return list(buckets.values())
|
||||
Reference in New Issue
Block a user