Initial commit: LogHive centralized log management system

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
v6ole
2026-05-09 14:55:14 +08:00
commit abfd07331e
54 changed files with 3816 additions and 0 deletions
View File
View File
+180
View File
@@ -0,0 +1,180 @@
"""Alert management API routes."""
from datetime import datetime, timezone
from typing import List, Optional
from fastapi import APIRouter, Depends, HTTPException, Query, status
from sqlalchemy import select, func
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.models.alert import AlertRule, AlertHistory, AlertLevel, AlertOperator
from app.schemas.alert import (
AlertRuleCreate,
AlertRuleUpdate,
AlertRuleResponse,
AlertHistoryResponse,
)
router = APIRouter(prefix="/api/alerts", tags=["alerts"])
# ── Rule CRUD ──────────────────────────────────────────────────
@router.post("/rules", response_model=AlertRuleResponse, status_code=201)
async def create_alert_rule(
data: AlertRuleCreate,
db: AsyncSession = Depends(get_db),
):
"""Create a new alert rule."""
rule = AlertRule(
project_id=data.project_id,
name=data.name,
level=data.level,
field=data.field,
operator=data.operator,
threshold=data.threshold,
window_minutes=data.window_minutes,
notify_channels=",".join(data.notify_channels) if data.notify_channels else "",
)
db.add(rule)
await db.flush()
await db.refresh(rule)
return _rule_to_response(rule)
@router.get("/rules", response_model=List[AlertRuleResponse])
async def list_alert_rules(
project_id: Optional[str] = Query(None),
db: AsyncSession = Depends(get_db),
):
"""List alert rules, optionally filtered by project."""
stmt = select(AlertRule)
if project_id:
stmt = stmt.where(AlertRule.project_id == project_id)
stmt = stmt.order_by(AlertRule.created_at.desc())
result = await db.execute(stmt)
return [_rule_to_response(r) for r in result.scalars().all()]
@router.get("/rules/{rule_id}", response_model=AlertRuleResponse)
async def get_alert_rule(rule_id: str, db: AsyncSession = Depends(get_db)):
"""Get a single alert rule."""
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if not rule:
raise HTTPException(status_code=404, detail="Alert rule not found")
return _rule_to_response(rule)
@router.patch("/rules/{rule_id}", response_model=AlertRuleResponse)
async def update_alert_rule(
rule_id: str,
data: AlertRuleUpdate,
db: AsyncSession = Depends(get_db),
):
"""Update an alert rule."""
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if not rule:
raise HTTPException(status_code=404, detail="Alert rule not found")
if data.name is not None:
rule.name = data.name
if data.level is not None:
rule.level = data.level
if data.field is not None:
rule.field = data.field
if data.operator is not None:
rule.operator = data.operator
if data.threshold is not None:
rule.threshold = data.threshold
if data.window_minutes is not None:
rule.window_minutes = data.window_minutes
if data.is_enabled is not None:
rule.is_enabled = data.is_enabled
if data.notify_channels is not None:
rule.notify_channels = ",".join(data.notify_channels)
await db.flush()
await db.refresh(rule)
return _rule_to_response(rule)
@router.delete("/rules/{rule_id}", status_code=204)
async def delete_alert_rule(rule_id: str, db: AsyncSession = Depends(get_db)):
"""Delete an alert rule."""
result = await db.execute(select(AlertRule).where(AlertRule.id == rule_id))
rule = result.scalar_one_or_none()
if not rule:
raise HTTPException(status_code=404, detail="Alert rule not found")
await db.delete(rule)
# ── Alert History ──────────────────────────────────────────────
@router.get("/history", response_model=List[AlertHistoryResponse])
async def list_alert_history(
project_id: Optional[str] = Query(None),
rule_id: Optional[str] = Query(None),
limit: int = Query(100, ge=1, le=1000),
db: AsyncSession = Depends(get_db),
):
"""List alert trigger history."""
stmt = select(AlertHistory).order_by(AlertHistory.triggered_at.desc()).limit(limit)
if project_id:
stmt = stmt.where(AlertHistory.project_id == project_id)
if rule_id:
stmt = stmt.where(AlertHistory.rule_id == rule_id)
result = await db.execute(stmt)
return [_history_to_response(h) for h in result.scalars().all()]
@router.post("/history/{history_id}/acknowledge", response_model=AlertHistoryResponse)
async def acknowledge_alert(
history_id: str,
db: AsyncSession = Depends(get_db),
):
"""Mark an alert as acknowledged."""
result = await db.execute(select(AlertHistory).where(AlertHistory.id == history_id))
history = result.scalar_one_or_none()
if not history:
raise HTTPException(status_code=404, detail="Alert history not found")
history.is_acknowledged = True
await db.flush()
await db.refresh(history)
return _history_to_response(history)
# ── Helpers ────────────────────────────────────────────────────
def _rule_to_response(rule: AlertRule) -> AlertRuleResponse:
return AlertRuleResponse(
id=rule.id,
project_id=rule.project_id,
name=rule.name,
level=rule.level.value if hasattr(rule.level, "value") else rule.level,
field=rule.field,
operator=rule.operator.value if hasattr(rule.operator, "value") else rule.operator,
threshold=rule.threshold,
window_minutes=rule.window_minutes,
is_enabled=rule.is_enabled,
notify_channels=rule.notify_channels.split(",") if rule.notify_channels else [],
created_at=rule.created_at,
)
def _history_to_response(history: AlertHistory) -> AlertHistoryResponse:
return AlertHistoryResponse(
id=history.id,
rule_id=history.rule_id,
project_id=history.project_id,
level=history.level.value if hasattr(history.level, "value") else history.level,
message=history.message,
triggered_value=history.triggered_value,
is_acknowledged=history.is_acknowledged,
triggered_at=history.triggered_at,
)
+34
View File
@@ -0,0 +1,34 @@
"""Dashboard API routes."""
from typing import Optional
from fastapi import APIRouter, Depends, Query
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.services.analytics_service import AnalyticsService
router = APIRouter(prefix="/api/dashboard", tags=["dashboard"])
@router.get("/stats")
async def dashboard_stats(
project: Optional[str] = Query(None),
hours: int = Query(24, ge=1, le=720),
db: AsyncSession = Depends(get_db),
):
"""Get overview statistics for the dashboard."""
service = AnalyticsService(db)
return await service.get_dashboard_stats(project=project, hours=hours)
@router.get("/timeseries")
async def dashboard_timeseries(
project: Optional[str] = Query(None),
hours: int = Query(24, ge=1, le=720),
interval: str = Query("5m", max_length=16),
db: AsyncSession = Depends(get_db),
):
"""Get time-series data for chart rendering."""
service = AnalyticsService(db)
return await service.get_time_series(project=project, hours=hours, interval=interval)
+253
View File
@@ -0,0 +1,253 @@
"""Log ingestion and query API routes."""
import logging
from datetime import datetime, timezone, timedelta
from typing import Optional
from fastapi import APIRouter, Depends, Header, HTTPException, Query, status
from sqlalchemy import func, select, delete, and_
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings, to_display_time, from_query_time
from app.database import get_db
from app.schemas.log import (
LogBatchRequest,
LogEntry as LogEntrySchema,
LogQueryParams,
LogSearchResult,
LogStatsResult,
)
from app.models.project import Project
from app.models.log import LogEntry
from app.core.auth import verify_api_key, resolve_project_by_api_key
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/logs", tags=["logs"])
# ── Ingestion ─────────────────────────────────────────────────
@router.post("/ingest", status_code=201)
async def ingest_logs(
batch: LogBatchRequest,
authorization: Optional[str] = Header(None),
db: AsyncSession = Depends(get_db),
):
"""Receive a batch of log entries from a project."""
api_key = await verify_api_key(authorization)
project = await resolve_project_by_api_key(db, api_key)
now = datetime.now(timezone.utc)
rows = []
for entry in batch.entries:
row = LogEntry(
project_id=project.id,
project_name=project.name,
timestamp=entry.timestamp or now,
level=entry.level,
logger=entry.logger,
message=entry.message,
module=entry.module,
function=entry.function,
line_no=entry.line_no,
trace_id=entry.trace_id,
exception=entry.exception,
extra=entry.extra,
)
rows.append(row)
db.add_all(rows)
await db.flush()
return {"accepted": len(rows), "project": project.name}
@router.post("/ingest/single", status_code=201)
async def ingest_single_log(
entry: LogEntrySchema,
project_name: str = Query(..., description="Project name"),
authorization: Optional[str] = Header(None),
db: AsyncSession = Depends(get_db),
):
"""Receive a single log entry (useful for testing / simple clients)."""
api_key = await verify_api_key(authorization)
project = await resolve_project_by_api_key(db, api_key)
now = datetime.now(timezone.utc)
row = LogEntry(
project_id=project.id,
project_name=project.name,
timestamp=entry.timestamp or now,
level=entry.level,
logger=entry.logger,
message=entry.message,
module=entry.module,
function=entry.function,
line_no=entry.line_no,
trace_id=entry.trace_id,
exception=entry.exception,
extra=entry.extra,
)
db.add(row)
await db.flush()
return {"accepted": 1, "project": project.name}
# ── Query ──────────────────────────────────────────────────────
@router.get("/search", response_model=LogSearchResult)
async def search_logs(
project: Optional[str] = Query(None, description="Filter by project name"),
level: Optional[str] = Query(None, pattern="^(debug|info|warning|error|critical)$"),
query: Optional[str] = Query(None, max_length=1024),
trace_id: Optional[str] = Query(None, max_length=64),
start_time: Optional[datetime] = None,
end_time: Optional[datetime] = None,
page: int = Query(1, ge=1),
page_size: int = Query(50, ge=1, le=1000),
sort_by: str = Query("timestamp", max_length=32),
sort_order: str = Query("desc", pattern="^(asc|desc)$"),
db: AsyncSession = Depends(get_db),
):
"""Search logs with full-text query and filters."""
conditions = []
if project:
conditions.append(LogEntry.project_name == project)
if level:
conditions.append(LogEntry.level == level)
if trace_id:
conditions.append(LogEntry.trace_id == trace_id)
if start_time:
conditions.append(LogEntry.timestamp >= from_query_time(start_time))
if end_time:
conditions.append(LogEntry.timestamp <= from_query_time(end_time))
if query:
conditions.append(
func.to_tsvector(
"english",
func.concat(
LogEntry.message, " ",
func.coalesce(LogEntry.logger, ""), " ",
func.coalesce(LogEntry.module, ""),
),
).bool_op("@@")(
func.plainto_tsquery("english", query)
)
)
# Sort
sort_col = getattr(LogEntry, sort_by, LogEntry.timestamp)
order = sort_col.desc() if sort_order == "desc" else sort_col.asc()
# Count total
count_stmt = select(func.count(LogEntry.id))
if conditions:
count_stmt = count_stmt.where(and_(*conditions))
total = (await db.execute(count_stmt)).scalar() or 0
# Fetch page
stmt = select(LogEntry).order_by(order).offset((page - 1) * page_size).limit(page_size)
if conditions:
stmt = stmt.where(and_(*conditions))
rows = (await db.execute(stmt)).scalars().all()
hits = []
for r in rows:
hits.append({
"project_id": r.project_id,
"project_name": r.project_name,
"timestamp": to_display_time(r.timestamp),
"level": r.level,
"logger": r.logger,
"message": r.message,
"module": r.module,
"function": r.function,
"line_no": r.line_no,
"trace_id": r.trace_id,
"exception": r.exception,
"extra": r.extra,
})
return LogSearchResult(total=total, page=page, page_size=page_size, hits=hits)
@router.get("/stats", response_model=LogStatsResult)
async def log_stats(
project: Optional[str] = Query(None),
start_time: Optional[datetime] = None,
end_time: Optional[datetime] = None,
interval: str = Query("5m", max_length=16),
db: AsyncSession = Depends(get_db),
):
"""Get aggregated log statistics over a time range."""
conditions = []
if project:
conditions.append(LogEntry.project_name == project)
if start_time:
conditions.append(LogEntry.timestamp >= from_query_time(start_time))
if end_time:
conditions.append(LogEntry.timestamp <= from_query_time(end_time))
def apply(stmt):
if conditions:
stmt = stmt.where(and_(*conditions))
return stmt
# Time buckets (hourly)
stmt = apply(
select(
func.date_trunc("hour", LogEntry.timestamp).label("bucket"),
func.count(LogEntry.id).label("cnt"),
).group_by("bucket").order_by("bucket")
)
time_rows = (await db.execute(stmt)).all()
time_buckets = [
{"key": to_display_time(r.bucket), "doc_count": r.cnt}
for r in time_rows
]
# Level counts
stmt = apply(
select(LogEntry.level, func.count(LogEntry.id).label("cnt"))
.group_by(LogEntry.level)
)
level_rows = (await db.execute(stmt)).all()
level_counts = {r.level: r.cnt for r in level_rows}
# Top loggers
stmt = apply(
select(LogEntry.logger, func.count(LogEntry.id).label("cnt"))
.group_by(LogEntry.logger)
.order_by(func.count(LogEntry.id).desc())
.limit(20)
)
logger_rows = (await db.execute(stmt)).all()
top_loggers = [{"logger": r.logger, "count": r.cnt} for r in logger_rows]
# Total count
total = (await db.execute(apply(select(func.count(LogEntry.id))))).scalar() or 0
return LogStatsResult(
project=project,
time_buckets=time_buckets,
level_counts=level_counts,
top_loggers=top_loggers,
total=total,
)
@router.delete("/cleanup")
async def cleanup_old_logs(
db: AsyncSession = Depends(get_db),
):
"""Delete log entries older than the retention period."""
cutoff = datetime.now(timezone.utc) - timedelta(days=settings.LOG_RETENTION_DAYS)
result = await db.execute(
delete(LogEntry).where(LogEntry.timestamp < cutoff)
)
await db.flush()
return {"deleted_count": result.rowcount}
+123
View File
@@ -0,0 +1,123 @@
"""Project management API routes."""
import uuid
from datetime import datetime, timezone
from typing import List
from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.models.project import Project
from app.schemas.project import ProjectCreate, ProjectUpdate, ProjectResponse, ProjectWithToken
router = APIRouter(prefix="/api/projects", tags=["projects"])
@router.post("", response_model=ProjectWithToken, status_code=201)
async def create_project(
data: ProjectCreate,
db: AsyncSession = Depends(get_db),
):
"""Register a new project for log collection."""
existing = await db.execute(select(Project).where(Project.name == data.name))
if existing.scalar_one_or_none():
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"Project '{data.name}' already exists",
)
project = Project(
name=data.name,
description=data.description,
api_key=uuid.uuid4().hex,
)
db.add(project)
await db.flush()
await db.refresh(project)
return project
@router.get("", response_model=List[ProjectResponse])
async def list_projects(
db: AsyncSession = Depends(get_db),
):
"""List all registered projects."""
result = await db.execute(select(Project).order_by(Project.created_at.desc()))
return result.scalars().all()
@router.get("/{project_id}", response_model=ProjectResponse)
async def get_project(
project_id: str,
db: AsyncSession = Depends(get_db),
):
"""Get a single project by ID."""
result = await db.execute(select(Project).where(Project.id == project_id))
project = result.scalar_one_or_none()
if not project:
raise HTTPException(status_code=404, detail="Project not found")
return project
@router.patch("/{project_id}", response_model=ProjectResponse)
async def update_project(
project_id: str,
data: ProjectUpdate,
db: AsyncSession = Depends(get_db),
):
"""Update a project."""
result = await db.execute(select(Project).where(Project.id == project_id))
project = result.scalar_one_or_none()
if not project:
raise HTTPException(status_code=404, detail="Project not found")
if data.name is not None:
# Check uniqueness
existing = await db.execute(
select(Project).where(Project.name == data.name, Project.id != project_id)
)
if existing.scalar_one_or_none():
raise HTTPException(status_code=409, detail="Name already taken")
project.name = data.name
if data.description is not None:
project.description = data.description
if data.is_active is not None:
project.is_active = data.is_active
project.updated_at = datetime.now(timezone.utc)
await db.flush()
await db.refresh(project)
return project
@router.delete("/{project_id}", status_code=204)
async def delete_project(
project_id: str,
db: AsyncSession = Depends(get_db),
):
"""Delete a project."""
result = await db.execute(select(Project).where(Project.id == project_id))
project = result.scalar_one_or_none()
if not project:
raise HTTPException(status_code=404, detail="Project not found")
await db.delete(project)
@router.post("/{project_id}/rotate-key", response_model=ProjectWithToken)
async def rotate_api_key(
project_id: str,
db: AsyncSession = Depends(get_db),
):
"""Rotate a project's API key."""
result = await db.execute(select(Project).where(Project.id == project_id))
project = result.scalar_one_or_none()
if not project:
raise HTTPException(status_code=404, detail="Project not found")
project.api_key = uuid.uuid4().hex
project.updated_at = datetime.now(timezone.utc)
await db.flush()
await db.refresh(project)
return project
+79
View File
@@ -0,0 +1,79 @@
"""LogHive configuration."""
import os
from datetime import datetime, timezone, timedelta
from typing import Optional
# Beijing timezone offset
_BEIJING_TZ = timezone(timedelta(hours=8))
def to_display_time(dt: datetime) -> str:
"""Convert a UTC datetime to Beijing time for display.
Returns ISO format without timezone suffix (e.g. '2026-05-08T14:30:00')."""
if dt is None:
return None
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
local = dt.astimezone(_BEIJING_TZ)
return local.strftime("%Y-%m-%dT%H:%M:%S")
def from_query_time(dt: Optional[datetime]) -> Optional[datetime]:
"""Interpret a naive datetime query param as Beijing time, return UTC datetime."""
if dt is None:
return None
if dt.tzinfo is None:
dt = dt.replace(tzinfo=_BEIJING_TZ)
return dt.astimezone(timezone.utc)
class Settings:
"""Application settings loaded from environment variables."""
# Service name
SERVICE_NAME: str = "LogHive"
# API
API_HOST: str = os.getenv("API_HOST", "0.0.0.0")
API_PORT: int = int(os.getenv("API_PORT", "8000"))
DEBUG: bool = os.getenv("DEBUG", "false").lower() == "true"
CORS_ORIGINS: list[str] = os.getenv("CORS_ORIGINS", "*").split(",")
# Security
SECRET_KEY: str = os.getenv("SECRET_KEY", "change-me-in-production")
TOKEN_EXPIRE_HOURS: int = int(os.getenv("TOKEN_EXPIRE_HOURS", "720"))
# PostgreSQL
DATABASE_URL: str = os.getenv(
"DATABASE_URL",
"postgresql+asyncpg://LogHive:LogHive@localhost:5432/LogHive",
)
# Redis
REDIS_URL: str = os.getenv("REDIS_URL", "redis://localhost:6379/0")
# Celery
CELERY_BROKER_URL: str = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/1")
CELERY_RESULT_BACKEND: str = os.getenv(
"CELERY_RESULT_BACKEND", "redis://localhost:6379/2"
)
# Log retention (days)
LOG_RETENTION_DAYS: int = int(os.getenv("LOG_RETENTION_DAYS", "30"))
LOG_COLD_STORAGE_DAYS: int = int(os.getenv("LOG_COLD_STORAGE_DAYS", "7"))
# Rate limit
RATE_LIMIT_PER_SECOND: int = int(os.getenv("RATE_LIMIT_PER_SECOND", "1000"))
# Alert
ALERT_CHECK_INTERVAL_SECONDS: int = int(
os.getenv("ALERT_CHECK_INTERVAL_SECONDS", "60")
)
# Notification
FEISHU_WEBHOOK_URL: Optional[str] = os.getenv("FEISHU_WEBHOOK_URL", None)
DINGTALK_WEBHOOK_URL: Optional[str] = os.getenv("DINGTALK_WEBHOOK_URL", None)
settings = Settings()
+1
View File
@@ -0,0 +1 @@
from .auth import verify_api_key, resolve_project_by_api_key
+50
View File
@@ -0,0 +1,50 @@
"""Auth utilities — API key verification."""
import hmac
from typing import Optional
from fastapi import Header, HTTPException, status
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.models.project import Project
async def verify_api_key(
authorization: Optional[str] = Header(None),
) -> str:
"""Extract and verify the API key from the Authorization header.
Returns the project_id on success.
"""
if not authorization:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Missing Authorization header",
)
scheme, _, key = authorization.partition(" ")
if scheme.lower() != "bearer" or not key.strip():
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid Authorization format. Use: Bearer <api_key>",
)
return key.strip()
async def resolve_project_by_api_key(
db: AsyncSession, api_key: str
) -> Project:
"""Look up a project by its API key."""
result = await db.execute(
select(Project).where(Project.api_key == api_key, Project.is_active == True)
)
project = result.scalar_one_or_none()
if not project:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid or inactive API key",
)
return project
+1
View File
@@ -0,0 +1 @@
"""Core auth logic."""
+37
View File
@@ -0,0 +1,37 @@
"""Database connections and session management."""
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
from sqlalchemy.orm import DeclarativeBase
from app.config import settings
# ── PostgreSQL ────────────────────────────────────────────────────
engine = create_async_engine(settings.DATABASE_URL, echo=settings.DEBUG)
async_session_factory = async_sessionmaker(engine, expire_on_commit=False)
class Base(DeclarativeBase):
pass
async def get_db() -> AsyncSession:
"""Yield a DB session per request."""
async with async_session_factory() as session:
try:
yield session
await session.commit()
except Exception:
await session.rollback()
raise
finally:
await session.close()
async def init_db() -> None:
"""Create all tables (use Alembic in production)."""
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
async def close_db() -> None:
await engine.dispose()
+65
View File
@@ -0,0 +1,65 @@
"""LogHive — Centralized Log Management System.
FastAPI application that receives, stores, searches and analyzes logs
from multiple Python projects.
"""
import logging
from contextlib import asynccontextmanager
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from app.config import settings
from app.database import init_db, close_db
from app.api import logs, projects, alerts, dashboard
# ── Logging ────────────────────────────────────────────────────
logging.basicConfig(
level=logging.INFO if not settings.DEBUG else logging.DEBUG,
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger(__name__)
# ── Lifecycle ──────────────────────────────────────────────────
@asynccontextmanager
async def lifespan(app: FastAPI):
"""Startup and shutdown lifecycle."""
logger.info("🚀 LogHive starting up...")
await init_db()
yield
logger.info("🛑 LogHive shutting down...")
await close_db()
# ── App ────────────────────────────────────────────────────────
app = FastAPI(
title="LogHive API",
description="Centralized log management system — ingest, search, monitor, and alert on logs from your Python services.",
version="0.1.0",
lifespan=lifespan,
)
app.add_middleware(
CORSMiddleware,
allow_origins=settings.CORS_ORIGINS,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# ── Routes ─────────────────────────────────────────────────────
app.include_router(logs.router)
app.include_router(projects.router)
app.include_router(alerts.router)
app.include_router(dashboard.router)
@app.get("/api/health")
async def health():
return {"status": "ok", "service": settings.SERVICE_NAME, "version": "0.1.0"}
+5
View File
@@ -0,0 +1,5 @@
from app.models.project import Project
from app.models.alert import AlertRule, AlertHistory
from app.models.log import LogEntry
__all__ = ["Project", "AlertRule", "AlertHistory", "LogEntry"]
+71
View File
@@ -0,0 +1,71 @@
"""Alert models — rules and history."""
import uuid
from datetime import datetime, timezone
from sqlalchemy import String, Integer, Float, Boolean, DateTime, Text, Enum as SAEnum
from sqlalchemy.orm import Mapped, mapped_column
import enum
from app.database import Base
class AlertLevel(str, enum.Enum):
INFO = "info"
WARNING = "warning"
CRITICAL = "critical"
class AlertOperator(str, enum.Enum):
GT = "gt"
GTE = "gte"
LT = "lt"
LTE = "lte"
EQ = "eq"
class AlertRule(Base):
__tablename__ = "alert_rules"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
)
project_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
name: Mapped[str] = mapped_column(String(128), nullable=False)
level: Mapped[AlertLevel] = mapped_column(
SAEnum(AlertLevel), default=AlertLevel.WARNING
)
field: Mapped[str] = mapped_column(
String(64), nullable=False, default="level"
) # log field to evaluate
operator: Mapped[AlertOperator] = mapped_column(
SAEnum(AlertOperator), default=AlertOperator.GTE
)
threshold: Mapped[float] = mapped_column(Float, nullable=False)
window_minutes: Mapped[int] = mapped_column(Integer, default=5)
is_enabled: Mapped[bool] = mapped_column(Boolean, default=True)
notify_channels: Mapped[str] = mapped_column(
String(256), default=""
) # comma-separated
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
)
class AlertHistory(Base):
__tablename__ = "alert_history"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
)
rule_id: Mapped[str] = mapped_column(
String(36), nullable=False, index=True
)
project_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
level: Mapped[AlertLevel] = mapped_column(SAEnum(AlertLevel))
message: Mapped[str] = mapped_column(Text, nullable=False)
triggered_value: Mapped[float] = mapped_column(Float, nullable=True)
is_acknowledged: Mapped[bool] = mapped_column(Boolean, default=False)
triggered_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
)
+37
View File
@@ -0,0 +1,37 @@
"""Log entry model — stored in PostgreSQL."""
import uuid
from datetime import datetime, timezone
from sqlalchemy import DateTime, Index, Integer, String, Text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Mapped, mapped_column
from app.database import Base
class LogEntry(Base):
__tablename__ = "log_entries"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
)
project_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
project_name: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
timestamp: Mapped[datetime] = mapped_column(
DateTime(timezone=True), nullable=False, index=True,
default=lambda: datetime.now(timezone.utc),
)
level: Mapped[str] = mapped_column(String(16), nullable=False, index=True)
logger: Mapped[str] = mapped_column(String(128), default="root")
message: Mapped[str] = mapped_column(Text, nullable=False)
module: Mapped[str | None] = mapped_column(String(256), nullable=True)
function: Mapped[str | None] = mapped_column(String(128), nullable=True)
line_no: Mapped[int | None] = mapped_column(Integer, nullable=True)
trace_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
exception: Mapped[str | None] = mapped_column(Text, nullable=True)
extra: Mapped[dict] = mapped_column(JSONB, default=dict)
__table_args__ = (
Index("ix_log_entries_project_time", "project_id", timestamp.desc()),
)
+34
View File
@@ -0,0 +1,34 @@
"""Project model — represents an external service that pushes logs."""
import uuid
from datetime import datetime, timezone
from sqlalchemy import String, Boolean, DateTime, Text
from sqlalchemy.orm import Mapped, mapped_column
from app.database import Base
class Project(Base):
__tablename__ = "projects"
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=lambda: str(uuid.uuid4())
)
name: Mapped[str] = mapped_column(String(128), unique=True, nullable=False)
description: Mapped[str] = mapped_column(Text, default="")
api_key: Mapped[str] = mapped_column(
String(64), unique=True, nullable=False, default=lambda: uuid.uuid4().hex
)
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(timezone.utc),
onupdate=lambda: datetime.now(timezone.utc),
)
def __repr__(self) -> str:
return f"<Project {self.name}>"
+35
View File
@@ -0,0 +1,35 @@
from app.schemas.log import (
LogEntry,
LogBatchRequest,
LogQueryParams,
LogSearchResult,
LogStatsResult,
)
from app.schemas.project import (
ProjectCreate,
ProjectUpdate,
ProjectResponse,
ProjectWithToken,
)
from app.schemas.alert import (
AlertRuleCreate,
AlertRuleUpdate,
AlertRuleResponse,
AlertHistoryResponse,
)
__all__ = [
"LogEntry",
"LogBatchRequest",
"LogQueryParams",
"LogSearchResult",
"LogStatsResult",
"ProjectCreate",
"ProjectUpdate",
"ProjectResponse",
"ProjectWithToken",
"AlertRuleCreate",
"AlertRuleUpdate",
"AlertRuleResponse",
"AlertHistoryResponse",
]
+63
View File
@@ -0,0 +1,63 @@
"""Alert schemas."""
from datetime import datetime
from typing import List, Optional
from pydantic import BaseModel, Field
# ── Alert Rule ─────────────────────────────────────────────────
class AlertRuleCreate(BaseModel):
project_id: str = Field(..., min_length=1, max_length=36)
name: str = Field(..., min_length=1, max_length=128)
level: str = Field(default="warning", pattern="^(info|warning|critical)$")
field: str = Field(default="level", max_length=64)
operator: str = Field(default="gte", pattern="^(gt|gte|lt|lte|eq)$")
threshold: float = Field(..., ge=0)
window_minutes: int = Field(default=5, ge=1, le=1440)
notify_channels: List[str] = Field(default_factory=list)
class AlertRuleUpdate(BaseModel):
name: Optional[str] = Field(default=None, max_length=128)
level: Optional[str] = Field(default=None, pattern="^(info|warning|critical)$")
field: Optional[str] = Field(default=None, max_length=64)
operator: Optional[str] = Field(default=None, pattern="^(gt|gte|lt|lte|eq)$")
threshold: Optional[float] = Field(default=None, ge=0)
window_minutes: Optional[int] = Field(default=None, ge=1, le=1440)
is_enabled: Optional[bool] = None
notify_channels: Optional[List[str]] = None
class AlertRuleResponse(BaseModel):
id: str
project_id: str
name: str
level: str
field: str
operator: str
threshold: float
window_minutes: int
is_enabled: bool
notify_channels: List[str]
created_at: datetime
model_config = {"from_attributes": True}
# ── Alert History ──────────────────────────────────────────────
class AlertHistoryResponse(BaseModel):
id: str
rule_id: str
project_id: str
level: str
message: str
triggered_value: Optional[float] = None
is_acknowledged: bool
triggered_at: datetime
model_config = {"from_attributes": True}
+64
View File
@@ -0,0 +1,64 @@
"""Log entry schemas."""
from datetime import datetime
from typing import Any, Dict, List, Optional
from pydantic import BaseModel, Field
class LogEntry(BaseModel):
"""A single log entry pushed by a project."""
timestamp: Optional[datetime] = None
level: str = Field(default="info", pattern="^(debug|info|warning|error|critical)$")
logger: str = Field(default="root", max_length=128)
message: str = Field(..., min_length=1, max_length=65536)
module: Optional[str] = Field(default=None, max_length=256)
function: Optional[str] = Field(default=None, max_length=128)
line_no: Optional[int] = None
trace_id: Optional[str] = Field(default=None, max_length=64)
exception: Optional[str] = None
extra: Dict[str, Any] = Field(default_factory=dict)
class LogBatchRequest(BaseModel):
"""Batch of log entries pushed in a single request."""
project: str = Field(..., min_length=1, max_length=128)
entries: List[LogEntry] = Field(..., min_length=1, max_length=5000)
class LogQueryParams(BaseModel):
"""Parameters for log search / query."""
project: Optional[str] = None
level: Optional[str] = Field(
default=None, pattern="^(debug|info|warning|error|critical)$"
)
query: Optional[str] = Field(default=None, max_length=1024)
trace_id: Optional[str] = Field(default=None, max_length=64)
start_time: Optional[datetime] = None
end_time: Optional[datetime] = None
page: int = Field(default=1, ge=1)
page_size: int = Field(default=50, ge=1, le=1000)
sort_by: str = Field(default="timestamp", max_length=32)
sort_order: str = Field(default="desc", pattern="^(asc|desc)$")
class LogSearchResult(BaseModel):
"""Search result wrapper."""
total: int
page: int
page_size: int
hits: List[Dict[str, Any]]
class LogStatsResult(BaseModel):
"""Aggregated log statistics."""
project: Optional[str] = None
time_buckets: List[Dict[str, Any]]
level_counts: Dict[str, int]
top_loggers: List[Dict[str, Any]]
total: int
+34
View File
@@ -0,0 +1,34 @@
"""Project schemas."""
from datetime import datetime
from typing import Optional
from pydantic import BaseModel, Field
class ProjectCreate(BaseModel):
name: str = Field(..., min_length=1, max_length=128, description="Project name")
description: str = Field(default="", max_length=1024)
class ProjectUpdate(BaseModel):
name: Optional[str] = Field(default=None, max_length=128)
description: Optional[str] = Field(default=None, max_length=1024)
is_active: Optional[bool] = None
class ProjectResponse(BaseModel):
id: str
name: str
description: str
is_active: bool
created_at: datetime
updated_at: datetime
model_config = {"from_attributes": True}
class ProjectWithToken(ProjectResponse):
api_key: str
model_config = {"from_attributes": True}
View File
+142
View File
@@ -0,0 +1,142 @@
"""Alert evaluation and notification service."""
import logging
from datetime import datetime, timezone, timedelta
from typing import Optional
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.models.alert import AlertRule, AlertHistory, AlertLevel, AlertOperator
from app.models.log import LogEntry
logger = logging.getLogger(__name__)
class AlertService:
"""Evaluates alert rules against recent log data and triggers notifications."""
def __init__(self, db: AsyncSession):
self.db = db
async def evaluate_rules(self) -> list[AlertHistory]:
"""Check all enabled alert rules and trigger any that match."""
result = await self.db.execute(
select(AlertRule).where(AlertRule.is_enabled == True)
)
rules: list[AlertRule] = result.scalars().all()
triggered: list[AlertHistory] = []
for rule in rules:
try:
hit = await self._evaluate_single(rule)
if hit:
triggered.append(hit)
except Exception as e:
logger.error("Error evaluating rule %s: %s", rule.id, e)
return triggered
async def _evaluate_single(self, rule: AlertRule) -> Optional[AlertHistory]:
"""Evaluate a single rule. Returns AlertHistory if triggered."""
now = datetime.now(timezone.utc)
since = now - timedelta(minutes=rule.window_minutes)
stmt = (
select(func.count(LogEntry.id))
.where(LogEntry.project_id == rule.project_id)
.where(LogEntry.timestamp >= since)
.where(LogEntry.level == rule.level.value)
)
count = (await self.db.execute(stmt)).scalar() or 0
operator_map = {
AlertOperator.GT: lambda v, t: v > t,
AlertOperator.GTE: lambda v, t: v >= t,
AlertOperator.LT: lambda v, t: v < t,
AlertOperator.LTE: lambda v, t: v <= t,
AlertOperator.EQ: lambda v, t: v == t,
}
op_fn = operator_map.get(rule.operator)
if op_fn and op_fn(float(count), rule.threshold):
history = AlertHistory(
rule_id=rule.id,
project_id=rule.project_id,
level=rule.level,
message=(
f"Alert '{rule.name}': {rule.level.value} logs count ({count}) "
f"{rule.operator.value} threshold ({rule.threshold}) "
f"in last {rule.window_minutes} min"
),
triggered_value=float(count),
)
self.db.add(history)
await self.db.flush()
logger.info("Alert triggered: %s", history.message)
if rule.notify_channels:
await self._notify(rule, history)
return history
return None
async def _notify(self, rule: AlertRule, history: AlertHistory) -> None:
"""Send notification via configured channels."""
channels = [c.strip() for c in rule.notify_channels.split(",") if c.strip()]
for channel in channels:
if channel == "feishu" and settings.FEISHU_WEBHOOK_URL:
await self._send_feishu(rule, history)
elif channel == "dingtalk" and settings.DINGTALK_WEBHOOK_URL:
await self._send_dingtalk(rule, history)
async def _send_feishu(self, rule: AlertRule, history: AlertHistory) -> None:
"""Send a notification to Feishu webhook."""
import httpx
payload = {
"msg_type": "post",
"content": {
"post": {
"zh_cn": {
"title": f"[LogHive] Alert: {rule.name}",
"content": [
[{"tag": "text", "text": history.message}],
[
{
"tag": "text",
"text": f"Project: {rule.project_id} | Level: {rule.level.value}",
}
],
],
}
}
},
}
try:
async with httpx.AsyncClient() as client:
await client.post(settings.FEISHU_WEBHOOK_URL, json=payload, timeout=10)
except Exception as e:
logger.error("Feishu notification failed: %s", e)
async def _send_dingtalk(self, rule: AlertRule, history: AlertHistory) -> None:
"""Send a notification to DingTalk webhook."""
import httpx
payload = {
"msgtype": "text",
"text": {
"content": (
f"[LogHive] Alert: {rule.name}\n"
f"{history.message}\n"
f"Project: {rule.project_id} | Level: {rule.level.value}"
)
},
}
try:
async with httpx.AsyncClient() as client:
await client.post(settings.DINGTALK_WEBHOOK_URL, json=payload, timeout=10)
except Exception as e:
logger.error("DingTalk notification failed: %s", e)
+125
View File
@@ -0,0 +1,125 @@
"""Log analytics service — provides aggregated statistics and insights."""
import logging
from datetime import datetime, timezone, timedelta
from typing import Optional
from sqlalchemy import func, select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import to_display_time
from app.models.log import LogEntry
logger = logging.getLogger(__name__)
class AnalyticsService:
"""Time-series analytics over log data."""
def __init__(self, db: AsyncSession):
self.db = db
def _build_filters(self, project: Optional[str], since: datetime):
"""Build SQLAlchemy filter list."""
filters = [LogEntry.timestamp >= since]
if project:
filters.append(LogEntry.project_name == project)
return filters
async def get_dashboard_stats(
self, project: Optional[str] = None, hours: int = 24
) -> dict:
"""Get overview statistics for the dashboard."""
now = datetime.now(timezone.utc)
since = now - timedelta(hours=hours)
filters = self._build_filters(project, since)
# Total log count
total = (await self.db.execute(
select(func.count(LogEntry.id)).where(*filters)
)).scalar() or 0
# Error count
error_count = (await self.db.execute(
select(func.count(LogEntry.id)).where(
*filters, LogEntry.level.in_(["error", "critical"])
)
)).scalar() or 0
# Level breakdown
level_rows = (await self.db.execute(
select(LogEntry.level, func.count(LogEntry.id))
.where(*filters)
.group_by(LogEntry.level)
)).all()
level_breakdown = {row[0]: row[1] for row in level_rows}
# Error rate
error_rate = round((error_count / total * 100), 2) if total > 0 else 0
# Top error messages
top_rows = (await self.db.execute(
select(LogEntry.message, func.count(LogEntry.id).label("cnt"))
.where(*filters, LogEntry.level.in_(["error", "critical"]))
.group_by(LogEntry.message)
.order_by(func.count(LogEntry.id).desc())
.limit(10)
)).all()
top_errors = [{"message": row[0], "count": row[1]} for row in top_rows]
return {
"total_logs": total,
"error_count": error_count,
"error_rate": error_rate,
"level_breakdown": level_breakdown,
"time_range_hours": hours,
"top_errors": top_errors,
}
async def get_time_series(
self,
project: Optional[str] = None,
hours: int = 24,
interval: str = "5m",
) -> list[dict]:
"""Get time-series data for log volume charts."""
now = datetime.now(timezone.utc)
since = now - timedelta(hours=hours)
n = int("".join(c for c in interval if c.isdigit()) or "5")
unit = "".join(c for c in interval if c.isalpha()) or "m"
if unit == "h":
n = n * 60
n = max(n, 1)
# Build WHERE clauses
where_clauses = [f"timestamp >= '{since.isoformat()}'"]
params: dict = {}
if project:
where_clauses.append("project_name = :project")
params["project"] = project
where_sql = " AND ".join(where_clauses)
sql = text(
f"SELECT "
f" date_trunc('hour', timestamp) + "
f" ((floor(extract(minute from timestamp) / {n})::int * {n}) || ' min')::interval AS bucket, "
f" level, "
f" count(*) AS cnt "
f"FROM log_entries "
f"WHERE {where_sql} "
f"GROUP BY bucket, level "
f"ORDER BY bucket"
)
rows = (await self.db.execute(sql, params)).all()
buckets: dict[str, dict] = {}
for row in rows:
ts = to_display_time(row.bucket) if isinstance(row.bucket, datetime) else str(row.bucket)
if ts not in buckets:
buckets[ts] = {"timestamp": ts, "total": 0, "levels": {}}
buckets[ts]["total"] += row.cnt
buckets[ts]["levels"][row.level] = row.cnt
return list(buckets.values())