Initial commit: LogHive centralized log management system
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
v6ole
@@ -0,0 +1,253 @@
|
||||
"""Log ingestion and query API routes."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Query, status
|
||||
from sqlalchemy import func, select, delete, and_
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings, to_display_time, from_query_time
|
||||
from app.database import get_db
|
||||
from app.schemas.log import (
|
||||
LogBatchRequest,
|
||||
LogEntry as LogEntrySchema,
|
||||
LogQueryParams,
|
||||
LogSearchResult,
|
||||
LogStatsResult,
|
||||
)
|
||||
from app.models.project import Project
|
||||
from app.models.log import LogEntry
|
||||
from app.core.auth import verify_api_key, resolve_project_by_api_key
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/logs", tags=["logs"])
|
||||
|
||||
|
||||
# ── Ingestion ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.post("/ingest", status_code=201)
|
||||
async def ingest_logs(
|
||||
batch: LogBatchRequest,
|
||||
authorization: Optional[str] = Header(None),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Receive a batch of log entries from a project."""
|
||||
api_key = await verify_api_key(authorization)
|
||||
project = await resolve_project_by_api_key(db, api_key)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
rows = []
|
||||
for entry in batch.entries:
|
||||
row = LogEntry(
|
||||
project_id=project.id,
|
||||
project_name=project.name,
|
||||
timestamp=entry.timestamp or now,
|
||||
level=entry.level,
|
||||
logger=entry.logger,
|
||||
message=entry.message,
|
||||
module=entry.module,
|
||||
function=entry.function,
|
||||
line_no=entry.line_no,
|
||||
trace_id=entry.trace_id,
|
||||
exception=entry.exception,
|
||||
extra=entry.extra,
|
||||
)
|
||||
rows.append(row)
|
||||
|
||||
db.add_all(rows)
|
||||
await db.flush()
|
||||
return {"accepted": len(rows), "project": project.name}
|
||||
|
||||
|
||||
@router.post("/ingest/single", status_code=201)
|
||||
async def ingest_single_log(
|
||||
entry: LogEntrySchema,
|
||||
project_name: str = Query(..., description="Project name"),
|
||||
authorization: Optional[str] = Header(None),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Receive a single log entry (useful for testing / simple clients)."""
|
||||
api_key = await verify_api_key(authorization)
|
||||
project = await resolve_project_by_api_key(db, api_key)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
row = LogEntry(
|
||||
project_id=project.id,
|
||||
project_name=project.name,
|
||||
timestamp=entry.timestamp or now,
|
||||
level=entry.level,
|
||||
logger=entry.logger,
|
||||
message=entry.message,
|
||||
module=entry.module,
|
||||
function=entry.function,
|
||||
line_no=entry.line_no,
|
||||
trace_id=entry.trace_id,
|
||||
exception=entry.exception,
|
||||
extra=entry.extra,
|
||||
)
|
||||
db.add(row)
|
||||
await db.flush()
|
||||
return {"accepted": 1, "project": project.name}
|
||||
|
||||
|
||||
# ── Query ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("/search", response_model=LogSearchResult)
|
||||
async def search_logs(
|
||||
project: Optional[str] = Query(None, description="Filter by project name"),
|
||||
level: Optional[str] = Query(None, pattern="^(debug|info|warning|error|critical)$"),
|
||||
query: Optional[str] = Query(None, max_length=1024),
|
||||
trace_id: Optional[str] = Query(None, max_length=64),
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(50, ge=1, le=1000),
|
||||
sort_by: str = Query("timestamp", max_length=32),
|
||||
sort_order: str = Query("desc", pattern="^(asc|desc)$"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Search logs with full-text query and filters."""
|
||||
conditions = []
|
||||
|
||||
if project:
|
||||
conditions.append(LogEntry.project_name == project)
|
||||
if level:
|
||||
conditions.append(LogEntry.level == level)
|
||||
if trace_id:
|
||||
conditions.append(LogEntry.trace_id == trace_id)
|
||||
if start_time:
|
||||
conditions.append(LogEntry.timestamp >= from_query_time(start_time))
|
||||
if end_time:
|
||||
conditions.append(LogEntry.timestamp <= from_query_time(end_time))
|
||||
if query:
|
||||
conditions.append(
|
||||
func.to_tsvector(
|
||||
"english",
|
||||
func.concat(
|
||||
LogEntry.message, " ",
|
||||
func.coalesce(LogEntry.logger, ""), " ",
|
||||
func.coalesce(LogEntry.module, ""),
|
||||
),
|
||||
).bool_op("@@")(
|
||||
func.plainto_tsquery("english", query)
|
||||
)
|
||||
)
|
||||
|
||||
# Sort
|
||||
sort_col = getattr(LogEntry, sort_by, LogEntry.timestamp)
|
||||
order = sort_col.desc() if sort_order == "desc" else sort_col.asc()
|
||||
|
||||
# Count total
|
||||
count_stmt = select(func.count(LogEntry.id))
|
||||
if conditions:
|
||||
count_stmt = count_stmt.where(and_(*conditions))
|
||||
total = (await db.execute(count_stmt)).scalar() or 0
|
||||
|
||||
# Fetch page
|
||||
stmt = select(LogEntry).order_by(order).offset((page - 1) * page_size).limit(page_size)
|
||||
if conditions:
|
||||
stmt = stmt.where(and_(*conditions))
|
||||
rows = (await db.execute(stmt)).scalars().all()
|
||||
|
||||
hits = []
|
||||
for r in rows:
|
||||
hits.append({
|
||||
"project_id": r.project_id,
|
||||
"project_name": r.project_name,
|
||||
"timestamp": to_display_time(r.timestamp),
|
||||
"level": r.level,
|
||||
"logger": r.logger,
|
||||
"message": r.message,
|
||||
"module": r.module,
|
||||
"function": r.function,
|
||||
"line_no": r.line_no,
|
||||
"trace_id": r.trace_id,
|
||||
"exception": r.exception,
|
||||
"extra": r.extra,
|
||||
})
|
||||
|
||||
return LogSearchResult(total=total, page=page, page_size=page_size, hits=hits)
|
||||
|
||||
|
||||
@router.get("/stats", response_model=LogStatsResult)
|
||||
async def log_stats(
|
||||
project: Optional[str] = Query(None),
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
interval: str = Query("5m", max_length=16),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Get aggregated log statistics over a time range."""
|
||||
conditions = []
|
||||
if project:
|
||||
conditions.append(LogEntry.project_name == project)
|
||||
if start_time:
|
||||
conditions.append(LogEntry.timestamp >= from_query_time(start_time))
|
||||
if end_time:
|
||||
conditions.append(LogEntry.timestamp <= from_query_time(end_time))
|
||||
|
||||
def apply(stmt):
|
||||
if conditions:
|
||||
stmt = stmt.where(and_(*conditions))
|
||||
return stmt
|
||||
|
||||
# Time buckets (hourly)
|
||||
stmt = apply(
|
||||
select(
|
||||
func.date_trunc("hour", LogEntry.timestamp).label("bucket"),
|
||||
func.count(LogEntry.id).label("cnt"),
|
||||
).group_by("bucket").order_by("bucket")
|
||||
)
|
||||
time_rows = (await db.execute(stmt)).all()
|
||||
time_buckets = [
|
||||
{"key": to_display_time(r.bucket), "doc_count": r.cnt}
|
||||
for r in time_rows
|
||||
]
|
||||
|
||||
# Level counts
|
||||
stmt = apply(
|
||||
select(LogEntry.level, func.count(LogEntry.id).label("cnt"))
|
||||
.group_by(LogEntry.level)
|
||||
)
|
||||
level_rows = (await db.execute(stmt)).all()
|
||||
level_counts = {r.level: r.cnt for r in level_rows}
|
||||
|
||||
# Top loggers
|
||||
stmt = apply(
|
||||
select(LogEntry.logger, func.count(LogEntry.id).label("cnt"))
|
||||
.group_by(LogEntry.logger)
|
||||
.order_by(func.count(LogEntry.id).desc())
|
||||
.limit(20)
|
||||
)
|
||||
logger_rows = (await db.execute(stmt)).all()
|
||||
top_loggers = [{"logger": r.logger, "count": r.cnt} for r in logger_rows]
|
||||
|
||||
# Total count
|
||||
total = (await db.execute(apply(select(func.count(LogEntry.id))))).scalar() or 0
|
||||
|
||||
return LogStatsResult(
|
||||
project=project,
|
||||
time_buckets=time_buckets,
|
||||
level_counts=level_counts,
|
||||
top_loggers=top_loggers,
|
||||
total=total,
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/cleanup")
|
||||
async def cleanup_old_logs(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Delete log entries older than the retention period."""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=settings.LOG_RETENTION_DAYS)
|
||||
|
||||
result = await db.execute(
|
||||
delete(LogEntry).where(LogEntry.timestamp < cutoff)
|
||||
)
|
||||
await db.flush()
|
||||
|
||||
return {"deleted_count": result.rowcount}
|
||||
Reference in New Issue
Block a user