v6ole
254 lines
8.2 KiB
Python
254 lines
8.2 KiB
Python
"""Log ingestion and query API routes."""
|
|
|
|
import logging
|
|
from datetime import datetime, timezone, timedelta
|
|
from typing import Optional
|
|
|
|
from fastapi import APIRouter, Depends, Header, HTTPException, Query, status
|
|
from sqlalchemy import func, select, delete, and_
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.config import settings, to_display_time, from_query_time
|
|
from app.database import get_db
|
|
from app.schemas.log import (
|
|
LogBatchRequest,
|
|
LogEntry as LogEntrySchema,
|
|
LogQueryParams,
|
|
LogSearchResult,
|
|
LogStatsResult,
|
|
)
|
|
from app.models.project import Project
|
|
from app.models.log import LogEntry
|
|
from app.core.auth import verify_api_key, resolve_project_by_api_key
|
|
|
|
logger = logging.getLogger(__name__)
|
|
router = APIRouter(prefix="/api/logs", tags=["logs"])
|
|
|
|
|
|
# ── Ingestion ─────────────────────────────────────────────────
|
|
|
|
|
|
@router.post("/ingest", status_code=201)
|
|
async def ingest_logs(
|
|
batch: LogBatchRequest,
|
|
authorization: Optional[str] = Header(None),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
"""Receive a batch of log entries from a project."""
|
|
api_key = await verify_api_key(authorization)
|
|
project = await resolve_project_by_api_key(db, api_key)
|
|
|
|
now = datetime.now(timezone.utc)
|
|
rows = []
|
|
for entry in batch.entries:
|
|
row = LogEntry(
|
|
project_id=project.id,
|
|
project_name=project.name,
|
|
timestamp=entry.timestamp or now,
|
|
level=entry.level,
|
|
logger=entry.logger,
|
|
message=entry.message,
|
|
module=entry.module,
|
|
function=entry.function,
|
|
line_no=entry.line_no,
|
|
trace_id=entry.trace_id,
|
|
exception=entry.exception,
|
|
extra=entry.extra,
|
|
)
|
|
rows.append(row)
|
|
|
|
db.add_all(rows)
|
|
await db.flush()
|
|
return {"accepted": len(rows), "project": project.name}
|
|
|
|
|
|
@router.post("/ingest/single", status_code=201)
|
|
async def ingest_single_log(
|
|
entry: LogEntrySchema,
|
|
project_name: str = Query(..., description="Project name"),
|
|
authorization: Optional[str] = Header(None),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
"""Receive a single log entry (useful for testing / simple clients)."""
|
|
api_key = await verify_api_key(authorization)
|
|
project = await resolve_project_by_api_key(db, api_key)
|
|
|
|
now = datetime.now(timezone.utc)
|
|
row = LogEntry(
|
|
project_id=project.id,
|
|
project_name=project.name,
|
|
timestamp=entry.timestamp or now,
|
|
level=entry.level,
|
|
logger=entry.logger,
|
|
message=entry.message,
|
|
module=entry.module,
|
|
function=entry.function,
|
|
line_no=entry.line_no,
|
|
trace_id=entry.trace_id,
|
|
exception=entry.exception,
|
|
extra=entry.extra,
|
|
)
|
|
db.add(row)
|
|
await db.flush()
|
|
return {"accepted": 1, "project": project.name}
|
|
|
|
|
|
# ── Query ──────────────────────────────────────────────────────
|
|
|
|
|
|
@router.get("/search", response_model=LogSearchResult)
|
|
async def search_logs(
|
|
project: Optional[str] = Query(None, description="Filter by project name"),
|
|
level: Optional[str] = Query(None, pattern="^(debug|info|warning|error|critical)$"),
|
|
query: Optional[str] = Query(None, max_length=1024),
|
|
trace_id: Optional[str] = Query(None, max_length=64),
|
|
start_time: Optional[datetime] = None,
|
|
end_time: Optional[datetime] = None,
|
|
page: int = Query(1, ge=1),
|
|
page_size: int = Query(50, ge=1, le=1000),
|
|
sort_by: str = Query("timestamp", max_length=32),
|
|
sort_order: str = Query("desc", pattern="^(asc|desc)$"),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
"""Search logs with full-text query and filters."""
|
|
conditions = []
|
|
|
|
if project:
|
|
conditions.append(LogEntry.project_name == project)
|
|
if level:
|
|
conditions.append(LogEntry.level == level)
|
|
if trace_id:
|
|
conditions.append(LogEntry.trace_id == trace_id)
|
|
if start_time:
|
|
conditions.append(LogEntry.timestamp >= from_query_time(start_time))
|
|
if end_time:
|
|
conditions.append(LogEntry.timestamp <= from_query_time(end_time))
|
|
if query:
|
|
conditions.append(
|
|
func.to_tsvector(
|
|
"english",
|
|
func.concat(
|
|
LogEntry.message, " ",
|
|
func.coalesce(LogEntry.logger, ""), " ",
|
|
func.coalesce(LogEntry.module, ""),
|
|
),
|
|
).bool_op("@@")(
|
|
func.plainto_tsquery("english", query)
|
|
)
|
|
)
|
|
|
|
# Sort
|
|
sort_col = getattr(LogEntry, sort_by, LogEntry.timestamp)
|
|
order = sort_col.desc() if sort_order == "desc" else sort_col.asc()
|
|
|
|
# Count total
|
|
count_stmt = select(func.count(LogEntry.id))
|
|
if conditions:
|
|
count_stmt = count_stmt.where(and_(*conditions))
|
|
total = (await db.execute(count_stmt)).scalar() or 0
|
|
|
|
# Fetch page
|
|
stmt = select(LogEntry).order_by(order).offset((page - 1) * page_size).limit(page_size)
|
|
if conditions:
|
|
stmt = stmt.where(and_(*conditions))
|
|
rows = (await db.execute(stmt)).scalars().all()
|
|
|
|
hits = []
|
|
for r in rows:
|
|
hits.append({
|
|
"project_id": r.project_id,
|
|
"project_name": r.project_name,
|
|
"timestamp": to_display_time(r.timestamp),
|
|
"level": r.level,
|
|
"logger": r.logger,
|
|
"message": r.message,
|
|
"module": r.module,
|
|
"function": r.function,
|
|
"line_no": r.line_no,
|
|
"trace_id": r.trace_id,
|
|
"exception": r.exception,
|
|
"extra": r.extra,
|
|
})
|
|
|
|
return LogSearchResult(total=total, page=page, page_size=page_size, hits=hits)
|
|
|
|
|
|
@router.get("/stats", response_model=LogStatsResult)
|
|
async def log_stats(
|
|
project: Optional[str] = Query(None),
|
|
start_time: Optional[datetime] = None,
|
|
end_time: Optional[datetime] = None,
|
|
interval: str = Query("5m", max_length=16),
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
"""Get aggregated log statistics over a time range."""
|
|
conditions = []
|
|
if project:
|
|
conditions.append(LogEntry.project_name == project)
|
|
if start_time:
|
|
conditions.append(LogEntry.timestamp >= from_query_time(start_time))
|
|
if end_time:
|
|
conditions.append(LogEntry.timestamp <= from_query_time(end_time))
|
|
|
|
def apply(stmt):
|
|
if conditions:
|
|
stmt = stmt.where(and_(*conditions))
|
|
return stmt
|
|
|
|
# Time buckets (hourly)
|
|
stmt = apply(
|
|
select(
|
|
func.date_trunc("hour", LogEntry.timestamp).label("bucket"),
|
|
func.count(LogEntry.id).label("cnt"),
|
|
).group_by("bucket").order_by("bucket")
|
|
)
|
|
time_rows = (await db.execute(stmt)).all()
|
|
time_buckets = [
|
|
{"key": to_display_time(r.bucket), "doc_count": r.cnt}
|
|
for r in time_rows
|
|
]
|
|
|
|
# Level counts
|
|
stmt = apply(
|
|
select(LogEntry.level, func.count(LogEntry.id).label("cnt"))
|
|
.group_by(LogEntry.level)
|
|
)
|
|
level_rows = (await db.execute(stmt)).all()
|
|
level_counts = {r.level: r.cnt for r in level_rows}
|
|
|
|
# Top loggers
|
|
stmt = apply(
|
|
select(LogEntry.logger, func.count(LogEntry.id).label("cnt"))
|
|
.group_by(LogEntry.logger)
|
|
.order_by(func.count(LogEntry.id).desc())
|
|
.limit(20)
|
|
)
|
|
logger_rows = (await db.execute(stmt)).all()
|
|
top_loggers = [{"logger": r.logger, "count": r.cnt} for r in logger_rows]
|
|
|
|
# Total count
|
|
total = (await db.execute(apply(select(func.count(LogEntry.id))))).scalar() or 0
|
|
|
|
return LogStatsResult(
|
|
project=project,
|
|
time_buckets=time_buckets,
|
|
level_counts=level_counts,
|
|
top_loggers=top_loggers,
|
|
total=total,
|
|
)
|
|
|
|
|
|
@router.delete("/cleanup")
|
|
async def cleanup_old_logs(
|
|
db: AsyncSession = Depends(get_db),
|
|
):
|
|
"""Delete log entries older than the retention period."""
|
|
cutoff = datetime.now(timezone.utc) - timedelta(days=settings.LOG_RETENTION_DAYS)
|
|
|
|
result = await db.execute(
|
|
delete(LogEntry).where(LogEntry.timestamp < cutoff)
|
|
)
|
|
await db.flush()
|
|
|
|
return {"deleted_count": result.rowcount}
|