PingWatch 网络设备离线监控系统
- FastAPI 后端 + Vue 3 前端 - Docker Compose 一键部署 - Casdoor OAuth 认证集成 - LogHive 集中式日志 - 设备批量 CSV 导入/导出 - WebSocket 实时状态推送 - 企业微信告警通知 - fping 高性能并发 Ping 检测 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
# 数据库(开发用 SQLite,生产用 PostgreSQL)
|
||||
DATABASE_URL=sqlite+aiosqlite:///./pingwatch.db
|
||||
# DATABASE_URL=postgresql+asyncpg://user:pass@localhost/pingwatch
|
||||
|
||||
# 企业微信
|
||||
WECOM_CORP_ID=your_corp_id
|
||||
WECOM_AGENT_ID=1000001
|
||||
WECOM_APP_SECRET=your_app_secret
|
||||
|
||||
# Casdoor
|
||||
CASDOOR_ENDPOINT=https://casdoor.dhdx.fun
|
||||
CASDOOR_CLIENT_ID=e46b9e1eb893027bdf2a
|
||||
CASDOOR_CLIENT_SECRET=b12c7e1688ed51481f3b5c5dae4191b6edbba916
|
||||
CASDOOR_CERTIFICATE=
|
||||
CASDOOR_ORGANIZATION=dahua
|
||||
CASDOOR_APPLICATION=PingWatch
|
||||
CASDOOR_REDIRECT_URI=http://10.10.10.7:5173/login
|
||||
|
||||
# Ping 引擎
|
||||
PING_INTERVAL_SECONDS=30
|
||||
PING_TIMEOUT_SECONDS=5.0
|
||||
DEFAULT_ALERT_THRESHOLD=5
|
||||
PING_CONCURRENCY=50
|
||||
|
||||
# 上游心跳
|
||||
UPSTREAM_PING_TARGET=114.114.114.114
|
||||
UPSTREAM_PING_THRESHOLD=2
|
||||
OFFLINE_SUPPRESS_RATIO=0.9
|
||||
|
||||
# 数据保留
|
||||
PING_RECORD_RETENTION_DAYS=90
|
||||
ALERT_RETENTION_DAYS=365
|
||||
|
||||
# JWT 密钥(请改成随机字符串)
|
||||
SECRET_KEY=change-me-to-a-long-random-string
|
||||
@@ -0,0 +1,14 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
fping iputils-ping && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
EXPOSE 8000
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
@@ -0,0 +1 @@
|
||||
from . import devices, alerts, stats, auth, ws, users
|
||||
@@ -0,0 +1,133 @@
|
||||
"""告警记录 API"""
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.deps import get_db
|
||||
from app.core.auth import get_current_user
|
||||
from app.models.device import Device
|
||||
from app.models.alert_event import AlertEvent, AlertTypeEnum
|
||||
from app.models.user import User
|
||||
from app.schemas.alert import AlertEventOut
|
||||
|
||||
router = APIRouter(prefix="/api/alerts", tags=["告警记录"])
|
||||
|
||||
|
||||
@router.get("", response_model=dict)
|
||||
async def list_alerts(
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(20, ge=1, le=100),
|
||||
device_id: Optional[int] = None,
|
||||
alert_type: Optional[str] = None,
|
||||
is_resolved: Optional[bool] = None,
|
||||
start_time: Optional[datetime] = None,
|
||||
end_time: Optional[datetime] = None,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""查询告警记录,支持分页和过滤"""
|
||||
query = select(AlertEvent)
|
||||
count_query = select(func.count(AlertEvent.id))
|
||||
|
||||
# 过滤条件
|
||||
if device_id:
|
||||
query = query.where(AlertEvent.device_id == device_id)
|
||||
count_query = count_query.where(AlertEvent.device_id == device_id)
|
||||
if alert_type:
|
||||
query = query.where(AlertEvent.alert_type == alert_type)
|
||||
count_query = count_query.where(AlertEvent.alert_type == alert_type)
|
||||
if is_resolved is not None:
|
||||
query = query.where(AlertEvent.is_resolved == is_resolved)
|
||||
count_query = count_query.where(AlertEvent.is_resolved == is_resolved)
|
||||
if start_time:
|
||||
query = query.where(AlertEvent.created_at >= start_time)
|
||||
count_query = count_query.where(AlertEvent.created_at >= start_time)
|
||||
if end_time:
|
||||
query = query.where(AlertEvent.created_at <= end_time)
|
||||
count_query = count_query.where(AlertEvent.created_at <= end_time)
|
||||
|
||||
# 总数
|
||||
total_result = await db.execute(count_query)
|
||||
total = total_result.scalar() or 0
|
||||
|
||||
# 分页
|
||||
offset = (page - 1) * page_size
|
||||
query = query.order_by(AlertEvent.created_at.desc()).offset(offset).limit(page_size)
|
||||
result = await db.execute(query)
|
||||
events = list(result.scalars().all())
|
||||
|
||||
# 关联设备信息
|
||||
device_ids = {e.device_id for e in events if e.device_id > 0}
|
||||
devices_map = {}
|
||||
if device_ids:
|
||||
dev_result = await db.execute(select(Device).where(Device.id.in_(device_ids)))
|
||||
devices_map = {d.id: d for d in dev_result.scalars().all()}
|
||||
|
||||
items = []
|
||||
for e in events:
|
||||
dev = devices_map.get(e.device_id)
|
||||
items.append(AlertEventOut(
|
||||
id=e.id,
|
||||
device_id=e.device_id,
|
||||
device_name=dev.name if dev else "系统",
|
||||
device_ip=dev.ip if dev else "",
|
||||
device_type=dev.device_type.value if dev else "",
|
||||
location=dev.location if dev else "",
|
||||
project_name=dev.project_name if dev else "",
|
||||
alert_type=e.alert_type.value,
|
||||
message=e.message,
|
||||
start_at=e.start_at,
|
||||
end_at=e.end_at,
|
||||
duration_minutes=e.duration_minutes,
|
||||
is_resolved=e.is_resolved,
|
||||
notification_sent=e.notification_sent,
|
||||
created_at=e.created_at,
|
||||
))
|
||||
|
||||
return {"items": items, "total": total, "page": page, "page_size": page_size}
|
||||
|
||||
|
||||
@router.get("/latest", response_model=list[AlertEventOut])
|
||||
async def get_latest_alerts(
|
||||
limit: int = Query(10, ge=1, le=50),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""获取最近的告警"""
|
||||
result = await db.execute(
|
||||
select(AlertEvent)
|
||||
.order_by(AlertEvent.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
events = list(result.scalars().all())
|
||||
|
||||
device_ids = {e.device_id for e in events if e.device_id > 0}
|
||||
devices_map = {}
|
||||
if device_ids:
|
||||
dev_result = await db.execute(select(Device).where(Device.id.in_(device_ids)))
|
||||
devices_map = {d.id: d for d in dev_result.scalars().all()}
|
||||
|
||||
items = []
|
||||
for e in events:
|
||||
dev = devices_map.get(e.device_id)
|
||||
items.append(AlertEventOut(
|
||||
id=e.id,
|
||||
device_id=e.device_id,
|
||||
device_name=dev.name if dev else "系统",
|
||||
device_ip=dev.ip if dev else "",
|
||||
device_type=dev.device_type.value if dev else "",
|
||||
location=dev.location if dev else "",
|
||||
project_name=dev.project_name if dev else "",
|
||||
alert_type=e.alert_type.value,
|
||||
message=e.message,
|
||||
start_at=e.start_at,
|
||||
end_at=e.end_at,
|
||||
duration_minutes=e.duration_minutes,
|
||||
is_resolved=e.is_resolved,
|
||||
notification_sent=e.notification_sent,
|
||||
created_at=e.created_at,
|
||||
))
|
||||
return items
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Casdoor 认证 API"""
|
||||
from datetime import datetime
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.core.deps import get_db
|
||||
from app.core.auth import create_access_token, exchange_code_for_user, get_current_user
|
||||
from app.models.user import User, UserRoleEnum
|
||||
|
||||
router = APIRouter(prefix="/api/auth", tags=["认证"])
|
||||
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
code: str # Casdoor 返回的 OAuth code
|
||||
|
||||
|
||||
class LoginResponse(BaseModel):
|
||||
token: str
|
||||
user: dict
|
||||
|
||||
|
||||
@router.post("/login", response_model=LoginResponse)
|
||||
async def login(data: LoginRequest, db: AsyncSession = Depends(get_db)):
|
||||
"""用 Casdoor OAuth code 登录"""
|
||||
casdoor_user = await exchange_code_for_user(data.code)
|
||||
if not casdoor_user:
|
||||
raise HTTPException(status_code=401, detail="Casdoor 认证失败")
|
||||
|
||||
# 从 id_token 中提取用户信息
|
||||
# id_token payload 一般包含: sub, name, preferred_username, email, displayName 等
|
||||
casdoor_uid = casdoor_user.get("sub") or casdoor_user.get("name", "")
|
||||
username = casdoor_user.get("preferred_username") or casdoor_user.get("name", casdoor_uid)
|
||||
display_name = casdoor_user.get("displayName", "")
|
||||
|
||||
if not casdoor_uid:
|
||||
# 尝试用 name 作为备用标识
|
||||
casdoor_uid = casdoor_user.get("name", "")
|
||||
if not casdoor_uid:
|
||||
raise HTTPException(status_code=401, detail="无法从 Casdoor 获取用户标识")
|
||||
|
||||
# 查找或创建本地用户
|
||||
result = await db.execute(select(User).where(User.casdoor_uid == casdoor_uid))
|
||||
user = result.scalar_one_or_none()
|
||||
|
||||
if not user:
|
||||
# 首次登录,自动创建 viewer 账号
|
||||
user = User(
|
||||
casdoor_uid=casdoor_uid,
|
||||
username=username,
|
||||
display_name=display_name or username,
|
||||
role=UserRoleEnum.viewer,
|
||||
)
|
||||
db.add(user)
|
||||
await db.commit()
|
||||
await db.refresh(user)
|
||||
logger.info(f"新用户自动创建: {username} (uid: {casdoor_uid})")
|
||||
|
||||
# 更新最后登录时间
|
||||
user.last_login_at = datetime.now()
|
||||
await db.commit()
|
||||
|
||||
# 签发 PingWatch JWT
|
||||
token = create_access_token(
|
||||
data={"sub": user.casdoor_uid, "role": user.role.value}
|
||||
)
|
||||
|
||||
return LoginResponse(
|
||||
token=token,
|
||||
user={
|
||||
"id": user.id,
|
||||
"username": user.username,
|
||||
"display_name": user.display_name,
|
||||
"role": user.role.value,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
async def get_me(current_user: User = Depends(get_current_user)):
|
||||
"""获取当前登录用户信息"""
|
||||
return {
|
||||
"id": current_user.id,
|
||||
"username": current_user.username,
|
||||
"display_name": current_user.display_name,
|
||||
"role": current_user.role.value,
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
"""设备管理 API"""
|
||||
import csv
|
||||
import io
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, UploadFile, File
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy import select, func, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.deps import get_db
|
||||
from app.core.auth import get_current_user, require_admin
|
||||
from app.models.device import Device, DeviceTypeEnum
|
||||
from app.models.user import User
|
||||
from app.schemas.device import DeviceCreate, DeviceUpdate, DeviceOut
|
||||
|
||||
logger = logging.getLogger("pingwatch.devices")
|
||||
router = APIRouter(prefix="/api/devices", tags=["设备管理"])
|
||||
|
||||
# CSV 模板列
|
||||
IMPORT_COLUMNS = [
|
||||
("name", "设备名称(必填)"),
|
||||
("ip", "IP地址(必填)"),
|
||||
("device_type", "设备类型(server/olt/switch/firewall/other)"),
|
||||
("location", "物理位置"),
|
||||
("project_name", "所属项目"),
|
||||
("tags", "标签(逗号分隔)"),
|
||||
("ping_interval", "Ping间隔(秒)"),
|
||||
("alert_threshold", "告警阈值(次)"),
|
||||
]
|
||||
|
||||
|
||||
@router.get("", response_model=list[DeviceOut])
|
||||
async def list_devices(
|
||||
is_enabled: Optional[bool] = None,
|
||||
device_type: Optional[str] = None,
|
||||
search: Optional[str] = None,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""获取设备列表,支持过滤和搜索"""
|
||||
query = select(Device)
|
||||
|
||||
if is_enabled is not None:
|
||||
query = query.where(Device.is_enabled == is_enabled)
|
||||
if device_type:
|
||||
query = query.where(Device.device_type == device_type)
|
||||
if search:
|
||||
like = f"%{search}%"
|
||||
query = query.where(
|
||||
Device.name.ilike(like) | Device.ip.ilike(like) | Device.location.ilike(like)
|
||||
)
|
||||
|
||||
query = query.order_by(Device.id)
|
||||
result = await db.execute(query)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
@router.get("/{device_id}", response_model=DeviceOut)
|
||||
async def get_device(
|
||||
device_id: int,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
result = await db.execute(select(Device).where(Device.id == device_id))
|
||||
device = result.scalar_one_or_none()
|
||||
if not device:
|
||||
raise HTTPException(status_code=404, detail="设备不存在")
|
||||
return device
|
||||
|
||||
|
||||
@router.post("", response_model=DeviceOut)
|
||||
async def create_device(
|
||||
data: DeviceCreate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
admin: User = Depends(require_admin),
|
||||
):
|
||||
device = Device(**data.model_dump())
|
||||
db.add(device)
|
||||
await db.commit()
|
||||
await db.refresh(device)
|
||||
return device
|
||||
|
||||
|
||||
@router.put("/{device_id}", response_model=DeviceOut)
|
||||
async def update_device(
|
||||
device_id: int,
|
||||
data: DeviceUpdate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
admin: User = Depends(require_admin),
|
||||
):
|
||||
result = await db.execute(select(Device).where(Device.id == device_id))
|
||||
device = result.scalar_one_or_none()
|
||||
if not device:
|
||||
raise HTTPException(status_code=404, detail="设备不存在")
|
||||
|
||||
update_data = data.model_dump(exclude_unset=True)
|
||||
if update_data:
|
||||
update_data["updated_at"] = datetime.now()
|
||||
await db.execute(update(Device).where(Device.id == device_id).values(**update_data))
|
||||
await db.commit()
|
||||
await db.refresh(device)
|
||||
return device
|
||||
|
||||
|
||||
@router.delete("/{device_id}")
|
||||
async def delete_device(
|
||||
device_id: int,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
admin: User = Depends(require_admin),
|
||||
):
|
||||
result = await db.execute(select(Device).where(Device.id == device_id))
|
||||
device = result.scalar_one_or_none()
|
||||
if not device:
|
||||
raise HTTPException(status_code=404, detail="设备不存在")
|
||||
await db.delete(device)
|
||||
await db.commit()
|
||||
return {"message": "删除成功"}
|
||||
|
||||
|
||||
@router.get("/template/download")
|
||||
async def download_template(
|
||||
admin: User = Depends(require_admin),
|
||||
):
|
||||
"""下载 CSV 导入模板"""
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
writer.writerow([col[1] for col in IMPORT_COLUMNS])
|
||||
# 写入一行示例数据
|
||||
writer.writerow([
|
||||
"示例设备", "192.168.1.1", "server", "机房A", "项目1", "核心,生产", "30", "5"
|
||||
])
|
||||
output.seek(0)
|
||||
return StreamingResponse(
|
||||
iter([output.getvalue()]),
|
||||
media_type="text/csv; charset=utf-8-sig",
|
||||
headers={"Content-Disposition": "attachment; filename=device_template.csv"},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/import")
|
||||
async def import_devices(
|
||||
file: UploadFile = File(...),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
admin: User = Depends(require_admin),
|
||||
):
|
||||
"""批量导入设备(CSV)"""
|
||||
if not file.filename or not file.filename.endswith(".csv"):
|
||||
raise HTTPException(status_code=400, detail="请上传 .csv 文件")
|
||||
|
||||
content = await file.read()
|
||||
# 处理 BOM
|
||||
text = content.decode("utf-8-sig")
|
||||
reader = csv.reader(io.StringIO(text))
|
||||
rows = list(reader)
|
||||
|
||||
if len(rows) < 2:
|
||||
raise HTTPException(status_code=400, detail="CSV 至少需要表头行 + 一行数据")
|
||||
|
||||
devices_added = 0
|
||||
errors = []
|
||||
|
||||
for i, row in enumerate(rows[1:], start=1):
|
||||
if not any(cell.strip() for cell in row):
|
||||
continue # 跳过空行
|
||||
|
||||
if len(row) < 2:
|
||||
errors.append(f"第 {i + 1} 行: 缺少必填列")
|
||||
continue
|
||||
|
||||
name = row[0].strip() if len(row) > 0 else ""
|
||||
ip = row[1].strip() if len(row) > 1 else ""
|
||||
|
||||
if not name or not ip:
|
||||
errors.append(f"第 {i + 1} 行: 设备名称和 IP 地址为必填")
|
||||
continue
|
||||
|
||||
# 检查 IP 是否重复
|
||||
existing = await db.execute(select(Device).where(Device.ip == ip))
|
||||
if existing.scalar_one_or_none():
|
||||
errors.append(f"第 {i + 1} 行: IP {ip} 已存在")
|
||||
continue
|
||||
|
||||
device_type = row[2].strip().lower() if len(row) > 2 and row[2].strip() else "other"
|
||||
if device_type not in (e.value for e in DeviceTypeEnum):
|
||||
device_type = "other"
|
||||
|
||||
device = Device(
|
||||
name=name,
|
||||
ip=ip,
|
||||
device_type=device_type,
|
||||
location=row[3].strip() if len(row) > 3 else "",
|
||||
project_name=row[4].strip() if len(row) > 4 else "",
|
||||
tags=row[5].strip() if len(row) > 5 else "",
|
||||
ping_interval=int(row[6]) if len(row) > 6 and row[6].strip().isdigit() else 30,
|
||||
alert_threshold=int(row[7]) if len(row) > 7 and row[7].strip().isdigit() else 5,
|
||||
)
|
||||
db.add(device)
|
||||
devices_added += 1
|
||||
|
||||
await db.commit()
|
||||
|
||||
result = {"devices_added": devices_added}
|
||||
if errors:
|
||||
result["errors"] = errors
|
||||
logger.info(f"批量导入完成: 成功 {devices_added} 台, 错误 {len(errors)} 条")
|
||||
return result
|
||||
@@ -0,0 +1,183 @@
|
||||
"""统计 API"""
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import select, func, and_, case
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.deps import get_db
|
||||
from app.core.auth import get_current_user
|
||||
from app.models.device import Device
|
||||
from app.models.ping_record import PingRecord
|
||||
from app.models.alert_event import AlertEvent, AlertTypeEnum
|
||||
from app.models.user import User
|
||||
from app.schemas.stats import DeviceStatusSummary, DeviceStatsItem, TimeSeriesPoint, DashboardStats
|
||||
|
||||
router = APIRouter(prefix="/api/stats", tags=["统计"])
|
||||
|
||||
|
||||
@router.get("/summary", response_model=DeviceStatusSummary)
|
||||
async def get_summary(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""获取当前设备状态汇总"""
|
||||
result = await db.execute(select(Device))
|
||||
devices = list(result.scalars().all())
|
||||
|
||||
total = len(devices)
|
||||
online = sum(1 for d in devices if d.current_status == "online")
|
||||
offline = sum(1 for d in devices if d.current_status == "offline")
|
||||
checking = sum(1 for d in devices if d.current_status == "checking")
|
||||
unknown = sum(1 for d in devices if d.current_status == "unknown")
|
||||
online_rate = round(online / max(total, 1) * 100, 2)
|
||||
|
||||
return DeviceStatusSummary(
|
||||
total=total, online=online, offline=offline,
|
||||
checking=checking, unknown=unknown, online_rate=online_rate,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/offline-trend", response_model=list[TimeSeriesPoint])
|
||||
async def get_offline_trend(
|
||||
days: int = Query(7, ge=1, le=90),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""获取指定天数内的每日离线次数趋势"""
|
||||
start = datetime.now() - timedelta(days=days)
|
||||
result = await db.execute(
|
||||
select(
|
||||
func.date(AlertEvent.start_at).label("day"),
|
||||
func.count(AlertEvent.id).label("count"),
|
||||
)
|
||||
.where(AlertEvent.alert_type == AlertTypeEnum.offline)
|
||||
.where(AlertEvent.start_at >= start)
|
||||
.group_by(func.date(AlertEvent.start_at))
|
||||
.order_by("day")
|
||||
)
|
||||
rows = result.all()
|
||||
return [TimeSeriesPoint(time=datetime.strptime(r.day, "%Y-%m-%d"), value=r.count) for r in rows]
|
||||
|
||||
|
||||
@router.get("/online-rate-trend", response_model=list[TimeSeriesPoint])
|
||||
async def get_online_rate_trend(
|
||||
days: int = Query(7, ge=1, le=90),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""获取每日在线率趋势"""
|
||||
start = datetime.now() - timedelta(days=days)
|
||||
# 按天统计每轮的存活/总数比例
|
||||
result = await db.execute(
|
||||
select(
|
||||
func.date(PingRecord.created_at).label("day"),
|
||||
func.round(
|
||||
func.sum(case((PingRecord.is_alive == True, 1), else_=0)) /
|
||||
func.count(PingRecord.id) * 100, 2
|
||||
).label("rate"),
|
||||
)
|
||||
.where(PingRecord.created_at >= start)
|
||||
.group_by(func.date(PingRecord.created_at))
|
||||
.order_by("day")
|
||||
)
|
||||
rows = result.all()
|
||||
return [TimeSeriesPoint(time=datetime.strptime(r.day, "%Y-%m-%d"), value=r.rate) for r in rows]
|
||||
|
||||
|
||||
@router.get("/packet-loss-top", response_model=list[DeviceStatsItem])
|
||||
async def get_packet_loss_top(
|
||||
limit: int = Query(10, ge=1, le=50),
|
||||
days: int = Query(7, ge=1, le=90),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""获取丢包率最高的设备排名"""
|
||||
start = datetime.now() - timedelta(days=days)
|
||||
|
||||
subq = (
|
||||
select(
|
||||
PingRecord.device_id,
|
||||
func.count(PingRecord.id).label("total_pings"),
|
||||
func.sum(case((PingRecord.is_alive == True, 1), else_=0)).label("alive_pings"),
|
||||
func.avg(PingRecord.response_time_ms).label("avg_rtt"),
|
||||
)
|
||||
.where(PingRecord.created_at >= start)
|
||||
.group_by(PingRecord.device_id)
|
||||
.subquery()
|
||||
)
|
||||
|
||||
result = await db.execute(
|
||||
select(
|
||||
subq.c.device_id,
|
||||
subq.c.total_pings,
|
||||
subq.c.alive_pings,
|
||||
subq.c.avg_rtt,
|
||||
Device.name,
|
||||
Device.ip,
|
||||
Device.device_type,
|
||||
Device.location,
|
||||
Device.project_name,
|
||||
)
|
||||
.join(Device, subq.c.device_id == Device.id)
|
||||
.order_by(
|
||||
(subq.c.total_pings - subq.c.alive_pings) * 1.0 / subq.c.total_pings
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
rows = result.all()
|
||||
|
||||
items = []
|
||||
for r in rows:
|
||||
loss_rate = round((r.total_pings - r.alive_pings) / max(r.total_pings, 1) * 100, 2)
|
||||
|
||||
# 离线次数和总时长
|
||||
alert_result = await db.execute(
|
||||
select(
|
||||
func.count(AlertEvent.id),
|
||||
func.coalesce(func.sum(AlertEvent.duration_minutes), 0),
|
||||
func.coalesce(func.max(AlertEvent.duration_minutes), 0),
|
||||
)
|
||||
.where(AlertEvent.device_id == r.device_id)
|
||||
.where(AlertEvent.alert_type == AlertTypeEnum.offline)
|
||||
.where(AlertEvent.start_at >= start)
|
||||
)
|
||||
cnt, total_dur, max_dur = alert_result.one()
|
||||
|
||||
items.append(DeviceStatsItem(
|
||||
device_id=r.device_id,
|
||||
device_name=r.name,
|
||||
device_ip=r.ip,
|
||||
device_type=r.device_type.value if hasattr(r.device_type, 'value') else str(r.device_type),
|
||||
location=r.location or "",
|
||||
project_name=r.project_name or "",
|
||||
total_pings=r.total_pings,
|
||||
alive_pings=r.alive_pings,
|
||||
packet_loss_rate=loss_rate,
|
||||
avg_response_time=round(r.avg_rtt, 2) if r.avg_rtt else None,
|
||||
offline_count=cnt,
|
||||
total_offline_duration=total_dur,
|
||||
max_offline_duration=max_dur,
|
||||
))
|
||||
return items
|
||||
|
||||
|
||||
@router.get("/dashboard", response_model=DashboardStats)
|
||||
async def get_dashboard(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: User = Depends(get_current_user),
|
||||
):
|
||||
"""仪表盘聚合数据"""
|
||||
summary = await get_summary(db, current_user)
|
||||
recent = await get_latest_alerts(limit=10, db=db, current_user=current_user)
|
||||
trend = await get_offline_trend(7, db, current_user)
|
||||
top_loss = await get_packet_loss_top(10, 7, db, current_user)
|
||||
return DashboardStats(
|
||||
summary=summary, recent_offline=recent,
|
||||
offline_trend=trend, packet_loss_top=top_loss,
|
||||
)
|
||||
|
||||
|
||||
# 复用上面定义的函数
|
||||
from app.api.alerts import get_latest_alerts
|
||||
@@ -0,0 +1,57 @@
|
||||
"""用户管理 API(管理员)"""
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.deps import get_db
|
||||
from app.core.auth import get_current_user, require_admin
|
||||
from app.models.user import User, UserRoleEnum
|
||||
|
||||
router = APIRouter(prefix="/api/users", tags=["用户管理"])
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_users(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
admin: User = Depends(require_admin),
|
||||
):
|
||||
"""获取所有用户"""
|
||||
result = await db.execute(select(User).order_by(User.id))
|
||||
users = result.scalars().all()
|
||||
return [
|
||||
{
|
||||
"id": u.id,
|
||||
"username": u.username,
|
||||
"display_name": u.display_name,
|
||||
"role": u.role.value,
|
||||
"is_active": u.is_active,
|
||||
"last_login_at": u.last_login_at,
|
||||
"created_at": u.created_at,
|
||||
}
|
||||
for u in users
|
||||
]
|
||||
|
||||
|
||||
class UpdateUserRoleRequest(BaseModel):
|
||||
role: str # "admin" or "viewer"
|
||||
|
||||
|
||||
|
||||
@router.put("/{user_id}/role")
|
||||
async def update_user_role(
|
||||
user_id: int,
|
||||
data: UpdateUserRoleRequest,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
admin: User = Depends(require_admin),
|
||||
):
|
||||
"""更新用户角色"""
|
||||
if data.role not in ("admin", "viewer"):
|
||||
raise HTTPException(status_code=400, detail="无效角色")
|
||||
if user_id == admin.id:
|
||||
raise HTTPException(status_code=400, detail="不能修改自己的角色")
|
||||
|
||||
new_role = UserRoleEnum.admin if data.role == "admin" else UserRoleEnum.viewer
|
||||
await db.execute(update(User).where(User.id == user_id).values(role=new_role))
|
||||
await db.commit()
|
||||
return {"message": "更新成功"}
|
||||
@@ -0,0 +1,70 @@
|
||||
"""WebSocket 实时推送"""
|
||||
import json
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
|
||||
|
||||
logger = logging.getLogger("pingwatch.ws")
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
"""WebSocket 连接管理器"""
|
||||
|
||||
def __init__(self):
|
||||
self._connections: set[WebSocket] = set()
|
||||
|
||||
async def connect(self, ws: WebSocket):
|
||||
await ws.accept()
|
||||
self._connections.add(ws)
|
||||
|
||||
def disconnect(self, ws: WebSocket):
|
||||
self._connections.discard(ws)
|
||||
|
||||
async def broadcast(self, message: dict):
|
||||
"""向所有客户端广播消息"""
|
||||
dead = set()
|
||||
for ws in self._connections:
|
||||
try:
|
||||
await ws.send_json(message)
|
||||
except Exception:
|
||||
dead.add(ws)
|
||||
self._connections -= dead
|
||||
|
||||
@property
|
||||
def count(self) -> int:
|
||||
return len(self._connections)
|
||||
|
||||
|
||||
manager = ConnectionManager()
|
||||
|
||||
|
||||
@router.websocket("/ws")
|
||||
async def websocket_endpoint(ws: WebSocket):
|
||||
"""WebSocket 端点,用于前端实时接收状态更新"""
|
||||
await manager.connect(ws)
|
||||
try:
|
||||
while True:
|
||||
# 保持连接,接收心跳 pong
|
||||
data = await ws.receive_text()
|
||||
if data == "ping":
|
||||
await ws.send_text("pong")
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"WebSocket 异常: {e}")
|
||||
finally:
|
||||
manager.disconnect(ws)
|
||||
|
||||
|
||||
async def broadcast_status_change(device_id: int, status: str, name: str):
|
||||
"""广播设备状态变化"""
|
||||
await manager.broadcast({
|
||||
"type": "device_status_change",
|
||||
"device_id": device_id,
|
||||
"status": status,
|
||||
"name": name,
|
||||
"timestamp": asyncio.get_event_loop().time(),
|
||||
})
|
||||
@@ -0,0 +1,57 @@
|
||||
"""应用配置,通过环境变量注入,不支持 .env 文件"""
|
||||
|
||||
from pydantic_settings import BaseSettings
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
# ---------- 数据库 ----------
|
||||
DATABASE_URL: str = "sqlite+aiosqlite:///./pingwatch.db"
|
||||
# PostgreSQL: "postgresql+asyncpg://user:pass@localhost/pingwatch"
|
||||
|
||||
# ---------- 企业微信 ----------
|
||||
WECOM_CORP_ID: str = ""
|
||||
WECOM_AGENT_ID: int = 0
|
||||
WECOM_APP_SECRET: str = ""
|
||||
|
||||
# ---------- Casdoor ----------
|
||||
CASDOOR_ENDPOINT: str = "https://casdoor.dhdx.fun"
|
||||
CASDOOR_CLIENT_ID: str = ""
|
||||
CASDOOR_CLIENT_SECRET: str = ""
|
||||
CASDOOR_CERTIFICATE: str = "" # 可选,用于验证 id_token 签名
|
||||
CASDOOR_ORGANIZATION: str = "dahua"
|
||||
CASDOOR_APPLICATION: str = "PingWatch"
|
||||
CASDOOR_REDIRECT_URI: str = "http://10.10.10.7:5173/login" # 前端回调地址
|
||||
|
||||
# ---------- Ping 引擎 ----------
|
||||
PING_INTERVAL_SECONDS: int = 30
|
||||
PING_TIMEOUT_SECONDS: float = 5.0
|
||||
DEFAULT_ALERT_THRESHOLD: int = 5
|
||||
PING_CONCURRENCY: int = 50
|
||||
FPING_PATH: str = "/usr/bin/fping"
|
||||
|
||||
# ---------- 上游心跳 ----------
|
||||
UPSTREAM_PING_TARGET: str = "114.114.114.114"
|
||||
UPSTREAM_PING_THRESHOLD: int = 2
|
||||
OFFLINE_SUPPRESS_RATIO: float = 0.9
|
||||
|
||||
# ---------- 数据保留 ----------
|
||||
PING_RECORD_RETENTION_DAYS: int = 90
|
||||
ALERT_RETENTION_DAYS: int = 365
|
||||
|
||||
# ---------- JWT ----------
|
||||
SECRET_KEY: str = "change-me-to-a-long-random-string"
|
||||
ACCESS_TOKEN_EXPIRE_MINUTES: int = 480
|
||||
|
||||
# ---------- LogHive ----------
|
||||
LOGHIVE_ENDPOINT: str = "http://10.10.10.14:8000"
|
||||
LOGHIVE_PROJECT: str = "PingWatch"
|
||||
LOGHIVE_API_KEY: str = ""
|
||||
|
||||
# ---------- CORS ----------
|
||||
CORS_ORIGINS: str = "http://localhost:5173,http://10.10.10.7:5173,http://10.10.10.7"
|
||||
|
||||
model_config = {"env_file": ".env", "env_file_encoding": "utf-8"}
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,128 @@
|
||||
"""
|
||||
Casdoor OAuth 认证集成 + JWT 会话管理
|
||||
|
||||
流程:
|
||||
1. 前端跳转到 Casdoor 登录页 → 用户登录
|
||||
2. Casdoor 回调到前端(带 code 参数)
|
||||
3. 前端将 code 发到后端 /api/auth/login
|
||||
4. 后端用 code 向 Casdoor 换取 access_token + id_token
|
||||
5. 后端从 id_token (JWT) 解析用户信息
|
||||
6. 后端签发自己的 JWT,返回给前端
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
from jose import JWTError, jwt
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.models.user import User, UserRoleEnum
|
||||
from app.core.deps import get_db
|
||||
|
||||
logger = logging.getLogger("pingwatch.auth")
|
||||
security = HTTPBearer()
|
||||
|
||||
|
||||
def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -> str:
|
||||
"""签发 PingWatch 自己的 JWT"""
|
||||
to_encode = data.copy()
|
||||
expire = datetime.now() + (expires_delta or timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES))
|
||||
to_encode.update({"exp": expire})
|
||||
return jwt.encode(to_encode, settings.SECRET_KEY, algorithm="HS256")
|
||||
|
||||
|
||||
def _load_casdoor_certificate() -> str:
|
||||
"""加载 Casdoor 证书:优先读取 dahua.pem,否则用环境变量"""
|
||||
pem_path = os.path.join(os.path.dirname(__file__), "..", "..", "dahua.pem")
|
||||
pem_path = os.path.normpath(pem_path)
|
||||
if os.path.isfile(pem_path):
|
||||
with open(pem_path, "r") as f:
|
||||
return f.read()
|
||||
return settings.CASDOOR_CERTIFICATE
|
||||
|
||||
|
||||
async def exchange_code_for_user(code: str) -> Optional[dict]:
|
||||
"""
|
||||
用 OAuth code 向 Casdoor 换取用户信息。
|
||||
|
||||
步骤:
|
||||
1. POST → /api/login/oauth/access_token 换取 id_token
|
||||
2. 解码 id_token (JWT) 得到用户信息
|
||||
"""
|
||||
token_url = f"{settings.CASDOOR_ENDPOINT}/api/login/oauth/access_token"
|
||||
data = {
|
||||
"grant_type": "authorization_code",
|
||||
"client_id": settings.CASDOOR_CLIENT_ID,
|
||||
"client_secret": settings.CASDOOR_CLIENT_SECRET,
|
||||
"code": code,
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=15, verify=False) as client:
|
||||
try:
|
||||
resp = await client.post(token_url, data=data)
|
||||
if resp.status_code != 200:
|
||||
logger.error(f"Casdoor token 换取失败: {resp.status_code} {resp.text}")
|
||||
return None
|
||||
|
||||
token_data = resp.json()
|
||||
id_token = token_data.get("id_token")
|
||||
if not id_token:
|
||||
logger.error("Casdoor 返回中没有 id_token")
|
||||
return None
|
||||
|
||||
# 解码 id_token (JWT) payload,不验证签名(HTTPS 已保证传输安全)
|
||||
# 生产环境建议验证 Casdoor 证书
|
||||
cert = _load_casdoor_certificate()
|
||||
try:
|
||||
payload = jwt.decode(
|
||||
id_token,
|
||||
key=cert or None,
|
||||
options={"verify_signature": bool(cert)},
|
||||
audience=settings.CASDOOR_CLIENT_ID,
|
||||
)
|
||||
except JWTError:
|
||||
# 不验证签名的方式解码
|
||||
payload = jwt.get_unverified_claims(id_token)
|
||||
|
||||
return payload
|
||||
|
||||
except httpx.TimeoutException:
|
||||
logger.error("Casdoor token 请求超时")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Casdoor token 请求异常: {e}", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
async def get_current_user(
|
||||
credentials: HTTPAuthorizationCredentials = Depends(security),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> User:
|
||||
"""从 PingWatch JWT 中解析当前登录用户"""
|
||||
token = credentials.credentials
|
||||
try:
|
||||
payload = jwt.decode(token, settings.SECRET_KEY, algorithms=["HS256"])
|
||||
casdoor_uid: str = payload.get("sub", "")
|
||||
if not casdoor_uid:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="无效 token")
|
||||
except JWTError:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="无效 token")
|
||||
|
||||
result = await db.execute(select(User).where(User.casdoor_uid == casdoor_uid))
|
||||
user = result.scalar_one_or_none()
|
||||
if not user or not user.is_active:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="用户不存在或已禁用")
|
||||
return user
|
||||
|
||||
|
||||
async def require_admin(current_user: User = Depends(get_current_user)) -> User:
|
||||
if current_user.role != UserRoleEnum.admin:
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="仅管理员可执行此操作")
|
||||
return current_user
|
||||
@@ -0,0 +1,28 @@
|
||||
"""数据库会话依赖"""
|
||||
|
||||
from typing import AsyncGenerator
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
|
||||
from app.config import settings
|
||||
|
||||
# 处理 sqlite 协议兼容
|
||||
db_url = settings.DATABASE_URL
|
||||
if db_url.startswith("sqlite"):
|
||||
db_url = db_url.replace("sqlite://", "sqlite+aiosqlite://")
|
||||
|
||||
engine = create_async_engine(db_url, echo=False, pool_pre_ping=True)
|
||||
async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
|
||||
|
||||
async def get_db() -> AsyncGenerator[AsyncSession, None]:
|
||||
async with async_session() as session:
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def init_db():
|
||||
"""创建所有表"""
|
||||
from app.models.device import Base
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
@@ -0,0 +1,137 @@
|
||||
"""
|
||||
LogHive 日志 Handler
|
||||
|
||||
基于标准 logging.Handler,通过 REST API 将日志异步批量发送到 LogHive。
|
||||
不依赖外部包,后台线程发送,失败不影响主业务。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import atexit
|
||||
import json
|
||||
import logging
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
import traceback as tb
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
class LogHiveHandler(logging.Handler):
|
||||
"""异步批量发送日志到 LogHive"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
endpoint: str,
|
||||
project: str,
|
||||
api_key: str,
|
||||
level: int = logging.INFO,
|
||||
batch_size: int = 50,
|
||||
flush_interval: float = 2.0,
|
||||
max_retries: int = 3,
|
||||
):
|
||||
super().__init__(level=level)
|
||||
self._endpoint = endpoint.rstrip("/") + "/api/logs/ingest"
|
||||
self._project = project
|
||||
self._api_key = api_key
|
||||
self._batch_size = batch_size
|
||||
self._flush_interval = flush_interval
|
||||
self._max_retries = max_retries
|
||||
|
||||
self._queue: queue.Queue = queue.Queue()
|
||||
self._client: httpx.Client | None = None
|
||||
self._thread: threading.Thread | None = None
|
||||
self._running = False
|
||||
|
||||
def _ensure_client(self):
|
||||
if self._client is None:
|
||||
self._client = httpx.Client(timeout=10)
|
||||
|
||||
def _ensure_thread(self):
|
||||
if self._thread is None or not self._thread.is_alive():
|
||||
self._running = True
|
||||
self._thread = threading.Thread(target=self._send_loop, daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def emit(self, record: logging.LogRecord):
|
||||
"""接收日志记录,放入队列"""
|
||||
if not self._api_key:
|
||||
return
|
||||
self._ensure_thread()
|
||||
try:
|
||||
entry = {
|
||||
"level": record.levelname.lower(),
|
||||
"message": self.format(record),
|
||||
"logger": record.name,
|
||||
"timestamp": record.created,
|
||||
}
|
||||
if record.exc_info and record.exc_info[1]:
|
||||
entry["exception"] = "".join(
|
||||
tb.format_exception(*record.exc_info)
|
||||
)
|
||||
self._queue.put_nowait(entry)
|
||||
except Exception:
|
||||
pass # 日志发送失败不能影响主业务
|
||||
|
||||
def _send_loop(self):
|
||||
"""后台线程:定时批量发送"""
|
||||
while self._running:
|
||||
batch = []
|
||||
deadline = time.monotonic() + self._flush_interval
|
||||
|
||||
while len(batch) < self._batch_size:
|
||||
try:
|
||||
remaining = max(0, deadline - time.monotonic())
|
||||
batch.append(self._queue.get(timeout=remaining))
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
if batch:
|
||||
payload = {"project": self._project, "entries": batch}
|
||||
for attempt in range(self._max_retries):
|
||||
try:
|
||||
self._ensure_client()
|
||||
resp = self._client.post(
|
||||
self._endpoint,
|
||||
json=payload,
|
||||
headers={"Authorization": f"Bearer {self._api_key}"},
|
||||
)
|
||||
if resp.status_code < 500:
|
||||
break
|
||||
except Exception:
|
||||
if attempt == self._max_retries - 1:
|
||||
pass # 最终丢弃
|
||||
else:
|
||||
time.sleep(0.5 * (attempt + 1))
|
||||
|
||||
def close(self):
|
||||
"""关闭 handler,flush 剩余日志"""
|
||||
self._running = False
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._thread.join(timeout=5)
|
||||
if self._client:
|
||||
self._client.close()
|
||||
super().close()
|
||||
|
||||
|
||||
class AsyncLogHiveHandler:
|
||||
"""
|
||||
用于 asyncio 事件循环的异步 handler。
|
||||
在独立的线程中运行同步 LogHiveHandler,通过 asyncio 队列桥接。
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self._handler = LogHiveHandler(**kwargs)
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
def setup(self, loop: asyncio.AbstractEventLoop):
|
||||
self._loop = loop
|
||||
atexit.register(self._handler.close)
|
||||
|
||||
async def emit(self, record: logging.LogRecord):
|
||||
"""异步安全地提交日志记录"""
|
||||
# LogHiveHandler.emit 已经把日志放入内部队列,这里只需确保线程运行
|
||||
self._handler.emit(record)
|
||||
|
||||
def close(self):
|
||||
self._handler.close()
|
||||
@@ -0,0 +1,99 @@
|
||||
"""
|
||||
PingWatch — 网络设备离线监控系统
|
||||
FastAPI 后端入口
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app.config import settings
|
||||
from app.core.deps import init_db
|
||||
from app.core.loghive import LogHiveHandler
|
||||
from app.services.scheduler import scheduler
|
||||
from app.services.cleanup import cleanup_old_data
|
||||
from app.api import devices, alerts, stats, auth, ws, users
|
||||
|
||||
# 日志配置
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(name)s] %(levelname)s: %(message)s",
|
||||
)
|
||||
|
||||
# 接入 LogHive 日志系统
|
||||
loghive_handler = LogHiveHandler(
|
||||
endpoint=settings.LOGHIVE_ENDPOINT,
|
||||
project=settings.LOGHIVE_PROJECT,
|
||||
api_key=settings.LOGHIVE_API_KEY,
|
||||
)
|
||||
if settings.LOGHIVE_API_KEY:
|
||||
logging.getLogger().addHandler(loghive_handler)
|
||||
|
||||
logger = logging.getLogger("pingwatch")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""应用生命周期"""
|
||||
logger.info("PingWatch 启动中...")
|
||||
|
||||
# 初始化数据库
|
||||
await init_db()
|
||||
logger.info("数据库初始化完成")
|
||||
|
||||
# 启动 Ping 调度器
|
||||
scheduler.start()
|
||||
logger.info("Ping 调度器已启动")
|
||||
|
||||
# 启动定时清理任务
|
||||
async def cleanup_loop():
|
||||
while True:
|
||||
await asyncio.sleep(3600 * 6) # 每 6 小时
|
||||
try:
|
||||
await cleanup_old_data()
|
||||
except Exception as e:
|
||||
logger.error(f"清理任务异常: {e}")
|
||||
|
||||
cleanup_task = asyncio.create_task(cleanup_loop())
|
||||
|
||||
yield
|
||||
|
||||
# 关闭
|
||||
await scheduler.stop()
|
||||
cleanup_task.cancel()
|
||||
loghive_handler.close()
|
||||
logger.info("PingWatch 已关闭")
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="PingWatch",
|
||||
description="网络设备离线监控系统",
|
||||
version="1.0.0",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
# CORS
|
||||
origins = [o.strip() for o in settings.CORS_ORIGINS.split(",")]
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# 注册路由
|
||||
app.include_router(auth.router)
|
||||
app.include_router(devices.router)
|
||||
app.include_router(alerts.router)
|
||||
app.include_router(stats.router)
|
||||
app.include_router(ws.router)
|
||||
app.include_router(users.router)
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
async def health():
|
||||
return {"status": "ok"}
|
||||
@@ -0,0 +1,11 @@
|
||||
from .device import Device, DeviceTypeEnum
|
||||
from .ping_record import PingRecord
|
||||
from .alert_event import AlertEvent, AlertTypeEnum
|
||||
from .user import User, UserRoleEnum
|
||||
|
||||
__all__ = [
|
||||
"Device", "DeviceTypeEnum",
|
||||
"PingRecord",
|
||||
"AlertEvent", "AlertTypeEnum",
|
||||
"User", "UserRoleEnum",
|
||||
]
|
||||
@@ -0,0 +1,34 @@
|
||||
import enum
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, Integer, String, Boolean, DateTime, BigInteger, Enum
|
||||
from .device import Base
|
||||
|
||||
|
||||
class AlertTypeEnum(str, enum.Enum):
|
||||
offline = "offline" # 设备离线
|
||||
recovered = "recovered" # 设备恢复
|
||||
system = "system" # 系统告警(如上游断网检测)
|
||||
|
||||
|
||||
class AlertEvent(Base):
|
||||
"""告警事件表"""
|
||||
__tablename__ = "alert_events"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
device_id = Column(Integer, nullable=False, index=True, comment="关联设备 ID")
|
||||
alert_type = Column(Enum(AlertTypeEnum), nullable=False, comment="告警类型")
|
||||
message = Column(String(1024), default="", comment="告警消息摘要")
|
||||
|
||||
# 离线起止
|
||||
start_at = Column(DateTime, nullable=False, comment="离线开始时间")
|
||||
end_at = Column(DateTime, nullable=True, comment="恢复时间")
|
||||
duration_minutes = Column(Integer, nullable=True, comment="离线时长(分钟)")
|
||||
|
||||
is_resolved = Column(Boolean, default=False, comment="是否已恢复")
|
||||
notification_sent = Column(Boolean, default=False, comment="是否已发送通知")
|
||||
acknowledged_at = Column(DateTime, nullable=True, comment="用户确认时间")
|
||||
|
||||
created_at = Column(DateTime, default=datetime.now, comment="创建时间")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<AlertEvent(id={self.id}, device={self.device_id}, type={self.alert_type})>"
|
||||
@@ -0,0 +1,46 @@
|
||||
import enum
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, Integer, String, Float, Boolean, DateTime, Enum, Text
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class DeviceTypeEnum(str, enum.Enum):
|
||||
server = "server" # 服务器
|
||||
olt = "olt" # OLT
|
||||
switch = "switch" # 交换机
|
||||
firewall = "firewall" # 防火墙
|
||||
other = "other" # 其他
|
||||
|
||||
|
||||
class Device(Base):
|
||||
__tablename__ = "devices"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
name = Column(String(128), nullable=False, comment="设备名称")
|
||||
ip = Column(String(45), nullable=False, index=True, comment="IP 地址")
|
||||
device_type = Column(Enum(DeviceTypeEnum), nullable=False, default=DeviceTypeEnum.other, comment="设备类型")
|
||||
location = Column(String(256), default="", comment="物理位置/地址")
|
||||
project_name = Column(String(256), default="", comment="所属项目")
|
||||
tags = Column(String(512), default="", comment="标签,逗号分隔")
|
||||
|
||||
# Ping 设置
|
||||
ping_interval = Column(Integer, default=30, comment="ping 间隔(秒)")
|
||||
alert_threshold = Column(Integer, default=5, comment="连续失败次数判离线")
|
||||
is_enabled = Column(Boolean, default=True, comment="是否启用监控")
|
||||
|
||||
# 运行状态
|
||||
current_status = Column(String(16), default="unknown", comment="当前状态: online/offline/unknown")
|
||||
consecutive_failures = Column(Integer, default=0, comment="当前连续失败次数")
|
||||
last_ping_time = Column(DateTime, nullable=True, comment="最后一次 ping 时间")
|
||||
last_online_time = Column(DateTime, nullable=True, comment="最后一次在线时间")
|
||||
last_offline_time = Column(DateTime, nullable=True, comment="最后一次离线时间")
|
||||
|
||||
created_at = Column(DateTime, default=datetime.now, comment="创建时间")
|
||||
updated_at = Column(DateTime, default=datetime.now, onupdate=datetime.now, comment="更新时间")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<Device(id={self.id}, name={self.name}, ip={self.ip})>"
|
||||
@@ -0,0 +1,18 @@
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, Integer, Float, Boolean, DateTime, BigInteger
|
||||
from .device import Base
|
||||
|
||||
|
||||
class PingRecord(Base):
|
||||
"""单次 ping 结果记录"""
|
||||
__tablename__ = "ping_records"
|
||||
|
||||
id = Column(BigInteger, primary_key=True, autoincrement=True)
|
||||
device_id = Column(Integer, nullable=False, index=True)
|
||||
is_alive = Column(Boolean, nullable=False, comment="是否通")
|
||||
response_time_ms = Column(Float, nullable=True, comment="响应时间毫秒,不通则为 NULL")
|
||||
round_num = Column(Integer, nullable=False, comment="轮次编号(从 1 递增)")
|
||||
created_at = Column(DateTime, default=datetime.now, index=True, comment="记录时间")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<PingRecord(device={self.device_id}, alive={self.is_alive}, rtt={self.response_time_ms})>"
|
||||
@@ -0,0 +1,27 @@
|
||||
import enum
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Boolean, Column, Integer, String, DateTime, Enum
|
||||
from .device import Base
|
||||
|
||||
|
||||
class UserRoleEnum(str, enum.Enum):
|
||||
admin = "admin" # 管理员:可管理设备
|
||||
viewer = "viewer" # 查看者:仅查看数据
|
||||
|
||||
|
||||
class User(Base):
|
||||
"""本地用户表,关联 Casdoor"""
|
||||
__tablename__ = "users"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
casdoor_uid = Column(String(128), unique=True, nullable=False, comment="Casdoor 中的用户 ID")
|
||||
username = Column(String(128), nullable=False, comment="用户名")
|
||||
display_name = Column(String(128), default="", comment="显示名称")
|
||||
role = Column(Enum(UserRoleEnum), nullable=False, default=UserRoleEnum.viewer, comment="角色")
|
||||
wecom_userid = Column(String(128), default="", comment="企业微信 UserID(用于个人通知)")
|
||||
is_active = Column(Boolean, default=True, comment="是否启用")
|
||||
last_login_at = Column(DateTime, nullable=True, comment="最后登录时间")
|
||||
created_at = Column(DateTime, default=datetime.now, comment="创建时间")
|
||||
|
||||
def __repr__(self):
|
||||
return f"<User(id={self.id}, name={self.username}, role={self.role})>"
|
||||
@@ -0,0 +1,33 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class AlertEventOut(BaseModel):
|
||||
id: int
|
||||
device_id: int
|
||||
device_name: str = ""
|
||||
device_ip: str = ""
|
||||
device_type: str = ""
|
||||
location: str = ""
|
||||
project_name: str = ""
|
||||
alert_type: str
|
||||
message: str
|
||||
start_at: datetime
|
||||
end_at: Optional[datetime] = None
|
||||
duration_minutes: Optional[int] = None
|
||||
is_resolved: bool
|
||||
notification_sent: bool
|
||||
created_at: datetime
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class AlertEventQuery(BaseModel):
|
||||
page: int = 1
|
||||
page_size: int = 20
|
||||
device_id: Optional[int] = None
|
||||
alert_type: Optional[str] = None
|
||||
is_resolved: Optional[bool] = None
|
||||
start_time: Optional[datetime] = None
|
||||
end_time: Optional[datetime] = None
|
||||
@@ -0,0 +1,49 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel, IPvAnyAddress
|
||||
|
||||
|
||||
class DeviceCreate(BaseModel):
|
||||
name: str
|
||||
ip: str
|
||||
device_type: str = "other"
|
||||
location: str = ""
|
||||
project_name: str = ""
|
||||
tags: str = ""
|
||||
ping_interval: int = 30
|
||||
alert_threshold: int = 5
|
||||
is_enabled: bool = True
|
||||
|
||||
|
||||
class DeviceUpdate(BaseModel):
|
||||
name: Optional[str] = None
|
||||
ip: Optional[str] = None
|
||||
device_type: Optional[str] = None
|
||||
location: Optional[str] = None
|
||||
project_name: Optional[str] = None
|
||||
tags: Optional[str] = None
|
||||
ping_interval: Optional[int] = None
|
||||
alert_threshold: Optional[int] = None
|
||||
is_enabled: Optional[bool] = None
|
||||
|
||||
|
||||
class DeviceOut(BaseModel):
|
||||
id: int
|
||||
name: str
|
||||
ip: str
|
||||
device_type: str
|
||||
location: str
|
||||
project_name: str
|
||||
tags: str
|
||||
ping_interval: int
|
||||
alert_threshold: int
|
||||
is_enabled: bool
|
||||
current_status: str
|
||||
consecutive_failures: int
|
||||
last_ping_time: Optional[datetime] = None
|
||||
last_online_time: Optional[datetime] = None
|
||||
last_offline_time: Optional[datetime] = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
@@ -0,0 +1,40 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class DeviceStatusSummary(BaseModel):
|
||||
total: int
|
||||
online: int
|
||||
offline: int
|
||||
checking: int
|
||||
unknown: int
|
||||
online_rate: float
|
||||
|
||||
|
||||
class DeviceStatsItem(BaseModel):
|
||||
device_id: int
|
||||
device_name: str
|
||||
device_ip: str
|
||||
device_type: str
|
||||
location: str
|
||||
project_name: str
|
||||
total_pings: int
|
||||
alive_pings: int
|
||||
packet_loss_rate: float
|
||||
avg_response_time: Optional[float] = None
|
||||
offline_count: int
|
||||
total_offline_duration: int # 分钟
|
||||
max_offline_duration: int
|
||||
|
||||
|
||||
class TimeSeriesPoint(BaseModel):
|
||||
time: datetime
|
||||
value: float
|
||||
|
||||
|
||||
class DashboardStats(BaseModel):
|
||||
summary: DeviceStatusSummary
|
||||
recent_offline: list # list[AlertEventOut]
|
||||
offline_trend: list[TimeSeriesPoint]
|
||||
packet_loss_top: list[DeviceStatsItem]
|
||||
@@ -0,0 +1,319 @@
|
||||
"""
|
||||
告警服务
|
||||
|
||||
职责:
|
||||
1. 接收 Pinger 的状态变化事件
|
||||
2. 判断是否需要发送告警
|
||||
3. 上游心跳检测 + 全量离线抑制
|
||||
4. 企业微信消息推送(聚合告警)
|
||||
5. 记录告警事件到数据库
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from collections import defaultdict
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select, func
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.models.device import Device
|
||||
from app.models.alert_event import AlertEvent, AlertTypeEnum
|
||||
from app.services.pinger import DeviceStateChange
|
||||
|
||||
logger = logging.getLogger("pingwatch.alerter")
|
||||
|
||||
|
||||
class PendingOfflineAlert:
|
||||
"""等待发送的离线告警(用于聚合)"""
|
||||
def __init__(self, device: Device, alert_time: datetime):
|
||||
self.device = device
|
||||
self.alert_time = alert_time
|
||||
|
||||
|
||||
class Alerter:
|
||||
"""
|
||||
告警处理器。
|
||||
|
||||
核心逻辑:
|
||||
- 设备 offline → 收集到待发送队列
|
||||
- 每轮结束后检查:
|
||||
a) 上游心跳是否正常?
|
||||
b) 离线率是否 < 90%?
|
||||
c) 满足条件 → 聚合所有待发告警 → 一条企业微信消息
|
||||
d) 不满足 → 丢弃本轮告警,记录系统日志
|
||||
- 设备 recovered → 单独发送恢复通知
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._pending_alerts: list[PendingOfflineAlert] = []
|
||||
self._lock = asyncio.Lock()
|
||||
# 上游心跳状态
|
||||
self._upstream_failures = 0
|
||||
self._upstream_available = True
|
||||
|
||||
async def on_state_change(self, change: DeviceStateChange, db: AsyncSession):
|
||||
"""Pinger 状态变化回调"""
|
||||
if change.new_status == "offline":
|
||||
async with self._lock:
|
||||
self._pending_alerts.append(
|
||||
PendingOfflineAlert(device=change.device, alert_time=datetime.now())
|
||||
)
|
||||
|
||||
elif change.new_status == "online" and change.old_status == "offline":
|
||||
# 设备恢复,立即记录并发送恢复通知
|
||||
await self._handle_recovery(change.device, db)
|
||||
|
||||
async def flush_pending(self, db: AsyncSession, total_device_count: int):
|
||||
"""
|
||||
每轮结束时调用:处理待发送的离线告警。
|
||||
判断是否应该抑制告警,然后发送或丢弃。
|
||||
"""
|
||||
async with self._lock:
|
||||
if not self._pending_alerts:
|
||||
return
|
||||
pending = self._pending_alerts.copy()
|
||||
self._pending_alerts.clear()
|
||||
|
||||
# 1. 检查上游心跳
|
||||
upstream_ok = await self._check_upstream()
|
||||
|
||||
# 2. 计算离线率
|
||||
offline_count = len(pending)
|
||||
offline_ratio = offline_count / max(total_device_count, 1)
|
||||
|
||||
# 3. 抑制条件
|
||||
suppressed = False
|
||||
suppress_reason = ""
|
||||
|
||||
if not upstream_ok:
|
||||
suppressed = True
|
||||
suppress_reason = "上游网络不可达(监控节点可能断网)"
|
||||
elif offline_ratio >= settings.OFFLINE_SUPPRESS_RATIO:
|
||||
suppressed = True
|
||||
suppress_reason = f"离线率 {offline_ratio:.0%} >= {settings.OFFLINE_SUPPRESS_RATIO:.0%},疑似监控节点断网"
|
||||
|
||||
if suppressed:
|
||||
logger.warning(
|
||||
f"告警抑制: {suppress_reason},"
|
||||
f"本轮 {offline_count} 条告警已丢弃"
|
||||
)
|
||||
# 记录系统告警
|
||||
db.add(AlertEvent(
|
||||
device_id=0,
|
||||
alert_type=AlertTypeEnum.system,
|
||||
message=f"告警抑制: {suppress_reason},丢弃 {offline_count} 条离线告警",
|
||||
start_at=datetime.now(),
|
||||
is_resolved=True,
|
||||
notification_sent=False,
|
||||
))
|
||||
await db.commit()
|
||||
return
|
||||
|
||||
# 4. 发送聚合告警
|
||||
if pending:
|
||||
await self._send_aggregated_alert(pending, db)
|
||||
|
||||
async def _check_upstream(self) -> bool:
|
||||
"""
|
||||
上游心跳检测。
|
||||
连续 UPSTREAM_PING_THRESHOLD 次失败才判定为上游断网。
|
||||
"""
|
||||
try:
|
||||
import subprocess
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"ping", "-c", "1", "-W", "3",
|
||||
settings.UPSTREAM_PING_TARGET,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
await proc.wait()
|
||||
if proc.returncode == 0:
|
||||
self._upstream_failures = 0
|
||||
self._upstream_available = True
|
||||
return True
|
||||
else:
|
||||
self._upstream_failures += 1
|
||||
if self._upstream_failures >= settings.UPSTREAM_PING_THRESHOLD:
|
||||
self._upstream_available = False
|
||||
return False
|
||||
# 未达到阈值,认为上游还可用
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"上游心跳检测异常: {e}")
|
||||
return True # 异常时保守地允许告警
|
||||
|
||||
async def _send_aggregated_alert(self, alerts: list[PendingOfflineAlert], db: AsyncSession):
|
||||
"""发送聚合离线告警"""
|
||||
now = datetime.now()
|
||||
|
||||
# 构建企业微信消息
|
||||
if len(alerts) == 1:
|
||||
a = alerts[0]
|
||||
msg = self._build_offline_message_single(a.device, a.alert_time)
|
||||
else:
|
||||
msg = self._build_offline_message_batch(alerts)
|
||||
|
||||
# 发送
|
||||
success = await self._send_wecom_message(msg)
|
||||
|
||||
# 记录告警事件
|
||||
for a in alerts:
|
||||
db.add(AlertEvent(
|
||||
device_id=a.device.id,
|
||||
alert_type=AlertTypeEnum.offline,
|
||||
message=a.device.name,
|
||||
start_at=a.alert_time,
|
||||
is_resolved=False,
|
||||
notification_sent=success,
|
||||
))
|
||||
|
||||
await db.commit()
|
||||
|
||||
if success:
|
||||
logger.info(f"已推送离线告警: {len(alerts)} 台设备")
|
||||
else:
|
||||
logger.error(f"企业微信推送失败: {len(alerts)} 台设备")
|
||||
|
||||
async def _handle_recovery(self, device: Device, db: AsyncSession):
|
||||
"""处理设备恢复"""
|
||||
now = datetime.now()
|
||||
|
||||
# 查找未解决的离线事件
|
||||
result = await db.execute(
|
||||
select(AlertEvent)
|
||||
.where(AlertEvent.device_id == device.id)
|
||||
.where(AlertEvent.alert_type == AlertTypeEnum.offline)
|
||||
.where(AlertEvent.is_resolved == False)
|
||||
.order_by(AlertEvent.created_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
event = result.scalar_one_or_none()
|
||||
|
||||
duration_minutes = None
|
||||
if event:
|
||||
delta = now - event.start_at
|
||||
duration_minutes = int(delta.total_seconds() / 60)
|
||||
event.end_at = now
|
||||
event.duration_minutes = duration_minutes
|
||||
event.is_resolved = True
|
||||
|
||||
# 发送恢复通知
|
||||
msg = self._build_recovery_message(device, now, duration_minutes)
|
||||
success = await self._send_wecom_message(msg)
|
||||
|
||||
if event:
|
||||
event.notification_sent = success
|
||||
|
||||
await db.commit()
|
||||
|
||||
if success:
|
||||
logger.info(f"已推送恢复通知: {device.name}")
|
||||
else:
|
||||
logger.error(f"恢复通知推送失败: {device.name}")
|
||||
|
||||
# ---------- 消息格式化 ----------
|
||||
|
||||
def _build_offline_message_single(self, device: Device, alert_time: datetime) -> str:
|
||||
"""单台设备离线消息"""
|
||||
time_str = alert_time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
return (
|
||||
f"⛔ 设备离线啦!\n"
|
||||
f"地址:{device.location or '未知'}\n"
|
||||
f"时间:{time_str}\n"
|
||||
f"项目:{device.project_name or '未分组'}\n"
|
||||
f"设备类型:{self._fmt_device_type(device.device_type)}\n"
|
||||
f"IP地址:{device.ip}"
|
||||
)
|
||||
|
||||
def _build_offline_message_batch(self, alerts: list[PendingOfflineAlert]) -> str:
|
||||
"""多台设备聚合离线消息"""
|
||||
now = alerts[0].alert_time
|
||||
time_str = now.strftime("%Y-%m-%d %H:%M:%S")
|
||||
lines = [f"⛔ 设备离线啦!(共 {len(alerts)} 台)\n"]
|
||||
|
||||
for a in alerts:
|
||||
dev = a.device
|
||||
lines.append(
|
||||
f"地址:{dev.location or '未知'}\n"
|
||||
f"时间:{a.alert_time.strftime('%Y-%m-%d %H:%M:%S')}\n"
|
||||
f"项目:{dev.project_name or '未分组'}\n"
|
||||
f"设备类型:{self._fmt_device_type(dev.device_type)}\n"
|
||||
f"IP地址:{dev.ip}\n"
|
||||
f"{'---' if len(alerts) > 1 else ''}"
|
||||
)
|
||||
|
||||
return "\n".join(lines).rstrip("---\n")
|
||||
|
||||
def _build_recovery_message(self, device: Device, recover_time: datetime, duration: Optional[int]) -> str:
|
||||
"""设备恢复消息"""
|
||||
time_str = recover_time.strftime("%Y-%m-%d %H:%M:%S")
|
||||
duration_str = f"{duration}分钟" if duration is not None else "未知"
|
||||
return (
|
||||
f"✅ 设备恢复在线!\n"
|
||||
f"地址:{device.location or '未知'}\n"
|
||||
f"时间:{time_str}\n"
|
||||
f"项目:{device.project_name or '未分组'}\n"
|
||||
f"设备类型:{self._fmt_device_type(device.device_type)}\n"
|
||||
f"IP地址:{device.ip}\n"
|
||||
f"离线时长:{duration_str}"
|
||||
)
|
||||
|
||||
def _fmt_device_type(self, dtype) -> str:
|
||||
mapping = {
|
||||
"server": "服务器",
|
||||
"olt": "OLT",
|
||||
"switch": "交换机",
|
||||
"firewall": "防火墙",
|
||||
"other": "其他设备",
|
||||
}
|
||||
return mapping.get(str(dtype), str(dtype))
|
||||
|
||||
# ---------- 企业微信推送 ----------
|
||||
|
||||
async def _get_wecom_token(self) -> Optional[str]:
|
||||
"""获取企业微信 access_token"""
|
||||
url = (
|
||||
f"https://qyapi.weixin.qq.com/cgi-bin/gettoken"
|
||||
f"?corpid={settings.WECOM_CORP_ID}"
|
||||
f"&corpsecret={settings.WECOM_APP_SECRET}"
|
||||
)
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
resp = await client.get(url)
|
||||
data = resp.json()
|
||||
if data.get("errcode") == 0:
|
||||
return data["access_token"]
|
||||
else:
|
||||
logger.error(f"获取企业微信 token 失败: {data}")
|
||||
return None
|
||||
|
||||
async def _send_wecom_message(self, content: str) -> bool:
|
||||
"""发送企业微信应用消息"""
|
||||
if not settings.WECOM_CORP_ID or not settings.WECOM_APP_SECRET:
|
||||
logger.warning("企业微信未配置,跳过推送")
|
||||
logger.info(f"[模拟推送] {content}")
|
||||
return True # 开发模式
|
||||
|
||||
token = await self._get_wecom_token()
|
||||
if not token:
|
||||
return False
|
||||
|
||||
url = f"https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={token}"
|
||||
payload = {
|
||||
"touser": "@all",
|
||||
"msgtype": "text",
|
||||
"agentid": settings.WECOM_AGENT_ID,
|
||||
"text": {"content": content},
|
||||
"safe": 0,
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
resp = await client.post(url, json=payload)
|
||||
data = resp.json()
|
||||
if data.get("errcode") != 0:
|
||||
logger.error(f"发送企业微信消息失败: {data}")
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1,38 @@
|
||||
"""数据清理服务"""
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy import delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.models.ping_record import PingRecord
|
||||
from app.models.alert_event import AlertEvent
|
||||
from app.core.deps import async_session
|
||||
|
||||
logger = logging.getLogger("pingwatch.cleanup")
|
||||
|
||||
|
||||
async def cleanup_old_data():
|
||||
"""清理超过保留期限的旧数据"""
|
||||
now = datetime.now()
|
||||
|
||||
# 清理 ping_records
|
||||
ping_cutoff = now - timedelta(days=settings.PING_RECORD_RETENTION_DAYS)
|
||||
async with async_session() as db:
|
||||
result = await db.execute(
|
||||
delete(PingRecord).where(PingRecord.created_at < ping_cutoff)
|
||||
)
|
||||
await db.commit()
|
||||
if result.rowcount > 0:
|
||||
logger.info(f"清理 ping_records: {result.rowcount} 条")
|
||||
|
||||
# 清理 alert_events
|
||||
alert_cutoff = now - timedelta(days=settings.ALERT_RETENTION_DAYS)
|
||||
async with async_session() as db:
|
||||
result = await db.execute(
|
||||
delete(AlertEvent).where(AlertEvent.created_at < alert_cutoff)
|
||||
)
|
||||
await db.commit()
|
||||
if result.rowcount > 0:
|
||||
logger.info(f"清理 alert_events: {result.rowcount} 条")
|
||||
@@ -0,0 +1,242 @@
|
||||
"""
|
||||
异步 Ping 引擎
|
||||
|
||||
核心逻辑:
|
||||
1. 每轮从数据库加载所有启用设备,批量 fping
|
||||
2. 记录每台设备本轮 ping 结果(存活/响应时间)
|
||||
3. 状态机管理设备状态,判断是否从 online→offline 或 offline→online
|
||||
4. 结果通过回调或队列通知 alerter
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import subprocess
|
||||
import time
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional, Callable, Awaitable
|
||||
from collections import defaultdict
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.models.device import Device, DeviceTypeEnum
|
||||
from app.models.ping_record import PingRecord
|
||||
|
||||
logger = logging.getLogger("pingwatch.pinger")
|
||||
|
||||
|
||||
class PingResult:
|
||||
"""单台设备一轮 ping 的结果"""
|
||||
def __init__(self, device_id: int, is_alive: bool, response_time_ms: Optional[float] = None):
|
||||
self.device_id = device_id
|
||||
self.is_alive = is_alive
|
||||
self.response_time_ms = response_time_ms
|
||||
|
||||
|
||||
class DeviceStateChange:
|
||||
"""设备状态变化事件"""
|
||||
def __init__(self, device: Device, old_status: str, new_status: str, consecutive_failures: int):
|
||||
self.device = device
|
||||
self.old_status = old_status
|
||||
self.new_status = new_status
|
||||
self.consecutive_failures = consecutive_failures
|
||||
|
||||
|
||||
class Pinger:
|
||||
"""
|
||||
Ping 引擎,使用 fping 批量并发检测。
|
||||
对所有设备进 ping,返回存活状态和响应时间。
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._round_num = 0
|
||||
self._on_state_change: Optional[Callable[[DeviceStateChange], Awaitable[None]]] = None
|
||||
|
||||
def on_state_change(self, callback: Callable[[DeviceStateChange], Awaitable[None]]):
|
||||
"""注册状态变化回调"""
|
||||
self._on_state_change = callback
|
||||
|
||||
async def run_one_round(self, db: AsyncSession) -> list[PingResult]:
|
||||
"""
|
||||
执行一轮 ping 检测:
|
||||
1. 加载所有启用设备
|
||||
2. 批量 fping
|
||||
3. 记录结果
|
||||
4. 更新设备状态
|
||||
"""
|
||||
self._round_num += 1
|
||||
round_num = self._round_num
|
||||
|
||||
# 1. 加载启用设备
|
||||
result = await db.execute(
|
||||
select(Device).where(Device.is_enabled == True)
|
||||
)
|
||||
devices = list(result.scalars().all())
|
||||
|
||||
if not devices:
|
||||
logger.info(f"[Round {round_num}] 没有启用的设备")
|
||||
return []
|
||||
|
||||
logger.info(f"[Round {round_num}] 开始检测 {len(devices)} 台设备")
|
||||
|
||||
# 2. 批量 ping
|
||||
start_time = time.time()
|
||||
ip_to_device = {d.ip: d for d in devices}
|
||||
ip_list = list(ip_to_device.keys())
|
||||
|
||||
ping_results_map = await self._batch_ping(ip_list)
|
||||
|
||||
# 3. 构造结果
|
||||
results: list[PingResult] = []
|
||||
for ip, dev in ip_to_device.items():
|
||||
is_alive, rtt = ping_results_map.get(ip, (False, None))
|
||||
results.append(PingResult(device_id=dev.id, is_alive=is_alive, response_time_ms=rtt))
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
alive_count = sum(1 for r in results if r.is_alive)
|
||||
|
||||
# 4. 批量写入 ping_records
|
||||
now = datetime.now()
|
||||
records = [
|
||||
PingRecord(
|
||||
device_id=r.device_id,
|
||||
is_alive=r.is_alive,
|
||||
response_time_ms=r.response_time_ms,
|
||||
round_num=round_num,
|
||||
created_at=now,
|
||||
)
|
||||
for r in results
|
||||
]
|
||||
db.add_all(records)
|
||||
await db.flush()
|
||||
|
||||
# 5. 更新设备状态(状态机)
|
||||
device_map = {d.id: d for d in devices}
|
||||
for r in results:
|
||||
dev = device_map.get(r.device_id)
|
||||
if not dev:
|
||||
continue
|
||||
|
||||
old_status = dev.current_status
|
||||
if r.is_alive:
|
||||
dev.consecutive_failures = 0
|
||||
dev.last_ping_time = now
|
||||
dev.last_online_time = now
|
||||
dev.current_status = "online"
|
||||
else:
|
||||
dev.consecutive_failures = (dev.consecutive_failures or 0) + 1
|
||||
dev.last_ping_time = now
|
||||
if dev.consecutive_failures >= dev.alert_threshold:
|
||||
if dev.current_status != "offline":
|
||||
dev.current_status = "offline"
|
||||
dev.last_offline_time = now
|
||||
else:
|
||||
if dev.current_status == "online":
|
||||
dev.current_status = "checking"
|
||||
|
||||
# 状态变化回调
|
||||
if old_status != dev.current_status and self._on_state_change:
|
||||
change = DeviceStateChange(
|
||||
device=dev,
|
||||
old_status=old_status,
|
||||
new_status=dev.current_status,
|
||||
consecutive_failures=dev.consecutive_failures,
|
||||
)
|
||||
await self._on_state_change(change)
|
||||
|
||||
await db.commit()
|
||||
|
||||
logger.info(
|
||||
f"[Round {round_num}] 完成: {alive_count}/{len(devices)} 在线, "
|
||||
f"耗时 {elapsed:.2f}s"
|
||||
)
|
||||
return results
|
||||
|
||||
async def _batch_ping(self, ip_list: list[str]) -> dict[str, tuple[bool, Optional[float]]]:
|
||||
"""
|
||||
使用 fping 批量 ping
|
||||
返回: { ip: (is_alive, response_time_ms) }
|
||||
"""
|
||||
if not ip_list:
|
||||
return {}
|
||||
|
||||
try:
|
||||
# fping 一次性 ping 多个 IP
|
||||
# -c 1: 每个 IP 发 1 个包
|
||||
# -t: 超时毫秒
|
||||
timeout_ms = int(settings.PING_TIMEOUT_SECONDS * 1000)
|
||||
cmd = [
|
||||
settings.FPING_PATH,
|
||||
"-c", "1",
|
||||
"-t", str(timeout_ms),
|
||||
"-e", # 显示响应时间
|
||||
] + ip_list
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await proc.communicate()
|
||||
|
||||
result_map: dict[str, tuple[bool, Optional[float]]] = {}
|
||||
|
||||
# fping 标准输出逐行: "IP : xmt/rcv/%loss = 1/1/0%, rtt min/avg/max = 0.12/0.12/0.12"
|
||||
# 或 "IP : xmt/rcv/%loss = 1/0/100%"
|
||||
for line in stdout.decode("utf-8", errors="replace").splitlines():
|
||||
line = line.strip()
|
||||
if ":" not in line:
|
||||
continue
|
||||
ip = line.split(":")[0].strip()
|
||||
# 解析响应时间
|
||||
if "rtt" in line:
|
||||
try:
|
||||
# 提取 avg rtt
|
||||
rtt_part = line.split("rtt")[1]
|
||||
# 格式: min/avg/max = 0.12/0.12/0.12
|
||||
if "=" in rtt_part:
|
||||
avg_rtt_str = rtt_part.split("=")[1].strip().split("/")[1]
|
||||
rtt_ms = float(avg_rtt_str)
|
||||
else:
|
||||
rtt_ms = None
|
||||
except (IndexError, ValueError):
|
||||
rtt_ms = None
|
||||
result_map[ip] = (True, rtt_ms)
|
||||
else:
|
||||
result_map[ip] = (False, None)
|
||||
|
||||
return result_map
|
||||
|
||||
except FileNotFoundError:
|
||||
logger.warning("fping 未找到,回退到系统 ping (串行)")
|
||||
return await self._fallback_ping(ip_list)
|
||||
except Exception as e:
|
||||
logger.error(f"fping 异常: {e}")
|
||||
return await self._fallback_ping(ip_list)
|
||||
|
||||
async def _fallback_ping(self, ip_list: list[str]) -> dict[str, tuple[bool, Optional[float]]]:
|
||||
"""回退方案:使用系统 ping,并发执行"""
|
||||
async def ping_one(ip: str) -> tuple[str, bool, Optional[float]]:
|
||||
try:
|
||||
timeout = settings.PING_TIMEOUT_SECONDS
|
||||
cmd = ["ping", "-c", "1", "-W", str(int(timeout)), ip]
|
||||
start = time.time()
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
)
|
||||
await proc.wait()
|
||||
elapsed = (time.time() - start) * 1000
|
||||
return ip, proc.returncode == 0, round(elapsed, 2)
|
||||
except Exception:
|
||||
return ip, False, None
|
||||
|
||||
tasks = [ping_one(ip) for ip in ip_list]
|
||||
sem = asyncio.Semaphore(settings.PING_CONCURRENCY)
|
||||
|
||||
async def bounded_ping(ip: str):
|
||||
async with sem:
|
||||
return await ping_one(ip)
|
||||
|
||||
results = await asyncio.gather(*[bounded_ping(ip) for ip in ip_list])
|
||||
return {ip: (alive, rtt) for ip, alive, rtt in results}
|
||||
@@ -0,0 +1,94 @@
|
||||
"""
|
||||
定时任务调度器
|
||||
|
||||
使用 asyncio 循环驱动 Ping 引擎,协调 Pinger 和 Alerter。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.config import settings
|
||||
from app.services.pinger import Pinger
|
||||
from app.services.alerter import Alerter
|
||||
from app.core.deps import async_session
|
||||
|
||||
logger = logging.getLogger("pingwatch.scheduler")
|
||||
|
||||
|
||||
class PingScheduler:
|
||||
"""
|
||||
调度器职责:
|
||||
1. 按间隔驱动 Ping 引擎
|
||||
2. 每轮结束后触发 Alerter 处理待发送告警
|
||||
3. 控制并发和清理
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._pinger = Pinger()
|
||||
self._alerter = Alerter()
|
||||
self._running = False
|
||||
self._task: asyncio.Task | None = None
|
||||
|
||||
# 注册状态变化回调
|
||||
self._pinger.on_state_change(self._on_state_change)
|
||||
|
||||
async def _on_state_change(self, change):
|
||||
"""收到设备状态变化,转给 alerter"""
|
||||
async with async_session() as db:
|
||||
try:
|
||||
await self._alerter.on_state_change(change, db)
|
||||
except Exception as e:
|
||||
logger.error(f"告警处理异常: {e}", exc_info=True)
|
||||
|
||||
async def _run_loop(self):
|
||||
"""主循环"""
|
||||
logger.info("Ping 调度器已启动")
|
||||
self._running = True
|
||||
|
||||
while self._running:
|
||||
try:
|
||||
async with async_session() as db:
|
||||
# 执行一轮 ping
|
||||
results = await self._pinger.run_one_round(db)
|
||||
|
||||
if results:
|
||||
# 直接用启用的设备数
|
||||
total = len(results)
|
||||
|
||||
# 处理待发送告警
|
||||
await self._alerter.flush_pending(db, total)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"调度器异常: {e}", exc_info=True)
|
||||
|
||||
# 等待下一轮
|
||||
await asyncio.sleep(settings.PING_INTERVAL_SECONDS)
|
||||
|
||||
logger.info("Ping 调度器已停止")
|
||||
|
||||
def start(self):
|
||||
"""启动调度器(后台任务)"""
|
||||
if self._running:
|
||||
logger.warning("调度器已在运行")
|
||||
return
|
||||
self._task = asyncio.create_task(self._run_loop())
|
||||
|
||||
async def stop(self):
|
||||
"""停止调度器"""
|
||||
self._running = False
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._task = None
|
||||
|
||||
|
||||
# 全局调度器实例
|
||||
scheduler = PingScheduler()
|
||||
@@ -0,0 +1,28 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIE2TCCAsGgAwIBAgIDAeJAMA0GCSqGSIb3DQEBCwUAMCYxDjAMBgNVBAoTBWFk
|
||||
bWluMRQwEgYDVQQDDAtjZXJ0X3drMWVlZjAeFw0yNTEyMTAxMDAzMzVaFw00NTEy
|
||||
MTAxMDAzMzVaMCYxDjAMBgNVBAoTBWFkbWluMRQwEgYDVQQDDAtjZXJ0X3drMWVl
|
||||
ZjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK78ajnNnwnutdZ48l92
|
||||
hmqa2c8lP1IcpyB7CYVTKurQxSz5iQorYOVhR1UzluSpU8yiPPeFyTRD0pH+DzqG
|
||||
otc+Alvxnka5DfP1z/P0asogkALJXouRR+YtUY6i1oo5tKlDbIGJNO2aQcOOak9b
|
||||
YoqSRd9y/pq/bPjte7oww29hGQc03LgbNXmIb9n5NGznWCte1c88NUrTz9Dlpn2H
|
||||
r3ncOmHpqpzg5NtXughnHsF4YCF+pPIgWlC0C6MtKk0fuJysCY5wpuA620pGL4zO
|
||||
6QFv920MiWsdVWxcNo0aNQkXKGjEcy3LraXux2k/sH+E0e3GRXGhWYnkrK7i8kg+
|
||||
V+Nm1YFgyGbY7V3yZ0mXPxb+iMlIvz885ViGsRnKqlR0pN1v8NbmzXPu9EkAO3wi
|
||||
T/L4fxnETW1hd/ph+AdQ0jEpeyAMRjcl3kMjauOBqfU/THl7L6aUMXB3d05+JK0Y
|
||||
uTc1nrZ0Qjh/5EG5XyvRSuKNVxpCB1XcAlAaBTuE5art6jQkCJINvFoOjoHZB65K
|
||||
HJVvfMtMuVr7dLSbYOPHH6YJB2fUijNKoXnclAbxldX3fEALsC/SW7zMFF+3FThg
|
||||
Seq3OEpNdbkRlQp5sORHeTkWtMO9A60GHvTqDamIqppC4fk71zIRpmEPlfGsOno5
|
||||
gNhH8+NKH5Cvmfj4s7/S98D/AgMBAAGjEDAOMAwGA1UdEwEB/wQCMAAwDQYJKoZI
|
||||
hvcNAQELBQADggIBAJLW6zjDKZdkfq00r88lM1IxaxRWmUEPQjINOa0GZH/DyAtd
|
||||
fgYf35AN+NDEiIiDnJQTR3N8jbX1JySDClEOzv3wLXRiKhWcX8z4HOoieNmMu9lZ
|
||||
6pf1lEaMX8WNxB475nfOueEEyHegbsbfpmYM/aVAiSxaKb87if7uDnxJeGqK2Ba0
|
||||
c3LJnQ+H87gcAb84G78KXn/XwjRtwaf+Gy06EzycEnckEeN0vni0pRdw8coSudcX
|
||||
726Kb+kZ4961G/XxQx4fKX+E4lXkx1CdxPBAws/uOjfciLnuVdT4yp6URXJKRw+/
|
||||
R+flPtfvHXNDXB8BHj9ul4UJ4oSHGppx0BR80CmWWNDVdaoO+o0WieFOXG2U2PB+
|
||||
p2pleJzckTDuOh5wIe2GFj9WKZTIn4S7ZA6hS5j5ea4bOX5m6cxXVaS5FPHHfHcN
|
||||
FyUOEE4f61tv9omsFyl23YD/gmQzotFY+5YPS8DTb1soSzydKQODCq7SobyK8QFo
|
||||
3FnOw1cnKuWDZbqkZS6TYHFc3doNaH77bBxgSgIBPlJewd98eZZCv+m8/mCOXCbo
|
||||
+7HOgmOHJ1cOEtweCtgrCc2VNvV0w6RSwcZ+WmUAZMajvyEqikVVQzR2LtOOhOyr
|
||||
REkkNRAX3PX69GDQLM9QSgGLv0f6nA5uTX9SFj71Iik6YAuUsvIK6uPpn/2B
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,11 @@
|
||||
fastapi>=0.110.0
|
||||
uvicorn[standard]>=0.29.0
|
||||
sqlalchemy[asyncio]>=2.0.30
|
||||
aiosqlite>=0.20.0
|
||||
asyncpg>=0.29.0
|
||||
pydantic>=2.7.0
|
||||
pydantic-settings>=2.2.0
|
||||
python-jose[cryptography]>=3.3.0
|
||||
httpx>=0.27.0
|
||||
python-multipart>=0.0.9
|
||||
websockets>=12.0
|
||||
Reference in New Issue
Block a user