368 lines
12 KiB
Bash
Executable File
368 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
||
|
||
# H3C ONU设备管理系统 - 系统监控脚本
|
||
# 用于监控系统状态、资源使用和性能指标
|
||
|
||
set -e
|
||
|
||
# 颜色输出
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
BLUE='\033[0;34m'
|
||
PURPLE='\033[0;35m'
|
||
CYAN='\033[0;36m'
|
||
NC='\033[0m' # No Color
|
||
|
||
# 默认值
|
||
INTERVAL=${1:-5} # 监控间隔（秒）
|
||
DURATION=${2:-0} # 监控时长（0表示无限）
|
||
COMPOSE_FILE="docker-compose.yml"
|
||
|
||
# 加载环境变量
|
||
if [ -f ".env" ]; then
|
||
set -a
|
||
source .env
|
||
set +a
|
||
fi
|
||
|
||
# 函数：显示帮助信息
|
||
show_help() {
|
||
echo "H3C ONU设备管理系统 - 系统监控脚本"
|
||
echo ""
|
||
echo "使用方法: $0 [间隔] [时长]"
|
||
echo ""
|
||
echo "参数:"
|
||
echo " 间隔 监控间隔（秒，默认: 5）"
|
||
echo " 时长 监控总时长（秒，0表示无限，默认: 0）"
|
||
echo ""
|
||
echo "示例:"
|
||
echo " $0 # 每5秒监控一次，无限时长"
|
||
echo " $0 10 60 # 每10秒监控一次，持续60秒"
|
||
echo ""
|
||
}
|
||
|
||
# 函数：检查Docker Compose服务状态
|
||
check_services_status() {
|
||
echo -e "${BLUE}=== 服务状态 ===${NC}"
|
||
|
||
if ! command -v docker-compose &> /dev/null; then
|
||
echo -e "${RED}错误: docker-compose 未安装${NC}"
|
||
return
|
||
fi
|
||
|
||
# 获取服务状态
|
||
SERVICES=$(docker-compose -f "$COMPOSE_FILE" ps --services)
|
||
|
||
for service in $SERVICES; do
|
||
STATUS=$(docker-compose -f "$COMPOSE_FILE" ps --filter "name=$service" --format "table {{.Status}}" | tail -1)
|
||
CONTAINER_ID=$(docker-compose -f "$COMPOSE_FILE" ps -q "$service")
|
||
|
||
if [ -n "$CONTAINER_ID" ]; then
|
||
# 检查容器是否运行
|
||
if docker inspect --format='{{.State.Status}}' "$CONTAINER_ID" | grep -q "running"; then
|
||
# 检查健康状态
|
||
HEALTH=$(docker inspect --format='{{if .State.Health}}{{.State.Health.Status}}{{else}}no healthcheck{{end}}' "$CONTAINER_ID")
|
||
|
||
if [ "$HEALTH" = "healthy" ]; then
|
||
echo -e "${GREEN}✓ $service: 运行中 (健康)${NC}"
|
||
elif [ "$HEALTH" = "no healthcheck" ]; then
|
||
echo -e "${GREEN}✓ $service: 运行中${NC}"
|
||
else
|
||
echo -e "${YELLOW}⚠ $service: 运行中 ($HEALTH)${NC}"
|
||
fi
|
||
else
|
||
echo -e "${RED}✗ $service: 未运行${NC}"
|
||
fi
|
||
else
|
||
echo -e "${RED}✗ $service: 容器不存在${NC}"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# 函数：检查系统资源使用
|
||
check_system_resources() {
|
||
echo -e "\n${BLUE}=== 系统资源 ===${NC}"
|
||
|
||
# CPU使用率
|
||
CPU_USAGE=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1)
|
||
echo -e "CPU使用率: ${CYAN}${CPU_USAGE}%${NC}"
|
||
|
||
# 内存使用
|
||
MEM_TOTAL=$(free -m | awk '/Mem:/ {print $2}')
|
||
MEM_USED=$(free -m | awk '/Mem:/ {print $3}')
|
||
MEM_PERCENT=$((MEM_USED * 100 / MEM_TOTAL))
|
||
|
||
if [ $MEM_PERCENT -lt 70 ]; then
|
||
echo -e "内存使用: ${GREEN}${MEM_USED}MB/${MEM_TOTAL}MB (${MEM_PERCENT}%)${NC}"
|
||
elif [ $MEM_PERCENT -lt 90 ]; then
|
||
echo -e "内存使用: ${YELLOW}${MEM_USED}MB/${MEM_TOTAL}MB (${MEM_PERCENT}%)${NC}"
|
||
else
|
||
echo -e "内存使用: ${RED}${MEM_USED}MB/${MEM_TOTAL}MB (${MEM_PERCENT}%)${NC}"
|
||
fi
|
||
|
||
# 磁盘使用
|
||
DISK_USAGE=$(df -h / | awk 'NR==2 {print $5}' | cut -d'%' -f1)
|
||
DISK_TOTAL=$(df -h / | awk 'NR==2 {print $2}')
|
||
DISK_USED=$(df -h / | awk 'NR==2 {print $3}')
|
||
|
||
if [ $DISK_USAGE -lt 70 ]; then
|
||
echo -e "磁盘使用: ${GREEN}${DISK_USED}/${DISK_TOTAL} (${DISK_USAGE}%)${NC}"
|
||
elif [ $DISK_USAGE -lt 90 ]; then
|
||
echo -e "磁盘使用: ${YELLOW}${DISK_USED}/${DISK_TOTAL} (${DISK_USAGE}%)${NC}"
|
||
else
|
||
echo -e "磁盘使用: ${RED}${DISK_USED}/${DISK_TOTAL} (${DISK_USAGE}%)${NC}"
|
||
fi
|
||
}
|
||
|
||
# 函数：检查容器资源使用
|
||
check_container_resources() {
|
||
echo -e "\n${BLUE}=== 容器资源 ===${NC}"
|
||
|
||
if ! command -v docker &> /dev/null; then
|
||
echo -e "${RED}错误: docker 未安装${NC}"
|
||
return
|
||
fi
|
||
|
||
# 获取所有运行中的容器
|
||
CONTAINERS=$(docker ps --format "{{.Names}}")
|
||
|
||
for container in $CONTAINERS; do
|
||
# 获取容器资源使用
|
||
STATS=$(docker stats --no-stream --format "{{.CPUPerc}} {{.MemUsage}} {{.MemPerc}}" "$container" 2>/dev/null || echo "0% 0B/0B 0%")
|
||
|
||
CPU=$(echo "$STATS" | awk '{print $1}')
|
||
MEM_USAGE=$(echo "$STATS" | awk '{print $2}')
|
||
MEM_PERCENT=$(echo "$STATS" | awk '{print $3}' | cut -d'%' -f1)
|
||
|
||
# 检查是否属于本项目
|
||
if echo "$container" | grep -q "h3c-onu-ms"; then
|
||
COLOR="$CYAN"
|
||
else
|
||
COLOR="$PURPLE"
|
||
fi
|
||
|
||
echo -e "${COLOR}$container${NC}: CPU=${CPU}, 内存=${MEM_USAGE} (${MEM_PERCENT}%)"
|
||
done
|
||
}
|
||
|
||
# 函数：检查应用健康状态
|
||
check_application_health() {
|
||
echo -e "\n${BLUE}=== 应用健康 ===${NC}"
|
||
|
||
# 检查后端API
|
||
if curl -s -f http://localhost:8000/health > /dev/null 2>&1; then
|
||
echo -e "${GREEN}✓ 后端API: 健康${NC}"
|
||
|
||
# 获取详细健康信息
|
||
HEALTH_INFO=$(curl -s http://localhost:8000/health)
|
||
echo " 详细信息: $HEALTH_INFO"
|
||
else
|
||
echo -e "${RED}✗ 后端API: 不可用${NC}"
|
||
fi
|
||
|
||
# 检查前端
|
||
if curl -s -f http://localhost:8080 > /dev/null 2>&1; then
|
||
echo -e "${GREEN}✓ 前端应用: 可访问${NC}"
|
||
else
|
||
echo -e "${RED}✗ 前端应用: 不可访问${NC}"
|
||
fi
|
||
|
||
# 检查Redis
|
||
REDIS_CONTAINER=$(docker-compose -f "$COMPOSE_FILE" ps -q redis 2>/dev/null || echo "")
|
||
if [ -n "$REDIS_CONTAINER" ]; then
|
||
if docker exec "$REDIS_CONTAINER" redis-cli ping > /dev/null 2>&1; then
|
||
echo -e "${GREEN}✓ Redis: 运行正常${NC}"
|
||
else
|
||
echo -e "${RED}✗ Redis: 连接失败${NC}"
|
||
fi
|
||
fi
|
||
|
||
# 检查数据库连接
|
||
if [ -n "$DATABASE_URL" ]; then
|
||
# 简单检查数据库连接
|
||
if command -v pg_isready &> /dev/null; then
|
||
# 从DATABASE_URL解析主机和端口
|
||
DB_HOST=$(echo "$DATABASE_URL" | sed -n 's/.*@\([^:]*\).*/\1/p')
|
||
DB_PORT=$(echo "$DATABASE_URL" | sed -n 's/.*:\([0-9]*\)\/.*/\1/p')
|
||
|
||
if pg_isready -h "$DB_HOST" -p "$DB_PORT" > /dev/null 2>&1; then
|
||
echo -e "${GREEN}✓ 数据库: 可连接${NC}"
|
||
else
|
||
echo -e "${RED}✗ 数据库: 连接失败${NC}"
|
||
fi
|
||
fi
|
||
fi
|
||
}
|
||
|
||
# 函数：检查网络连接
|
||
check_network() {
|
||
echo -e "\n${BLUE}=== 网络连接 ===${NC}"
|
||
|
||
# 检查互联网连接
|
||
if ping -c 1 -W 2 8.8.8.8 > /dev/null 2>&1; then
|
||
echo -e "${GREEN}✓ 互联网: 可连接${NC}"
|
||
else
|
||
echo -e "${YELLOW}⚠ 互联网: 连接失败${NC}"
|
||
fi
|
||
|
||
# 检查DNS
|
||
if nslookup google.com > /dev/null 2>&1; then
|
||
echo -e "${GREEN}✓ DNS: 工作正常${NC}"
|
||
else
|
||
echo -e "${YELLOW}⚠ DNS: 解析失败${NC}"
|
||
fi
|
||
|
||
# 检查端口监听
|
||
echo -e "端口监听:"
|
||
PORTS="80 443 8000 8080 5432 6379"
|
||
for port in $PORTS; do
|
||
if ss -tuln | grep -q ":$port "; then
|
||
SERVICE=$(ss -tuln | grep ":$port " | awk '{print $5}' | cut -d':' -f2)
|
||
echo -e " ${GREEN}✓ 端口 $port: 被监听 ($SERVICE)${NC}"
|
||
else
|
||
echo -e " ${YELLOW}⚠ 端口 $port: 未监听${NC}"
|
||
fi
|
||
done
|
||
}
|
||
|
||
# 函数：检查日志错误
|
||
check_log_errors() {
|
||
echo -e "\n${BLUE}=== 日志检查 ===${NC}"
|
||
|
||
# 检查最近错误日志
|
||
LOG_FILES="/var/log/syslog /var/log/messages /var/log/docker/*.log ../../backend/logs/*.log"
|
||
|
||
for log_file in $LOG_FILES; do
|
||
if [ -f "$log_file" ]; then
|
||
ERROR_COUNT=$(tail -100 "$log_file" | grep -i "error\|exception\|failed\|critical" | wc -l)
|
||
if [ "$ERROR_COUNT" -gt 0 ]; then
|
||
echo -e "${YELLOW}⚠ $log_file: 最近有 $ERROR_COUNT 个错误${NC}"
|
||
|
||
# 显示最近3个错误
|
||
tail -100 "$log_file" | grep -i "error\|exception\|failed\|critical" | tail -3 | while read line; do
|
||
echo " - $line"
|
||
done
|
||
fi
|
||
fi
|
||
done
|
||
}
|
||
|
||
# 函数：检查定时任务
|
||
check_scheduled_tasks() {
|
||
echo -e "\n${BLUE}=== 定时任务 ===${NC}"
|
||
|
||
# 检查Celery Beat
|
||
BEAT_CONTAINER=$(docker-compose -f "$COMPOSE_FILE" ps -q celery-beat 2>/dev/null || echo "")
|
||
if [ -n "$BEAT_CONTAINER" ]; then
|
||
if docker logs --tail 10 "$BEAT_CONTAINER" 2>/dev/null | grep -q "beat"; then
|
||
echo -e "${GREEN}✓ Celery Beat: 运行中${NC}"
|
||
else
|
||
echo -e "${YELLOW}⚠ Celery Beat: 可能异常${NC}"
|
||
fi
|
||
fi
|
||
|
||
# 检查Celery Worker
|
||
WORKER_CONTAINER=$(docker-compose -f "$COMPOSE_FILE" ps -q celery-worker 2>/dev/null || echo "")
|
||
if [ -n "$WORKER_CONTAINER" ]; then
|
||
ACTIVE_TASKS=$(docker exec "$WORKER_CONTAINER" celery -A app.core.celery_app inspect active 2>/dev/null | grep -c "worker")
|
||
if [ "$ACTIVE_TASKS" -gt 0 ]; then
|
||
echo -e "${GREEN}✓ Celery Worker: 有 $ACTIVE_TASKS 个活跃任务${NC}"
|
||
else
|
||
echo -e "${CYAN}ℹ Celery Worker: 无活跃任务${NC}"
|
||
fi
|
||
fi
|
||
}
|
||
|
||
# 函数：生成监控报告
|
||
generate_report() {
|
||
local timestamp=$(date "+%Y-%m-%d %H:%M:%S")
|
||
|
||
echo -e "\n${PURPLE}========================================${NC}"
|
||
echo -e "${PURPLE} 监控报告 - $timestamp${NC}"
|
||
echo -e "${PURPLE}========================================${NC}"
|
||
|
||
check_services_status
|
||
check_system_resources
|
||
check_container_resources
|
||
check_application_health
|
||
check_network
|
||
check_log_errors
|
||
check_scheduled_tasks
|
||
|
||
echo -e "\n${PURPLE}========================================${NC}"
|
||
}
|
||
|
||
# 函数：连续监控
|
||
continuous_monitoring() {
|
||
local start_time=$(date +%s)
|
||
local iteration=1
|
||
|
||
echo -e "${GREEN}开始监控，间隔: ${INTERVAL}秒${NC}"
|
||
if [ "$DURATION" -gt 0 ]; then
|
||
echo -e "监控时长: ${DURATION}秒${NC}"
|
||
else
|
||
echo -e "监控时长: 无限${NC}"
|
||
fi
|
||
echo "按 Ctrl+C 停止监控"
|
||
echo ""
|
||
|
||
while true; do
|
||
# 检查是否达到监控时长
|
||
if [ "$DURATION" -gt 0 ]; then
|
||
local current_time=$(date +%s)
|
||
local elapsed=$((current_time - start_time))
|
||
|
||
if [ $elapsed -ge $DURATION ]; then
|
||
echo -e "\n${GREEN}监控时长达到，停止监控${NC}"
|
||
break
|
||
fi
|
||
fi
|
||
|
||
# 清屏并显示报告
|
||
clear
|
||
echo -e "${BLUE}监控迭代 #$iteration - $(date "+%H:%M:%S")${NC}"
|
||
generate_report
|
||
|
||
# 等待间隔
|
||
sleep "$INTERVAL"
|
||
iteration=$((iteration + 1))
|
||
done
|
||
}
|
||
|
||
# 函数：单次监控
|
||
single_monitoring() {
|
||
generate_report
|
||
}
|
||
|
||
# 主程序
|
||
main() {
|
||
# 显示标题
|
||
echo -e "${GREEN}========================================${NC}"
|
||
echo -e "${GREEN} H3C ONU设备管理系统 - 系统监控${NC}"
|
||
echo -e "${GREEN}========================================${NC}"
|
||
echo ""
|
||
|
||
# 检查参数
|
||
if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
|
||
show_help
|
||
exit 0
|
||
fi
|
||
|
||
# 检查Docker Compose文件
|
||
if [ ! -f "$COMPOSE_FILE" ]; then
|
||
echo -e "${YELLOW}警告: $COMPOSE_FILE 不存在，使用默认配置${NC}"
|
||
fi
|
||
|
||
# 选择监控模式
|
||
if [ "$DURATION" -eq 0 ] && [ "$INTERVAL" -eq 5 ]; then
|
||
# 默认单次监控
|
||
single_monitoring
|
||
else
|
||
# 连续监控
|
||
continuous_monitoring
|
||
fi
|
||
}
|
||
|
||
# 执行主程序
|
||
main "$@" |