#!/bin/bash # H3C ONU设备管理系统 - 系统监控脚本 # 用于监控系统状态、资源使用和性能指标 set -e # 颜色输出 RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' PURPLE='\033[0;35m' CYAN='\033[0;36m' NC='\033[0m' # No Color # 默认值 INTERVAL=${1:-5} # 监控间隔(秒) DURATION=${2:-0} # 监控时长(0表示无限) COMPOSE_FILE="docker-compose.yml" # 加载环境变量 if [ -f ".env" ]; then set -a source .env set +a fi # 函数:显示帮助信息 show_help() { echo "H3C ONU设备管理系统 - 系统监控脚本" echo "" echo "使用方法: $0 [间隔] [时长]" echo "" echo "参数:" echo " 间隔 监控间隔(秒,默认: 5)" echo " 时长 监控总时长(秒,0表示无限,默认: 0)" echo "" echo "示例:" echo " $0 # 每5秒监控一次,无限时长" echo " $0 10 60 # 每10秒监控一次,持续60秒" echo "" } # 函数:检查Docker Compose服务状态 check_services_status() { echo -e "${BLUE}=== 服务状态 ===${NC}" if ! command -v docker-compose &> /dev/null; then echo -e "${RED}错误: docker-compose 未安装${NC}" return fi # 获取服务状态 SERVICES=$(docker-compose -f "$COMPOSE_FILE" ps --services) for service in $SERVICES; do STATUS=$(docker-compose -f "$COMPOSE_FILE" ps --filter "name=$service" --format "table {{.Status}}" | tail -1) CONTAINER_ID=$(docker-compose -f "$COMPOSE_FILE" ps -q "$service") if [ -n "$CONTAINER_ID" ]; then # 检查容器是否运行 if docker inspect --format='{{.State.Status}}' "$CONTAINER_ID" | grep -q "running"; then # 检查健康状态 HEALTH=$(docker inspect --format='{{if .State.Health}}{{.State.Health.Status}}{{else}}no healthcheck{{end}}' "$CONTAINER_ID") if [ "$HEALTH" = "healthy" ]; then echo -e "${GREEN}✓ $service: 运行中 (健康)${NC}" elif [ "$HEALTH" = "no healthcheck" ]; then echo -e "${GREEN}✓ $service: 运行中${NC}" else echo -e "${YELLOW}⚠ $service: 运行中 ($HEALTH)${NC}" fi else echo -e "${RED}✗ $service: 未运行${NC}" fi else echo -e "${RED}✗ $service: 容器不存在${NC}" fi done } # 函数:检查系统资源使用 check_system_resources() { echo -e "\n${BLUE}=== 系统资源 ===${NC}" # CPU使用率 CPU_USAGE=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1) echo -e "CPU使用率: ${CYAN}${CPU_USAGE}%${NC}" # 内存使用 MEM_TOTAL=$(free -m | awk '/Mem:/ {print $2}') MEM_USED=$(free -m | awk '/Mem:/ {print $3}') MEM_PERCENT=$((MEM_USED * 100 / MEM_TOTAL)) if [ $MEM_PERCENT -lt 70 ]; then echo -e "内存使用: ${GREEN}${MEM_USED}MB/${MEM_TOTAL}MB (${MEM_PERCENT}%)${NC}" elif [ $MEM_PERCENT -lt 90 ]; then echo -e "内存使用: ${YELLOW}${MEM_USED}MB/${MEM_TOTAL}MB (${MEM_PERCENT}%)${NC}" else echo -e "内存使用: ${RED}${MEM_USED}MB/${MEM_TOTAL}MB (${MEM_PERCENT}%)${NC}" fi # 磁盘使用 DISK_USAGE=$(df -h / | awk 'NR==2 {print $5}' | cut -d'%' -f1) DISK_TOTAL=$(df -h / | awk 'NR==2 {print $2}') DISK_USED=$(df -h / | awk 'NR==2 {print $3}') if [ $DISK_USAGE -lt 70 ]; then echo -e "磁盘使用: ${GREEN}${DISK_USED}/${DISK_TOTAL} (${DISK_USAGE}%)${NC}" elif [ $DISK_USAGE -lt 90 ]; then echo -e "磁盘使用: ${YELLOW}${DISK_USED}/${DISK_TOTAL} (${DISK_USAGE}%)${NC}" else echo -e "磁盘使用: ${RED}${DISK_USED}/${DISK_TOTAL} (${DISK_USAGE}%)${NC}" fi } # 函数:检查容器资源使用 check_container_resources() { echo -e "\n${BLUE}=== 容器资源 ===${NC}" if ! command -v docker &> /dev/null; then echo -e "${RED}错误: docker 未安装${NC}" return fi # 获取所有运行中的容器 CONTAINERS=$(docker ps --format "{{.Names}}") for container in $CONTAINERS; do # 获取容器资源使用 STATS=$(docker stats --no-stream --format "{{.CPUPerc}} {{.MemUsage}} {{.MemPerc}}" "$container" 2>/dev/null || echo "0% 0B/0B 0%") CPU=$(echo "$STATS" | awk '{print $1}') MEM_USAGE=$(echo "$STATS" | awk '{print $2}') MEM_PERCENT=$(echo "$STATS" | awk '{print $3}' | cut -d'%' -f1) # 检查是否属于本项目 if echo "$container" | grep -q "h3c-onu-ms"; then COLOR="$CYAN" else COLOR="$PURPLE" fi echo -e "${COLOR}$container${NC}: CPU=${CPU}, 内存=${MEM_USAGE} (${MEM_PERCENT}%)" done } # 函数:检查应用健康状态 check_application_health() { echo -e "\n${BLUE}=== 应用健康 ===${NC}" # 检查后端API if curl -s -f http://localhost:8000/health > /dev/null 2>&1; then echo -e "${GREEN}✓ 后端API: 健康${NC}" # 获取详细健康信息 HEALTH_INFO=$(curl -s http://localhost:8000/health) echo " 详细信息: $HEALTH_INFO" else echo -e "${RED}✗ 后端API: 不可用${NC}" fi # 检查前端 if curl -s -f http://localhost:8080 > /dev/null 2>&1; then echo -e "${GREEN}✓ 前端应用: 可访问${NC}" else echo -e "${RED}✗ 前端应用: 不可访问${NC}" fi # 检查Redis REDIS_CONTAINER=$(docker-compose -f "$COMPOSE_FILE" ps -q redis 2>/dev/null || echo "") if [ -n "$REDIS_CONTAINER" ]; then if docker exec "$REDIS_CONTAINER" redis-cli ping > /dev/null 2>&1; then echo -e "${GREEN}✓ Redis: 运行正常${NC}" else echo -e "${RED}✗ Redis: 连接失败${NC}" fi fi # 检查数据库连接 if [ -n "$DATABASE_URL" ]; then # 简单检查数据库连接 if command -v pg_isready &> /dev/null; then # 从DATABASE_URL解析主机和端口 DB_HOST=$(echo "$DATABASE_URL" | sed -n 's/.*@\([^:]*\).*/\1/p') DB_PORT=$(echo "$DATABASE_URL" | sed -n 's/.*:\([0-9]*\)\/.*/\1/p') if pg_isready -h "$DB_HOST" -p "$DB_PORT" > /dev/null 2>&1; then echo -e "${GREEN}✓ 数据库: 可连接${NC}" else echo -e "${RED}✗ 数据库: 连接失败${NC}" fi fi fi } # 函数:检查网络连接 check_network() { echo -e "\n${BLUE}=== 网络连接 ===${NC}" # 检查互联网连接 if ping -c 1 -W 2 8.8.8.8 > /dev/null 2>&1; then echo -e "${GREEN}✓ 互联网: 可连接${NC}" else echo -e "${YELLOW}⚠ 互联网: 连接失败${NC}" fi # 检查DNS if nslookup google.com > /dev/null 2>&1; then echo -e "${GREEN}✓ DNS: 工作正常${NC}" else echo -e "${YELLOW}⚠ DNS: 解析失败${NC}" fi # 检查端口监听 echo -e "端口监听:" PORTS="80 443 8000 8080 5432 6379" for port in $PORTS; do if ss -tuln | grep -q ":$port "; then SERVICE=$(ss -tuln | grep ":$port " | awk '{print $5}' | cut -d':' -f2) echo -e " ${GREEN}✓ 端口 $port: 被监听 ($SERVICE)${NC}" else echo -e " ${YELLOW}⚠ 端口 $port: 未监听${NC}" fi done } # 函数:检查日志错误 check_log_errors() { echo -e "\n${BLUE}=== 日志检查 ===${NC}" # 检查最近错误日志 LOG_FILES="/var/log/syslog /var/log/messages /var/log/docker/*.log ../../backend/logs/*.log" for log_file in $LOG_FILES; do if [ -f "$log_file" ]; then ERROR_COUNT=$(tail -100 "$log_file" | grep -i "error\|exception\|failed\|critical" | wc -l) if [ "$ERROR_COUNT" -gt 0 ]; then echo -e "${YELLOW}⚠ $log_file: 最近有 $ERROR_COUNT 个错误${NC}" # 显示最近3个错误 tail -100 "$log_file" | grep -i "error\|exception\|failed\|critical" | tail -3 | while read line; do echo " - $line" done fi fi done } # 函数:检查定时任务 check_scheduled_tasks() { echo -e "\n${BLUE}=== 定时任务 ===${NC}" # 检查Celery Beat BEAT_CONTAINER=$(docker-compose -f "$COMPOSE_FILE" ps -q celery-beat 2>/dev/null || echo "") if [ -n "$BEAT_CONTAINER" ]; then if docker logs --tail 10 "$BEAT_CONTAINER" 2>/dev/null | grep -q "beat"; then echo -e "${GREEN}✓ Celery Beat: 运行中${NC}" else echo -e "${YELLOW}⚠ Celery Beat: 可能异常${NC}" fi fi # 检查Celery Worker WORKER_CONTAINER=$(docker-compose -f "$COMPOSE_FILE" ps -q celery-worker 2>/dev/null || echo "") if [ -n "$WORKER_CONTAINER" ]; then ACTIVE_TASKS=$(docker exec "$WORKER_CONTAINER" celery -A app.core.celery_app inspect active 2>/dev/null | grep -c "worker") if [ "$ACTIVE_TASKS" -gt 0 ]; then echo -e "${GREEN}✓ Celery Worker: 有 $ACTIVE_TASKS 个活跃任务${NC}" else echo -e "${CYAN}ℹ Celery Worker: 无活跃任务${NC}" fi fi } # 函数:生成监控报告 generate_report() { local timestamp=$(date "+%Y-%m-%d %H:%M:%S") echo -e "\n${PURPLE}========================================${NC}" echo -e "${PURPLE} 监控报告 - $timestamp${NC}" echo -e "${PURPLE}========================================${NC}" check_services_status check_system_resources check_container_resources check_application_health check_network check_log_errors check_scheduled_tasks echo -e "\n${PURPLE}========================================${NC}" } # 函数:连续监控 continuous_monitoring() { local start_time=$(date +%s) local iteration=1 echo -e "${GREEN}开始监控,间隔: ${INTERVAL}秒${NC}" if [ "$DURATION" -gt 0 ]; then echo -e "监控时长: ${DURATION}秒${NC}" else echo -e "监控时长: 无限${NC}" fi echo "按 Ctrl+C 停止监控" echo "" while true; do # 检查是否达到监控时长 if [ "$DURATION" -gt 0 ]; then local current_time=$(date +%s) local elapsed=$((current_time - start_time)) if [ $elapsed -ge $DURATION ]; then echo -e "\n${GREEN}监控时长达到,停止监控${NC}" break fi fi # 清屏并显示报告 clear echo -e "${BLUE}监控迭代 #$iteration - $(date "+%H:%M:%S")${NC}" generate_report # 等待间隔 sleep "$INTERVAL" iteration=$((iteration + 1)) done } # 函数:单次监控 single_monitoring() { generate_report } # 主程序 main() { # 显示标题 echo -e "${GREEN}========================================${NC}" echo -e "${GREEN} H3C ONU设备管理系统 - 系统监控${NC}" echo -e "${GREEN}========================================${NC}" echo "" # 检查参数 if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then show_help exit 0 fi # 检查Docker Compose文件 if [ ! -f "$COMPOSE_FILE" ]; then echo -e "${YELLOW}警告: $COMPOSE_FILE 不存在,使用默认配置${NC}" fi # 选择监控模式 if [ "$DURATION" -eq 0 ] && [ "$INTERVAL" -eq 5 ]; then # 默认单次监控 single_monitoring else # 连续监控 continuous_monitoring fi } # 执行主程序 main "$@"