手动模式

gx_gp_monitor/core/__init__.py

@@ -0,0 +1 @@
+"""核心模块"""

gx_gp_monitor/core/config_manager.py

@@ -0,0 +1,394 @@
+"""
+配置管理模块
+负责加载、验证和管理系统配置
+"""
+
+import os
+import yaml
+from typing import Dict, Any, Optional
+from pathlib import Path
+from dataclasses import dataclass
+from enum import Enum
+
+
+class LogLevel(Enum):
+    """日志级别枚举"""
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+
+@dataclass
+class CrawlerConfig:
+    """爬虫配置"""
+    base_url: str
+    timeout: int
+    max_retries: int
+    retry_delay: float
+    max_retry_delay: float
+    backoff_factor: float
+    user_agents: list
+    proxies: list
+    request_delay: float
+    request_delay_max: float
+    keyword: list
+    start_date: str
+    end_date: str
+    max_pages: int
+    page_size: int
+
+
+@dataclass
+class DatabaseConfig:
+    """数据库配置"""
+    enabled: bool
+    type: str
+    host: str
+    port: int
+    name: str
+    user: str
+    password: str
+    pool_size: int
+    max_overflow: int
+    pool_timeout: int
+    pool_recycle: int
+    data_retention_days: int
+    auto_cleanup: bool
+
+
+@dataclass
+class WeChatConfig:
+    """企业微信配置"""
+    enabled: bool
+    corp_id: str
+    agent_id: str
+    secret: str
+    token: str
+    encoding_aes_key: str
+    port: int
+    host: str
+    debug: bool
+    use_proxy: bool
+    proxy_api_url: str
+
+
+@dataclass
+class MarkdownConfig:
+    """Markdown配置"""
+    enabled: bool
+    output_file: str
+    max_entries: int
+    include_today_highlight: bool
+    template_file: str
+
+
+@dataclass
+class SchedulerConfig:
+    """调度配置"""
+    enabled: bool
+    timezone: str
+    jobs: list
+
+
+@dataclass
+class MonitoringConfig:
+    """监控配置"""
+    enabled: bool
+    health_check_interval: int
+    alert_on_failure: bool
+    max_consecutive_failures: int
+    metrics_enabled: bool
+
+
+@dataclass
+class SystemConfig:
+    """系统配置"""
+    debug: bool
+    log_level: LogLevel
+    log_file: str
+    log_max_size: int
+    log_backup_count: int
+    crawler: CrawlerConfig
+    database: DatabaseConfig
+    wechat_app: WeChatConfig
+    markdown: MarkdownConfig
+    scheduler: Optional[SchedulerConfig]
+    monitoring: MonitoringConfig
+    sources: Dict[str, Dict[str, Any]]
+
+
+class ConfigManager:
+    """配置管理器"""
+
+    def __init__(self, config_file: Optional[str] = None):
+        """
+        初始化配置管理器
+
+        Args:
+            config_file: 配置文件路径，如果为None则使用默认路径
+        """
+        if config_file is None:
+            # 默认配置文件路径
+            current_dir = Path(__file__).parent.parent
+            self.config_file = current_dir / "config" / "config.yaml"
+        else:
+            self.config_file = Path(config_file)
+
+        self._config_data = {}
+        self._config = None
+
+    def load_config(self) -> SystemConfig:
+        """
+        加载配置文件
+
+        Returns:
+            SystemConfig: 系统配置对象
+
+        Raises:
+            FileNotFoundError: 配置文件不存在
+            yaml.YAMLError: 配置文件格式错误
+            ValueError: 配置验证失败
+        """
+        if not self.config_file.exists():
+            raise FileNotFoundError(f"配置文件不存在: {self.config_file}")
+
+        try:
+            with open(self.config_file, 'r', encoding='utf-8') as f:
+                self._config_data = yaml.safe_load(f)
+        except yaml.YAMLError as e:
+            raise yaml.YAMLError(f"配置文件格式错误: {e}")
+
+        # 验证配置
+        self._validate_config()
+
+        # 解析配置
+        self._config = self._parse_config()
+        return self._config
+
+    def _validate_config(self):
+        """验证配置完整性"""
+        required_keys = [
+            'debug', 'log_level', 'log_file', 'log_max_size', 'log_backup_count',
+            'crawler', 'database', 'wechat_app', 'markdown',
+            'monitoring', 'sources'
+        ]
+
+        for key in required_keys:
+            if key not in self._config_data:
+                raise ValueError(f"配置文件缺少必需的配置项: {key}")
+
+        # 验证爬虫配置
+        crawler_required = [
+            'base_url', 'timeout', 'max_retries', 'retry_delay', 'max_retry_delay',
+            'backoff_factor', 'user_agents', 'proxies', 'request_delay',
+            'request_delay_max', 'keyword', 'max_pages', 'page_size'
+        ]
+
+        for key in crawler_required:
+            if key not in self._config_data['crawler']:
+                raise ValueError(f"爬虫配置缺少必需项: {key}")
+
+        # 验证数据库配置
+        if self._config_data.get('database', {}).get('enabled', False):
+            db_required = ['type', 'host', 'port', 'name', 'user', 'password']
+            for key in db_required:
+                if key not in self._config_data['database']:
+                    raise ValueError(f"数据库配置缺少必需项: {key}")
+
+        # 验证企业微信配置
+        if self._config_data.get('wechat_app', {}).get('enabled', False):
+            wechat_required = ['corp_id', 'agent_id', 'secret', 'token', 'encoding_aes_key']
+            for key in wechat_required:
+                if key not in self._config_data['wechat_app']:
+                    raise ValueError(f"企业微信配置缺少必需项: {key}")
+
+    def _parse_config(self) -> SystemConfig:
+        """解析配置数据"""
+        crawler_data = self._config_data['crawler']
+        crawler = CrawlerConfig(
+            base_url=crawler_data['base_url'],
+            timeout=crawler_data['timeout'],
+            max_retries=crawler_data['max_retries'],
+            retry_delay=crawler_data['retry_delay'],
+            max_retry_delay=crawler_data['max_retry_delay'],
+            backoff_factor=crawler_data['backoff_factor'],
+            user_agents=crawler_data['user_agents'],
+            proxies=crawler_data['proxies'],
+            request_delay=crawler_data['request_delay'],
+            request_delay_max=crawler_data['request_delay_max'],
+            keyword=crawler_data['keyword'],
+            start_date=crawler_data.get('start_date', ''),
+            end_date=crawler_data.get('end_date', ''),
+            max_pages=crawler_data['max_pages'],
+            page_size=crawler_data['page_size']
+        )
+
+        db_data = self._config_data['database']
+        database = DatabaseConfig(
+            enabled=db_data.get('enabled', False),
+            type=db_data.get('type', 'postgresql'),
+            host=db_data.get('host', 'localhost'),
+            port=db_data.get('port', 5432),
+            name=db_data.get('name', ''),
+            user=db_data.get('user', ''),
+            password=db_data.get('password', ''),
+            pool_size=db_data.get('pool_size', 5),
+            max_overflow=db_data.get('max_overflow', 10),
+            pool_timeout=db_data.get('pool_timeout', 30),
+            pool_recycle=db_data.get('pool_recycle', 3600),
+            data_retention_days=db_data.get('data_retention_days', 90),
+            auto_cleanup=db_data.get('auto_cleanup', True)
+        )
+
+        wechat_data = self._config_data['wechat_app']
+        wechat_app = WeChatConfig(
+            enabled=wechat_data.get('enabled', False),
+            corp_id=wechat_data.get('corp_id', ''),
+            agent_id=wechat_data.get('agent_id', ''),
+            secret=wechat_data.get('secret', ''),
+            token=wechat_data.get('token', ''),
+            encoding_aes_key=wechat_data.get('encoding_aes_key', ''),
+            port=wechat_data.get('port', 18001),
+            host=wechat_data.get('host', '0.0.0.0'),
+            debug=wechat_data.get('debug', False),
+            use_proxy=wechat_data.get('use_proxy', False),
+            proxy_api_url=wechat_data.get('proxy_api_url', 'https://api.v6ole.top')
+        )
+
+        md_data = self._config_data['markdown']
+        markdown = MarkdownConfig(
+            enabled=md_data.get('enabled', True),
+            output_file=md_data.get('output_file', 'onu.md'),
+            max_entries=md_data.get('max_entries', 1000),
+            include_today_highlight=md_data.get('include_today_highlight', True),
+            template_file=md_data.get('template_file', 'templates/announcement.md')
+        )
+
+        # scheduler配置为可选
+        scheduler = None
+        if 'scheduler' in self._config_data:
+            scheduler_data = self._config_data['scheduler']
+            scheduler = SchedulerConfig(
+                enabled=scheduler_data.get('enabled', False),
+                timezone=scheduler_data.get('timezone', 'Asia/Shanghai'),
+                jobs=scheduler_data.get('jobs', [])
+            )
+
+        monitoring_data = self._config_data['monitoring']
+        monitoring = MonitoringConfig(
+            enabled=monitoring_data.get('enabled', True),
+            health_check_interval=monitoring_data.get('health_check_interval', 300),
+            alert_on_failure=monitoring_data.get('alert_on_failure', True),
+            max_consecutive_failures=monitoring_data.get('max_consecutive_failures', 3),
+            metrics_enabled=monitoring_data.get('metrics_enabled', True)
+        )
+
+        return SystemConfig(
+            debug=self._config_data['debug'],
+            log_level=LogLevel(self._config_data['log_level']),
+            log_file=self._config_data['log_file'],
+            log_max_size=self._config_data['log_max_size'],
+            log_backup_count=self._config_data['log_backup_count'],
+            crawler=crawler,
+            database=database,
+            wechat_app=wechat_app,
+            markdown=markdown,
+            scheduler=scheduler,
+            monitoring=monitoring,
+            sources=self._config_data['sources']
+        )
+
+    def get_config(self) -> SystemConfig:
+        """
+        获取配置对象
+
+        Returns:
+            SystemConfig: 系统配置对象
+
+        Raises:
+            RuntimeError: 配置未加载
+        """
+        if self._config is None:
+            raise RuntimeError("配置未加载，请先调用 load_config()")
+        return self._config
+
+    def reload_config(self) -> SystemConfig:
+        """
+        重新加载配置
+
+        Returns:
+            SystemConfig: 重新加载的系统配置对象
+        """
+        self._config = None
+        return self.load_config()
+
+    def get_value(self, key_path: str, default=None) -> Any:
+        """
+        通过路径获取配置值
+
+        Args:
+            key_path: 配置路径，如 'crawler.timeout' 或 'database.host'
+            default: 默认值
+
+        Returns:
+            配置值或默认值
+        """
+        keys = key_path.split('.')
+        value = self._config_data
+
+        try:
+            for key in keys:
+                value = value[key]
+            return value
+        except (KeyError, TypeError):
+            return default
+
+
+# 全局配置管理器实例
+_config_manager = None
+
+
+def get_config_manager(config_file: Optional[str] = None) -> ConfigManager:
+    """
+    获取全局配置管理器实例
+
+    Args:
+        config_file: 配置文件路径
+
+    Returns:
+        ConfigManager: 配置管理器实例
+    """
+    global _config_manager
+    if _config_manager is None:
+        _config_manager = ConfigManager(config_file)
+    return _config_manager
+
+
+def load_config(config_file: Optional[str] = None) -> SystemConfig:
+    """
+    加载系统配置
+
+    Args:
+        config_file: 配置文件路径
+
+    Returns:
+        SystemConfig: 系统配置对象
+    """
+    manager = get_config_manager(config_file)
+    return manager.load_config()
+
+
+def get_config() -> SystemConfig:
+    """
+    获取当前加载的配置
+
+    Returns:
+        SystemConfig: 系统配置对象
+
+    Raises:
+        RuntimeError: 配置未加载
+    """
+    manager = get_config_manager()
+    return manager.get_config()

gx_gp_monitor/core/database.py

@@ -0,0 +1,612 @@
+"""
+PostgreSQL数据库连接和操作模块
+提供数据库连接池、CRUD操作、数据清理等功能
+"""
+
+import psycopg2
+from psycopg2 import pool, extras
+from psycopg2.extras import RealDictCursor
+from contextlib import contextmanager
+from typing import List, Dict, Any, Optional, Generator
+from datetime import datetime, timedelta
+import threading
+from dataclasses import asdict
+
+from .models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus
+from .config_manager import get_config
+from .logger import get_logger
+from .reliability import retry_on_exception, RetryConfig
+
+
+logger = get_logger(__name__)
+
+
+class DatabaseConnectionPool:
+    """数据库连接池管理器"""
+
+    _instance = None
+    _pool = None
+    _lock = threading.Lock()
+
+    def __new__(cls):
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self):
+        if self._pool is None:
+            self._pool = None
+            self._config = None
+
+    def init_pool(self, config):
+        """
+        初始化连接池
+
+        Args:
+            config: 数据库配置
+        """
+        if self._pool is not None:
+            return
+
+        try:
+            self._config = config
+            self._pool = psycopg2.pool.SimpleConnectionPool(
+                minconn=config.pool_size,
+                maxconn=config.pool_size + config.max_overflow,
+                host=config.host,
+                port=config.port,
+                database=config.name,
+                user=config.user,
+                password=config.password,
+                connect_timeout=config.pool_timeout
+            )
+            logger.info("数据库连接池初始化成功")
+        except Exception as e:
+            logger.error(f"数据库连接池初始化失败: {str(e)}")
+            raise
+
+    def get_connection(self):
+        """
+        获取数据库连接
+
+        Returns:
+            数据库连接对象
+
+        Raises:
+            Exception: 获取连接失败
+        """
+        if self._pool is None:
+            raise Exception("数据库连接池未初始化")
+
+        try:
+            conn = self._pool.getconn()
+            # 设置自动提交为False，需要手动提交
+            conn.autocommit = False
+            return conn
+        except Exception as e:
+            logger.error(f"获取数据库连接失败: {str(e)}")
+            raise
+
+    def return_connection(self, conn):
+        """
+        返回数据库连接到连接池
+
+        Args:
+            conn: 数据库连接对象
+        """
+        if self._pool and conn:
+            try:
+                self._pool.putconn(conn)
+            except Exception as e:
+                logger.warning(f"返回数据库连接失败: {str(e)}")
+
+    def close_all(self):
+        """关闭所有连接"""
+        if self._pool:
+            try:
+                self._pool.closeall()
+                logger.info("数据库连接池已关闭")
+            except Exception as e:
+                logger.error(f"关闭数据库连接池失败: {str(e)}")
+
+
+# 全局连接池实例
+_connection_pool = DatabaseConnectionPool()
+
+
+@contextmanager
+def get_db_connection():
+    """
+    获取数据库连接的上下文管理器
+
+    Yields:
+        数据库连接对象
+    """
+    conn = None
+    try:
+        conn = _connection_pool.get_connection()
+        yield conn
+    except Exception as e:
+        logger.error(f"数据库连接错误: {str(e)}")
+        raise
+    finally:
+        if conn:
+            _connection_pool.return_connection(conn)
+
+
+@contextmanager
+def get_db_cursor(commit: bool = True):
+    """
+    获取数据库游标的上下文管理器
+
+    Args:
+        commit: 是否自动提交事务
+
+    Yields:
+        数据库游标对象
+    """
+    with get_db_connection() as conn:
+        cursor = None
+        try:
+            cursor = conn.cursor(cursor_factory=RealDictCursor)
+            yield cursor
+            if commit:
+                conn.commit()
+        except Exception as e:
+            conn.rollback()
+            logger.error(f"数据库操作错误: {str(e)}")
+            raise
+        finally:
+            if cursor:
+                cursor.close()
+
+
+class DatabaseManager:
+    """数据库管理器"""
+
+    def __init__(self):
+        self.config = get_config()
+        if self.config.database.enabled:
+            _connection_pool.init_pool(self.config.database)
+        else:
+            logger.warning("数据库功能已禁用")
+
+    def init_database(self):
+        """初始化数据库表结构"""
+        if not self.config.database.enabled:
+            return
+
+        logger.info("开始初始化数据库表结构")
+
+        # 创建表的SQL语句
+        create_tables_sql = """
+        -- 公告表
+        CREATE TABLE IF NOT EXISTS announcements (
+            id SERIAL PRIMARY KEY,
+            title VARCHAR(500) NOT NULL,
+            publish_date TIMESTAMP NOT NULL,
+            purchase_name VARCHAR(200),
+            content_url TEXT,
+            source_code VARCHAR(50) NOT NULL,
+            source_name VARCHAR(100) NOT NULL,
+            announcement_type VARCHAR(50) NOT NULL,
+            crawled_at TIMESTAMP,
+            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            content_hash VARCHAR(32) UNIQUE,
+            keyword_matched BOOLEAN DEFAULT FALSE,
+            date_filtered BOOLEAN DEFAULT TRUE,
+            is_new BOOLEAN DEFAULT TRUE,
+            is_today BOOLEAN DEFAULT FALSE
+        );
+
+        -- 公告来源表
+        CREATE TABLE IF NOT EXISTS announcement_sources (
+            code VARCHAR(50) PRIMARY KEY,
+            category_id INTEGER NOT NULL,
+            name VARCHAR(100) NOT NULL,
+            type VARCHAR(50) NOT NULL
+        );
+
+        -- 爬取结果表
+        CREATE TABLE IF NOT EXISTS crawl_results (
+            id SERIAL PRIMARY KEY,
+            source_code VARCHAR(50) NOT NULL,
+            status VARCHAR(20) NOT NULL,
+            total_count INTEGER DEFAULT 0,
+            new_count INTEGER DEFAULT 0,
+            error_message TEXT,
+            crawled_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            duration FLOAT DEFAULT 0.0,
+            FOREIGN KEY (source_code) REFERENCES announcement_sources(code)
+        );
+
+        -- 创建索引
+        CREATE INDEX IF NOT EXISTS idx_announcements_publish_date ON announcements(publish_date DESC);
+        CREATE INDEX IF NOT EXISTS idx_announcements_source_code ON announcements(source_code);
+        CREATE INDEX IF NOT EXISTS idx_announcements_content_hash ON announcements(content_hash);
+        CREATE INDEX IF NOT EXISTS idx_announcements_created_at ON announcements(created_at DESC);
+        CREATE INDEX IF NOT EXISTS idx_crawl_results_crawled_at ON crawl_results(crawled_at DESC);
+
+        -- 创建更新时间触发器
+        CREATE OR REPLACE FUNCTION update_updated_at_column()
+        RETURNS TRIGGER AS $$
+        BEGIN
+            NEW.updated_at = CURRENT_TIMESTAMP;
+            RETURN NEW;
+        END;
+        $$ language 'plpgsql';
+
+        DROP TRIGGER IF EXISTS update_announcements_updated_at ON announcements;
+        CREATE TRIGGER update_announcements_updated_at
+            BEFORE UPDATE ON announcements
+            FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
+        """
+
+        with get_db_cursor() as cursor:
+            cursor.execute(create_tables_sql)
+            logger.info("数据库表结构初始化完成")
+
+    @retry_on_exception(RetryConfig(max_retries=3))
+    def save_announcement(self, announcement: Announcement) -> bool:
+        """
+        保存公告到数据库
+
+        Args:
+            announcement: 公告对象
+
+        Returns:
+            bool: 保存是否成功
+        """
+        if not self.config.database.enabled:
+            return False
+
+        # 生成内容哈希（如果还没有）
+        if not announcement.content_hash:
+            announcement.generate_content_hash()
+
+        sql = """
+        INSERT INTO announcements (
+            title, publish_date, purchase_name, content_url, source_code, source_name,
+            announcement_type, crawled_at, content_hash, keyword_matched,
+            date_filtered, is_new, is_today
+        ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+        ON CONFLICT (content_hash) DO NOTHING
+        """
+
+        values = (
+            announcement.title,
+            announcement.publish_date,
+            announcement.purchase_name,
+            announcement.content_url,
+            announcement.source_code,
+            announcement.source_name,
+            announcement.announcement_type.value,
+            announcement.crawled_at,
+            announcement.content_hash,
+            announcement.keyword_matched,
+            announcement.date_filtered,
+            announcement.is_new,
+            announcement.is_today
+        )
+
+        try:
+            with get_db_cursor() as cursor:
+                cursor.execute(sql, values)
+                affected_rows = cursor.rowcount
+                if affected_rows > 0:
+                    logger.debug(f"成功保存公告: {announcement.title[:50]}...")
+                    return True
+                else:
+                    logger.debug(f"公告已存在，跳过保存: {announcement.title[:50]}...")
+                    return False
+        except Exception as e:
+            logger.error(f"保存公告失败: {str(e)}")
+            return False
+
+    @retry_on_exception(RetryConfig(max_retries=3))
+    def save_announcements_batch(self, announcements: List[Announcement]) -> int:
+        """
+        批量保存公告
+
+        Args:
+            announcements: 公告列表
+
+        Returns:
+            int: 成功保存的数量
+        """
+        if not self.config.database.enabled:
+            return 0
+
+        if not announcements:
+            return 0
+
+        # 为没有哈希的公告生成哈希
+        for announcement in announcements:
+            if not announcement.content_hash:
+                announcement.generate_content_hash()
+
+        sql = """
+        INSERT INTO announcements (
+            title, publish_date, purchase_name, content_url, source_code, source_name,
+            announcement_type, crawled_at, content_hash, keyword_matched,
+            date_filtered, is_new, is_today
+        ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+        ON CONFLICT (content_hash) DO NOTHING
+        """
+
+        values = []
+        for announcement in announcements:
+            values.append((
+                announcement.title,
+                announcement.publish_date,
+                announcement.purchase_name,
+                announcement.content_url,
+                announcement.source_code,
+                announcement.source_name,
+                announcement.announcement_type.value,
+                announcement.crawled_at,
+                announcement.content_hash,
+                announcement.keyword_matched,
+                announcement.date_filtered,
+                announcement.is_new,
+                announcement.is_today
+            ))
+
+        try:
+            with get_db_cursor() as cursor:
+                extras.execute_batch(cursor, sql, values)
+                affected_rows = cursor.rowcount
+                logger.info(f"批量保存公告完成，成功保存 {affected_rows} 条")
+                return affected_rows
+        except Exception as e:
+            logger.error(f"批量保存公告失败: {str(e)}")
+            return 0
+
+    @retry_on_exception(RetryConfig(max_retries=3))
+    def get_announcements(self,
+                         source_code: Optional[str] = None,
+                         start_date: Optional[datetime] = None,
+                         end_date: Optional[datetime] = None,
+                         limit: int = 100,
+                         offset: int = 0) -> List[Announcement]:
+        """
+        查询公告
+
+        Args:
+            source_code: 来源代码过滤
+            start_date: 开始日期
+            end_date: 结束日期
+            limit: 限制数量
+            offset: 偏移量
+
+        Returns:
+            List[Announcement]: 公告列表
+        """
+        if not self.config.database.enabled:
+            return []
+
+        sql = """
+        SELECT * FROM announcements
+        WHERE 1=1
+        """
+        params = []
+
+        if source_code:
+            sql += " AND source_code = %s"
+            params.append(source_code)
+
+        if start_date:
+            sql += " AND publish_date >= %s"
+            params.append(start_date)
+
+        if end_date:
+            sql += " AND publish_date <= %s"
+            params.append(end_date)
+
+        sql += " ORDER BY publish_date DESC LIMIT %s OFFSET %s"
+        params.extend([limit, offset])
+
+        try:
+            with get_db_cursor() as cursor:
+                cursor.execute(sql, params)
+                rows = cursor.fetchall()
+
+                announcements = []
+                for row in rows:
+                    # 转换数据类型
+                    row_dict = dict(row)
+                    row_dict['announcement_type'] = AnnouncementType(row_dict['announcement_type'])
+                    announcements.append(Announcement.from_dict(row_dict))
+
+                return announcements
+        except Exception as e:
+            logger.error(f"查询公告失败: {str(e)}")
+            return []
+
+    @retry_on_exception(RetryConfig(max_retries=3))
+    def save_crawl_result(self, result: CrawlResult) -> bool:
+        """
+        保存爬取结果
+
+        Args:
+            result: 爬取结果对象
+
+        Returns:
+            bool: 保存是否成功
+        """
+        if not self.config.database.enabled:
+            return False
+
+        sql = """
+        INSERT INTO crawl_results (
+            source_code, status, total_count, new_count, error_message,
+            crawled_at, duration
+        ) VALUES (%s, %s, %s, %s, %s, %s, %s)
+        """
+
+        values = (
+            result.source.code,
+            result.status.value,
+            result.total_count,
+            result.new_count,
+            result.error_message,
+            result.crawled_at,
+            result.duration
+        )
+
+        try:
+            with get_db_cursor() as cursor:
+                cursor.execute(sql, values)
+                logger.debug(f"保存爬取结果: {result.source.name}")
+                return True
+        except Exception as e:
+            logger.error(f"保存爬取结果失败: {str(e)}")
+            return False
+
+    @retry_on_exception(RetryConfig(max_retries=3))
+    def cleanup_expired_data(self, days: int = 90) -> int:
+        """
+        清理过期数据
+
+        Args:
+            days: 保留天数
+
+        Returns:
+            int: 清理的记录数
+        """
+        if not self.config.database.enabled:
+            return 0
+
+        cutoff_date = datetime.now() - timedelta(days=days)
+
+        sql = "DELETE FROM announcements WHERE created_at < %s"
+        try:
+            with get_db_cursor() as cursor:
+                cursor.execute(sql, (cutoff_date,))
+                deleted_count = cursor.rowcount
+                logger.info(f"清理过期数据完成，删除 {deleted_count} 条记录")
+                return deleted_count
+        except Exception as e:
+            logger.error(f"清理过期数据失败: {str(e)}")
+            return 0
+
+    @retry_on_exception(RetryConfig(max_retries=3))
+    def get_statistics(self) -> Dict[str, Any]:
+        """
+        获取统计信息
+
+        Returns:
+            Dict[str, Any]: 统计数据
+        """
+        if not self.config.database.enabled:
+            return {}
+
+        sql = """
+        SELECT
+            COUNT(*) as total_announcements,
+            COUNT(CASE WHEN is_today THEN 1 END) as today_announcements,
+            COUNT(CASE WHEN is_new THEN 1 END) as new_announcements,
+            COUNT(DISTINCT source_code) as sources_count,
+            MAX(crawled_at) as last_crawl_time
+        FROM announcements
+        """
+
+        try:
+            with get_db_cursor() as cursor:
+                cursor.execute(sql)
+                result = cursor.fetchone()
+                return dict(result) if result else {}
+        except Exception as e:
+            logger.error(f"获取统计信息失败: {str(e)}")
+            return {}
+
+    def is_announcement_exists(self, content_hash: str) -> bool:
+        """
+        检查公告是否已存在
+
+        Args:
+            content_hash: 内容哈希
+
+        Returns:
+            bool: 是否存在
+        """
+        if not self.config.database.enabled:
+            return False
+
+        sql = "SELECT 1 FROM announcements WHERE content_hash = %s LIMIT 1"
+
+        try:
+            with get_db_cursor() as cursor:
+                cursor.execute(sql, (content_hash,))
+                return cursor.fetchone() is not None
+        except Exception as e:
+            logger.error(f"检查公告存在性失败: {str(e)}")
+            return False
+
+    def get_recent_announcements(self, hours: int = 24) -> List[Announcement]:
+        """
+        获取最近的公告
+
+        Args:
+            hours: 最近小时数
+
+        Returns:
+            List[Announcement]: 公告列表
+        """
+        if not self.config.database.enabled:
+            return []
+
+        cutoff_time = datetime.now() - timedelta(hours=hours)
+
+        sql = """
+        SELECT * FROM announcements
+        WHERE crawled_at >= %s
+        ORDER BY crawled_at DESC
+        """
+
+        try:
+            with get_db_cursor() as cursor:
+                cursor.execute(sql, (cutoff_time,))
+                rows = cursor.fetchall()
+
+                announcements = []
+                for row in rows:
+                    row_dict = dict(row)
+                    row_dict['announcement_type'] = AnnouncementType(row_dict['announcement_type'])
+                    announcements.append(Announcement.from_dict(row_dict))
+
+                return announcements
+        except Exception as e:
+            logger.error(f"获取最近公告失败: {str(e)}")
+            return []
+
+
+# 全局数据库管理器实例
+_db_manager = None
+
+
+def get_database_manager() -> DatabaseManager:
+    """
+    获取数据库管理器实例
+
+    Returns:
+        DatabaseManager: 数据库管理器实例
+    """
+    global _db_manager
+    if _db_manager is None:
+        _db_manager = DatabaseManager()
+    return _db_manager
+
+
+def init_database():
+    """初始化数据库"""
+    manager = get_database_manager()
+    manager.init_database()
+
+
+def cleanup_database():
+    """清理数据库连接"""
+    _connection_pool.close_all()

gx_gp_monitor/core/logger.py

@@ -0,0 +1,354 @@
+"""
+统一日志管理模块
+提供结构化日志记录功能，支持控制台和文件输出
+"""
+
+import os
+import sys
+import logging
+import logging.handlers
+from pathlib import Path
+from typing import Optional, Dict, Any
+from datetime import datetime
+
+from .config_manager import get_config
+
+
+class ColoredFormatter(logging.Formatter):
+    """带颜色的日志格式化器"""
+
+    # ANSI颜色代码
+    COLORS = {
+        'DEBUG': '\033[36m',      # 青色
+        'INFO': '\033[32m',       # 绿色
+        'WARNING': '\033[33m',    # 黄色
+        'ERROR': '\033[31m',      # 红色
+        'CRITICAL': '\033[35m',   # 紫色
+    }
+    RESET = '\033[0m'  # 重置颜色
+
+    def format(self, record):
+        # 检查是否已经包含ANSI颜色代码
+        if '\033[' in record.levelname:
+            # 如果已经着色，直接返回原始格式
+            return super().format(record)
+
+        # 添加颜色
+        if record.levelname in self.COLORS:
+            # 为levelname添加颜色
+            colored_levelname = f"{self.COLORS[record.levelname]}{record.levelname}{self.RESET}"
+            # 使用原始levelname来确定消息的颜色
+            record.msg = f"{self.COLORS[record.levelname]}{record.msg}{self.RESET}"
+            record.levelname = colored_levelname
+
+        return super().format(record)
+
+
+class Logger:
+    """统一日志管理器"""
+
+    _instance = None
+    _initialized = False
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self):
+        if not self._initialized:
+            self._loggers = {}
+            self._config = None
+            self._initialized = True
+
+    def init_logger(self, name: str = "gx_gp_monitor", config=None) -> logging.Logger:
+        """
+        初始化日志器
+
+        Args:
+            name: 日志器名称
+            config: 配置对象，如果为None则从全局配置加载
+
+        Returns:
+            logging.Logger: 配置好的日志器实例
+        """
+        if name in self._loggers:
+            return self._loggers[name]
+
+        # 获取配置
+        if config is None:
+            try:
+                self._config = get_config()
+            except RuntimeError:
+                # 配置未加载，使用默认配置
+                self._config = self._get_default_config()
+        else:
+            self._config = config
+
+        # 创建日志器
+        logger = logging.getLogger(name)
+        logger.setLevel(getattr(logging, self._config.log_level.value))
+
+        # 避免重复添加处理器
+        if logger.handlers:
+            return logger
+
+        # 创建格式化器
+        formatter = logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+            datefmt='%Y-%m-%d %H:%M:%S'
+        )
+
+        # 控制台处理器
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.setLevel(getattr(logging, self._config.log_level.value))
+
+        # 使用彩色格式化器（如果支持）
+        if sys.platform != 'win32' and 'TERM' in os.environ:
+            colored_formatter = ColoredFormatter(
+                '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+                datefmt='%Y-%m-%d %H:%M:%S'
+            )
+            console_handler.setFormatter(colored_formatter)
+        else:
+            console_handler.setFormatter(formatter)
+
+        logger.addHandler(console_handler)
+
+        # 文件处理器（如果配置了日志文件）
+        if self._config.log_file:
+            log_dir = Path(self._config.log_file).parent
+            log_dir.mkdir(parents=True, exist_ok=True)
+
+            file_handler = logging.handlers.RotatingFileHandler(
+                self._config.log_file,
+                maxBytes=self._config.log_max_size,
+                backupCount=self._config.log_backup_count,
+                encoding='utf-8'
+            )
+            file_handler.setLevel(getattr(logging, self._config.log_level.value))
+            file_handler.setFormatter(formatter)
+            logger.addHandler(file_handler)
+
+        self._loggers[name] = logger
+        return logger
+
+    def _get_default_config(self):
+        """获取默认配置"""
+        from .config_manager import LogLevel
+
+        class DefaultConfig:
+            def __init__(self):
+                self.log_level = LogLevel.INFO
+                self.log_file = "logs/gx_gp_monitor.log"
+                self.log_max_size = 10485760  # 10MB
+                self.log_backup_count = 5
+
+        return DefaultConfig()
+
+    def get_logger(self, name: str = "gx_gp_monitor") -> logging.Logger:
+        """
+        获取日志器
+
+        Args:
+            name: 日志器名称
+
+        Returns:
+            logging.Logger: 日志器实例
+        """
+        if name not in self._loggers:
+            return self.init_logger(name)
+        return self._loggers[name]
+
+    def log_crawl_start(self, source_name: str, logger: Optional[logging.Logger] = None):
+        """记录爬取开始"""
+        if logger is None:
+            logger = self.get_logger()
+        logger.info(f"开始爬取 {source_name}")
+
+    def log_crawl_success(self, source_name: str, count: int, duration: float,
+                         logger: Optional[logging.Logger] = None):
+        """记录爬取成功"""
+        if logger is None:
+            logger = self.get_logger()
+        logger.info(f"{source_name} 爬取完成，共获取 {count} 条公告，耗时 {duration:.2f}秒")
+
+    def log_crawl_error(self, source_name: str, error: str,
+                       logger: Optional[logging.Logger] = None):
+        """记录爬取错误"""
+        if logger is None:
+            logger = self.get_logger()
+        logger.error(f"{source_name} 爬取失败: {error}")
+
+    def log_announcement_filtered(self, reason: str, count: int,
+                                 logger: Optional[logging.Logger] = None):
+        """记录公告筛选信息"""
+        if logger is None:
+            logger = self.get_logger()
+        logger.info(f"公告筛选 - {reason}: {count} 条")
+
+    def log_database_operation(self, operation: str, table: str, count: int = 0,
+                              logger: Optional[logging.Logger] = None):
+        """记录数据库操作"""
+        if logger is None:
+            logger = self.get_logger()
+        if count > 0:
+            logger.info(f"数据库操作 - {operation} {table}: {count} 条记录")
+        else:
+            logger.info(f"数据库操作 - {operation} {table}")
+
+    def log_notification_sent(self, channel: str, recipient_count: int,
+                             logger: Optional[logging.Logger] = None):
+        """记录通知发送"""
+        if logger is None:
+            logger = self.get_logger()
+        logger.info(f"通知发送 - {channel}: 向 {recipient_count} 个接收者发送")
+
+    def log_system_metrics(self, metrics: Dict[str, Any],
+                          logger: Optional[logging.Logger] = None):
+        """记录系统指标"""
+        if logger is None:
+            logger = self.get_logger()
+        metrics_str = ", ".join([f"{k}={v}" for k, v in metrics.items()])
+        logger.info(f"系统指标: {metrics_str}")
+
+    def log_performance_warning(self, operation: str, duration: float, threshold: float,
+                               logger: Optional[logging.Logger] = None):
+        """记录性能警告"""
+        if logger is None:
+            logger = self.get_logger()
+        logger.warning(f"性能警告 - {operation} 耗时 {duration:.2f}秒，超过阈值 {threshold:.2f}秒")
+
+
+# 全局日志管理器实例
+_logger_manager = Logger()
+
+
+def get_logger(name: str = "gx_gp_monitor") -> logging.Logger:
+    """
+    获取日志器
+
+    Args:
+        name: 日志器名称
+
+    Returns:
+        logging.Logger: 日志器实例
+    """
+    return _logger_manager.get_logger(name)
+
+
+def init_logger(name: str = "gx_gp_monitor", config=None) -> logging.Logger:
+    """
+    初始化并获取日志器
+
+    Args:
+        name: 日志器名称
+        config: 配置对象
+
+    Returns:
+        logging.Logger: 日志器实例
+    """
+    return _logger_manager.init_logger(name, config)
+
+
+def log_function_call(func_name: str, args: Optional[Dict[str, Any]] = None,
+                     logger: Optional[logging.Logger] = None):
+    """
+    装饰器：记录函数调用
+
+    Args:
+        func_name: 函数名称
+        args: 函数参数
+        logger: 日志器实例
+    """
+    def decorator(func):
+        def wrapper(*args, **kwargs):
+            nonlocal logger
+            if logger is None:
+                logger = get_logger()
+
+            start_time = datetime.now()
+            logger.debug(f"调用函数: {func_name}")
+
+            try:
+                result = func(*args, **kwargs)
+                duration = (datetime.now() - start_time).total_seconds()
+                logger.debug(f"函数 {func_name} 执行完成，耗时 {duration:.3f}秒")
+                return result
+            except Exception as e:
+                duration = (datetime.now() - start_time).total_seconds()
+                logger.error(f"函数 {func_name} 执行失败，耗时 {duration:.3f}秒: {str(e)}")
+                raise
+
+        return wrapper
+    return decorator
+
+
+# 便捷函数
+def log_info(message: str, logger: Optional[logging.Logger] = None):
+    """记录信息日志"""
+    if logger is None:
+        logger = get_logger()
+    logger.info(message)
+
+
+def log_warning(message: str, logger: Optional[logging.Logger] = None):
+    """记录警告日志"""
+    if logger is None:
+        logger = get_logger()
+    logger.warning(message)
+
+
+def log_error(message: str, logger: Optional[logging.Logger] = None):
+    """记录错误日志"""
+    if logger is None:
+        logger = get_logger()
+    logger.error(message)
+
+
+def log_debug(message: str, logger: Optional[logging.Logger] = None):
+    """记录调试日志"""
+    if logger is None:
+        logger = get_logger()
+    logger.debug(message)
+
+
+# 便捷的爬取日志记录函数
+def log_crawl_start(source_name: str):
+    """记录爬取开始"""
+    _logger_manager.log_crawl_start(source_name)
+
+
+def log_crawl_success(source_name: str, count: int, duration: float):
+    """记录爬取成功"""
+    _logger_manager.log_crawl_success(source_name, count, duration)
+
+
+def log_crawl_error(source_name: str, error: str):
+    """记录爬取错误"""
+    _logger_manager.log_crawl_error(source_name, error)
+
+
+def log_announcement_filtered(reason: str, count: int):
+    """记录公告筛选信息"""
+    _logger_manager.log_announcement_filtered(reason, count)
+
+
+def log_database_operation(operation: str, table: str, count: int = 0):
+    """记录数据库操作"""
+    _logger_manager.log_database_operation(operation, table, count)
+
+
+def log_notification_sent(channel: str, recipient_count: int):
+    """记录通知发送"""
+    _logger_manager.log_notification_sent(channel, recipient_count)
+
+
+def log_system_metrics(metrics: Dict[str, Any]):
+    """记录系统指标"""
+    _logger_manager.log_system_metrics(metrics)
+
+
+def log_performance_warning(operation: str, duration: float, threshold: float):
+    """记录性能警告"""
+    _logger_manager.log_performance_warning(operation, duration, threshold)

gx_gp_monitor/core/models.py

@@ -0,0 +1,255 @@
+"""
+数据模型定义
+定义系统使用的数据结构和模型
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Optional, List, Dict, Any
+from enum import Enum
+
+
+class AnnouncementType(Enum):
+    """公告类型枚举"""
+    PURCHASE = "purchase"                    # 采购公告
+    RESULT = "result"                        # 结果公告
+    CONTRACT = "contract"                    # 合同公告
+    CORRECTION = "correction"                # 更正公告
+    PRE_ANNOUNCEMENT = "pre_announcement"    # 招标文件预公示
+    SINGLE_SOURCE = "single_source"          # 单一来源公示
+    ELECTRONIC_MARKET = "electronic_market"  # 电子卖场公示
+    ACCEPTANCE = "acceptance"                # 履约验收公示
+    ENGINEERING = "engineering"              # 工程类公告
+    FRAMEWORK_AGREEMENT = "framework_agreement"  # 框架协议征集公告
+    FRAMEWORK_RESULT = "framework_result"    # 框架协议入围结果公告
+    FRAMEWORK_SUMMARY = "framework_summary"  # 框架协议成交结果汇总公告
+    INTENTION = "intention"                  # 采购意向公开
+
+
+class CrawlStatus(Enum):
+    """爬取状态枚举"""
+    PENDING = "pending"      # 待爬取
+    RUNNING = "running"      # 爬取中
+    SUCCESS = "success"      # 成功
+    FAILED = "failed"        # 失败
+    PARTIAL = "partial"      # 部分成功
+
+
+@dataclass
+class AnnouncementSource:
+    """公告来源"""
+    code: str                    # 来源代码，如 "ZcyAnnouncement1"
+    category_id: int            # 分类ID
+    name: str                   # 显示名称，如 "采购公告"
+    type: AnnouncementType      # 公告类型
+
+
+@dataclass
+class Announcement:
+    """公告数据模型"""
+    id: Optional[int] = None                          # 数据库ID
+    title: str = ""                                   # 公告标题
+    publish_date: datetime = field(default_factory=datetime.now)  # 发布时间
+    purchase_name: str = ""                           # 发布单位
+    content_url: str = ""                            # 内容链接
+    source_code: str = ""                            # 来源代码
+    source_name: str = ""                            # 来源名称
+    announcement_type: AnnouncementType = AnnouncementType.PURCHASE  # 公告类型
+
+    # 爬取相关字段
+    crawled_at: Optional[datetime] = None            # 爬取时间
+    created_at: Optional[datetime] = None            # 创建时间
+    updated_at: Optional[datetime] = None            # 更新时间
+
+    # 去重字段
+    content_hash: Optional[str] = None               # 内容哈希，用于去重
+
+    # 筛选相关
+    keyword_matched: bool = False                    # 是否匹配关键词
+    date_filtered: bool = True                       # 是否在日期范围内
+
+    # 业务字段
+    is_new: bool = True                              # 是否为新公告
+    is_today: bool = False                           # 是否为今日公告
+
+    def __post_init__(self):
+        """后初始化处理"""
+        if isinstance(self.announcement_type, str):
+            self.announcement_type = AnnouncementType(self.announcement_type)
+
+        if self.publish_date and isinstance(self.publish_date, str):
+            try:
+                self.publish_date = datetime.fromisoformat(self.publish_date.replace('Z', '+00:00'))
+            except ValueError:
+                # 如果解析失败，使用当前时间
+                self.publish_date = datetime.now()
+
+        # 判断是否为今日公告
+        today = datetime.now().date()
+        if self.publish_date:
+            self.is_today = self.publish_date.date() == today
+
+    @property
+    def publish_date_str(self) -> str:
+        """获取发布日期字符串"""
+        return self.publish_date.strftime("%Y-%m-%d") if self.publish_date else ""
+
+    @property
+    def crawled_at_str(self) -> str:
+        """获取爬取时间字符串"""
+        return self.crawled_at.strftime("%Y-%m-%d %H:%M:%S") if self.crawled_at else ""
+
+    def to_dict(self) -> Dict[str, Any]:
+        """转换为字典"""
+        return {
+            "id": self.id,
+            "title": self.title,
+            "publish_date": self.publish_date.isoformat() if self.publish_date else None,
+            "purchase_name": self.purchase_name,
+            "content_url": self.content_url,
+            "source_code": self.source_code,
+            "source_name": self.source_name,
+            "announcement_type": self.announcement_type.value,
+            "crawled_at": self.crawled_at.isoformat() if self.crawled_at else None,
+            "created_at": self.created_at.isoformat() if self.created_at else None,
+            "updated_at": self.updated_at.isoformat() if self.updated_at else None,
+            "content_hash": self.content_hash,
+            "keyword_matched": self.keyword_matched,
+            "date_filtered": self.date_filtered,
+            "is_new": self.is_new,
+            "is_today": self.is_today
+        }
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> 'Announcement':
+        """从字典创建实例"""
+        # 处理枚举类型
+        if 'announcement_type' in data and isinstance(data['announcement_type'], str):
+            data['announcement_type'] = AnnouncementType(data['announcement_type'])
+
+        # 处理日期时间
+        for date_field in ['publish_date', 'crawled_at', 'created_at', 'updated_at']:
+            if date_field in data and data[date_field] and isinstance(data[date_field], str):
+                try:
+                    data[date_field] = datetime.fromisoformat(data[date_field].replace('Z', '+00:00'))
+                except ValueError:
+                    data[date_field] = None
+
+        return cls(**data)
+
+    def generate_content_hash(self) -> str:
+        """生成内容哈希用于去重"""
+        import hashlib
+        content = f"{self.title}|{self.publish_date_str}|{self.purchase_name}|{self.content_url}|{self.source_code}"
+        self.content_hash = hashlib.md5(content.encode('utf-8')).hexdigest()
+        return self.content_hash
+
+    def matches_keywords(self, keywords: List[str]) -> bool:
+        """检查是否匹配关键词"""
+        if not keywords:
+            return True
+
+        search_text = f"{self.title} {self.purchase_name}".lower()
+        for keyword in keywords:
+            if keyword.lower() in search_text:
+                self.keyword_matched = True
+                return True
+
+        self.keyword_matched = False
+        return False
+
+    def in_date_range(self, start_date: Optional[str], end_date: Optional[str]) -> bool:
+        """检查是否在日期范围内"""
+        if not self.publish_date:
+            self.date_filtered = False
+            return False
+
+        publish_date = self.publish_date.date()
+
+        try:
+            if start_date:
+                start = datetime.fromisoformat(start_date).date()
+                if publish_date < start:
+                    self.date_filtered = False
+                    return False
+
+            if end_date:
+                end = datetime.fromisoformat(end_date).date()
+                if publish_date > end:
+                    self.date_filtered = False
+                    return False
+
+            self.date_filtered = True
+            return True
+        except ValueError:
+            # 日期格式错误时，默认通过
+            self.date_filtered = True
+            return True
+
+
+@dataclass
+class CrawlResult:
+    """爬取结果"""
+    source: AnnouncementSource               # 公告来源
+    status: CrawlStatus                      # 爬取状态
+    total_count: int = 0                     # 总公告数
+    new_count: int = 0                       # 新增公告数
+    error_message: Optional[str] = None      # 错误信息
+    announcements: List[Announcement] = field(default_factory=list)  # 公告列表
+    crawled_at: datetime = field(default_factory=datetime.now)       # 爬取时间
+    duration: float = 0.0                    # 爬取耗时（秒）
+
+
+@dataclass
+class CrawlSession:
+    """爬取会话"""
+    session_id: str                          # 会话ID
+    start_time: datetime                     # 开始时间
+    end_time: Optional[datetime] = None      # 结束时间
+    status: CrawlStatus = CrawlStatus.PENDING  # 会话状态
+    total_sources: int = 0                   # 总来源数
+    completed_sources: int = 0               # 已完成来源数
+    total_announcements: int = 0             # 总公告数
+    new_announcements: int = 0               # 新增公告数
+    results: List[CrawlResult] = field(default_factory=list)  # 各来源结果
+
+    @property
+    def duration(self) -> float:
+        """获取会话持续时间"""
+        if self.end_time and self.start_time:
+            return (self.end_time - self.start_time).total_seconds()
+        elif self.start_time:
+            return (datetime.now() - self.start_time).total_seconds()
+        return 0.0
+
+    @property
+    def progress(self) -> float:
+        """获取完成进度（0-1）"""
+        if self.total_sources == 0:
+            return 0.0
+        return self.completed_sources / self.total_sources
+
+
+@dataclass
+class NotificationMessage:
+    """通知消息"""
+    title: str                               # 消息标题
+    content: str                             # 消息内容
+    message_type: str = "text"               # 消息类型：text, markdown, card
+    recipients: List[str] = field(default_factory=lambda: ["@all"])  # 接收者列表
+    attachments: Optional[Dict[str, Any]] = None  # 附件信息
+    created_at: datetime = field(default_factory=datetime.now)  # 创建时间
+
+
+@dataclass
+class SystemMetrics:
+    """系统指标"""
+    timestamp: datetime = field(default_factory=datetime.now)  # 时间戳
+    total_announcements: int = 0               # 总公告数
+    today_announcements: int = 0               # 今日公告数
+    new_announcements_today: int = 0           # 今日新增公告数
+    crawl_sessions_today: int = 0              # 今日爬取会话数
+    last_crawl_duration: float = 0.0           # 最后一次爬取耗时
+    database_size: int = 0                     # 数据库大小（字节）
+    memory_usage: float = 0.0                  # 内存使用率
+    disk_usage: float = 0.0                    # 磁盘使用率

gx_gp_monitor/core/reliability.py

@@ -0,0 +1,489 @@
+"""
+高可用性模块
+提供重试机制、超时控制、幂等操作、异常处理和恢复功能
+"""
+
+import time
+import random
+import hashlib
+from contextlib import contextmanager
+from functools import wraps
+from typing import Callable, Any, Optional, Type, Union, List
+from datetime import datetime, timedelta
+import threading
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+from .logger import get_logger
+from .config_manager import get_config
+
+
+logger = get_logger(__name__)
+
+
+class RetryConfig:
+    """重试配置"""
+
+    def __init__(self,
+                 max_retries: int = 3,
+                 initial_delay: float = 1.0,
+                 max_delay: float = 60.0,
+                 backoff_factor: float = 2.0,
+                 jitter: bool = True):
+        """
+        初始化重试配置
+
+        Args:
+            max_retries: 最大重试次数
+            initial_delay: 初始延迟时间（秒）
+            max_delay: 最大延迟时间（秒）
+            backoff_factor: 退避因子
+            jitter: 是否添加随机抖动
+        """
+        self.max_retries = max_retries
+        self.initial_delay = initial_delay
+        self.max_delay = max_delay
+        self.backoff_factor = backoff_factor
+        self.jitter = jitter
+
+
+class TimeoutConfig:
+    """超时配置"""
+
+    def __init__(self,
+                 connect_timeout: float = 10.0,
+                 read_timeout: float = 30.0,
+                 total_timeout: Optional[float] = None):
+        """
+        初始化超时配置
+
+        Args:
+            connect_timeout: 连接超时时间（秒）
+            read_timeout: 读取超时时间（秒）
+            total_timeout: 总超时时间（秒）
+        """
+        self.connect_timeout = connect_timeout
+        self.read_timeout = read_timeout
+        self.total_timeout = total_timeout or (connect_timeout + read_timeout)
+
+
+class CircuitBreakerState:
+    """熔断器状态"""
+    CLOSED = "closed"       # 关闭状态，正常工作
+    OPEN = "open"           # 打开状态，快速失败
+    HALF_OPEN = "half_open" # 半开状态，测试恢复
+
+
+class CircuitBreaker:
+    """熔断器实现"""
+
+    def __init__(self,
+                 failure_threshold: int = 5,
+                 recovery_timeout: int = 60,
+                 expected_exception: Type[Exception] = Exception):
+        """
+        初始化熔断器
+
+        Args:
+            failure_threshold: 失败阈值
+            recovery_timeout: 恢复超时时间（秒）
+            expected_exception: 期望的异常类型
+        """
+        self.failure_threshold = failure_threshold
+        self.recovery_timeout = recovery_timeout
+        self.expected_exception = expected_exception
+
+        self.state = CircuitBreakerState.CLOSED
+        self.failure_count = 0
+        self.last_failure_time = None
+        self._lock = threading.Lock()
+
+    def __call__(self, func: Callable) -> Callable:
+        """装饰器实现"""
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            return self._execute_with_circuit_breaker(func, *args, **kwargs)
+        return wrapper
+
+    def _execute_with_circuit_breaker(self, func: Callable, *args, **kwargs) -> Any:
+        """使用熔断器执行函数"""
+        if self.state == CircuitBreakerState.OPEN:
+            if self._should_attempt_reset():
+                self.state = CircuitBreakerState.HALF_OPEN
+                logger.info("熔断器半开，尝试恢复")
+            else:
+                raise CircuitBreakerOpenException("熔断器已打开")
+
+        try:
+            result = func(*args, **kwargs)
+            self._on_success()
+            return result
+        except self.expected_exception as e:
+            self._on_failure()
+            raise
+
+    def _should_attempt_reset(self) -> bool:
+        """检查是否应该尝试重置"""
+        if self.last_failure_time is None:
+            return True
+        return (datetime.now() - self.last_failure_time).total_seconds() >= self.recovery_timeout
+
+    def _on_success(self):
+        """成功时的处理"""
+        with self._lock:
+            if self.state == CircuitBreakerState.HALF_OPEN:
+                self.state = CircuitBreakerState.CLOSED
+                self.failure_count = 0
+                logger.info("熔断器关闭，服务恢复正常")
+
+    def _on_failure(self):
+        """失败时的处理"""
+        with self._lock:
+            self.failure_count += 1
+            self.last_failure_time = datetime.now()
+
+            if self.failure_count >= self.failure_threshold:
+                self.state = CircuitBreakerState.OPEN
+                logger.warning(f"熔断器打开，失败次数达到阈值: {self.failure_count}")
+
+
+class CircuitBreakerOpenException(Exception):
+    """熔断器打开异常"""
+    pass
+
+
+class IdempotencyKey:
+    """幂等性键生成器"""
+
+    @staticmethod
+    def generate(*args, **kwargs) -> str:
+        """
+        生成幂等性键
+
+        Args:
+            *args: 位置参数
+            **kwargs: 关键字参数
+
+        Returns:
+            str: 幂等性键
+        """
+        # 将参数转换为字符串并排序
+        key_parts = []
+
+        # 处理位置参数
+        for i, arg in enumerate(args):
+            key_parts.append(f"arg_{i}:{str(arg)}")
+
+        # 处理关键字参数（排序以保证一致性）
+        for key in sorted(kwargs.keys()):
+            key_parts.append(f"{key}:{str(kwargs[key])}")
+
+        # 生成哈希
+        key_string = "|".join(key_parts)
+        return hashlib.md5(key_string.encode('utf-8')).hexdigest()
+
+
+class IdempotencyManager:
+    """幂等性管理器"""
+
+    def __init__(self):
+        self._executed_keys = set()
+        self._lock = threading.Lock()
+
+    def is_executed(self, key: str) -> bool:
+        """
+        检查操作是否已执行
+
+        Args:
+            key: 幂等性键
+
+        Returns:
+            bool: 是否已执行
+        """
+        with self._lock:
+            return key in self._executed_keys
+
+    def mark_executed(self, key: str):
+        """
+        标记操作已执行
+
+        Args:
+            key: 幂等性键
+        """
+        with self._lock:
+            self._executed_keys.add(key)
+
+    def clear_expired_keys(self, max_age_seconds: int = 3600):
+        """
+        清理过期的键（简化实现，实际应该使用时间戳）
+
+        Args:
+            max_age_seconds: 最大年龄（秒）
+        """
+        # 这里简化实现，实际项目中应该记录时间戳
+        pass
+
+
+def retry_on_exception(retry_config: Optional[RetryConfig] = None,
+                      exceptions: tuple = (Exception,),
+                      logger: Optional[Any] = None) -> Callable:
+    """
+    重试装饰器
+
+    Args:
+        retry_config: 重试配置
+        exceptions: 需要重试的异常类型
+        logger: 日志器
+
+    Returns:
+        Callable: 装饰器函数
+    """
+    if retry_config is None:
+        retry_config = RetryConfig()
+
+    if logger is None:
+        logger = get_logger()
+
+    def decorator(func: Callable) -> Callable:
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            last_exception = None
+
+            for attempt in range(retry_config.max_retries + 1):
+                try:
+                    return func(*args, **kwargs)
+                except exceptions as e:
+                    last_exception = e
+
+                    if attempt < retry_config.max_retries:
+                        # 计算延迟时间
+                        delay = min(
+                            retry_config.initial_delay * (retry_config.backoff_factor ** attempt),
+                            retry_config.max_delay
+                        )
+
+                        # 添加随机抖动
+                        if retry_config.jitter:
+                            delay = delay * (0.5 + random.random() * 0.5)
+
+                        logger.warning(
+                            f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{retry_config.max_retries + 1}): {str(e)}，"
+                            f"等待 {delay:.2f} 秒后重试"
+                        )
+                        time.sleep(delay)
+                    else:
+                        logger.error(
+                            f"函数 {func.__name__} 在 {retry_config.max_retries + 1} 次尝试后仍然失败: {str(e)}"
+                        )
+
+            raise last_exception
+
+        return wrapper
+    return decorator
+
+
+def timeout_wrapper(timeout_config: Optional[TimeoutConfig] = None) -> Callable:
+    """
+    超时装饰器
+
+    Args:
+        timeout_config: 超时配置
+
+    Returns:
+        Callable: 装饰器函数
+    """
+    if timeout_config is None:
+        timeout_config = TimeoutConfig()
+
+    def decorator(func: Callable) -> Callable:
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            import signal
+
+            def timeout_handler(signum, frame):
+                raise TimeoutError(f"函数 {func.__name__} 执行超时")
+
+            # 设置信号处理器
+            old_handler = signal.signal(signal.SIGALRM, timeout_handler)
+            signal.alarm(int(timeout_config.total_timeout))
+
+            try:
+                result = func(*args, **kwargs)
+                signal.alarm(0)  # 取消闹钟
+                return result
+            finally:
+                signal.signal(signal.SIGALRM, old_handler)
+
+        return wrapper
+    return decorator
+
+
+@contextmanager
+def session_with_retry(timeout_config: Optional[TimeoutConfig] = None,
+                      retry_config: Optional[RetryConfig] = None):
+    """
+    创建带有重试机制的HTTP会话
+
+    Args:
+        timeout_config: 超时配置
+        retry_config: 重试配置
+
+    Yields:
+        requests.Session: 配置好的会话对象
+    """
+    if timeout_config is None:
+        timeout_config = TimeoutConfig()
+
+    if retry_config is None:
+        retry_config = RetryConfig()
+
+    session = requests.Session()
+
+    # 配置重试策略
+    retry_strategy = Retry(
+        total=retry_config.max_retries,
+        backoff_factor=retry_config.backoff_factor,
+        status_forcelist=[429, 500, 502, 503, 504],
+    )
+
+    adapter = HTTPAdapter(max_retries=retry_strategy)
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+
+    # 设置默认超时
+    session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout)
+
+    try:
+        yield session
+    finally:
+        session.close()
+
+
+def safe_execute(func: Callable,
+                fallback: Optional[Callable] = None,
+                exceptions: tuple = (Exception,),
+                logger: Optional[Any] = None) -> Any:
+    """
+    安全执行函数，提供降级处理
+
+    Args:
+        func: 要执行的函数
+        fallback: 降级函数
+        exceptions: 需要捕获的异常类型
+        logger: 日志器
+
+    Returns:
+        Any: 函数执行结果或降级结果
+    """
+    if logger is None:
+        logger = get_logger()
+
+    try:
+        return func()
+    except exceptions as e:
+        logger.error(f"函数执行失败: {str(e)}")
+        if fallback:
+            try:
+                logger.info("执行降级函数")
+                return fallback()
+            except Exception as fallback_e:
+                logger.error(f"降级函数也执行失败: {str(fallback_e)}")
+        return None
+
+
+class HealthChecker:
+    """健康检查器"""
+
+    def __init__(self, check_interval: int = 300):
+        """
+        初始化健康检查器
+
+        Args:
+            check_interval: 检查间隔（秒）
+        """
+        self.check_interval = check_interval
+        self.last_check = None
+        self.is_healthy = True
+        self.consecutive_failures = 0
+        self.max_consecutive_failures = 3
+
+    def check_health(self) -> bool:
+        """
+        执行健康检查
+
+        Returns:
+            bool: 健康状态
+        """
+        current_time = datetime.now()
+
+        # 检查是否需要执行检查
+        if (self.last_check and
+            (current_time - self.last_check).total_seconds() < self.check_interval):
+            return self.is_healthy
+
+        self.last_check = current_time
+
+        try:
+            # 执行健康检查逻辑
+            self._perform_health_check()
+            self.is_healthy = True
+            self.consecutive_failures = 0
+            logger.info("健康检查通过")
+            return True
+        except Exception as e:
+            self.consecutive_failures += 1
+            logger.warning(f"健康检查失败 ({self.consecutive_failures}/{self.max_consecutive_failures}): {str(e)}")
+
+            if self.consecutive_failures >= self.max_consecutive_failures:
+                self.is_healthy = False
+                logger.error("连续健康检查失败，系统标记为不健康")
+
+            return False
+
+    def _perform_health_check(self):
+        """执行具体的健康检查逻辑"""
+        # 这里可以添加数据库连接检查、外部服务检查等
+        config = get_config()
+
+        # 检查数据库连接（如果启用）
+        if config.database.enabled:
+            # 这里应该检查数据库连接
+            pass
+
+        # 检查网络连接
+        try:
+            requests.get("https://www.baidu.com", timeout=5)
+        except:
+            raise Exception("网络连接检查失败")
+
+
+# 全局实例
+_circuit_breaker = CircuitBreaker()
+_idempotency_manager = IdempotencyManager()
+_health_checker = HealthChecker()
+
+
+def get_circuit_breaker() -> CircuitBreaker:
+    """获取全局熔断器实例"""
+    return _circuit_breaker
+
+
+def get_idempotency_manager() -> IdempotencyManager:
+    """获取全局幂等性管理器实例"""
+    return _idempotency_manager
+
+
+def get_health_checker() -> HealthChecker:
+    """获取全局健康检查器实例"""
+    return _health_checker
+
+
+def check_system_health() -> bool:
+    """
+    检查系统健康状态
+
+    Returns:
+        bool: 系统是否健康
+    """
+    return _health_checker.check_health()