""" 调度器模块 提供定时任务调度功能，支持cron表达式和间隔执行 """ import time import threading from datetime import datetime, timedelta from typing import Callable, Dict, Any, Optional, List import schedule from croniter import croniter try: from ..core.config_manager import get_config from ..core.logger import get_logger except ImportError: from core.config_manager import get_config from core.logger import get_logger logger = get_logger(__name__) class TaskScheduler: """任务调度器""" def __init__(self): self.config = get_config().scheduler self._running = False self._thread = None self._jobs = {} self._job_stats = {} # 初始化调度 if self.config.enabled: self._load_jobs_from_config() def _load_jobs_from_config(self): """从配置加载定时任务""" for job_config in self.config.jobs: if job_config.get("enabled", False): job_name = job_config["name"] cron_expr = job_config["cron"] # 根据任务名称创建对应的任务函数 if job_name == "daily_crawl": func = self._create_crawl_job() elif job_name == "data_cleanup": func = self._create_cleanup_job() else: logger.warning(f"未知的任务类型: {job_name}") continue self.add_cron_job(job_name, cron_expr, func) def _create_crawl_job(self) -> Callable: """创建爬取任务""" def crawl_job(): try: logger.info("开始执行定时爬取任务") # 导入这里避免循环导入 try: from ..crawler.spider import crawl_announcements from ..filters.filters import filter_from_config from ..storage.postgresql import save_announcements_to_storage from ..storage.md_generator import generate_onu_md from ..notification.wechat import send_announcements_notification except ImportError: from crawler.spider import crawl_announcements from filters.filters import filter_from_config from storage.postgresql import save_announcements_to_storage from storage.md_generator import generate_onu_md from notification.wechat import send_announcements_notification # 执行爬取 crawl_results = crawl_announcements() if not crawl_results: logger.info("定时爬取任务完成：无数据") return # 收集所有公告 all_announcements = [] for result in crawl_results: if result.announcements: all_announcements.extend(result.announcements) if not all_announcements: logger.info("定时爬取任务完成：无新公告") return # 筛选公告 filter_obj = filter_from_config() filtered_announcements, filter_stats = filter_obj.filter(all_announcements) logger.info(f"筛选结果: {len(all_announcements)} -> {len(filtered_announcements)}") # 保存到数据库 saved_count = save_announcements_to_storage(filtered_announcements) # 生成Markdown文件 generate_onu_md(filtered_announcements) # 发送通知 if filtered_announcements: send_announcements_notification(filtered_announcements) logger.info(f"定时爬取任务完成：处理 {len(filtered_announcements)} 条公告，保存 {saved_count} 条") except Exception as e: logger.error(f"定时爬取任务执行失败: {str(e)}") # 发送错误通知 from ..notification.wechat import send_error_alert send_error_alert("定时爬取任务失败", str(e)) return crawl_job def _create_cleanup_job(self) -> Callable: """创建数据清理任务""" def cleanup_job(): try: logger.info("开始执行数据清理任务") try: from ..storage.postgresql import cleanup_storage except ImportError: from storage.postgresql import cleanup_storage # 执行清理 deleted_count = cleanup_storage() logger.info(f"数据清理任务完成：删除 {deleted_count} 条过期数据") # 发送通知（如果删除的数据较多） if deleted_count > 0: try: from ..notification.wechat import send_system_notification except ImportError: from notification.wechat import send_system_notification send_system_notification( "数据清理完成", f"已清理 {deleted_count} 条过期数据" ) except Exception as e: logger.error(f"数据清理任务执行失败: {str(e)}") return cleanup_job def add_cron_job(self, name: str, cron_expr: str, func: Callable) -> bool: """ 添加cron定时任务 Args: name: 任务名称 cron_expr: cron表达式 func: 任务函数 Returns: bool: 添加是否成功 """ try: # 验证cron表达式 croniter(cron_expr) # 添加到schedule schedule.every().day.at("00:00").do(func) # 临时设置，会被替换 # 存储任务信息 self._jobs[name] = { "func": func, "cron": cron_expr, "next_run": None, "last_run": None, "run_count": 0, "error_count": 0 } logger.info(f"添加定时任务: {name} ({cron_expr})") return True except Exception as e: logger.error(f"添加定时任务失败 {name}: {str(e)}") return False def add_interval_job(self, name: str, interval_seconds: int, func: Callable) -> bool: """ 添加间隔执行任务 Args: name: 任务名称 interval_seconds: 执行间隔（秒） func: 任务函数 Returns: bool: 添加是否成功 """ try: schedule.every(interval_seconds).seconds.do(func) self._jobs[name] = { "func": func, "interval": interval_seconds, "next_run": None, "last_run": None, "run_count": 0, "error_count": 0 } logger.info(f"添加间隔任务: {name} ({interval_seconds}秒)") return True except Exception as e: logger.error(f"添加间隔任务失败 {name}: {str(e)}") return False def remove_job(self, name: str) -> bool: """ 移除任务 Args: name: 任务名称 Returns: bool: 移除是否成功 """ if name in self._jobs: # 注意：schedule库没有直接的移除方法 # 这里只是从我们的记录中移除 del self._jobs[name] logger.info(f"移除任务: {name}") return True return False def start(self): """启动调度器""" if self._running: logger.warning("调度器已经在运行中") return if not self.config.enabled: logger.info("调度器已禁用") return self._running = True self._thread = threading.Thread(target=self._run_scheduler, daemon=True) self._thread.start() logger.info("调度器已启动") def stop(self): """停止调度器""" if not self._running: return self._running = False if self._thread and self._thread.is_alive(): self._thread.join(timeout=5) logger.info("调度器已停止") def _run_scheduler(self): """运行调度器主循环""" logger.info("调度器主循环开始") while self._running: try: schedule.run_pending() time.sleep(1) except Exception as e: logger.error(f"调度器运行异常: {str(e)}") time.sleep(5) # 出错后等待5秒再继续 logger.info("调度器主循环结束") def run_once(self, job_name: Optional[str] = None): """ 手动执行任务一次 Args: job_name: 任务名称，如果为None则执行所有任务 """ if job_name: if job_name in self._jobs: job_info = self._jobs[job_name] logger.info(f"手动执行任务: {job_name}") try: job_info["func"]() job_info["run_count"] += 1 job_info["last_run"] = datetime.now() logger.info(f"任务 {job_name} 执行完成") except Exception as e: job_info["error_count"] += 1 logger.error(f"任务 {job_name} 执行失败: {str(e)}") else: logger.error(f"任务不存在: {job_name}") else: # 执行所有任务 for name in list(self._jobs.keys()): self.run_once(name) def get_status(self) -> Dict[str, Any]: """ 获取调度器状态 Returns: Dict[str, Any]: 状态信息 """ jobs_status = {} for name, job_info in self._jobs.items(): jobs_status[name] = { "enabled": True, "run_count": job_info.get("run_count", 0), "error_count": job_info.get("error_count", 0), "last_run": job_info.get("last_run").isoformat() if job_info.get("last_run") else None, "next_run": job_info.get("next_run").isoformat() if job_info.get("next_run") else None } return { "enabled": self.config.enabled, "running": self._running, "timezone": self.config.timezone, "jobs": jobs_status } def is_running(self) -> bool: """检查调度器是否正在运行""" return self._running # 全局调度器实例 _scheduler = None def get_scheduler() -> TaskScheduler: """ 获取调度器实例 Returns: TaskScheduler: 调度器实例 """ global _scheduler if _scheduler is None: _scheduler = TaskScheduler() return _scheduler def start_scheduler(): """启动调度器""" scheduler = get_scheduler() scheduler.start() def stop_scheduler(): """停止调度器""" scheduler = get_scheduler() scheduler.stop() def run_scheduled_jobs(job_name: Optional[str] = None): """ 手动执行定时任务 Args: job_name: 任务名称 """ scheduler = get_scheduler() scheduler.run_once(job_name)