#!/usr/bin/env python3 """ 广西政府采购网公告监控系统主程序 广西政府采购网公告爬取和监控的智能系统 """ import sys import argparse import signal from pathlib import Path # 添加项目根目录到Python路径 project_root = Path(__file__).parent sys.path.insert(0, str(project_root)) try: # 尝试相对导入 from .core.config_manager import load_config, get_config from .core.logger import init_logger, get_logger from .core.reliability import check_system_health from .crawler.spider import crawl_announcements from .filters.filters import filter_from_config from .storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, cleanup_storage from .storage.md_generator import generate_onu_md from .notification.wechat import send_announcements_notification, send_system_notification except ImportError: try: # 尝试绝对导入（直接运行脚本时） from core.config_manager import load_config, get_config from core.logger import init_logger, get_logger from core.reliability import check_system_health from crawler.spider import crawl_announcements from filters.filters import filter_from_config from storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, cleanup_storage from storage.md_generator import generate_onu_md from notification.wechat import send_announcements_notification, send_system_notification except ImportError as e: print(f"导入错误: {e}", file=sys.stderr) print("请确保依赖已正确安装: pip install -r requirements.txt", file=sys.stderr) sys.exit(1) logger = get_logger(__name__) class GXGPMonitorApp: """广西政府采购网监控系统应用""" def __init__(self): self.config = None self.logger = None self.running = False def initialize(self, config_file: str = None): """初始化应用""" try: # 加载配置 self.config = load_config(config_file) # 初始化日志 self.logger = init_logger(config=self.config) logger.info("=== 广西政府采购网公告监控系统启动 ===") logger.info(f"版本: 1.0.0") logger.info(f"配置文件: {config_file or '默认配置'}") # 系统健康检查 if not check_system_health(): logger.warning("系统健康检查失败，但继续运行") # 初始化存储 init_storage() logger.info("应用初始化完成") return True except Exception as e: print(f"应用初始化失败: {str(e)}", file=sys.stderr) return False def run_crawl(self, keywords: list = None, sources: list = None, max_pages: int = None): """执行爬取任务""" try: logger.info("开始执行爬取任务") # 执行爬取 crawl_results = crawl_announcements(keywords, sources) if not crawl_results: logger.info("爬取完成：无数据") return {"success": True, "results": []} # 收集所有公告 all_announcements = [] for result in crawl_results: if result.announcements: all_announcements.extend(result.announcements) logger.info(f"爬取到 {len(all_announcements)} 条原始公告") # 先保存所有公告（按来源分组，每源保留最新100条） all_saved_stats = save_all_announcements_by_source_to_storage(all_announcements, max_per_source=100) all_saved_count = sum(all_saved_stats.values()) logger.info(f"保存所有公告完成：共保存 {all_saved_count} 条，按来源统计: {all_saved_stats}") # 筛选公告 filter_obj = filter_from_config() filtered_announcements, filter_stats = filter_obj.filter(all_announcements) logger.info(f"筛选后剩余 {len(filtered_announcements)} 条公告") # 保存筛选后的公告（用于标记关键词匹配等） saved_count = save_announcements_to_storage(filtered_announcements) # 生成Markdown文件 md_success = generate_onu_md(filtered_announcements) # 发送通知 notify_success = False if filtered_announcements and self.config.wechat_app.enabled: notify_success = send_announcements_notification(filtered_announcements) result = { "success": True, "total_crawled": len(all_announcements), "filtered": len(filtered_announcements), "saved": saved_count, "markdown_generated": md_success, "notification_sent": notify_success, "filter_stats": { "keyword_filtered": filter_stats.keyword_filtered, "date_filtered": filter_stats.date_filtered, "duplicate_filtered": filter_stats.duplicate_filtered, "source_filtered": filter_stats.source_filtered } } logger.info(f"爬取任务完成: {result}") return result except Exception as e: logger.error(f"爬取任务执行失败: {str(e)}") return {"success": False, "error": str(e)} def run_cleanup(self, days: int = None): """执行数据清理任务""" try: logger.info("开始执行数据清理任务") deleted_count = cleanup_storage(days) logger.info(f"数据清理完成：删除 {deleted_count} 条过期数据") # 发送通知 if deleted_count > 0 and self.config.wechat_app.enabled: send_system_notification( "数据清理完成", f"已清理 {deleted_count} 条过期数据" ) return {"success": True, "deleted": deleted_count} except Exception as e: logger.error(f"数据清理任务执行失败: {str(e)}") return {"success": False, "error": str(e)} def show_status(self): """显示系统状态""" try: status = { "system": { "version": "1.0.0", "healthy": check_system_health() }, "config": { "debug": self.config.debug, "log_level": self.config.log_level.value }, "database": { "enabled": self.config.database.enabled, "type": self.config.database.type }, "wechat": { "enabled": self.config.wechat_app.enabled } } # 格式化输出 print("\n=== 系统状态 ===") print(f"系统健康: {'正常' if status['system']['healthy'] else '异常'}") print(f"调试模式: {'开启' if status['config']['debug'] else '关闭'}") print(f"日志级别: {status['config']['log_level']}") print(f"数据库: {'启用' if status['database']['enabled'] else '禁用'} ({status['database']['type']})") print(f"企业微信: {'启用' if status['wechat']['enabled'] else '禁用'}") return status except Exception as e: logger.error(f"获取系统状态失败: {str(e)}") return {"error": str(e)} def create_argument_parser(): """创建命令行参数解析器""" parser = argparse.ArgumentParser( description="广西政府采购网公告监控系统", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" 使用示例: python main.py crawl # 执行一次爬取 python main.py crawl --keywords "大化" # 爬取指定关键词 python main.py cleanup # 执行数据清理 python main.py status # 查看系统状态 """ ) parser.add_argument( 'command', choices=['crawl', 'cleanup', 'status'], help='要执行的命令' ) parser.add_argument( '--config', '-c', help='配置文件路径' ) # crawl命令的参数 parser.add_argument( '--keywords', '-k', nargs='+', help='关键词过滤（多个关键词用空格分隔）' ) parser.add_argument( '--sources', '-s', nargs='+', help='来源代码过滤（多个来源用空格分隔）' ) parser.add_argument( '--max-pages', type=int, help='最大爬取页数' ) # cleanup命令的参数 parser.add_argument( '--days', '-d', type=int, help='清理多少天前的过期数据' ) return parser def main(): """主函数""" parser = create_argument_parser() args = parser.parse_args() # 创建应用实例 app = GXGPMonitorApp() # 初始化应用 if not app.initialize(args.config): sys.exit(1) try: if args.command == 'crawl': # 执行爬取 result = app.run_crawl( keywords=args.keywords, sources=args.sources, max_pages=args.max_pages ) if result["success"]: print("✅ 爬取任务执行成功") print(f" 爬取公告: {result['total_crawled']}") print(f" 筛选后: {result['filtered']}") print(f" 保存数量: {result['saved']}") if result.get("markdown_generated"): print(" Markdown文件: 已生成") if result.get("notification_sent"): print(" 通知发送: 已发送") else: print(f"❌ 爬取任务执行失败: {result.get('error', '未知错误')}") sys.exit(1) elif args.command == 'cleanup': # 执行清理 result = app.run_cleanup(days=args.days) if result["success"]: print(f"✅ 数据清理完成，删除 {result['deleted']} 条记录") else: print(f"❌ 数据清理失败: {result.get('error', '未知错误')}") sys.exit(1) elif args.command == 'status': # 显示状态 app.show_status() except KeyboardInterrupt: logger.info("收到中断信号，正在退出...") except Exception as e: logger.error(f"程序执行异常: {str(e)}") print(f"❌ 程序执行异常: {str(e)}", file=sys.stderr) sys.exit(1) if __name__ == "__main__": main()