Enhance manual crawling functionality in GXGPMonitorApp to skip database saving for both announcements and filtered results. Update logging to reflect manual crawl mode actions, ensuring clarity in the process. Adjusted comments for better understanding of the manual crawling logic.

2026-01-08 19:12:37 +08:00
parent 04d254cfef
commit 85cd7e9d3e
10 changed files with 1115 additions and 6 deletions
@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""
+定时爬取脚本
+执行爬取、筛选关键词、保存到数据库并发送企业微信卡片通知
+"""
+
+import sys
+import os
+from pathlib import Path
+
+# 添加项目根目录到路径（gx_gp_monitor的父目录）
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+try:
+    from gx_gp_monitor.core.config_manager import load_config, get_config
+    from gx_gp_monitor.core.logger import init_logger, get_logger
+    from gx_gp_monitor.crawler.spider import crawl_announcements
+    from gx_gp_monitor.filters.filters import filter_from_config
+    from gx_gp_monitor.storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage
+    from gx_gp_monitor.notification.wechat import send_announcements_notification, send_system_notification
+
+    logger = get_logger(__name__)
+
+    def main():
+        """主函数：执行定时爬取任务"""
+        try:
+            # 加载配置
+            config = load_config()
+            if not config:
+                logger.error("无法加载配置")
+                return False
+
+            # 初始化日志
+            init_logger(config=config)
+
+            logger.info("=== 开始定时爬取任务 ===")
+
+            # 初始化存储
+            init_storage()
+
+            # 执行爬取
+            logger.info("开始执行爬取...")
+            crawl_results = crawl_announcements()
+
+            if not crawl_results:
+                logger.info("爬取完成：无数据")
+                return True
+
+            # 收集所有公告
+            all_announcements = []
+            for result in crawl_results:
+                if result.announcements:
+                    all_announcements.extend(result.announcements)
+
+            total_crawled = len(all_announcements)
+            logger.info(f"爬取到 {total_crawled} 条原始公告")
+
+            # 保存所有公告（按来源分组，每源保留最新100条）
+            all_saved_stats = save_all_announcements_by_source_to_storage(all_announcements, max_per_source=100)
+            all_saved_count = sum(all_saved_stats.values())
+            logger.info(f"保存所有公告完成：共保存 {all_saved_count} 条，按来源统计: {all_saved_stats}")
+
+            # 筛选出新公告（数据库中没有的）
+            new_announcements = [ann for ann in all_announcements if ann.is_new]
+            logger.info(f"筛选出 {len(new_announcements)} 条新公告")
+
+            if not new_announcements:
+                logger.info("没有新的公告，任务完成")
+                return True
+
+            # 对新公告应用关键词筛选
+            filter_obj = filter_from_config()
+            filtered_announcements, filter_stats = filter_obj.filter(new_announcements)
+
+            logger.info(f"关键词筛选后剩余 {len(filtered_announcements)} 条公告")
+
+            if not filtered_announcements:
+                logger.info("没有匹配关键词的公告，任务完成")
+                return True
+
+            # 保存筛选后的公告（用于标记关键词匹配等）
+            saved_count = save_announcements_to_storage(filtered_announcements)
+            logger.info(f"保存筛选后公告完成：{saved_count} 条")
+
+            # 发送企业微信卡片通知
+            if config.wechat_app.enabled:
+                logger.info("开始发送企业微信卡片通知...")
+                notify_success = send_announcements_notification(filtered_announcements)
+
+                if notify_success:
+                    logger.info("企业微信卡片通知发送成功")
+                else:
+                    logger.error("企业微信卡片通知发送失败")
+            else:
+                logger.info("企业微信通知未启用，跳过发送")
+
+            # 输出统计信息
+            print("\n=== 定时爬取任务完成 ===")
+            print(f"总共爬取: {total_crawled} 条公告")
+            print(f"新增公告: {len(new_announcements)} 条")
+            print(f"关键词筛选: {len(filtered_announcements)} 条")
+            print(f"保存到数据库: {saved_count} 条")
+            print(f"企业微信通知: {'成功' if notify_success else '失败' if config.wechat_app.enabled else '未启用'}")
+
+            logger.info("=== 定时爬取任务完成 ===")
+            return True
+
+        except Exception as e:
+            logger.error(f"定时爬取任务执行失败: {str(e)}")
+            print(f"❌ 定时爬取任务失败: {str(e)}", file=sys.stderr)
+
+            # 尝试发送错误通知
+            try:
+                if config and config.wechat_app.enabled:
+                    send_system_notification(
+                        "定时爬取任务失败",
+                        f"错误信息: {str(e)}"
+                    )
+            except Exception as notify_error:
+                logger.error(f"发送错误通知失败: {notify_error}")
+
+            return False
+
+    if __name__ == "__main__":
+        success = main()
+        sys.exit(0 if success else 1)
+
+except ImportError as e:
+    print(f"导入失败: {e}", file=sys.stderr)
+    print("请确保已安装所有依赖: pip install -r gx_gp_monitor/requirements.txt", file=sys.stderr)
+    sys.exit(1)
+except Exception as e:
+    print(f"脚本执行失败: {e}", file=sys.stderr)
+    sys.exit(1)