删除无用文件

gx_gp_monitor/cron_crawl.py

@@ -17,7 +17,7 @@ try:
     from gx_gp_monitor.core.logger import init_logger, get_logger
     from gx_gp_monitor.crawler.spider import crawl_announcements
     from gx_gp_monitor.filters.filters import filter_from_config
-    from gx_gp_monitor.storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage
+    from gx_gp_monitor.storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, save_auto_announcements_to_storage
     from gx_gp_monitor.notification.wechat import send_announcements_notification, send_system_notification
 
     logger = get_logger(__name__)
@@ -39,8 +39,8 @@ try:
             # 初始化存储
             init_storage()
 
-            # 执行搜索
-            logger.info("开始执行搜索...")
+            # 执行搜索（爬取所有公告，然后进行关键词筛选）
+            logger.info("开始执行定时搜索任务")
             crawl_results = crawl_announcements()
 
             if not crawl_results:
@@ -56,22 +56,42 @@ try:
             total_crawled = len(all_announcements)
             logger.info(f"搜索到 {total_crawled} 条原始公告")
 
-            # 保存所有公告（按来源分组，每源保留最新100条）
-            all_saved_stats = save_all_announcements_by_source_to_storage(all_announcements, max_per_source=100)
-            all_saved_count = sum(all_saved_stats.values())
-            logger.info(f"保存所有公告完成：共保存 {all_saved_count} 条，按来源统计: {all_saved_stats}")
-
-            # 筛选出新公告（数据库中没有的）
-            new_announcements = [ann for ann in all_announcements if ann.is_new]
-            logger.info(f"筛选出 {len(new_announcements)} 条新公告")
-
-            if not new_announcements:
-                logger.info("没有新的公告，任务完成")
+            if not all_announcements:
+                logger.info("没有获取到任何公告")
                 return True
 
-            # 对新公告应用关键词筛选
-            filter_obj = filter_from_config()
-            filtered_announcements, filter_stats = filter_obj.filter(new_announcements)
+            # 对所有公告进行关键词筛选
+            from gx_gp_monitor.filters.filters import KeywordFilter, DateFilter
+            from datetime import date
+
+            # 1. 关键词筛选
+            keyword_filter = KeywordFilter()
+            keyword_filtered = keyword_filter.filter_announcements(all_announcements, keywords=config.crawler.keyword)
+
+            # 2. 日期筛选（只保留今天的）
+            date_filter = DateFilter()
+            today_announcements = date_filter.filter_announcements(
+                keyword_filtered,
+                start_date=date.today(),
+                end_date=date.today()
+            )
+
+            logger.info(f"关键词筛选后剩余 {len(keyword_filtered)} 条公告")
+            logger.info(f"筛选出今天 {len(today_announcements)} 条匹配公告")
+
+            if not today_announcements:
+                logger.info("今天没有匹配关键词的公告")
+                return True
+
+            # 构造筛选统计信息
+            filter_stats = type('FilterResult', (), {
+                "keyword_filtered": len(all_announcements) - len(keyword_filtered),
+                "date_filtered": len(keyword_filtered) - len(today_announcements),
+                "duplicate_filtered": 0,
+                "source_filtered": 0
+            })()
+
+            filtered_announcements = today_announcements
 
             logger.info(f"关键词筛选后剩余 {len(filtered_announcements)} 条公告")
 
@@ -79,9 +99,9 @@ try:
                 logger.info("没有匹配关键词的公告，任务完成")
                 return True
 
-            # 保存筛选后的公告（用于标记关键词匹配等）
-            saved_count = save_announcements_to_storage(filtered_announcements)
-            logger.info(f"保存筛选后公告完成：{saved_count} 条")
+            # 保存筛选后的公告到定时搜索专用表
+            saved_count = save_auto_announcements_to_storage(filtered_announcements)
+            logger.info(f"保存定时搜索公告完成：{saved_count} 条")
 
             # 发送企业微信卡片通知
             if config.wechat_app.enabled:
@@ -96,10 +116,11 @@ try:
                 logger.info("企业微信通知未启用，跳过发送")
 
             # 输出统计信息
-            print("\n=== 定时爬取任务完成 ===")
+            print("\n=== 定时搜索任务完成 ===")
             print(f"总共爬取: {total_crawled} 条公告")
-            print(f"新增公告: {len(new_announcements)} 条")
-            print(f"关键词筛选: {len(filtered_announcements)} 条")
+            print(f"关键词筛选: {len(keyword_filtered)} 条")
+            print(f"今日匹配公告: {len(today_announcements)} 条")
+            print(f"筛选后公告: {len(filtered_announcements)} 条")
             print(f"保存到数据库: {saved_count} 条")
             print(f"企业微信通知: {'成功' if notify_success else '失败' if config.wechat_app.enabled else '未启用'}")