Remove obsolete scripts related to WeChat menu creation, server functionality, and cron job scheduling. Update project structure by deleting unnecessary files to streamline the codebase.

gx_gp_monitor/cron_crawl.py

@@ -39,7 +39,7 @@ try:
             # 初始化存储
             init_storage()
 
-            # 执行搜索（爬取所有公告，然后进行关键词筛选）
+            # 执行搜索（爬取所有公告，筛选出新增的关键词匹配公告）
             logger.info("开始执行定时搜索任务")
             crawl_results = crawl_announcements()
 
@@ -64,34 +64,58 @@ try:
             from gx_gp_monitor.filters.filters import KeywordFilter, DateFilter
             from datetime import date
 
-            # 1. 关键词筛选
             keyword_filter = KeywordFilter()
             keyword_filtered = keyword_filter.filter_announcements(all_announcements, keywords=config.crawler.keyword)
 
-            # 2. 日期筛选（只保留今天的）
+            # 对关键词筛选结果进行日期筛选（只处理今天的）
             date_filter = DateFilter()
-            today_announcements = date_filter.filter_announcements(
+            today_keyword_announcements = date_filter.filter_announcements(
                 keyword_filtered,
                 start_date=date.today(),
                 end_date=date.today()
             )
 
             logger.info(f"关键词筛选后剩余 {len(keyword_filtered)} 条公告")
-            logger.info(f"筛选出今天 {len(today_announcements)} 条匹配公告")
+            logger.info(f"筛选出今天关键词匹配 {len(today_keyword_announcements)} 条公告")
 
-            if not today_announcements:
-                logger.info("今天没有匹配关键词的公告")
+            if not today_keyword_announcements:
+                logger.info("今天没有关键词匹配的公告")
+                return True
+
+            # 检查auto_announcements表，筛选出真正新增的公告
+            truly_new_announcements = []
+            for ann in today_keyword_announcements:
+                try:
+                    # 直接导入，避免相对导入问题
+                    import gx_gp_monitor.core.database as db_module
+                    with db_module.get_db_cursor() as cursor:
+                        cursor.execute(
+                            "SELECT 1 FROM auto_announcements WHERE content_hash = %s LIMIT 1",
+                            (ann.content_hash,)
+                        )
+                        exists = cursor.fetchone() is not None
+                        if not exists:
+                            truly_new_announcements.append(ann)
+                except Exception as e:
+                    logger.warning(f"检查公告是否存在失败: {str(e)}")
+                    # 如果检查失败，为了安全起见，不添加到新公告列表
+                    pass
+
+            logger.info(f"从今天关键词匹配公告中筛选出 {len(truly_new_announcements)} 条auto_announcements表中不存在的新公告")
+
+            if not truly_new_announcements:
+                logger.info("没有真正新增的关键词匹配公告，任务完成")
                 return True
 
             # 构造筛选统计信息
             filter_stats = type('FilterResult', (), {
                 "keyword_filtered": len(all_announcements) - len(keyword_filtered),
-                "date_filtered": len(keyword_filtered) - len(today_announcements),
-                "duplicate_filtered": 0,
+                "date_filtered": len(keyword_filtered) - len(today_keyword_announcements),
+                "duplicate_filtered": len(today_keyword_announcements) - len(truly_new_announcements),
                 "source_filtered": 0
             })()
 
-            filtered_announcements = today_announcements
+            filtered_announcements = truly_new_announcements
 
             logger.info(f"关键词筛选后剩余 {len(filtered_announcements)} 条公告")
 
@@ -119,7 +143,8 @@ try:
             print("\n=== 定时搜索任务完成 ===")
             print(f"总共爬取: {total_crawled} 条公告")
             print(f"关键词筛选: {len(keyword_filtered)} 条")
-            print(f"今日匹配公告: {len(today_announcements)} 条")
+            print(f"今日关键词匹配: {len(today_keyword_announcements)} 条")
+            print(f"真正新增公告: {len(truly_new_announcements)} 条")
             print(f"筛选后公告: {len(filtered_announcements)} 条")
             print(f"保存到数据库: {saved_count} 条")
             print(f"企业微信通知: {'成功' if notify_success else '失败' if config.wechat_app.enabled else '未启用'}")