删除无用文件

gx_gp_monitor/storage/postgresql.py

@@ -62,6 +62,95 @@ class PostgreSQLStorage:
             # 尝试逐个保存
             return self._save_announcements_fallback(announcements)
 
+    def save_auto_announcements(self, announcements: List[Announcement]) -> int:
+        """
+        保存定时搜索公告到专用表（关键词匹配专用）
+
+        Args:
+            announcements: 公告列表
+
+        Returns:
+            int: 成功保存的数量
+        """
+        if not announcements:
+            return 0
+
+        logger.info(f"开始保存 {len(announcements)} 条定时搜索公告到专用表")
+
+        try:
+            # 批量保存到auto_announcements表
+            saved_count = self.db_manager.save_announcements_batch_to_table(announcements, "auto_announcements")
+
+            if saved_count > 0:
+                logger.info(f"成功保存 {saved_count} 条定时搜索公告到专用表")
+
+            return saved_count
+
+        except Exception as e:
+            logger.error(f"保存定时搜索公告到专用表失败: {str(e)}")
+            return 0
+
+    def save_manual_announcements_by_source(self, announcements: List[Announcement],
+                                           max_per_source: int = 100) -> Dict[str, int]:
+        """
+        按来源保存手动搜索公告到专用表，每个来源保留最新的max_per_source条
+
+        Args:
+            announcements: 所有公告列表（未经关键词筛选）
+            max_per_source: 每个来源最大保留数量
+
+        Returns:
+            Dict[str, int]: 各来源保存的数量
+        """
+        if not announcements:
+            return {}
+
+        logger.info(f"开始按来源保存 {len(announcements)} 条手动搜索公告到专用表，每个来源最多保留 {max_per_source} 条")
+
+        try:
+            # 按来源分组
+            source_groups = {}
+            for announcement in announcements:
+                source_code = announcement.source_code
+                if source_code not in source_groups:
+                    source_groups[source_code] = []
+                source_groups[source_code].append(announcement)
+
+            saved_stats = {}
+
+            for source_code, source_announcements in source_groups.items():
+                # 对每个来源的公告按发布时间排序（最新的在前）
+                sorted_announcements = sorted(
+                    source_announcements,
+                    key=lambda x: x.publish_date or datetime.min,
+                    reverse=True
+                )
+
+                # 为没有哈希的公告生成哈希
+                for announcement in sorted_announcements:
+                    if not announcement.content_hash:
+                        announcement.generate_content_hash()
+
+                # 批量保存
+                to_save = sorted_announcements[:max_per_source]
+                saved_count = self.db_manager.save_announcements_batch_to_table(to_save, "manual_announcements")
+                saved_stats[source_code] = saved_count
+
+                # 清理该来源超出限制的旧数据
+                if len(sorted_announcements) > max_per_source:
+                    self._cleanup_old_announcements_by_source_in_table(source_code, max_per_source, "manual_announcements")
+
+                logger.info(f"来源 {source_code} 保存了 {saved_count} 条手动搜索公告")
+
+            total_saved = sum(saved_stats.values())
+            logger.info(f"按来源保存手动搜索公告完成，总计保存 {total_saved} 条公告")
+
+            return saved_stats
+
+        except Exception as e:
+            logger.error(f"按来源保存手动搜索公告失败: {str(e)}")
+            return {}
+
     def save_all_announcements_by_source(self, announcements: List[Announcement],
                                        max_per_source: int = 100) -> Dict[str, int]:
         """
@@ -186,6 +275,41 @@ class PostgreSQLStorage:
         # 由于我们在爬取时已经标记，这里主要是确保数据库中的标记正确
         pass
 
+    def _mark_new_announcements_in_table(self, announcements: List[Announcement], table_name: str):
+        """在指定表中标记新公告"""
+        # 这里可以添加新公告标记逻辑
+        pass
+
+    def _cleanup_old_announcements_by_source_in_table(self, source_code: str, max_per_source: int, table_name: str):
+        """在指定表中清理来源的旧公告"""
+        try:
+            with self.db_manager.get_db_cursor() as cursor:
+                # 获取该来源当前保存的公告数量
+                cursor.execute(f"""
+                    SELECT COUNT(*) FROM {table_name}
+                    WHERE source_code = %s
+                """, (source_code,))
+
+                current_count = cursor.fetchone()[0]
+
+                if current_count > max_per_source:
+                    # 删除超出数量的旧公告
+                    delete_count = current_count - max_per_source
+                    cursor.execute(f"""
+                        DELETE FROM {table_name}
+                        WHERE id IN (
+                            SELECT id FROM {table_name}
+                            WHERE source_code = %s
+                            ORDER BY publish_date DESC, created_at DESC
+                            OFFSET %s
+                        )
+                    """, (source_code, max_per_source))
+
+                    logger.info(f"清理了 {cursor.rowcount} 条{table_name}表中来源{source_code}的旧公告")
+
+        except Exception as e:
+            logger.error(f"清理{table_name}表中来源{source_code}的旧公告失败: {str(e)}")
+
     def save_crawl_results(self, results: List[CrawlResult]) -> int:
         """
         保存爬取结果
@@ -389,6 +513,15 @@ class StorageManager:
         """按来源保存所有公告"""
         return self._current_storage.save_all_announcements_by_source(announcements, max_per_source)
 
+    def save_auto_announcements(self, announcements: List[Announcement]) -> int:
+        """保存定时搜索公告到专用表"""
+        return self._current_storage.save_auto_announcements(announcements)
+
+    def save_manual_announcements_by_source(self, announcements: List[Announcement],
+                                           max_per_source: int = 100) -> Dict[str, int]:
+        """按来源保存手动搜索公告到专用表"""
+        return self._current_storage.save_manual_announcements_by_source(announcements, max_per_source)
+
     def save_crawl_results(self, results: List[CrawlResult]) -> int:
         """保存爬取结果"""
         return self._current_storage.save_crawl_results(results)
@@ -476,6 +609,34 @@ def save_all_announcements_by_source_to_storage(announcements: List[Announcement
     return get_storage_manager().save_all_announcements_by_source(announcements, max_per_source)
 
 
+def save_auto_announcements_to_storage(announcements: List[Announcement]) -> int:
+    """
+    保存定时搜索公告到专用表
+
+    Args:
+        announcements: 公告列表
+
+    Returns:
+        int: 保存成功的数量
+    """
+    return get_storage_manager().save_auto_announcements(announcements)
+
+
+def save_manual_announcements_by_source_to_storage(announcements: List[Announcement],
+                                                  max_per_source: int = 100) -> Dict[str, int]:
+    """
+    按来源保存手动搜索公告到专用表
+
+    Args:
+        announcements: 所有公告列表
+        max_per_source: 每个来源最大保留数量
+
+    Returns:
+        Dict[str, int]: 各来源保存的数量
+    """
+    return get_storage_manager().save_manual_announcements_by_source(announcements, max_per_source)
+
+
 def cleanup_storage(days: Optional[int] = None) -> int:
     """
     清理存储中的过期数据