Enhance announcement storage functionality by adding a method to save all announcements by source, retaining the latest 100 per source. Update related logging and comments for clarity. Adjusted the main script to utilize the new storage method and log the saving process. Updated the markdown file timestamp.

2026-01-08 09:31:47 +08:00
parent 28c57a040a
commit 317dbf3c58
37 changed files with 3065 additions and 10 deletions
@@ -32,7 +32,7 @@ class PostgreSQLStorage:

    def save_announcements(self, announcements: List[Announcement]) -> int:
        """
-        保存公告列表到数据库
+        保存公告列表到数据库（经过筛选的公告）

        Args:
            announcements: 公告列表
@@ -43,14 +43,14 @@ class PostgreSQLStorage:
        if not announcements:
            return 0

-        logger.info(f"开始保存 {len(announcements)} 条公告到数据库")
+        logger.info(f"开始保存 {len(announcements)} 条筛选后公告到数据库")

        try:
            # 批量保存
            saved_count = self.db_manager.save_announcements_batch(announcements)

            if saved_count > 0:
-                logger.info(f"成功保存 {saved_count} 条公告到数据库")
+                logger.info(f"成功保存 {saved_count} 条筛选后公告到数据库")

                # 标记新公告
                self._mark_new_announcements(announcements[:saved_count])
@@ -58,10 +58,112 @@ class PostgreSQLStorage:
            return saved_count

        except Exception as e:
-            logger.error(f"保存公告到数据库失败: {str(e)}")
+            logger.error(f"保存筛选后公告到数据库失败: {str(e)}")
            # 尝试逐个保存
            return self._save_announcements_fallback(announcements)

+    def save_all_announcements_by_source(self, announcements: List[Announcement],
+                                       max_per_source: int = 100) -> Dict[str, int]:
+        """
+        按来源保存所有公告，每个来源保留最新的max_per_source条
+
+        Args:
+            announcements: 所有公告列表（未经关键词筛选）
+            max_per_source: 每个来源最大保留数量
+
+        Returns:
+            Dict[str, int]: 各来源保存的数量
+        """
+        if not announcements:
+            return {}
+
+        logger.info(f"开始按来源保存 {len(announcements)} 条公告，每个来源最多保留 {max_per_source} 条")
+
+        try:
+            # 按来源分组
+            source_groups = {}
+            for announcement in announcements:
+                source_code = announcement.source_code
+                if source_code not in source_groups:
+                    source_groups[source_code] = []
+                source_groups[source_code].append(announcement)
+
+            saved_stats = {}
+
+            for source_code, source_announcements in source_groups.items():
+                # 对每个来源的公告按发布时间排序（最新的在前）
+                sorted_announcements = sorted(
+                    source_announcements,
+                    key=lambda x: x.publish_date or x.crawled_at or datetime.min,
+                    reverse=True
+                )
+
+                # 取最新的max_per_source条
+                to_save = sorted_announcements[:max_per_source]
+
+                # 为这些公告生成哈希
+                for announcement in to_save:
+                    if not announcement.content_hash:
+                        announcement.generate_content_hash()
+
+                # 批量保存
+                saved_count = self.db_manager.save_announcements_batch(to_save)
+
+                saved_stats[source_code] = saved_count
+
+                # 清理该来源超出限制的旧数据
+                if len(sorted_announcements) > max_per_source:
+                    self._cleanup_old_announcements_by_source(source_code, max_per_source)
+
+                logger.info(f"来源 {source_code} 保存了 {saved_count} 条公告")
+
+            total_saved = sum(saved_stats.values())
+            logger.info(f"按来源保存完成，总计保存 {total_saved} 条公告")
+
+            return saved_stats
+
+        except Exception as e:
+            logger.error(f"按来源保存公告失败: {str(e)}")
+            return {}
+
+    def _cleanup_old_announcements_by_source(self, source_code: str, keep_count: int):
+        """
+        清理指定来源超出限制的旧公告
+
+        Args:
+            source_code: 来源代码
+            keep_count: 保留数量
+        """
+        try:
+            # 使用窗口函数删除超出限制的记录
+            sql = """
+            DELETE FROM announcements
+            WHERE source_code = %s
+              AND id IN (
+                SELECT id FROM (
+                  SELECT id,
+                         ROW_NUMBER() OVER (ORDER BY publish_date DESC, crawled_at DESC) as rn
+                  FROM announcements
+                  WHERE source_code = %s
+                ) ranked
+                WHERE rn > %s
+              )
+            """
+
+            with get_db_cursor() as cursor:
+                cursor.execute(sql, (source_code, source_code, keep_count))
+                deleted_count = cursor.rowcount
+
+                if deleted_count > 0:
+                    logger.debug(f"清理来源 {source_code} 的 {deleted_count} 条旧公告")
+
+        except Exception as e:
+            logger.warning(f"清理来源 {source_code} 旧公告失败: {str(e)}")
+
+        except Exception as e:
+            logger.error(f"按来源保存公告失败: {str(e)}")
+            return {}
+
    def _save_announcements_fallback(self, announcements: List[Announcement]) -> int:
        """逐个保存公告的降级方案"""
        logger.info("使用降级方案逐个保存公告")
@@ -279,9 +381,14 @@ class StorageManager:
        self._current_storage = self.postgresql  # 默认使用PostgreSQL

    def save_announcements(self, announcements: List[Announcement]) -> int:
-        """保存公告"""
+        """保存筛选后的公告"""
        return self._current_storage.save_announcements(announcements)

+    def save_all_announcements_by_source(self, announcements: List[Announcement],
+                                       max_per_source: int = 100) -> Dict[str, int]:
+        """按来源保存所有公告"""
+        return self._current_storage.save_all_announcements_by_source(announcements, max_per_source)
+
    def save_crawl_results(self, results: List[CrawlResult]) -> int:
        """保存爬取结果"""
        return self._current_storage.save_crawl_results(results)
@@ -343,7 +450,7 @@ def init_storage():

 def save_announcements_to_storage(announcements: List[Announcement]) -> int:
    """
-    保存公告到存储
+    保存筛选后的公告到存储

    Args:
        announcements: 公告列表
@@ -354,6 +461,21 @@ def save_announcements_to_storage(announcements: List[Announcement]) -> int:
    return get_storage_manager().save_announcements(announcements)


+def save_all_announcements_by_source_to_storage(announcements: List[Announcement],
+                                              max_per_source: int = 100) -> Dict[str, int]:
+    """
+    按来源保存所有公告到存储
+
+    Args:
+        announcements: 所有公告列表
+        max_per_source: 每个来源最大保留数量
+
+    Returns:
+        Dict[str, int]: 各来源保存的数量
+    """
+    return get_storage_manager().save_all_announcements_by_source(announcements, max_per_source)
+
+
 def cleanup_storage(days: Optional[int] = None) -> int:
    """
    清理存储中的过期数据