Enhance announcement storage functionality by adding a method to save all announcements by source, retaining the latest 100 per source. Update related logging and comments for clarity. Adjusted the main script to utilize the new storage method and log the saving process. Updated the markdown file timestamp.
This commit is contained in:
@@ -20,7 +20,7 @@ try:
|
||||
from .core.reliability import check_system_health
|
||||
from .crawler.spider import crawl_announcements
|
||||
from .filters.filters import filter_from_config
|
||||
from .storage.postgresql import init_storage, save_announcements_to_storage, cleanup_storage
|
||||
from .storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, cleanup_storage
|
||||
from .storage.md_generator import generate_onu_md
|
||||
from .notification.wechat import send_announcements_notification, send_system_notification
|
||||
except ImportError:
|
||||
@@ -31,7 +31,7 @@ except ImportError:
|
||||
from core.reliability import check_system_health
|
||||
from crawler.spider import crawl_announcements
|
||||
from filters.filters import filter_from_config
|
||||
from storage.postgresql import init_storage, save_announcements_to_storage, cleanup_storage
|
||||
from storage.postgresql import init_storage, save_announcements_to_storage, save_all_announcements_by_source_to_storage, cleanup_storage
|
||||
from storage.md_generator import generate_onu_md
|
||||
from notification.wechat import send_announcements_notification, send_system_notification
|
||||
except ImportError as e:
|
||||
@@ -98,13 +98,18 @@ class GXGPMonitorApp:
|
||||
|
||||
logger.info(f"爬取到 {len(all_announcements)} 条原始公告")
|
||||
|
||||
# 先保存所有公告(按来源分组,每源保留最新100条)
|
||||
all_saved_stats = save_all_announcements_by_source_to_storage(all_announcements, max_per_source=100)
|
||||
all_saved_count = sum(all_saved_stats.values())
|
||||
logger.info(f"保存所有公告完成:共保存 {all_saved_count} 条,按来源统计: {all_saved_stats}")
|
||||
|
||||
# 筛选公告
|
||||
filter_obj = filter_from_config()
|
||||
filtered_announcements, filter_stats = filter_obj.filter(all_announcements)
|
||||
|
||||
logger.info(f"筛选后剩余 {len(filtered_announcements)} 条公告")
|
||||
|
||||
# 保存到数据库
|
||||
# 保存筛选后的公告(用于标记关键词匹配等)
|
||||
saved_count = save_announcements_to_storage(filtered_announcements)
|
||||
|
||||
# 生成Markdown文件
|
||||
|
||||
Reference in New Issue
Block a user