Enhance manual crawling functionality in GXGPMonitorApp to skip database saving for both announcements and filtered results. Update logging to reflect manual crawl mode actions, ensuring clarity in the process. Adjusted comments for better understanding of the manual crawling logic.
This commit is contained in:
+17
-6
@@ -116,10 +116,16 @@ class GXGPMonitorApp:
|
||||
|
||||
logger.info(f"爬取到 {len(all_announcements)} 条原始公告")
|
||||
|
||||
# 先保存所有公告(按来源分组,每源保留最新100条)
|
||||
all_saved_stats = save_all_announcements_by_source_to_storage(all_announcements, max_per_source=100)
|
||||
all_saved_count = sum(all_saved_stats.values())
|
||||
logger.info(f"保存所有公告完成:共保存 {all_saved_count} 条,按来源统计: {all_saved_stats}")
|
||||
# 对于手动爬取,不保存公告到数据库,只进行筛选和返回结果
|
||||
if not manual_crawl:
|
||||
# 先保存所有公告(按来源分组,每源保留最新100条)
|
||||
all_saved_stats = save_all_announcements_by_source_to_storage(all_announcements, max_per_source=100)
|
||||
all_saved_count = sum(all_saved_stats.values())
|
||||
logger.info(f"保存所有公告完成:共保存 {all_saved_count} 条,按来源统计: {all_saved_stats}")
|
||||
else:
|
||||
all_saved_count = 0
|
||||
all_saved_stats = {}
|
||||
logger.info("手动爬取模式:跳过数据库保存")
|
||||
|
||||
# 筛选公告
|
||||
if manual_crawl:
|
||||
@@ -169,8 +175,13 @@ class GXGPMonitorApp:
|
||||
|
||||
logger.info(f"筛选后剩余 {len(filtered_announcements)} 条公告")
|
||||
|
||||
# 保存筛选后的公告(用于标记关键词匹配等)
|
||||
saved_count = save_announcements_to_storage(filtered_announcements)
|
||||
# 对于手动爬取,不保存筛选后的公告到数据库
|
||||
if not manual_crawl:
|
||||
# 保存筛选后的公告(用于标记关键词匹配等)
|
||||
saved_count = save_announcements_to_storage(filtered_announcements)
|
||||
else:
|
||||
saved_count = 0
|
||||
logger.info("手动爬取模式:跳过筛选后公告的数据库保存")
|
||||
|
||||
# 生成Markdown文件(只在自动爬取时生成)
|
||||
md_success = False
|
||||
|
||||
Reference in New Issue
Block a user