Refactor the cron job functionality to transition from a crawling to a searching model. Update related documentation and scripts to reflect the new terminology and functionality, including changes in the README and script comments. Enhance WeChat menu and message handling to support the new search features, ensuring clarity in user interactions and logging.
This commit is contained in:
@@ -90,22 +90,22 @@ class GXGPMonitorApp:
|
||||
return False
|
||||
|
||||
def run_crawl(self, keywords: list = None, sources: list = None, max_pages: int = None, manual_crawl: bool = False):
|
||||
"""执行爬取任务
|
||||
"""执行搜索任务
|
||||
|
||||
Args:
|
||||
keywords: 关键词列表
|
||||
sources: 来源列表
|
||||
max_pages: 最大页数
|
||||
manual_crawl: 是否为手动爬取(只筛选今天的数据)
|
||||
manual_crawl: 是否为手动搜索(只筛选今天的数据)
|
||||
"""
|
||||
try:
|
||||
logger.info(f"开始执行爬取任务 (手动爬取: {manual_crawl})")
|
||||
logger.info(f"开始执行搜索任务 (手动搜索: {manual_crawl})")
|
||||
|
||||
# 执行爬取
|
||||
crawl_results = crawl_announcements(keywords, sources)
|
||||
|
||||
if not crawl_results:
|
||||
logger.info("爬取完成:无数据")
|
||||
logger.info("搜索完成:无数据")
|
||||
return {"success": True, "results": []}
|
||||
|
||||
# 收集所有公告
|
||||
@@ -114,9 +114,9 @@ class GXGPMonitorApp:
|
||||
if result.announcements:
|
||||
all_announcements.extend(result.announcements)
|
||||
|
||||
logger.info(f"爬取到 {len(all_announcements)} 条原始公告")
|
||||
logger.info(f"搜索到 {len(all_announcements)} 条原始公告")
|
||||
|
||||
# 对于手动爬取,不保存公告到数据库,只进行筛选和返回结果
|
||||
# 对于手动搜索,不保存公告到数据库,只进行筛选和返回结果
|
||||
if not manual_crawl:
|
||||
# 先保存所有公告(按来源分组,每源保留最新100条)
|
||||
all_saved_stats = save_all_announcements_by_source_to_storage(all_announcements, max_per_source=100)
|
||||
@@ -125,11 +125,11 @@ class GXGPMonitorApp:
|
||||
else:
|
||||
all_saved_count = 0
|
||||
all_saved_stats = {}
|
||||
logger.info("手动爬取模式:跳过数据库保存")
|
||||
logger.info("手动搜索模式:跳过数据库保存")
|
||||
|
||||
# 筛选公告
|
||||
if manual_crawl:
|
||||
# 手动爬取:只筛选今天的公告和用户指定的关键词,不进行去重
|
||||
# 手动搜索:只筛选今天的公告和用户指定的关键词,不进行去重
|
||||
from .filters.filters import KeywordFilter, DateFilter, SourceFilter
|
||||
from datetime import date
|
||||
|
||||
@@ -149,7 +149,7 @@ class GXGPMonitorApp:
|
||||
filter_stats = type('FilterResult', (), {
|
||||
"keyword_filtered": len(date_filtered) - len(keyword_filtered),
|
||||
"date_filtered": len(all_announcements) - len(date_filtered),
|
||||
"duplicate_filtered": 0, # 手动爬取不进行去重
|
||||
"duplicate_filtered": 0, # 手动搜索不进行去重
|
||||
"source_filtered": len(keyword_filtered) - len(filtered_announcements)
|
||||
})()
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user