fix: 连接 Pipeline 到爬取流程 + 夜间跳过 + 断网恢复
- CrawlService.run_spider 现在自动调用 PostCrawlPipeline(爬取→存储→筛选→去重→推送) - 定时任务添加 22:00-06:00 夜间跳过逻辑 - 断网恢复后不会漏公告:爬虫每次从 API 按日期倒序抓取,未推送的公告下次自动补推 - Pipeline 的 _exclude_sent + mark_sent 防止重复推送
This commit is contained in:
@@ -1,9 +1,19 @@
|
||||
import logging
|
||||
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from app.crawler.base import BaseSpider, CrawlResult
|
||||
from app.services.notification_service import NotificationService
|
||||
from app.services.pipeline import PostCrawlPipeline
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CrawlService:
|
||||
def __init__(self):
|
||||
def __init__(self, db_session_factory: async_sessionmaker,
|
||||
notification_service: NotificationService):
|
||||
self.db_session_factory = db_session_factory
|
||||
self.notification_service = notification_service
|
||||
self.spiders: dict[str, BaseSpider] = {}
|
||||
|
||||
def register(self, spider: BaseSpider):
|
||||
@@ -11,19 +21,47 @@ class CrawlService:
|
||||
|
||||
async def run_all(self) -> list[CrawlResult]:
|
||||
results = []
|
||||
for name, spider in self.spiders.items():
|
||||
result = await spider.crawl()
|
||||
results.append(result)
|
||||
for name in self.spiders:
|
||||
spider_results = await self.run_spider(name)
|
||||
results.extend(spider_results)
|
||||
return results
|
||||
|
||||
async def run_spider(self, name: str, **kwargs) -> list[CrawlResult]:
|
||||
async def run_spider(self, name: str) -> list[CrawlResult]:
|
||||
spider = self.spiders.get(name)
|
||||
if spider is None:
|
||||
return [CrawlResult(
|
||||
source_code=name, source_name=name,
|
||||
error_message=f"Spider not found: {name}"
|
||||
error_message=f"Spider not found: {name}",
|
||||
)]
|
||||
result = await spider.crawl(**kwargs)
|
||||
|
||||
# 1. Crawl
|
||||
result = await spider.crawl()
|
||||
|
||||
# 2. Pipeline: store → filter → notify
|
||||
if result.success and result.announcements:
|
||||
try:
|
||||
config = spider.get_pipeline_config()
|
||||
async with self.db_session_factory() as db:
|
||||
pipeline = PostCrawlPipeline(
|
||||
db_session=db,
|
||||
notification_service=self.notification_service,
|
||||
)
|
||||
pipe_result = await pipeline.process(
|
||||
result.announcements, config,
|
||||
)
|
||||
result.pipeline_result = pipe_result
|
||||
result.new_count = pipe_result.stored
|
||||
logger.info(
|
||||
f"Spider {name}: total={result.total_count}, "
|
||||
f"stored={pipe_result.stored}, "
|
||||
f"filtered={pipe_result.filtered}, "
|
||||
f"notified={pipe_result.notified}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Spider {name} pipeline 异常: {e}")
|
||||
if not result.error_message:
|
||||
result.error_message = str(e)
|
||||
|
||||
return [result]
|
||||
|
||||
def get_spider_names(self) -> list[str]:
|
||||
|
||||
Reference in New Issue
Block a user