import logging from sqlalchemy.ext.asyncio import async_sessionmaker from app.crawler.base import BaseSpider, CrawlResult from app.services.notification_service import NotificationService from app.services.pipeline import PostCrawlPipeline logger = logging.getLogger(__name__) class CrawlService: def __init__(self, db_session_factory: async_sessionmaker, notification_service: NotificationService): self.db_session_factory = db_session_factory self.notification_service = notification_service self.spiders: dict[str, BaseSpider] = {} def register(self, spider: BaseSpider): self.spiders[spider.name] = spider async def run_all(self) -> list[CrawlResult]: results = [] for name in self.spiders: spider_results = await self.run_spider(name) results.extend(spider_results) return results async def run_spider(self, name: str) -> list[CrawlResult]: spider = self.spiders.get(name) if spider is None: return [CrawlResult( source_code=name, source_name=name, error_message=f"Spider not found: {name}", )] # 1. Crawl result = await spider.crawl() # 2. Pipeline: store → filter → notify if result.success and result.announcements: try: config = spider.get_pipeline_config() async with self.db_session_factory() as db: pipeline = PostCrawlPipeline( db_session=db, notification_service=self.notification_service, ) pipe_result = await pipeline.process( result.announcements, config, ) result.pipeline_result = pipe_result result.new_count = pipe_result.stored logger.info( f"Spider {name}: total={result.total_count}, " f"stored={pipe_result.stored}, " f"filtered={pipe_result.filtered}, " f"notified={pipe_result.notified}" ) except Exception as e: logger.error(f"Spider {name} pipeline 异常: {e}") if not result.error_message: result.error_message = str(e) return [result] def get_spider_names(self) -> list[str]: return list(self.spiders.keys()) def get_pipeline_config(self, name: str): spider = self.spiders.get(name) if spider: return spider.get_pipeline_config() return None