37 lines
1.1 KiB
Python
37 lines
1.1 KiB
Python
from typing import Dict, List
|
|
from app.crawler.base import BaseSpider, CrawlResult
|
|
|
|
|
|
class CrawlService:
|
|
def __init__(self):
|
|
self.spiders: Dict[str, BaseSpider] = {}
|
|
|
|
def register(self, spider: BaseSpider):
|
|
self.spiders[spider.name] = spider
|
|
|
|
async def run_all(self) -> List[CrawlResult]:
|
|
results = []
|
|
for name, spider in self.spiders.items():
|
|
result = await spider.crawl()
|
|
results.append(result)
|
|
return results
|
|
|
|
async def run_spider(self, name: str, **kwargs) -> List[CrawlResult]:
|
|
spider = self.spiders.get(name)
|
|
if spider is None:
|
|
return [CrawlResult(
|
|
source_code=name, source_name=name,
|
|
error_message=f"Spider not found: {name}"
|
|
)]
|
|
result = await spider.crawl(**kwargs)
|
|
return [result]
|
|
|
|
def get_spider_names(self) -> List[str]:
|
|
return list(self.spiders.keys())
|
|
|
|
def get_pipeline_config(self, name: str):
|
|
spider = self.spiders.get(name)
|
|
if spider:
|
|
return spider.get_pipeline_config()
|
|
return None
|