v6ole
cbeb3c3504
- CrawlService.run_spider 现在自动调用 PostCrawlPipeline（爬取→存储→筛选→去重→推送） - 定时任务添加 22:00-06:00 夜间跳过逻辑 - 断网恢复后不会漏公告：爬虫每次从 API 按日期倒序抓取，未推送的公告下次自动补推 - Pipeline 的 _exclude_sent + mark_sent 防止重复推送
75 lines
2.6 KiB
Python
75 lines
2.6 KiB
Python
import logging
|
|
|
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
|
|
|
from app.crawler.base import BaseSpider, CrawlResult
|
|
from app.services.notification_service import NotificationService
|
|
from app.services.pipeline import PostCrawlPipeline
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class CrawlService:
|
|
def __init__(self, db_session_factory: async_sessionmaker,
|
|
notification_service: NotificationService):
|
|
self.db_session_factory = db_session_factory
|
|
self.notification_service = notification_service
|
|
self.spiders: dict[str, BaseSpider] = {}
|
|
|
|
def register(self, spider: BaseSpider):
|
|
self.spiders[spider.name] = spider
|
|
|
|
async def run_all(self) -> list[CrawlResult]:
|
|
results = []
|
|
for name in self.spiders:
|
|
spider_results = await self.run_spider(name)
|
|
results.extend(spider_results)
|
|
return results
|
|
|
|
async def run_spider(self, name: str) -> list[CrawlResult]:
|
|
spider = self.spiders.get(name)
|
|
if spider is None:
|
|
return [CrawlResult(
|
|
source_code=name, source_name=name,
|
|
error_message=f"Spider not found: {name}",
|
|
)]
|
|
|
|
# 1. Crawl
|
|
result = await spider.crawl()
|
|
|
|
# 2. Pipeline: store → filter → notify
|
|
if result.success and result.announcements:
|
|
try:
|
|
config = spider.get_pipeline_config()
|
|
async with self.db_session_factory() as db:
|
|
pipeline = PostCrawlPipeline(
|
|
db_session=db,
|
|
notification_service=self.notification_service,
|
|
)
|
|
pipe_result = await pipeline.process(
|
|
result.announcements, config,
|
|
)
|
|
result.pipeline_result = pipe_result
|
|
result.new_count = pipe_result.stored
|
|
logger.info(
|
|
f"Spider {name}: total={result.total_count}, "
|
|
f"stored={pipe_result.stored}, "
|
|
f"filtered={pipe_result.filtered}, "
|
|
f"notified={pipe_result.notified}"
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"Spider {name} pipeline 异常: {e}")
|
|
if not result.error_message:
|
|
result.error_message = str(e)
|
|
|
|
return [result]
|
|
|
|
def get_spider_names(self) -> list[str]:
|
|
return list(self.spiders.keys())
|
|
|
|
def get_pipeline_config(self, name: str):
|
|
spider = self.spiders.get(name)
|
|
if spider:
|
|
return spider.get_pipeline_config()
|
|
return None
|