v6ole
cbeb3c3504
- CrawlService.run_spider 现在自动调用 PostCrawlPipeline（爬取→存储→筛选→去重→推送） - 定时任务添加 22:00-06:00 夜间跳过逻辑 - 断网恢复后不会漏公告：爬虫每次从 API 按日期倒序抓取，未推送的公告下次自动补推 - Pipeline 的 _exclude_sent + mark_sent 防止重复推送
29 lines
872 B
Python
29 lines
872 B
Python
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.crawler.dahuagov_spider import DahuagovSpider
|
|
from app.crawler.gxgp_spider import GXGPSpider
|
|
from app.services.crawl_service import CrawlService
|
|
from app.services.notification_service import NotificationService
|
|
|
|
|
|
async def get_db() -> AsyncSession:
|
|
from app.main import async_session
|
|
async with async_session() as session:
|
|
yield session
|
|
|
|
|
|
_crawl_service: CrawlService | None = None
|
|
|
|
|
|
def get_crawl_service() -> CrawlService:
|
|
global _crawl_service
|
|
if _crawl_service is None:
|
|
from app.main import async_session
|
|
_crawl_service = CrawlService(
|
|
db_session_factory=async_session,
|
|
notification_service=NotificationService(),
|
|
)
|
|
_crawl_service.register(GXGPSpider())
|
|
_crawl_service.register(DahuagovSpider())
|
|
return _crawl_service
|