v6ole
cbeb3c3504
- CrawlService.run_spider 现在自动调用 PostCrawlPipeline（爬取→存储→筛选→去重→推送） - 定时任务添加 22:00-06:00 夜间跳过逻辑 - 断网恢复后不会漏公告：爬虫每次从 API 按日期倒序抓取，未推送的公告下次自动补推 - Pipeline 的 _exclude_sent + mark_sent 防止重复推送
55 lines
1.4 KiB
Python
55 lines
1.4 KiB
Python
from fastapi import APIRouter
|
|
|
|
from app.api.deps import get_crawl_service
|
|
from app.models.schemas import CrawlTriggerRequest
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
@router.post("/crawl/trigger")
|
|
async def trigger_crawl(request: CrawlTriggerRequest):
|
|
service = get_crawl_service()
|
|
names = service.get_spider_names()
|
|
|
|
all_results = []
|
|
total_stored = 0
|
|
total_notified = 0
|
|
for name in names:
|
|
results = await service.run_spider(name)
|
|
all_results.extend(results)
|
|
for r in results:
|
|
if r.pipeline_result:
|
|
total_stored += r.pipeline_result.stored
|
|
total_notified += r.pipeline_result.notified
|
|
|
|
return {
|
|
"spiders_run": names,
|
|
"total_announcements": sum(r.total_count for r in all_results),
|
|
"total_stored": total_stored,
|
|
"total_notified": total_notified,
|
|
"errors": [r.error_message for r in all_results if not r.success],
|
|
}
|
|
|
|
|
|
@router.get("/crawl/status")
|
|
async def crawl_status():
|
|
service = get_crawl_service()
|
|
return {
|
|
"spiders": service.get_spider_names(),
|
|
"running": False,
|
|
}
|
|
|
|
|
|
@router.get("/crawl/sources")
|
|
async def crawl_sources():
|
|
import json
|
|
|
|
from app.config import settings
|
|
sources = json.loads(settings.announcement_sources)
|
|
return {
|
|
"sources": [
|
|
{"code": code, "name": info["name"], "type": info["type"]}
|
|
for code, info in sources.items()
|
|
]
|
|
}
|