69 lines
2.0 KiB
Python
69 lines
2.0 KiB
Python
from fastapi import APIRouter
|
||
|
||
from app.api.deps import get_crawl_service
|
||
from app.models.schemas import CrawlTriggerRequest
|
||
|
||
router = APIRouter()
|
||
|
||
|
||
@router.post("/crawl/trigger")
|
||
async def trigger_crawl(request: CrawlTriggerRequest):
|
||
service = get_crawl_service()
|
||
if request.spider_name:
|
||
names = [request.spider_name]
|
||
else:
|
||
names = service.get_spider_names()
|
||
|
||
all_results = []
|
||
total_stored = 0
|
||
total_notified = 0
|
||
for name in names:
|
||
results = await service.run_spider(name)
|
||
all_results.extend(results)
|
||
for r in results:
|
||
if r.pipeline_result:
|
||
total_stored += r.pipeline_result.stored
|
||
total_notified += r.pipeline_result.notified
|
||
|
||
return {
|
||
"spiders_run": names,
|
||
"total_announcements": sum(r.total_count for r in all_results),
|
||
"total_stored": total_stored,
|
||
"total_notified": total_notified,
|
||
"errors": [r.error_message for r in all_results if not r.success],
|
||
}
|
||
|
||
|
||
@router.get("/crawl/status")
|
||
async def crawl_status():
|
||
service = get_crawl_service()
|
||
return {
|
||
"spiders": service.get_spider_names(),
|
||
"running": False,
|
||
}
|
||
|
||
|
||
@router.get("/crawl/sources")
|
||
async def crawl_sources():
|
||
import json
|
||
|
||
from app.config import settings
|
||
sources = json.loads(settings.announcement_sources)
|
||
result = [
|
||
{"code": code, "name": info["name"], "type": info["type"]}
|
||
for code, info in sources.items()
|
||
]
|
||
# 加入独立爬虫来源（非 GXGP 子来源的独立 Spider）
|
||
service = get_crawl_service()
|
||
for name in service.get_spider_names():
|
||
if name == "gxgp":
|
||
continue # gxgp 的子来源已在上面列出
|
||
spider = service.spiders.get(name)
|
||
if spider:
|
||
result.append({
|
||
"code": spider.source_code,
|
||
"name": spider.source_name,
|
||
"type": "independent",
|
||
})
|
||
return {"sources": result}
|