from fastapi import APIRouter from app.api.deps import get_crawl_service from app.models.schemas import CrawlTriggerRequest router = APIRouter() @router.post("/crawl/trigger") async def trigger_crawl(request: CrawlTriggerRequest): service = get_crawl_service() if request.spider_name: names = [request.spider_name] else: names = service.get_spider_names() all_results = [] total_stored = 0 total_notified = 0 for name in names: results = await service.run_spider(name) all_results.extend(results) for r in results: if r.pipeline_result: total_stored += r.pipeline_result.stored total_notified += r.pipeline_result.notified return { "spiders_run": names, "total_announcements": sum(r.total_count for r in all_results), "total_stored": total_stored, "total_notified": total_notified, "errors": [r.error_message for r in all_results if not r.success], } @router.get("/crawl/status") async def crawl_status(): service = get_crawl_service() return { "spiders": service.get_spider_names(), "running": False, } @router.get("/crawl/sources") async def crawl_sources(): import json from app.config import settings sources = json.loads(settings.announcement_sources) result = [ {"code": code, "name": info["name"], "type": info["type"]} for code, info in sources.items() ] # 加入独立爬虫来源(非 GXGP 子来源的独立 Spider) service = get_crawl_service() for name in service.get_spider_names(): if name == "gxgp": continue # gxgp 的子来源已在上面列出 spider = service.spiders.get(name) if spider: result.append({ "code": spider.source_code, "name": spider.source_name, "type": "independent", }) return {"sources": result}