from fastapi import APIRouter from app.api.deps import get_crawl_service from app.models.schemas import CrawlTriggerRequest router = APIRouter() @router.post("/crawl/trigger") async def trigger_crawl(request: CrawlTriggerRequest): service = get_crawl_service() names = service.get_spider_names() all_results = [] total_stored = 0 total_notified = 0 for name in names: results = await service.run_spider(name) all_results.extend(results) for r in results: if r.pipeline_result: total_stored += r.pipeline_result.stored total_notified += r.pipeline_result.notified return { "spiders_run": names, "total_announcements": sum(r.total_count for r in all_results), "total_stored": total_stored, "total_notified": total_notified, "errors": [r.error_message for r in all_results if not r.success], } @router.get("/crawl/status") async def crawl_status(): service = get_crawl_service() return { "spiders": service.get_spider_names(), "running": False, } @router.get("/crawl/sources") async def crawl_sources(): import json from app.config import settings sources = json.loads(settings.announcement_sources) return { "sources": [ {"code": code, "name": info["name"], "type": info["type"]} for code, info in sources.items() ] }