diff --git a/app/api/crawl.py b/app/api/crawl.py index 9590fc2..391f133 100644 --- a/app/api/crawl.py +++ b/app/api/crawl.py @@ -9,7 +9,10 @@ router = APIRouter() @router.post("/crawl/trigger") async def trigger_crawl(request: CrawlTriggerRequest): service = get_crawl_service() - names = service.get_spider_names() + if request.spider_name: + names = [request.spider_name] + else: + names = service.get_spider_names() all_results = [] total_stored = 0 @@ -46,9 +49,20 @@ async def crawl_sources(): from app.config import settings sources = json.loads(settings.announcement_sources) - return { - "sources": [ - {"code": code, "name": info["name"], "type": info["type"]} - for code, info in sources.items() - ] - } + result = [ + {"code": code, "name": info["name"], "type": info["type"]} + for code, info in sources.items() + ] + # 加入独立爬虫来源(非 GXGP 子来源的独立 Spider) + service = get_crawl_service() + for name in service.get_spider_names(): + if name == "gxgp": + continue # gxgp 的子来源已在上面列出 + spider = service.spiders.get(name) + if spider: + result.append({ + "code": spider.source_code, + "name": spider.source_name, + "type": "independent", + }) + return {"sources": result} diff --git a/app/models/schemas.py b/app/models/schemas.py index 611c853..c2464b7 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -29,6 +29,7 @@ class AnnouncementListResponse(BaseModel): class CrawlTriggerRequest(BaseModel): + spider_name: str | None = None keywords: list[str] | None = None sources: list[str] | None = None manual: bool = False