fix: /crawl/sources 包含独立爬虫来源 + trigger 支持 spider_name

- sources 接口原来只返回 GXGP 子来源,遗漏了大化县政府网
- trigger 接口现在支持 spider_name 参数指定单个爬虫
This commit is contained in:
2026-05-10 14:22:52 +08:00
parent 9f90e16661
commit 24d844bf44
2 changed files with 22 additions and 7 deletions
+21 -7
View File
@@ -9,7 +9,10 @@ router = APIRouter()
@router.post("/crawl/trigger") @router.post("/crawl/trigger")
async def trigger_crawl(request: CrawlTriggerRequest): async def trigger_crawl(request: CrawlTriggerRequest):
service = get_crawl_service() service = get_crawl_service()
names = service.get_spider_names() if request.spider_name:
names = [request.spider_name]
else:
names = service.get_spider_names()
all_results = [] all_results = []
total_stored = 0 total_stored = 0
@@ -46,9 +49,20 @@ async def crawl_sources():
from app.config import settings from app.config import settings
sources = json.loads(settings.announcement_sources) sources = json.loads(settings.announcement_sources)
return { result = [
"sources": [ {"code": code, "name": info["name"], "type": info["type"]}
{"code": code, "name": info["name"], "type": info["type"]} for code, info in sources.items()
for code, info in sources.items() ]
] # 加入独立爬虫来源(非 GXGP 子来源的独立 Spider
} service = get_crawl_service()
for name in service.get_spider_names():
if name == "gxgp":
continue # gxgp 的子来源已在上面列出
spider = service.spiders.get(name)
if spider:
result.append({
"code": spider.source_code,
"name": spider.source_name,
"type": "independent",
})
return {"sources": result}
+1
View File
@@ -29,6 +29,7 @@ class AnnouncementListResponse(BaseModel):
class CrawlTriggerRequest(BaseModel): class CrawlTriggerRequest(BaseModel):
spider_name: str | None = None
keywords: list[str] | None = None keywords: list[str] | None = None
sources: list[str] | None = None sources: list[str] | None = None
manual: bool = False manual: bool = False