fix: /crawl/sources 包含独立爬虫来源 + trigger 支持 spider_name
- sources 接口原来只返回 GXGP 子来源,遗漏了大化县政府网 - trigger 接口现在支持 spider_name 参数指定单个爬虫
This commit is contained in:
+17
-3
@@ -9,6 +9,9 @@ router = APIRouter()
|
||||
@router.post("/crawl/trigger")
|
||||
async def trigger_crawl(request: CrawlTriggerRequest):
|
||||
service = get_crawl_service()
|
||||
if request.spider_name:
|
||||
names = [request.spider_name]
|
||||
else:
|
||||
names = service.get_spider_names()
|
||||
|
||||
all_results = []
|
||||
@@ -46,9 +49,20 @@ async def crawl_sources():
|
||||
|
||||
from app.config import settings
|
||||
sources = json.loads(settings.announcement_sources)
|
||||
return {
|
||||
"sources": [
|
||||
result = [
|
||||
{"code": code, "name": info["name"], "type": info["type"]}
|
||||
for code, info in sources.items()
|
||||
]
|
||||
}
|
||||
# 加入独立爬虫来源(非 GXGP 子来源的独立 Spider)
|
||||
service = get_crawl_service()
|
||||
for name in service.get_spider_names():
|
||||
if name == "gxgp":
|
||||
continue # gxgp 的子来源已在上面列出
|
||||
spider = service.spiders.get(name)
|
||||
if spider:
|
||||
result.append({
|
||||
"code": spider.source_code,
|
||||
"name": spider.source_name,
|
||||
"type": "independent",
|
||||
})
|
||||
return {"sources": result}
|
||||
|
||||
@@ -29,6 +29,7 @@ class AnnouncementListResponse(BaseModel):
|
||||
|
||||
|
||||
class CrawlTriggerRequest(BaseModel):
|
||||
spider_name: str | None = None
|
||||
keywords: list[str] | None = None
|
||||
sources: list[str] | None = None
|
||||
manual: bool = False
|
||||
|
||||
Reference in New Issue
Block a user