fix: /crawl/sources 包含独立爬虫来源 + trigger 支持 spider_name
- sources 接口原来只返回 GXGP 子来源,遗漏了大化县政府网 - trigger 接口现在支持 spider_name 参数指定单个爬虫
This commit is contained in:
+21
-7
@@ -9,7 +9,10 @@ router = APIRouter()
|
|||||||
@router.post("/crawl/trigger")
|
@router.post("/crawl/trigger")
|
||||||
async def trigger_crawl(request: CrawlTriggerRequest):
|
async def trigger_crawl(request: CrawlTriggerRequest):
|
||||||
service = get_crawl_service()
|
service = get_crawl_service()
|
||||||
names = service.get_spider_names()
|
if request.spider_name:
|
||||||
|
names = [request.spider_name]
|
||||||
|
else:
|
||||||
|
names = service.get_spider_names()
|
||||||
|
|
||||||
all_results = []
|
all_results = []
|
||||||
total_stored = 0
|
total_stored = 0
|
||||||
@@ -46,9 +49,20 @@ async def crawl_sources():
|
|||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
sources = json.loads(settings.announcement_sources)
|
sources = json.loads(settings.announcement_sources)
|
||||||
return {
|
result = [
|
||||||
"sources": [
|
{"code": code, "name": info["name"], "type": info["type"]}
|
||||||
{"code": code, "name": info["name"], "type": info["type"]}
|
for code, info in sources.items()
|
||||||
for code, info in sources.items()
|
]
|
||||||
]
|
# 加入独立爬虫来源(非 GXGP 子来源的独立 Spider)
|
||||||
}
|
service = get_crawl_service()
|
||||||
|
for name in service.get_spider_names():
|
||||||
|
if name == "gxgp":
|
||||||
|
continue # gxgp 的子来源已在上面列出
|
||||||
|
spider = service.spiders.get(name)
|
||||||
|
if spider:
|
||||||
|
result.append({
|
||||||
|
"code": spider.source_code,
|
||||||
|
"name": spider.source_name,
|
||||||
|
"type": "independent",
|
||||||
|
})
|
||||||
|
return {"sources": result}
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ class AnnouncementListResponse(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class CrawlTriggerRequest(BaseModel):
|
class CrawlTriggerRequest(BaseModel):
|
||||||
|
spider_name: str | None = None
|
||||||
keywords: list[str] | None = None
|
keywords: list[str] | None = None
|
||||||
sources: list[str] | None = None
|
sources: list[str] | None = None
|
||||||
manual: bool = False
|
manual: bool = False
|
||||||
|
|||||||
Reference in New Issue
Block a user