chore: ruff 代码检查与修复
130 issues auto-fixed (import ordering, UP045/UP006 type annotations), 33 issues manually fixed (E712/E501/E402/E722 + N818 rename + per-file wechat ignore for N8xx naming conventions). All 33 tests pass.
This commit is contained in:
@@ -2,11 +2,12 @@ import json
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
||||
from app.crawler.parsers import parse_gxgp_api_response, extract_pagination
|
||||
from app.crawler.parsers import extract_pagination, parse_gxgp_api_response
|
||||
|
||||
|
||||
class GXGPSpider(BaseSpider):
|
||||
@@ -27,8 +28,8 @@ class GXGPSpider(BaseSpider):
|
||||
mark_sent=False,
|
||||
)
|
||||
|
||||
async def crawl(self, sources: Optional[List[str]] = None,
|
||||
max_pages: Optional[int] = None) -> CrawlResult:
|
||||
async def crawl(self, sources: list[str] | None = None,
|
||||
max_pages: int | None = None) -> CrawlResult:
|
||||
if max_pages is None:
|
||||
max_pages = settings.crawler_max_pages
|
||||
if sources is None:
|
||||
@@ -88,7 +89,7 @@ class GXGPSpider(BaseSpider):
|
||||
)
|
||||
|
||||
async def _fetch_page(self, client: httpx.AsyncClient, source_code: str,
|
||||
category_id: int, page_no: int) -> Optional[dict]:
|
||||
category_id: int, page_no: int) -> dict | None:
|
||||
payload = {
|
||||
"keyword": "",
|
||||
"publishDateBegin": "",
|
||||
@@ -102,7 +103,7 @@ class GXGPSpider(BaseSpider):
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||
"Content-Type": "application/json;charset=UTF-8",
|
||||
"Origin": self.base_url,
|
||||
"Referer": f"{self.base_url}/site/category?parentId={category_id}&childrenCode={source_code}",
|
||||
"Referer": f"{self.base_url}/site/category?parentId={category_id}&childrenCode={source_code}", # noqa: E501
|
||||
}
|
||||
response = await client.post(
|
||||
self.announcement_api, json=payload, headers=headers
|
||||
|
||||
Reference in New Issue
Block a user