v6ole
d851dafea9
- Pipeline 新增 _exclude_sent() 方法，通知前查询数据库跳过 is_sent=True 的公告 - GXGP Spider mark_sent 改为 True，配合去重逻辑防止每次爬取重复通知
119 lines
4.2 KiB
Python
119 lines
4.2 KiB
Python
import json
|
|
import random
|
|
import time
|
|
from datetime import datetime
|
|
|
|
import httpx
|
|
|
|
from app.config import settings
|
|
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
|
from app.crawler.parsers import extract_pagination, parse_gxgp_api_response
|
|
|
|
|
|
class GXGPSpider(BaseSpider):
|
|
name = "gxgp"
|
|
source_code = "gxgp"
|
|
source_name = "广西政府采购网"
|
|
|
|
def __init__(self):
|
|
self.base_url = settings.crawler_base_url
|
|
self.announcement_api = f"{self.base_url}/portal/category"
|
|
|
|
def get_pipeline_config(self) -> PipelineConfig:
|
|
return PipelineConfig(
|
|
filter_enabled=True,
|
|
keywords=list(settings.crawler_keywords),
|
|
dedup_enabled=True,
|
|
notify_mode="filtered",
|
|
mark_sent=True,
|
|
)
|
|
|
|
async def crawl(self, sources: list[str] | None = None,
|
|
max_pages: int | None = None) -> CrawlResult:
|
|
if max_pages is None:
|
|
max_pages = settings.crawler_max_pages
|
|
if sources is None:
|
|
source_map = json.loads(settings.announcement_sources)
|
|
sources = list(source_map.keys())
|
|
|
|
start_time = datetime.now()
|
|
all_announcements = []
|
|
error_messages = []
|
|
|
|
async with httpx.AsyncClient(timeout=settings.crawler_timeout) as client:
|
|
for source_code in sources:
|
|
source_info = json.loads(settings.announcement_sources).get(source_code)
|
|
if not source_info:
|
|
continue
|
|
|
|
category_id = source_info["category_id"]
|
|
source_name = source_info["name"]
|
|
|
|
for page_no in range(1, max_pages + 1):
|
|
if page_no > 1:
|
|
await self._delay()
|
|
|
|
try:
|
|
data = await self._fetch_page(
|
|
client, source_code, category_id, page_no
|
|
)
|
|
if data is None:
|
|
break
|
|
|
|
records = parse_gxgp_api_response(
|
|
data, source_code, source_name,
|
|
start_time, category_id
|
|
)
|
|
if not records:
|
|
break
|
|
|
|
all_announcements.extend(records)
|
|
|
|
pagination = extract_pagination(data)
|
|
if not pagination["has_next"]:
|
|
break
|
|
except Exception as e:
|
|
error_messages.append(f"{source_code} page {page_no}: {e}")
|
|
break
|
|
|
|
duration = (datetime.now() - start_time).total_seconds()
|
|
return CrawlResult(
|
|
source_code=self.source_code,
|
|
source_name=self.source_name,
|
|
total_count=len(all_announcements),
|
|
new_count=len(all_announcements),
|
|
announcements=all_announcements,
|
|
error_message="; ".join(error_messages) if error_messages else None,
|
|
crawled_at=start_time,
|
|
duration=duration,
|
|
)
|
|
|
|
async def _fetch_page(self, client: httpx.AsyncClient, source_code: str,
|
|
category_id: int, page_no: int) -> dict | None:
|
|
payload = {
|
|
"keyword": "",
|
|
"publishDateBegin": "",
|
|
"publishDateEnd": "",
|
|
"pageNo": page_no,
|
|
"pageSize": settings.crawler_page_size,
|
|
"categoryCode": source_code,
|
|
"_t": int(time.time() * 1000),
|
|
}
|
|
headers = {
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
|
"Content-Type": "application/json;charset=UTF-8",
|
|
"Origin": self.base_url,
|
|
"Referer": f"{self.base_url}/site/category?parentId={category_id}&childrenCode={source_code}", # noqa: E501
|
|
}
|
|
response = await client.post(
|
|
self.announcement_api, json=payload, headers=headers
|
|
)
|
|
if response.status_code != 200:
|
|
return None
|
|
return response.json()
|
|
|
|
async def _delay(self):
|
|
import asyncio
|
|
delay = random.uniform(1.0, 3.0)
|
|
await asyncio.sleep(delay)
|