459 lines
15 KiB
Python
459 lines
15 KiB
Python
"""
|
|
爬虫核心模块
|
|
实现广西政府采购网公告的智能爬取功能
|
|
"""
|
|
|
|
import time
|
|
import random
|
|
import json
|
|
from typing import List, Dict, Any, Optional, Tuple
|
|
from datetime import datetime
|
|
from urllib.parse import urljoin
|
|
import requests
|
|
from fake_useragent import UserAgent
|
|
|
|
try:
|
|
# 尝试相对导入
|
|
from ..core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus
|
|
from ..core.config_manager import get_config
|
|
from ..core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error
|
|
from ..core.reliability import (
|
|
retry_on_exception, RetryConfig, session_with_retry,
|
|
TimeoutConfig, safe_execute, check_system_health
|
|
)
|
|
from .parsers import AnnouncementParser, SensitiveWordChecker, ErrorResponseParser
|
|
except ImportError:
|
|
# 尝试绝对导入
|
|
from core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus
|
|
from core.config_manager import get_config
|
|
from core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error
|
|
from core.reliability import (
|
|
retry_on_exception, RetryConfig, session_with_retry,
|
|
TimeoutConfig, safe_execute, check_system_health
|
|
)
|
|
from crawler.parsers import AnnouncementParser, SensitiveWordChecker, ErrorResponseParser
|
|
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
class GXGPSpider:
|
|
"""广西政府采购网爬虫"""
|
|
|
|
def __init__(self):
|
|
self.config = get_config()
|
|
self.ua = UserAgent()
|
|
|
|
# API端点
|
|
self.base_url = self.config.crawler.base_url
|
|
self.announcement_api = urljoin(self.base_url, "/portal/category")
|
|
self.sensitive_check_api = urljoin(self.base_url, "/portal/sensitiveWords/check")
|
|
|
|
# 会话管理
|
|
self.session = None
|
|
|
|
# 统计信息
|
|
self.request_count = 0
|
|
self.error_count = 0
|
|
|
|
def init_session(self):
|
|
"""初始化会话"""
|
|
if self.session is None:
|
|
timeout_config = TimeoutConfig(
|
|
connect_timeout=self.config.crawler.timeout,
|
|
read_timeout=self.config.crawler.timeout
|
|
)
|
|
|
|
retry_config = RetryConfig(
|
|
max_retries=self.config.crawler.max_retries,
|
|
initial_delay=self.config.crawler.retry_delay,
|
|
max_delay=self.config.crawler.max_retry_delay,
|
|
backoff_factor=self.config.crawler.backoff_factor
|
|
)
|
|
|
|
self.session = requests.Session()
|
|
|
|
# 配置重试和超时
|
|
adapter = requests.adapters.HTTPAdapter(
|
|
pool_connections=10,
|
|
pool_maxsize=20,
|
|
max_retries=0 # 我们使用自己的重试逻辑
|
|
)
|
|
self.session.mount('http://', adapter)
|
|
self.session.mount('https://', adapter)
|
|
|
|
# 设置默认超时
|
|
self.session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout)
|
|
|
|
return self.session
|
|
|
|
def close_session(self):
|
|
"""关闭会话"""
|
|
if self.session:
|
|
self.session.close()
|
|
self.session = None
|
|
|
|
def get_random_user_agent(self) -> str:
|
|
"""获取随机User-Agent"""
|
|
try:
|
|
return self.ua.random
|
|
except:
|
|
# fallback到配置的user agents
|
|
return random.choice(self.config.crawler.user_agents)
|
|
|
|
def get_random_proxy(self) -> Optional[Dict[str, str]]:
|
|
"""获取随机代理"""
|
|
if not self.config.crawler.proxies:
|
|
return None
|
|
|
|
proxy = random.choice(self.config.crawler.proxies)
|
|
return {
|
|
"http": proxy,
|
|
"https": proxy
|
|
}
|
|
|
|
def check_sensitive_words(self, payload: Dict[str, Any],
|
|
category_code: str, childrencode: str) -> bool:
|
|
"""
|
|
执行敏感词检查
|
|
|
|
Args:
|
|
payload: 请求参数
|
|
category_code: 分类代码
|
|
childrencode: 子分类代码
|
|
|
|
Returns:
|
|
bool: 检查是否通过
|
|
"""
|
|
try:
|
|
session = self.init_session()
|
|
user_agent = self.get_random_user_agent()
|
|
|
|
headers = {
|
|
"User-Agent": user_agent,
|
|
"Content-Type": "application/json;charset=UTF-8",
|
|
"Origin": self.base_url,
|
|
"Referer": f"{self.base_url}/site/category?parentId={category_code}&childrenCode={childrencode}",
|
|
"Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64"
|
|
}
|
|
|
|
proxies = self.get_random_proxy()
|
|
|
|
logger.debug(f"执行敏感词检查: {category_code}/{childrencode}")
|
|
|
|
response = session.post(
|
|
self.sensitive_check_api,
|
|
json=payload,
|
|
headers=headers,
|
|
proxies=proxies,
|
|
timeout=self.session.timeout
|
|
)
|
|
|
|
self.request_count += 1
|
|
|
|
if response.status_code == 200:
|
|
result = response.json()
|
|
return SensitiveWordChecker.parse_check_response(result)
|
|
else:
|
|
logger.warning(f"敏感词检查请求失败, 状态码: {response.status_code}")
|
|
return False
|
|
|
|
except Exception as e:
|
|
logger.error(f"敏感词检查异常: {str(e)}")
|
|
self.error_count += 1
|
|
return False
|
|
|
|
def fetch_announcements_page(self, source: AnnouncementSource,
|
|
page_no: int = 1) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
|
"""
|
|
获取公告列表页数据
|
|
|
|
Args:
|
|
source: 公告来源
|
|
page_no: 页码
|
|
|
|
Returns:
|
|
Tuple[Optional[Dict[str, Any]], Optional[str]]: (响应数据, 错误信息)
|
|
"""
|
|
try:
|
|
session = self.init_session()
|
|
|
|
# 构建请求参数
|
|
payload = {
|
|
"keyword": "", # 关键词筛选，我们在筛选模块处理
|
|
"publishDateBegin": self.config.crawler.start_date or "",
|
|
"publishDateEnd": self.config.crawler.end_date or "",
|
|
"pageNo": page_no,
|
|
"pageSize": self.config.crawler.page_size,
|
|
"categoryCode": source.code,
|
|
"_t": int(time.time() * 1000)
|
|
}
|
|
|
|
# 先执行敏感词检查
|
|
if not self.check_sensitive_words(payload, str(source.category_id), source.code):
|
|
return None, "敏感词检查失败"
|
|
|
|
# 执行公告数据请求
|
|
user_agent = self.get_random_user_agent()
|
|
headers = {
|
|
"User-Agent": user_agent,
|
|
"Content-Type": "application/json;charset=UTF-8",
|
|
"Origin": self.base_url,
|
|
"Referer": f"{self.base_url}/site/category?parentId={source.category_id}&childrenCode={source.code}",
|
|
"Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64"
|
|
}
|
|
|
|
proxies = self.get_random_proxy()
|
|
|
|
logger.debug(f"请求公告数据: {source.name} 第{page_no}页")
|
|
|
|
# 添加请求间延迟
|
|
if page_no > 1:
|
|
delay = random.uniform(
|
|
self.config.crawler.request_delay,
|
|
self.config.crawler.request_delay_max
|
|
)
|
|
time.sleep(delay)
|
|
|
|
response = session.post(
|
|
self.announcement_api,
|
|
json=payload,
|
|
headers=headers,
|
|
proxies=proxies,
|
|
timeout=self.session.timeout
|
|
)
|
|
|
|
self.request_count += 1
|
|
|
|
if response.status_code == 200:
|
|
data = response.json()
|
|
if data.get("success", False):
|
|
return data, None
|
|
else:
|
|
error_msg = ErrorResponseParser.parse_error(data)
|
|
return None, f"API返回失败: {error_msg}"
|
|
else:
|
|
return None, f"请求失败, 状态码: {response.status_code}"
|
|
|
|
except requests.exceptions.Timeout as e:
|
|
self.error_count += 1
|
|
return None, f"请求超时: {str(e)}"
|
|
except requests.exceptions.ProxyError as e:
|
|
self.error_count += 1
|
|
return None, f"代理错误: {str(e)}"
|
|
except Exception as e:
|
|
self.error_count += 1
|
|
logger.error(f"获取公告数据异常: {str(e)}")
|
|
return None, f"请求异常: {str(e)}"
|
|
|
|
def crawl_source(self, source: AnnouncementSource,
|
|
max_pages: Optional[int] = None) -> CrawlResult:
|
|
"""
|
|
爬取单个来源的公告
|
|
|
|
Args:
|
|
source: 公告来源
|
|
max_pages: 最大页数限制
|
|
|
|
Returns:
|
|
CrawlResult: 爬取结果
|
|
"""
|
|
if max_pages is None:
|
|
max_pages = self.config.crawler.max_pages
|
|
|
|
log_crawl_start(source.name)
|
|
|
|
start_time = datetime.now()
|
|
result = CrawlResult(
|
|
source=source,
|
|
status=CrawlStatus.RUNNING,
|
|
crawled_at=start_time
|
|
)
|
|
|
|
try:
|
|
page = 1
|
|
all_announcements = []
|
|
|
|
while page <= max_pages:
|
|
# 获取页面数据
|
|
response_data, error_msg = self.fetch_announcements_page(source, page)
|
|
|
|
if error_msg:
|
|
logger.warning(f"{source.name} 第{page}页获取失败: {error_msg}")
|
|
result.status = CrawlStatus.FAILED
|
|
result.error_message = error_msg
|
|
break
|
|
|
|
if not response_data:
|
|
logger.info(f"{source.name} 第{page}页无数据")
|
|
break
|
|
|
|
# 解析分页信息
|
|
pagination = AnnouncementParser.extract_pagination_info(response_data)
|
|
result.total_count = pagination["total"]
|
|
|
|
# 解析公告数据
|
|
announcements = AnnouncementParser.parse_api_response(
|
|
response_data, source, start_time)
|
|
|
|
if not announcements:
|
|
logger.info(f"{source.name} 第{page}页解析到0条公告")
|
|
break
|
|
|
|
all_announcements.extend(announcements)
|
|
|
|
# 检查是否还有下一页
|
|
if not pagination["has_next"] or pagination["empty"]:
|
|
break
|
|
|
|
page += 1
|
|
|
|
# 更新结果
|
|
result.announcements = all_announcements
|
|
result.new_count = len(all_announcements) # 这里的新增数需要在筛选后确定
|
|
|
|
if result.status != CrawlStatus.FAILED:
|
|
result.status = CrawlStatus.SUCCESS
|
|
|
|
duration = (datetime.now() - start_time).total_seconds()
|
|
result.duration = duration
|
|
|
|
log_crawl_success(source.name, len(all_announcements), duration)
|
|
|
|
except Exception as e:
|
|
duration = (datetime.now() - start_time).total_seconds()
|
|
result.duration = duration
|
|
result.status = CrawlStatus.FAILED
|
|
result.error_message = str(e)
|
|
|
|
log_crawl_error(source.name, str(e))
|
|
|
|
return result
|
|
|
|
@retry_on_exception(RetryConfig(max_retries=2))
|
|
def crawl_all_sources(self, sources: Optional[List[AnnouncementSource]] = None) -> List[CrawlResult]:
|
|
"""
|
|
爬取所有来源的公告
|
|
|
|
Args:
|
|
sources: 指定的来源列表，如果为None则使用配置中的所有来源
|
|
|
|
Returns:
|
|
List[CrawlResult]: 所有来源的爬取结果
|
|
"""
|
|
# 系统健康检查
|
|
if not check_system_health():
|
|
logger.error("系统健康检查失败，跳过爬取")
|
|
return []
|
|
|
|
if sources is None:
|
|
sources = self._load_sources_from_config()
|
|
|
|
logger.info(f"开始爬取 {len(sources)} 个公告来源")
|
|
|
|
results = []
|
|
|
|
for source in sources:
|
|
try:
|
|
result = self.crawl_source(source)
|
|
results.append(result)
|
|
|
|
# 检查是否需要暂停
|
|
if result.status == CrawlStatus.FAILED:
|
|
logger.warning(f"来源 {source.name} 爬取失败，继续下一个来源")
|
|
continue
|
|
|
|
except Exception as e:
|
|
logger.error(f"爬取来源 {source.name} 时发生未预期错误: {str(e)}")
|
|
# 创建失败结果
|
|
failed_result = CrawlResult(
|
|
source=source,
|
|
status=CrawlStatus.FAILED,
|
|
error_message=str(e),
|
|
crawled_at=datetime.now()
|
|
)
|
|
results.append(failed_result)
|
|
|
|
# 统计总结果
|
|
total_announcements = sum(len(r.announcements) for r in results if r.status == CrawlStatus.SUCCESS)
|
|
success_count = sum(1 for r in results if r.status == CrawlStatus.SUCCESS)
|
|
failed_count = len(results) - success_count
|
|
|
|
logger.info(
|
|
f"爬取完成: 共处理 {len(results)} 个来源，"
|
|
f"成功 {success_count} 个，失败 {failed_count} 个，"
|
|
f"获取 {total_announcements} 条公告"
|
|
)
|
|
|
|
return results
|
|
|
|
def _load_sources_from_config(self) -> List[AnnouncementSource]:
|
|
"""从配置加载公告来源"""
|
|
sources = []
|
|
|
|
for code, source_config in self.config.sources.items():
|
|
try:
|
|
source = AnnouncementSource(
|
|
code=code,
|
|
category_id=source_config["category_id"],
|
|
name=source_config["name"],
|
|
type=AnnouncementType(source_config["type"])
|
|
)
|
|
sources.append(source)
|
|
except Exception as e:
|
|
logger.warning(f"加载来源配置失败 {code}: {str(e)}")
|
|
continue
|
|
|
|
return sources
|
|
|
|
def get_stats(self) -> Dict[str, Any]:
|
|
"""获取爬虫统计信息"""
|
|
return {
|
|
"request_count": self.request_count,
|
|
"error_count": self.error_count,
|
|
"error_rate": self.error_count / max(self.request_count, 1),
|
|
"session_active": self.session is not None
|
|
}
|
|
|
|
def reset_stats(self):
|
|
"""重置统计信息"""
|
|
self.request_count = 0
|
|
self.error_count = 0
|
|
|
|
|
|
def create_spider() -> GXGPSpider:
|
|
"""
|
|
创建爬虫实例
|
|
|
|
Returns:
|
|
GXGPSpider: 爬虫实例
|
|
"""
|
|
return GXGPSpider()
|
|
|
|
|
|
def crawl_announcements(keywords: Optional[List[str]] = None,
|
|
sources: Optional[List[str]] = None) -> List[CrawlResult]:
|
|
"""
|
|
便捷函数：爬取公告
|
|
|
|
Args:
|
|
keywords: 关键词过滤（暂时未使用，在筛选模块处理）
|
|
sources: 来源代码列表
|
|
|
|
Returns:
|
|
List[CrawlResult]: 爬取结果
|
|
"""
|
|
spider = create_spider()
|
|
|
|
try:
|
|
# 过滤来源
|
|
if sources:
|
|
all_sources = spider._load_sources_from_config()
|
|
filtered_sources = [s for s in all_sources if s.code in sources]
|
|
else:
|
|
filtered_sources = None
|
|
|
|
return spider.crawl_all_sources(filtered_sources)
|
|
finally:
|
|
spider.close_session()
|