手动模式
This commit is contained in:
@@ -0,0 +1,458 @@
|
||||
"""
|
||||
爬虫核心模块
|
||||
实现广西政府采购网公告的智能爬取功能
|
||||
"""
|
||||
|
||||
import time
|
||||
import random
|
||||
import json
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from datetime import datetime
|
||||
from urllib.parse import urljoin
|
||||
import requests
|
||||
from fake_useragent import UserAgent
|
||||
|
||||
try:
|
||||
# 尝试相对导入
|
||||
from ..core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus
|
||||
from ..core.config_manager import get_config
|
||||
from ..core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error
|
||||
from ..core.reliability import (
|
||||
retry_on_exception, RetryConfig, session_with_retry,
|
||||
TimeoutConfig, safe_execute, check_system_health
|
||||
)
|
||||
from .parsers import AnnouncementParser, SensitiveWordChecker, ErrorResponseParser
|
||||
except ImportError:
|
||||
# 尝试绝对导入
|
||||
from core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus
|
||||
from core.config_manager import get_config
|
||||
from core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error
|
||||
from core.reliability import (
|
||||
retry_on_exception, RetryConfig, session_with_retry,
|
||||
TimeoutConfig, safe_execute, check_system_health
|
||||
)
|
||||
from crawler.parsers import AnnouncementParser, SensitiveWordChecker, ErrorResponseParser
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class GXGPSpider:
|
||||
"""广西政府采购网爬虫"""
|
||||
|
||||
def __init__(self):
|
||||
self.config = get_config()
|
||||
self.ua = UserAgent()
|
||||
|
||||
# API端点
|
||||
self.base_url = self.config.crawler.base_url
|
||||
self.announcement_api = urljoin(self.base_url, "/portal/category")
|
||||
self.sensitive_check_api = urljoin(self.base_url, "/portal/sensitiveWords/check")
|
||||
|
||||
# 会话管理
|
||||
self.session = None
|
||||
|
||||
# 统计信息
|
||||
self.request_count = 0
|
||||
self.error_count = 0
|
||||
|
||||
def init_session(self):
|
||||
"""初始化会话"""
|
||||
if self.session is None:
|
||||
timeout_config = TimeoutConfig(
|
||||
connect_timeout=self.config.crawler.timeout,
|
||||
read_timeout=self.config.crawler.timeout
|
||||
)
|
||||
|
||||
retry_config = RetryConfig(
|
||||
max_retries=self.config.crawler.max_retries,
|
||||
initial_delay=self.config.crawler.retry_delay,
|
||||
max_delay=self.config.crawler.max_retry_delay,
|
||||
backoff_factor=self.config.crawler.backoff_factor
|
||||
)
|
||||
|
||||
self.session = requests.Session()
|
||||
|
||||
# 配置重试和超时
|
||||
adapter = requests.adapters.HTTPAdapter(
|
||||
pool_connections=10,
|
||||
pool_maxsize=20,
|
||||
max_retries=0 # 我们使用自己的重试逻辑
|
||||
)
|
||||
self.session.mount('http://', adapter)
|
||||
self.session.mount('https://', adapter)
|
||||
|
||||
# 设置默认超时
|
||||
self.session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout)
|
||||
|
||||
return self.session
|
||||
|
||||
def close_session(self):
|
||||
"""关闭会话"""
|
||||
if self.session:
|
||||
self.session.close()
|
||||
self.session = None
|
||||
|
||||
def get_random_user_agent(self) -> str:
|
||||
"""获取随机User-Agent"""
|
||||
try:
|
||||
return self.ua.random
|
||||
except:
|
||||
# fallback到配置的user agents
|
||||
return random.choice(self.config.crawler.user_agents)
|
||||
|
||||
def get_random_proxy(self) -> Optional[Dict[str, str]]:
|
||||
"""获取随机代理"""
|
||||
if not self.config.crawler.proxies:
|
||||
return None
|
||||
|
||||
proxy = random.choice(self.config.crawler.proxies)
|
||||
return {
|
||||
"http": proxy,
|
||||
"https": proxy
|
||||
}
|
||||
|
||||
def check_sensitive_words(self, payload: Dict[str, Any],
|
||||
category_code: str, childrencode: str) -> bool:
|
||||
"""
|
||||
执行敏感词检查
|
||||
|
||||
Args:
|
||||
payload: 请求参数
|
||||
category_code: 分类代码
|
||||
childrencode: 子分类代码
|
||||
|
||||
Returns:
|
||||
bool: 检查是否通过
|
||||
"""
|
||||
try:
|
||||
session = self.init_session()
|
||||
user_agent = self.get_random_user_agent()
|
||||
|
||||
headers = {
|
||||
"User-Agent": user_agent,
|
||||
"Content-Type": "application/json;charset=UTF-8",
|
||||
"Origin": self.base_url,
|
||||
"Referer": f"{self.base_url}/site/category?parentId={category_code}&childrenCode={childrencode}",
|
||||
"Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64"
|
||||
}
|
||||
|
||||
proxies = self.get_random_proxy()
|
||||
|
||||
logger.debug(f"执行敏感词检查: {category_code}/{childrencode}")
|
||||
|
||||
response = session.post(
|
||||
self.sensitive_check_api,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
proxies=proxies,
|
||||
timeout=self.session.timeout
|
||||
)
|
||||
|
||||
self.request_count += 1
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
return SensitiveWordChecker.parse_check_response(result)
|
||||
else:
|
||||
logger.warning(f"敏感词检查请求失败, 状态码: {response.status_code}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"敏感词检查异常: {str(e)}")
|
||||
self.error_count += 1
|
||||
return False
|
||||
|
||||
def fetch_announcements_page(self, source: AnnouncementSource,
|
||||
page_no: int = 1) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
"""
|
||||
获取公告列表页数据
|
||||
|
||||
Args:
|
||||
source: 公告来源
|
||||
page_no: 页码
|
||||
|
||||
Returns:
|
||||
Tuple[Optional[Dict[str, Any]], Optional[str]]: (响应数据, 错误信息)
|
||||
"""
|
||||
try:
|
||||
session = self.init_session()
|
||||
|
||||
# 构建请求参数
|
||||
payload = {
|
||||
"keyword": "", # 关键词筛选，我们在筛选模块处理
|
||||
"publishDateBegin": self.config.crawler.start_date or "",
|
||||
"publishDateEnd": self.config.crawler.end_date or "",
|
||||
"pageNo": page_no,
|
||||
"pageSize": self.config.crawler.page_size,
|
||||
"categoryCode": source.code,
|
||||
"_t": int(time.time() * 1000)
|
||||
}
|
||||
|
||||
# 先执行敏感词检查
|
||||
if not self.check_sensitive_words(payload, str(source.category_id), source.code):
|
||||
return None, "敏感词检查失败"
|
||||
|
||||
# 执行公告数据请求
|
||||
user_agent = self.get_random_user_agent()
|
||||
headers = {
|
||||
"User-Agent": user_agent,
|
||||
"Content-Type": "application/json;charset=UTF-8",
|
||||
"Origin": self.base_url,
|
||||
"Referer": f"{self.base_url}/site/category?parentId={source.category_id}&childrenCode={source.code}",
|
||||
"Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64"
|
||||
}
|
||||
|
||||
proxies = self.get_random_proxy()
|
||||
|
||||
logger.debug(f"请求公告数据: {source.name} 第{page_no}页")
|
||||
|
||||
# 添加请求间延迟
|
||||
if page_no > 1:
|
||||
delay = random.uniform(
|
||||
self.config.crawler.request_delay,
|
||||
self.config.crawler.request_delay_max
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
response = session.post(
|
||||
self.announcement_api,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
proxies=proxies,
|
||||
timeout=self.session.timeout
|
||||
)
|
||||
|
||||
self.request_count += 1
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get("success", False):
|
||||
return data, None
|
||||
else:
|
||||
error_msg = ErrorResponseParser.parse_error(data)
|
||||
return None, f"API返回失败: {error_msg}"
|
||||
else:
|
||||
return None, f"请求失败, 状态码: {response.status_code}"
|
||||
|
||||
except requests.exceptions.Timeout as e:
|
||||
self.error_count += 1
|
||||
return None, f"请求超时: {str(e)}"
|
||||
except requests.exceptions.ProxyError as e:
|
||||
self.error_count += 1
|
||||
return None, f"代理错误: {str(e)}"
|
||||
except Exception as e:
|
||||
self.error_count += 1
|
||||
logger.error(f"获取公告数据异常: {str(e)}")
|
||||
return None, f"请求异常: {str(e)}"
|
||||
|
||||
def crawl_source(self, source: AnnouncementSource,
|
||||
max_pages: Optional[int] = None) -> CrawlResult:
|
||||
"""
|
||||
爬取单个来源的公告
|
||||
|
||||
Args:
|
||||
source: 公告来源
|
||||
max_pages: 最大页数限制
|
||||
|
||||
Returns:
|
||||
CrawlResult: 爬取结果
|
||||
"""
|
||||
if max_pages is None:
|
||||
max_pages = self.config.crawler.max_pages
|
||||
|
||||
log_crawl_start(source.name)
|
||||
|
||||
start_time = datetime.now()
|
||||
result = CrawlResult(
|
||||
source=source,
|
||||
status=CrawlStatus.RUNNING,
|
||||
crawled_at=start_time
|
||||
)
|
||||
|
||||
try:
|
||||
page = 1
|
||||
all_announcements = []
|
||||
|
||||
while page <= max_pages:
|
||||
# 获取页面数据
|
||||
response_data, error_msg = self.fetch_announcements_page(source, page)
|
||||
|
||||
if error_msg:
|
||||
logger.warning(f"{source.name} 第{page}页获取失败: {error_msg}")
|
||||
result.status = CrawlStatus.FAILED
|
||||
result.error_message = error_msg
|
||||
break
|
||||
|
||||
if not response_data:
|
||||
logger.info(f"{source.name} 第{page}页无数据")
|
||||
break
|
||||
|
||||
# 解析分页信息
|
||||
pagination = AnnouncementParser.extract_pagination_info(response_data)
|
||||
result.total_count = pagination["total"]
|
||||
|
||||
# 解析公告数据
|
||||
announcements = AnnouncementParser.parse_api_response(
|
||||
response_data, source, start_time)
|
||||
|
||||
if not announcements:
|
||||
logger.info(f"{source.name} 第{page}页解析到0条公告")
|
||||
break
|
||||
|
||||
all_announcements.extend(announcements)
|
||||
|
||||
# 检查是否还有下一页
|
||||
if not pagination["has_next"] or pagination["empty"]:
|
||||
break
|
||||
|
||||
page += 1
|
||||
|
||||
# 更新结果
|
||||
result.announcements = all_announcements
|
||||
result.new_count = len(all_announcements) # 这里的新增数需要在筛选后确定
|
||||
|
||||
if result.status != CrawlStatus.FAILED:
|
||||
result.status = CrawlStatus.SUCCESS
|
||||
|
||||
duration = (datetime.now() - start_time).total_seconds()
|
||||
result.duration = duration
|
||||
|
||||
log_crawl_success(source.name, len(all_announcements), duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = (datetime.now() - start_time).total_seconds()
|
||||
result.duration = duration
|
||||
result.status = CrawlStatus.FAILED
|
||||
result.error_message = str(e)
|
||||
|
||||
log_crawl_error(source.name, str(e))
|
||||
|
||||
return result
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=2))
|
||||
def crawl_all_sources(self, sources: Optional[List[AnnouncementSource]] = None) -> List[CrawlResult]:
|
||||
"""
|
||||
爬取所有来源的公告
|
||||
|
||||
Args:
|
||||
sources: 指定的来源列表，如果为None则使用配置中的所有来源
|
||||
|
||||
Returns:
|
||||
List[CrawlResult]: 所有来源的爬取结果
|
||||
"""
|
||||
# 系统健康检查
|
||||
if not check_system_health():
|
||||
logger.error("系统健康检查失败，跳过爬取")
|
||||
return []
|
||||
|
||||
if sources is None:
|
||||
sources = self._load_sources_from_config()
|
||||
|
||||
logger.info(f"开始爬取 {len(sources)} 个公告来源")
|
||||
|
||||
results = []
|
||||
|
||||
for source in sources:
|
||||
try:
|
||||
result = self.crawl_source(source)
|
||||
results.append(result)
|
||||
|
||||
# 检查是否需要暂停
|
||||
if result.status == CrawlStatus.FAILED:
|
||||
logger.warning(f"来源 {source.name} 爬取失败，继续下一个来源")
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"爬取来源 {source.name} 时发生未预期错误: {str(e)}")
|
||||
# 创建失败结果
|
||||
failed_result = CrawlResult(
|
||||
source=source,
|
||||
status=CrawlStatus.FAILED,
|
||||
error_message=str(e),
|
||||
crawled_at=datetime.now()
|
||||
)
|
||||
results.append(failed_result)
|
||||
|
||||
# 统计总结果
|
||||
total_announcements = sum(len(r.announcements) for r in results if r.status == CrawlStatus.SUCCESS)
|
||||
success_count = sum(1 for r in results if r.status == CrawlStatus.SUCCESS)
|
||||
failed_count = len(results) - success_count
|
||||
|
||||
logger.info(
|
||||
f"爬取完成: 共处理 {len(results)} 个来源，"
|
||||
f"成功 {success_count} 个，失败 {failed_count} 个，"
|
||||
f"获取 {total_announcements} 条公告"
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _load_sources_from_config(self) -> List[AnnouncementSource]:
|
||||
"""从配置加载公告来源"""
|
||||
sources = []
|
||||
|
||||
for code, source_config in self.config.sources.items():
|
||||
try:
|
||||
source = AnnouncementSource(
|
||||
code=code,
|
||||
category_id=source_config["category_id"],
|
||||
name=source_config["name"],
|
||||
type=AnnouncementType(source_config["type"])
|
||||
)
|
||||
sources.append(source)
|
||||
except Exception as e:
|
||||
logger.warning(f"加载来源配置失败 {code}: {str(e)}")
|
||||
continue
|
||||
|
||||
return sources
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""获取爬虫统计信息"""
|
||||
return {
|
||||
"request_count": self.request_count,
|
||||
"error_count": self.error_count,
|
||||
"error_rate": self.error_count / max(self.request_count, 1),
|
||||
"session_active": self.session is not None
|
||||
}
|
||||
|
||||
def reset_stats(self):
|
||||
"""重置统计信息"""
|
||||
self.request_count = 0
|
||||
self.error_count = 0
|
||||
|
||||
|
||||
def create_spider() -> GXGPSpider:
|
||||
"""
|
||||
创建爬虫实例
|
||||
|
||||
Returns:
|
||||
GXGPSpider: 爬虫实例
|
||||
"""
|
||||
return GXGPSpider()
|
||||
|
||||
|
||||
def crawl_announcements(keywords: Optional[List[str]] = None,
|
||||
sources: Optional[List[str]] = None) -> List[CrawlResult]:
|
||||
"""
|
||||
便捷函数：爬取公告
|
||||
|
||||
Args:
|
||||
keywords: 关键词过滤（暂时未使用，在筛选模块处理）
|
||||
sources: 来源代码列表
|
||||
|
||||
Returns:
|
||||
List[CrawlResult]: 爬取结果
|
||||
"""
|
||||
spider = create_spider()
|
||||
|
||||
try:
|
||||
# 过滤来源
|
||||
if sources:
|
||||
all_sources = spider._load_sources_from_config()
|
||||
filtered_sources = [s for s in all_sources if s.code in sources]
|
||||
else:
|
||||
filtered_sources = None
|
||||
|
||||
return spider.crawl_all_sources(filtered_sources)
|
||||
finally:
|
||||
spider.close_session()
|
||||
Reference in New Issue
Block a user