""" 爬虫核心模块 实现广西政府采购网公告的智能爬取功能 """ import time import random import json from typing import List, Dict, Any, Optional, Tuple from datetime import datetime from urllib.parse import urljoin import requests from fake_useragent import UserAgent try: # 尝试相对导入 from ..core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus from ..core.config_manager import get_config from ..core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error from ..core.reliability import ( retry_on_exception, RetryConfig, session_with_retry, TimeoutConfig, safe_execute, check_system_health ) from .parsers import AnnouncementParser, SensitiveWordChecker, ErrorResponseParser except ImportError: # 尝试绝对导入 from core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus from core.config_manager import get_config from core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error from core.reliability import ( retry_on_exception, RetryConfig, session_with_retry, TimeoutConfig, safe_execute, check_system_health ) from crawler.parsers import AnnouncementParser, SensitiveWordChecker, ErrorResponseParser logger = get_logger(__name__) class GXGPSpider: """广西政府采购网爬虫""" def __init__(self): self.config = get_config() self.ua = UserAgent() # API端点 self.base_url = self.config.crawler.base_url self.announcement_api = urljoin(self.base_url, "/portal/category") self.sensitive_check_api = urljoin(self.base_url, "/portal/sensitiveWords/check") # 会话管理 self.session = None # 统计信息 self.request_count = 0 self.error_count = 0 def init_session(self): """初始化会话""" if self.session is None: timeout_config = TimeoutConfig( connect_timeout=self.config.crawler.timeout, read_timeout=self.config.crawler.timeout ) retry_config = RetryConfig( max_retries=self.config.crawler.max_retries, initial_delay=self.config.crawler.retry_delay, max_delay=self.config.crawler.max_retry_delay, backoff_factor=self.config.crawler.backoff_factor ) self.session = requests.Session() # 配置重试和超时 adapter = requests.adapters.HTTPAdapter( pool_connections=10, pool_maxsize=20, max_retries=0 # 我们使用自己的重试逻辑 ) self.session.mount('http://', adapter) self.session.mount('https://', adapter) # 设置默认超时 self.session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout) return self.session def close_session(self): """关闭会话""" if self.session: self.session.close() self.session = None def get_random_user_agent(self) -> str: """获取随机User-Agent""" try: return self.ua.random except: # fallback到配置的user agents return random.choice(self.config.crawler.user_agents) def get_random_proxy(self) -> Optional[Dict[str, str]]: """获取随机代理""" if not self.config.crawler.proxies: return None proxy = random.choice(self.config.crawler.proxies) return { "http": proxy, "https": proxy } def check_sensitive_words(self, payload: Dict[str, Any], category_code: str, childrencode: str) -> bool: """ 执行敏感词检查 Args: payload: 请求参数 category_code: 分类代码 childrencode: 子分类代码 Returns: bool: 检查是否通过 """ try: session = self.init_session() user_agent = self.get_random_user_agent() headers = { "User-Agent": user_agent, "Content-Type": "application/json;charset=UTF-8", "Origin": self.base_url, "Referer": f"{self.base_url}/site/category?parentId={category_code}&childrenCode={childrencode}", "Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64" } proxies = self.get_random_proxy() logger.debug(f"执行敏感词检查: {category_code}/{childrencode}") response = session.post( self.sensitive_check_api, json=payload, headers=headers, proxies=proxies, timeout=self.session.timeout ) self.request_count += 1 if response.status_code == 200: result = response.json() return SensitiveWordChecker.parse_check_response(result) else: logger.warning(f"敏感词检查请求失败, 状态码: {response.status_code}") return False except Exception as e: logger.error(f"敏感词检查异常: {str(e)}") self.error_count += 1 return False def fetch_announcements_page(self, source: AnnouncementSource, page_no: int = 1) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: """ 获取公告列表页数据 Args: source: 公告来源 page_no: 页码 Returns: Tuple[Optional[Dict[str, Any]], Optional[str]]: (响应数据, 错误信息) """ try: session = self.init_session() # 构建请求参数 payload = { "keyword": "", # 关键词筛选，我们在筛选模块处理 "publishDateBegin": self.config.crawler.start_date or "", "publishDateEnd": self.config.crawler.end_date or "", "pageNo": page_no, "pageSize": self.config.crawler.page_size, "categoryCode": source.code, "_t": int(time.time() * 1000) } # 先执行敏感词检查 if not self.check_sensitive_words(payload, str(source.category_id), source.code): return None, "敏感词检查失败" # 执行公告数据请求 user_agent = self.get_random_user_agent() headers = { "User-Agent": user_agent, "Content-Type": "application/json;charset=UTF-8", "Origin": self.base_url, "Referer": f"{self.base_url}/site/category?parentId={source.category_id}&childrenCode={source.code}", "Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64" } proxies = self.get_random_proxy() logger.debug(f"请求公告数据: {source.name} 第{page_no}页") # 添加请求间延迟 if page_no > 1: delay = random.uniform( self.config.crawler.request_delay, self.config.crawler.request_delay_max ) time.sleep(delay) response = session.post( self.announcement_api, json=payload, headers=headers, proxies=proxies, timeout=self.session.timeout ) self.request_count += 1 if response.status_code == 200: data = response.json() if data.get("success", False): return data, None else: error_msg = ErrorResponseParser.parse_error(data) return None, f"API返回失败: {error_msg}" else: return None, f"请求失败, 状态码: {response.status_code}" except requests.exceptions.Timeout as e: self.error_count += 1 return None, f"请求超时: {str(e)}" except requests.exceptions.ProxyError as e: self.error_count += 1 return None, f"代理错误: {str(e)}" except Exception as e: self.error_count += 1 logger.error(f"获取公告数据异常: {str(e)}") return None, f"请求异常: {str(e)}" def crawl_source(self, source: AnnouncementSource, max_pages: Optional[int] = None) -> CrawlResult: """ 爬取单个来源的公告 Args: source: 公告来源 max_pages: 最大页数限制 Returns: CrawlResult: 爬取结果 """ if max_pages is None: max_pages = self.config.crawler.max_pages log_crawl_start(source.name) start_time = datetime.now() result = CrawlResult( source=source, status=CrawlStatus.RUNNING, crawled_at=start_time ) try: page = 1 all_announcements = [] while page <= max_pages: # 获取页面数据 response_data, error_msg = self.fetch_announcements_page(source, page) if error_msg: logger.warning(f"{source.name} 第{page}页获取失败: {error_msg}") result.status = CrawlStatus.FAILED result.error_message = error_msg break if not response_data: logger.info(f"{source.name} 第{page}页无数据") break # 解析分页信息 pagination = AnnouncementParser.extract_pagination_info(response_data) result.total_count = pagination["total"] # 解析公告数据 announcements = AnnouncementParser.parse_api_response( response_data, source, start_time) if not announcements: logger.info(f"{source.name} 第{page}页解析到0条公告") break all_announcements.extend(announcements) # 检查是否还有下一页 if not pagination["has_next"] or pagination["empty"]: break page += 1 # 更新结果 result.announcements = all_announcements result.new_count = len(all_announcements) # 这里的新增数需要在筛选后确定 if result.status != CrawlStatus.FAILED: result.status = CrawlStatus.SUCCESS duration = (datetime.now() - start_time).total_seconds() result.duration = duration log_crawl_success(source.name, len(all_announcements), duration) except Exception as e: duration = (datetime.now() - start_time).total_seconds() result.duration = duration result.status = CrawlStatus.FAILED result.error_message = str(e) log_crawl_error(source.name, str(e)) return result @retry_on_exception(RetryConfig(max_retries=2)) def crawl_all_sources(self, sources: Optional[List[AnnouncementSource]] = None) -> List[CrawlResult]: """ 爬取所有来源的公告 Args: sources: 指定的来源列表，如果为None则使用配置中的所有来源 Returns: List[CrawlResult]: 所有来源的爬取结果 """ # 系统健康检查 if not check_system_health(): logger.error("系统健康检查失败，跳过爬取") return [] if sources is None: sources = self._load_sources_from_config() logger.info(f"开始爬取 {len(sources)} 个公告来源") results = [] for source in sources: try: result = self.crawl_source(source) results.append(result) # 检查是否需要暂停 if result.status == CrawlStatus.FAILED: logger.warning(f"来源 {source.name} 爬取失败，继续下一个来源") continue except Exception as e: logger.error(f"爬取来源 {source.name} 时发生未预期错误: {str(e)}") # 创建失败结果 failed_result = CrawlResult( source=source, status=CrawlStatus.FAILED, error_message=str(e), crawled_at=datetime.now() ) results.append(failed_result) # 统计总结果 total_announcements = sum(len(r.announcements) for r in results if r.status == CrawlStatus.SUCCESS) success_count = sum(1 for r in results if r.status == CrawlStatus.SUCCESS) failed_count = len(results) - success_count logger.info( f"爬取完成: 共处理 {len(results)} 个来源，" f"成功 {success_count} 个，失败 {failed_count} 个，" f"获取 {total_announcements} 条公告" ) return results def _load_sources_from_config(self) -> List[AnnouncementSource]: """从配置加载公告来源""" sources = [] for code, source_config in self.config.sources.items(): try: source = AnnouncementSource( code=code, category_id=source_config["category_id"], name=source_config["name"], type=AnnouncementType(source_config["type"]) ) sources.append(source) except Exception as e: logger.warning(f"加载来源配置失败 {code}: {str(e)}") continue return sources def get_stats(self) -> Dict[str, Any]: """获取爬虫统计信息""" return { "request_count": self.request_count, "error_count": self.error_count, "error_rate": self.error_count / max(self.request_count, 1), "session_active": self.session is not None } def reset_stats(self): """重置统计信息""" self.request_count = 0 self.error_count = 0 def create_spider() -> GXGPSpider: """ 创建爬虫实例 Returns: GXGPSpider: 爬虫实例 """ return GXGPSpider() def crawl_announcements(keywords: Optional[List[str]] = None, sources: Optional[List[str]] = None) -> List[CrawlResult]: """ 便捷函数：爬取公告 Args: keywords: 关键词过滤（暂时未使用，在筛选模块处理） sources: 来源代码列表 Returns: List[CrawlResult]: 爬取结果 """ spider = create_spider() try: # 过滤来源 if sources: all_sources = spider._load_sources_from_config() filtered_sources = [s for s in all_sources if s.code in sources] else: filtered_sources = None return spider.crawl_all_sources(filtered_sources) finally: spider.close_session()