手动模式
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""爬虫模块"""
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,301 @@
|
||||
"""
|
||||
数据解析器模块
|
||||
负责解析广西政府采购网的API响应数据
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import List, Dict, Any, Optional
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
from ..core.models import Announcement, AnnouncementSource, AnnouncementType
|
||||
from ..core.logger import get_logger
|
||||
except ImportError:
|
||||
from core.models import Announcement, AnnouncementSource, AnnouncementType
|
||||
from core.logger import get_logger
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class AnnouncementParser:
|
||||
"""公告数据解析器"""
|
||||
|
||||
@staticmethod
|
||||
def parse_api_response(response_data: Dict[str, Any],
|
||||
source: AnnouncementSource,
|
||||
crawled_at: datetime) -> List[Announcement]:
|
||||
"""
|
||||
解析API响应数据
|
||||
|
||||
Args:
|
||||
response_data: API响应数据
|
||||
source: 公告来源
|
||||
crawled_at: 爬取时间
|
||||
|
||||
Returns:
|
||||
List[Announcement]: 解析后的公告列表
|
||||
"""
|
||||
if not response_data or not isinstance(response_data, dict):
|
||||
logger.warning("API响应数据无效")
|
||||
return []
|
||||
|
||||
try:
|
||||
# 检查响应状态
|
||||
if not response_data.get("success", False):
|
||||
logger.warning(f"API响应失败: {response_data.get('message', '未知错误')}")
|
||||
return []
|
||||
|
||||
# 获取数据部分
|
||||
result = response_data.get("result", {})
|
||||
data = result.get("data", {})
|
||||
records = data.get("data", [])
|
||||
|
||||
if not records:
|
||||
logger.info(f"来源 {source.name} 没有新数据")
|
||||
return []
|
||||
|
||||
announcements = []
|
||||
for record in records:
|
||||
try:
|
||||
announcement = AnnouncementParser._parse_single_record(
|
||||
record, source, crawled_at)
|
||||
if announcement:
|
||||
announcements.append(announcement)
|
||||
except Exception as e:
|
||||
logger.warning(f"解析公告记录失败: {str(e)}, 记录: {record}")
|
||||
continue
|
||||
|
||||
logger.info(f"成功解析 {len(announcements)}/{len(records)} 条公告记录")
|
||||
return announcements
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"解析API响应数据失败: {str(e)}")
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def _parse_single_record(record: Dict[str, Any],
|
||||
source: AnnouncementSource,
|
||||
crawled_at: datetime) -> Optional[Announcement]:
|
||||
"""
|
||||
解析单个公告记录
|
||||
|
||||
Args:
|
||||
record: 公告记录数据
|
||||
source: 公告来源
|
||||
crawled_at: 爬取时间
|
||||
|
||||
Returns:
|
||||
Optional[Announcement]: 解析后的公告对象
|
||||
"""
|
||||
try:
|
||||
# 提取基本字段
|
||||
title_raw = record.get("title", "")
|
||||
title = str(title_raw).strip() if title_raw is not None else ""
|
||||
if not title:
|
||||
return None
|
||||
|
||||
# 解析发布时间
|
||||
publish_timestamp = record.get("publishDate")
|
||||
if not publish_timestamp:
|
||||
logger.warning(f"公告缺少发布时间: {title[:50]}...")
|
||||
return None
|
||||
|
||||
try:
|
||||
# 时间戳转换为datetime
|
||||
publish_date = datetime.fromtimestamp(int(publish_timestamp) / 1000)
|
||||
except (ValueError, TypeError) as e:
|
||||
logger.warning(f"发布时间格式错误: {publish_timestamp}, 错误: {str(e)}")
|
||||
return None
|
||||
|
||||
# 提取其他字段
|
||||
purchase_name_raw = record.get("purchaseName", "")
|
||||
purchase_name = str(purchase_name_raw).strip() if purchase_name_raw is not None else ""
|
||||
article_id = record.get("articleId")
|
||||
|
||||
if not article_id:
|
||||
logger.warning(f"公告缺少文章ID: {title[:50]}...")
|
||||
return None
|
||||
|
||||
# 构建内容链接
|
||||
content_url = AnnouncementParser._build_content_url(
|
||||
source.category_id, source.code, article_id)
|
||||
|
||||
# 创建公告对象
|
||||
announcement = Announcement(
|
||||
title=title,
|
||||
publish_date=publish_date,
|
||||
purchase_name=purchase_name,
|
||||
content_url=content_url,
|
||||
source_code=source.code,
|
||||
source_name=source.name,
|
||||
announcement_type=source.type,
|
||||
crawled_at=crawled_at,
|
||||
is_new=True # 默认标记为新公告
|
||||
)
|
||||
|
||||
# 生成内容哈希用于去重
|
||||
announcement.generate_content_hash()
|
||||
|
||||
return announcement
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"解析单个公告记录失败: {str(e)}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _build_content_url(category_id: int, source_code: str, article_id: int) -> str:
|
||||
"""
|
||||
构建公告内容链接
|
||||
|
||||
Args:
|
||||
category_id: 分类ID
|
||||
source_code: 来源代码
|
||||
article_id: 文章ID
|
||||
|
||||
Returns:
|
||||
str: 内容链接
|
||||
"""
|
||||
return f"https://zfcg.gxzf.gov.cn/site/detail?parentId={category_id}&articleId={article_id}"
|
||||
|
||||
@staticmethod
|
||||
def validate_response_structure(response_data: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
验证API响应数据结构
|
||||
|
||||
Args:
|
||||
response_data: API响应数据
|
||||
|
||||
Returns:
|
||||
bool: 结构是否有效
|
||||
"""
|
||||
try:
|
||||
if not isinstance(response_data, dict):
|
||||
return False
|
||||
|
||||
# 检查必需的字段
|
||||
if "success" not in response_data:
|
||||
return False
|
||||
|
||||
if not response_data.get("success", False):
|
||||
return False
|
||||
|
||||
result = response_data.get("result", {})
|
||||
if not isinstance(result, dict):
|
||||
return False
|
||||
|
||||
data = result.get("data", {})
|
||||
if not isinstance(data, dict):
|
||||
return False
|
||||
|
||||
records = data.get("data", [])
|
||||
if not isinstance(records, list):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"验证响应结构失败: {str(e)}")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def extract_pagination_info(response_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""
|
||||
提取分页信息
|
||||
|
||||
Args:
|
||||
response_data: API响应数据
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 分页信息
|
||||
"""
|
||||
try:
|
||||
result = response_data.get("result", {})
|
||||
data = result.get("data", {})
|
||||
|
||||
return {
|
||||
"total": data.get("total", 0),
|
||||
"page_no": data.get("pageNo", 1),
|
||||
"page_size": data.get("pageSize", 15),
|
||||
"pages": data.get("pages", 0),
|
||||
"empty": data.get("empty", True),
|
||||
"has_next": data.get("hasNext", False),
|
||||
"has_previous": data.get("hasPrevious", False)
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"提取分页信息失败: {str(e)}")
|
||||
return {
|
||||
"total": 0,
|
||||
"page_no": 1,
|
||||
"page_size": 15,
|
||||
"pages": 0,
|
||||
"empty": True,
|
||||
"has_next": False,
|
||||
"has_previous": False
|
||||
}
|
||||
|
||||
|
||||
class SensitiveWordChecker:
|
||||
"""敏感词检查器"""
|
||||
|
||||
@staticmethod
|
||||
def parse_check_response(response_data: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
解析敏感词检查响应
|
||||
|
||||
Args:
|
||||
response_data: 检查响应数据
|
||||
|
||||
Returns:
|
||||
bool: 检查是否通过
|
||||
"""
|
||||
try:
|
||||
if not isinstance(response_data, dict):
|
||||
logger.warning("敏感词检查响应格式无效")
|
||||
return False
|
||||
|
||||
success = response_data.get("success", False)
|
||||
if not success:
|
||||
message = response_data.get("message", "未知错误")
|
||||
logger.warning(f"敏感词检查失败: {message}")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"解析敏感词检查响应失败: {str(e)}")
|
||||
return False
|
||||
|
||||
|
||||
class ErrorResponseParser:
|
||||
"""错误响应解析器"""
|
||||
|
||||
@staticmethod
|
||||
def parse_error(response_data: Dict[str, Any]) -> str:
|
||||
"""
|
||||
解析错误响应
|
||||
|
||||
Args:
|
||||
response_data: 错误响应数据
|
||||
|
||||
Returns:
|
||||
str: 错误信息
|
||||
"""
|
||||
try:
|
||||
if not isinstance(response_data, dict):
|
||||
return "响应格式无效"
|
||||
|
||||
# 尝试提取错误信息
|
||||
error_msg = response_data.get("message") or response_data.get("msg")
|
||||
if error_msg:
|
||||
return str(error_msg)
|
||||
|
||||
# 检查状态码
|
||||
errcode = response_data.get("errcode")
|
||||
if errcode:
|
||||
return f"错误码: {errcode}"
|
||||
|
||||
return "未知错误"
|
||||
|
||||
except Exception as e:
|
||||
return f"解析错误响应失败: {str(e)}"
|
||||
@@ -0,0 +1,458 @@
|
||||
"""
|
||||
爬虫核心模块
|
||||
实现广西政府采购网公告的智能爬取功能
|
||||
"""
|
||||
|
||||
import time
|
||||
import random
|
||||
import json
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from datetime import datetime
|
||||
from urllib.parse import urljoin
|
||||
import requests
|
||||
from fake_useragent import UserAgent
|
||||
|
||||
try:
|
||||
# 尝试相对导入
|
||||
from ..core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus
|
||||
from ..core.config_manager import get_config
|
||||
from ..core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error
|
||||
from ..core.reliability import (
|
||||
retry_on_exception, RetryConfig, session_with_retry,
|
||||
TimeoutConfig, safe_execute, check_system_health
|
||||
)
|
||||
from .parsers import AnnouncementParser, SensitiveWordChecker, ErrorResponseParser
|
||||
except ImportError:
|
||||
# 尝试绝对导入
|
||||
from core.models import Announcement, AnnouncementSource, AnnouncementType, CrawlResult, CrawlStatus
|
||||
from core.config_manager import get_config
|
||||
from core.logger import get_logger, log_crawl_start, log_crawl_success, log_crawl_error
|
||||
from core.reliability import (
|
||||
retry_on_exception, RetryConfig, session_with_retry,
|
||||
TimeoutConfig, safe_execute, check_system_health
|
||||
)
|
||||
from crawler.parsers import AnnouncementParser, SensitiveWordChecker, ErrorResponseParser
|
||||
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class GXGPSpider:
|
||||
"""广西政府采购网爬虫"""
|
||||
|
||||
def __init__(self):
|
||||
self.config = get_config()
|
||||
self.ua = UserAgent()
|
||||
|
||||
# API端点
|
||||
self.base_url = self.config.crawler.base_url
|
||||
self.announcement_api = urljoin(self.base_url, "/portal/category")
|
||||
self.sensitive_check_api = urljoin(self.base_url, "/portal/sensitiveWords/check")
|
||||
|
||||
# 会话管理
|
||||
self.session = None
|
||||
|
||||
# 统计信息
|
||||
self.request_count = 0
|
||||
self.error_count = 0
|
||||
|
||||
def init_session(self):
|
||||
"""初始化会话"""
|
||||
if self.session is None:
|
||||
timeout_config = TimeoutConfig(
|
||||
connect_timeout=self.config.crawler.timeout,
|
||||
read_timeout=self.config.crawler.timeout
|
||||
)
|
||||
|
||||
retry_config = RetryConfig(
|
||||
max_retries=self.config.crawler.max_retries,
|
||||
initial_delay=self.config.crawler.retry_delay,
|
||||
max_delay=self.config.crawler.max_retry_delay,
|
||||
backoff_factor=self.config.crawler.backoff_factor
|
||||
)
|
||||
|
||||
self.session = requests.Session()
|
||||
|
||||
# 配置重试和超时
|
||||
adapter = requests.adapters.HTTPAdapter(
|
||||
pool_connections=10,
|
||||
pool_maxsize=20,
|
||||
max_retries=0 # 我们使用自己的重试逻辑
|
||||
)
|
||||
self.session.mount('http://', adapter)
|
||||
self.session.mount('https://', adapter)
|
||||
|
||||
# 设置默认超时
|
||||
self.session.timeout = (timeout_config.connect_timeout, timeout_config.read_timeout)
|
||||
|
||||
return self.session
|
||||
|
||||
def close_session(self):
|
||||
"""关闭会话"""
|
||||
if self.session:
|
||||
self.session.close()
|
||||
self.session = None
|
||||
|
||||
def get_random_user_agent(self) -> str:
|
||||
"""获取随机User-Agent"""
|
||||
try:
|
||||
return self.ua.random
|
||||
except:
|
||||
# fallback到配置的user agents
|
||||
return random.choice(self.config.crawler.user_agents)
|
||||
|
||||
def get_random_proxy(self) -> Optional[Dict[str, str]]:
|
||||
"""获取随机代理"""
|
||||
if not self.config.crawler.proxies:
|
||||
return None
|
||||
|
||||
proxy = random.choice(self.config.crawler.proxies)
|
||||
return {
|
||||
"http": proxy,
|
||||
"https": proxy
|
||||
}
|
||||
|
||||
def check_sensitive_words(self, payload: Dict[str, Any],
|
||||
category_code: str, childrencode: str) -> bool:
|
||||
"""
|
||||
执行敏感词检查
|
||||
|
||||
Args:
|
||||
payload: 请求参数
|
||||
category_code: 分类代码
|
||||
childrencode: 子分类代码
|
||||
|
||||
Returns:
|
||||
bool: 检查是否通过
|
||||
"""
|
||||
try:
|
||||
session = self.init_session()
|
||||
user_agent = self.get_random_user_agent()
|
||||
|
||||
headers = {
|
||||
"User-Agent": user_agent,
|
||||
"Content-Type": "application/json;charset=UTF-8",
|
||||
"Origin": self.base_url,
|
||||
"Referer": f"{self.base_url}/site/category?parentId={category_code}&childrenCode={childrencode}",
|
||||
"Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64"
|
||||
}
|
||||
|
||||
proxies = self.get_random_proxy()
|
||||
|
||||
logger.debug(f"执行敏感词检查: {category_code}/{childrencode}")
|
||||
|
||||
response = session.post(
|
||||
self.sensitive_check_api,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
proxies=proxies,
|
||||
timeout=self.session.timeout
|
||||
)
|
||||
|
||||
self.request_count += 1
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
return SensitiveWordChecker.parse_check_response(result)
|
||||
else:
|
||||
logger.warning(f"敏感词检查请求失败, 状态码: {response.status_code}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"敏感词检查异常: {str(e)}")
|
||||
self.error_count += 1
|
||||
return False
|
||||
|
||||
def fetch_announcements_page(self, source: AnnouncementSource,
|
||||
page_no: int = 1) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
||||
"""
|
||||
获取公告列表页数据
|
||||
|
||||
Args:
|
||||
source: 公告来源
|
||||
page_no: 页码
|
||||
|
||||
Returns:
|
||||
Tuple[Optional[Dict[str, Any]], Optional[str]]: (响应数据, 错误信息)
|
||||
"""
|
||||
try:
|
||||
session = self.init_session()
|
||||
|
||||
# 构建请求参数
|
||||
payload = {
|
||||
"keyword": "", # 关键词筛选，我们在筛选模块处理
|
||||
"publishDateBegin": self.config.crawler.start_date or "",
|
||||
"publishDateEnd": self.config.crawler.end_date or "",
|
||||
"pageNo": page_no,
|
||||
"pageSize": self.config.crawler.page_size,
|
||||
"categoryCode": source.code,
|
||||
"_t": int(time.time() * 1000)
|
||||
}
|
||||
|
||||
# 先执行敏感词检查
|
||||
if not self.check_sensitive_words(payload, str(source.category_id), source.code):
|
||||
return None, "敏感词检查失败"
|
||||
|
||||
# 执行公告数据请求
|
||||
user_agent = self.get_random_user_agent()
|
||||
headers = {
|
||||
"User-Agent": user_agent,
|
||||
"Content-Type": "application/json;charset=UTF-8",
|
||||
"Origin": self.base_url,
|
||||
"Referer": f"{self.base_url}/site/category?parentId={source.category_id}&childrenCode={source.code}",
|
||||
"Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64"
|
||||
}
|
||||
|
||||
proxies = self.get_random_proxy()
|
||||
|
||||
logger.debug(f"请求公告数据: {source.name} 第{page_no}页")
|
||||
|
||||
# 添加请求间延迟
|
||||
if page_no > 1:
|
||||
delay = random.uniform(
|
||||
self.config.crawler.request_delay,
|
||||
self.config.crawler.request_delay_max
|
||||
)
|
||||
time.sleep(delay)
|
||||
|
||||
response = session.post(
|
||||
self.announcement_api,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
proxies=proxies,
|
||||
timeout=self.session.timeout
|
||||
)
|
||||
|
||||
self.request_count += 1
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get("success", False):
|
||||
return data, None
|
||||
else:
|
||||
error_msg = ErrorResponseParser.parse_error(data)
|
||||
return None, f"API返回失败: {error_msg}"
|
||||
else:
|
||||
return None, f"请求失败, 状态码: {response.status_code}"
|
||||
|
||||
except requests.exceptions.Timeout as e:
|
||||
self.error_count += 1
|
||||
return None, f"请求超时: {str(e)}"
|
||||
except requests.exceptions.ProxyError as e:
|
||||
self.error_count += 1
|
||||
return None, f"代理错误: {str(e)}"
|
||||
except Exception as e:
|
||||
self.error_count += 1
|
||||
logger.error(f"获取公告数据异常: {str(e)}")
|
||||
return None, f"请求异常: {str(e)}"
|
||||
|
||||
def crawl_source(self, source: AnnouncementSource,
|
||||
max_pages: Optional[int] = None) -> CrawlResult:
|
||||
"""
|
||||
爬取单个来源的公告
|
||||
|
||||
Args:
|
||||
source: 公告来源
|
||||
max_pages: 最大页数限制
|
||||
|
||||
Returns:
|
||||
CrawlResult: 爬取结果
|
||||
"""
|
||||
if max_pages is None:
|
||||
max_pages = self.config.crawler.max_pages
|
||||
|
||||
log_crawl_start(source.name)
|
||||
|
||||
start_time = datetime.now()
|
||||
result = CrawlResult(
|
||||
source=source,
|
||||
status=CrawlStatus.RUNNING,
|
||||
crawled_at=start_time
|
||||
)
|
||||
|
||||
try:
|
||||
page = 1
|
||||
all_announcements = []
|
||||
|
||||
while page <= max_pages:
|
||||
# 获取页面数据
|
||||
response_data, error_msg = self.fetch_announcements_page(source, page)
|
||||
|
||||
if error_msg:
|
||||
logger.warning(f"{source.name} 第{page}页获取失败: {error_msg}")
|
||||
result.status = CrawlStatus.FAILED
|
||||
result.error_message = error_msg
|
||||
break
|
||||
|
||||
if not response_data:
|
||||
logger.info(f"{source.name} 第{page}页无数据")
|
||||
break
|
||||
|
||||
# 解析分页信息
|
||||
pagination = AnnouncementParser.extract_pagination_info(response_data)
|
||||
result.total_count = pagination["total"]
|
||||
|
||||
# 解析公告数据
|
||||
announcements = AnnouncementParser.parse_api_response(
|
||||
response_data, source, start_time)
|
||||
|
||||
if not announcements:
|
||||
logger.info(f"{source.name} 第{page}页解析到0条公告")
|
||||
break
|
||||
|
||||
all_announcements.extend(announcements)
|
||||
|
||||
# 检查是否还有下一页
|
||||
if not pagination["has_next"] or pagination["empty"]:
|
||||
break
|
||||
|
||||
page += 1
|
||||
|
||||
# 更新结果
|
||||
result.announcements = all_announcements
|
||||
result.new_count = len(all_announcements) # 这里的新增数需要在筛选后确定
|
||||
|
||||
if result.status != CrawlStatus.FAILED:
|
||||
result.status = CrawlStatus.SUCCESS
|
||||
|
||||
duration = (datetime.now() - start_time).total_seconds()
|
||||
result.duration = duration
|
||||
|
||||
log_crawl_success(source.name, len(all_announcements), duration)
|
||||
|
||||
except Exception as e:
|
||||
duration = (datetime.now() - start_time).total_seconds()
|
||||
result.duration = duration
|
||||
result.status = CrawlStatus.FAILED
|
||||
result.error_message = str(e)
|
||||
|
||||
log_crawl_error(source.name, str(e))
|
||||
|
||||
return result
|
||||
|
||||
@retry_on_exception(RetryConfig(max_retries=2))
|
||||
def crawl_all_sources(self, sources: Optional[List[AnnouncementSource]] = None) -> List[CrawlResult]:
|
||||
"""
|
||||
爬取所有来源的公告
|
||||
|
||||
Args:
|
||||
sources: 指定的来源列表，如果为None则使用配置中的所有来源
|
||||
|
||||
Returns:
|
||||
List[CrawlResult]: 所有来源的爬取结果
|
||||
"""
|
||||
# 系统健康检查
|
||||
if not check_system_health():
|
||||
logger.error("系统健康检查失败，跳过爬取")
|
||||
return []
|
||||
|
||||
if sources is None:
|
||||
sources = self._load_sources_from_config()
|
||||
|
||||
logger.info(f"开始爬取 {len(sources)} 个公告来源")
|
||||
|
||||
results = []
|
||||
|
||||
for source in sources:
|
||||
try:
|
||||
result = self.crawl_source(source)
|
||||
results.append(result)
|
||||
|
||||
# 检查是否需要暂停
|
||||
if result.status == CrawlStatus.FAILED:
|
||||
logger.warning(f"来源 {source.name} 爬取失败，继续下一个来源")
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"爬取来源 {source.name} 时发生未预期错误: {str(e)}")
|
||||
# 创建失败结果
|
||||
failed_result = CrawlResult(
|
||||
source=source,
|
||||
status=CrawlStatus.FAILED,
|
||||
error_message=str(e),
|
||||
crawled_at=datetime.now()
|
||||
)
|
||||
results.append(failed_result)
|
||||
|
||||
# 统计总结果
|
||||
total_announcements = sum(len(r.announcements) for r in results if r.status == CrawlStatus.SUCCESS)
|
||||
success_count = sum(1 for r in results if r.status == CrawlStatus.SUCCESS)
|
||||
failed_count = len(results) - success_count
|
||||
|
||||
logger.info(
|
||||
f"爬取完成: 共处理 {len(results)} 个来源，"
|
||||
f"成功 {success_count} 个，失败 {failed_count} 个，"
|
||||
f"获取 {total_announcements} 条公告"
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _load_sources_from_config(self) -> List[AnnouncementSource]:
|
||||
"""从配置加载公告来源"""
|
||||
sources = []
|
||||
|
||||
for code, source_config in self.config.sources.items():
|
||||
try:
|
||||
source = AnnouncementSource(
|
||||
code=code,
|
||||
category_id=source_config["category_id"],
|
||||
name=source_config["name"],
|
||||
type=AnnouncementType(source_config["type"])
|
||||
)
|
||||
sources.append(source)
|
||||
except Exception as e:
|
||||
logger.warning(f"加载来源配置失败 {code}: {str(e)}")
|
||||
continue
|
||||
|
||||
return sources
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""获取爬虫统计信息"""
|
||||
return {
|
||||
"request_count": self.request_count,
|
||||
"error_count": self.error_count,
|
||||
"error_rate": self.error_count / max(self.request_count, 1),
|
||||
"session_active": self.session is not None
|
||||
}
|
||||
|
||||
def reset_stats(self):
|
||||
"""重置统计信息"""
|
||||
self.request_count = 0
|
||||
self.error_count = 0
|
||||
|
||||
|
||||
def create_spider() -> GXGPSpider:
|
||||
"""
|
||||
创建爬虫实例
|
||||
|
||||
Returns:
|
||||
GXGPSpider: 爬虫实例
|
||||
"""
|
||||
return GXGPSpider()
|
||||
|
||||
|
||||
def crawl_announcements(keywords: Optional[List[str]] = None,
|
||||
sources: Optional[List[str]] = None) -> List[CrawlResult]:
|
||||
"""
|
||||
便捷函数：爬取公告
|
||||
|
||||
Args:
|
||||
keywords: 关键词过滤（暂时未使用，在筛选模块处理）
|
||||
sources: 来源代码列表
|
||||
|
||||
Returns:
|
||||
List[CrawlResult]: 爬取结果
|
||||
"""
|
||||
spider = create_spider()
|
||||
|
||||
try:
|
||||
# 过滤来源
|
||||
if sources:
|
||||
all_sources = spider._load_sources_from_config()
|
||||
filtered_sources = [s for s in all_sources if s.code in sources]
|
||||
else:
|
||||
filtered_sources = None
|
||||
|
||||
return spider.crawl_all_sources(filtered_sources)
|
||||
finally:
|
||||
spider.close_session()
|
||||
Reference in New Issue
Block a user