257 lines
10 KiB
Python
257 lines
10 KiB
Python
"""
|
||
数据模型定义
|
||
定义系统使用的数据结构和模型
|
||
"""
|
||
|
||
from dataclasses import dataclass, field
|
||
from datetime import datetime
|
||
from typing import Optional, List, Dict, Any
|
||
from enum import Enum
|
||
|
||
|
||
class AnnouncementType(Enum):
|
||
"""公告类型枚举"""
|
||
PURCHASE = "purchase" # 采购公告
|
||
RESULT = "result" # 结果公告
|
||
CONTRACT = "contract" # 合同公告
|
||
CORRECTION = "correction" # 更正公告
|
||
PRE_ANNOUNCEMENT = "pre_announcement" # 招标文件预公示
|
||
SINGLE_SOURCE = "single_source" # 单一来源公示
|
||
ELECTRONIC_MARKET = "electronic_market" # 电子卖场公示
|
||
ACCEPTANCE = "acceptance" # 履约验收公示
|
||
ENGINEERING = "engineering" # 工程类公告
|
||
FRAMEWORK_AGREEMENT = "framework_agreement" # 框架协议征集公告
|
||
FRAMEWORK_RESULT = "framework_result" # 框架协议入围结果公告
|
||
FRAMEWORK_SUMMARY = "framework_summary" # 框架协议成交结果汇总公告
|
||
INTENTION = "intention" # 采购意向公开
|
||
|
||
|
||
class CrawlStatus(Enum):
|
||
"""爬取状态枚举"""
|
||
PENDING = "pending" # 待爬取
|
||
RUNNING = "running" # 爬取中
|
||
SUCCESS = "success" # 成功
|
||
FAILED = "failed" # 失败
|
||
PARTIAL = "partial" # 部分成功
|
||
|
||
|
||
@dataclass
|
||
class AnnouncementSource:
|
||
"""公告来源"""
|
||
code: str # 来源代码，如 "ZcyAnnouncement1"
|
||
category_id: int # 分类ID
|
||
name: str # 显示名称，如 "采购公告"
|
||
type: AnnouncementType # 公告类型
|
||
|
||
|
||
@dataclass
|
||
class Announcement:
|
||
"""公告数据模型"""
|
||
id: Optional[int] = None # 数据库ID
|
||
title: str = "" # 公告标题
|
||
publish_date: datetime = field(default_factory=datetime.now) # 发布时间
|
||
purchase_name: str = "" # 发布单位
|
||
content_url: str = "" # 内容链接
|
||
source_code: str = "" # 来源代码
|
||
source_name: str = "" # 来源名称
|
||
announcement_type: AnnouncementType = AnnouncementType.PURCHASE # 公告类型
|
||
|
||
# 爬取相关字段
|
||
crawled_at: Optional[datetime] = None # 爬取时间
|
||
created_at: Optional[datetime] = None # 创建时间
|
||
updated_at: Optional[datetime] = None # 更新时间
|
||
crawl_mode: str = "auto" # 爬取模式：auto(自动)/manual(手动)
|
||
|
||
# 去重字段
|
||
content_hash: Optional[str] = None # 内容哈希，用于去重
|
||
|
||
# 筛选相关
|
||
keyword_matched: bool = False # 是否匹配关键词
|
||
date_filtered: bool = True # 是否在日期范围内
|
||
|
||
# 业务字段
|
||
is_new: bool = True # 是否为新公告
|
||
is_today: bool = False # 是否为今日公告
|
||
|
||
def __post_init__(self):
|
||
"""后初始化处理"""
|
||
if isinstance(self.announcement_type, str):
|
||
self.announcement_type = AnnouncementType(self.announcement_type)
|
||
|
||
if self.publish_date and isinstance(self.publish_date, str):
|
||
try:
|
||
self.publish_date = datetime.fromisoformat(self.publish_date.replace('Z', '+00:00'))
|
||
except ValueError:
|
||
# 如果解析失败，使用当前时间
|
||
self.publish_date = datetime.now()
|
||
|
||
# 判断是否为今日公告
|
||
today = datetime.now().date()
|
||
if self.publish_date:
|
||
self.is_today = self.publish_date.date() == today
|
||
|
||
@property
|
||
def publish_date_str(self) -> str:
|
||
"""获取发布日期字符串"""
|
||
return self.publish_date.strftime("%Y-%m-%d") if self.publish_date else ""
|
||
|
||
@property
|
||
def crawled_at_str(self) -> str:
|
||
"""获取爬取时间字符串"""
|
||
return self.crawled_at.strftime("%Y-%m-%d %H:%M:%S") if self.crawled_at else ""
|
||
|
||
def to_dict(self) -> Dict[str, Any]:
|
||
"""转换为字典"""
|
||
return {
|
||
"id": self.id,
|
||
"title": self.title,
|
||
"publish_date": self.publish_date.isoformat() if self.publish_date else None,
|
||
"purchase_name": self.purchase_name,
|
||
"content_url": self.content_url,
|
||
"source_code": self.source_code,
|
||
"source_name": self.source_name,
|
||
"announcement_type": self.announcement_type.value,
|
||
"crawled_at": self.crawled_at.isoformat() if self.crawled_at else None,
|
||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
|
||
"content_hash": self.content_hash,
|
||
"keyword_matched": self.keyword_matched,
|
||
"date_filtered": self.date_filtered,
|
||
"is_new": self.is_new,
|
||
"is_today": self.is_today
|
||
}
|
||
|
||
@classmethod
|
||
def from_dict(cls, data: Dict[str, Any]) -> 'Announcement':
|
||
"""从字典创建实例"""
|
||
# 处理枚举类型
|
||
if 'announcement_type' in data and isinstance(data['announcement_type'], str):
|
||
data['announcement_type'] = AnnouncementType(data['announcement_type'])
|
||
|
||
# 处理日期时间
|
||
for date_field in ['publish_date', 'crawled_at', 'created_at', 'updated_at']:
|
||
if date_field in data and data[date_field] and isinstance(data[date_field], str):
|
||
try:
|
||
data[date_field] = datetime.fromisoformat(data[date_field].replace('Z', '+00:00'))
|
||
except ValueError:
|
||
data[date_field] = None
|
||
|
||
return cls(**data)
|
||
|
||
def generate_content_hash(self) -> str:
|
||
"""生成内容哈希用于去重"""
|
||
import hashlib
|
||
content = f"{self.title}|{self.publish_date_str}|{self.purchase_name}|{self.content_url}|{self.source_code}"
|
||
self.content_hash = hashlib.md5(content.encode('utf-8')).hexdigest()
|
||
return self.content_hash
|
||
|
||
def matches_keywords(self, keywords: List[str]) -> bool:
|
||
"""检查是否匹配关键词"""
|
||
if not keywords:
|
||
return True
|
||
|
||
search_text = f"{self.title} {self.purchase_name}".lower()
|
||
for keyword in keywords:
|
||
if keyword.lower() in search_text:
|
||
self.keyword_matched = True
|
||
return True
|
||
|
||
self.keyword_matched = False
|
||
return False
|
||
|
||
def in_date_range(self, start_date: Optional[str], end_date: Optional[str]) -> bool:
|
||
"""检查是否在日期范围内"""
|
||
if not self.publish_date:
|
||
self.date_filtered = False
|
||
return False
|
||
|
||
publish_date = self.publish_date.date()
|
||
|
||
try:
|
||
if start_date:
|
||
start = datetime.fromisoformat(start_date).date()
|
||
if publish_date < start:
|
||
self.date_filtered = False
|
||
return False
|
||
|
||
if end_date:
|
||
end = datetime.fromisoformat(end_date).date()
|
||
if publish_date > end:
|
||
self.date_filtered = False
|
||
return False
|
||
|
||
self.date_filtered = True
|
||
return True
|
||
except ValueError:
|
||
# 日期格式错误时，默认通过
|
||
self.date_filtered = True
|
||
return True
|
||
|
||
|
||
@dataclass
|
||
class CrawlResult:
|
||
"""爬取结果"""
|
||
source: AnnouncementSource # 公告来源
|
||
status: CrawlStatus # 爬取状态
|
||
total_count: int = 0 # 总公告数
|
||
new_count: int = 0 # 新增公告数
|
||
error_message: Optional[str] = None # 错误信息
|
||
announcements: List[Announcement] = field(default_factory=list) # 公告列表
|
||
crawled_at: datetime = field(default_factory=datetime.now) # 爬取时间
|
||
duration: float = 0.0 # 爬取耗时（秒）
|
||
|
||
|
||
@dataclass
|
||
class CrawlSession:
|
||
"""爬取会话"""
|
||
session_id: str # 会话ID
|
||
start_time: datetime # 开始时间
|
||
end_time: Optional[datetime] = None # 结束时间
|
||
status: CrawlStatus = CrawlStatus.PENDING # 会话状态
|
||
total_sources: int = 0 # 总来源数
|
||
completed_sources: int = 0 # 已完成来源数
|
||
total_announcements: int = 0 # 总公告数
|
||
new_announcements: int = 0 # 新增公告数
|
||
results: List[CrawlResult] = field(default_factory=list) # 各来源结果
|
||
|
||
@property
|
||
def duration(self) -> float:
|
||
"""获取会话持续时间"""
|
||
if self.end_time and self.start_time:
|
||
return (self.end_time - self.start_time).total_seconds()
|
||
elif self.start_time:
|
||
return (datetime.now() - self.start_time).total_seconds()
|
||
return 0.0
|
||
|
||
@property
|
||
def progress(self) -> float:
|
||
"""获取完成进度（0-1）"""
|
||
if self.total_sources == 0:
|
||
return 0.0
|
||
return self.completed_sources / self.total_sources
|
||
|
||
|
||
@dataclass
|
||
class NotificationMessage:
|
||
"""通知消息"""
|
||
title: str # 消息标题
|
||
content: str # 消息内容
|
||
message_type: str = "text" # 消息类型：text, markdown, card
|
||
recipients: List[str] = field(default_factory=lambda: ["@all"]) # 接收者列表
|
||
attachments: Optional[Dict[str, Any]] = None # 附件信息
|
||
created_at: datetime = field(default_factory=datetime.now) # 创建时间
|
||
|
||
|
||
@dataclass
|
||
class SystemMetrics:
|
||
"""系统指标"""
|
||
timestamp: datetime = field(default_factory=datetime.now) # 时间戳
|
||
total_announcements: int = 0 # 总公告数
|
||
today_announcements: int = 0 # 今日公告数
|
||
new_announcements_today: int = 0 # 今日新增公告数
|
||
crawl_sessions_today: int = 0 # 今日爬取会话数
|
||
last_crawl_duration: float = 0.0 # 最后一次爬取耗时
|
||
database_size: int = 0 # 数据库大小（字节）
|
||
memory_usage: float = 0.0 # 内存使用率
|
||
disk_usage: float = 0.0 # 磁盘使用率
|