""" 数据模型定义 定义系统使用的数据结构和模型 """ from dataclasses import dataclass, field from datetime import datetime from typing import Optional, List, Dict, Any from enum import Enum class AnnouncementType(Enum): """公告类型枚举""" PURCHASE = "purchase" # 采购公告 RESULT = "result" # 结果公告 CONTRACT = "contract" # 合同公告 CORRECTION = "correction" # 更正公告 PRE_ANNOUNCEMENT = "pre_announcement" # 招标文件预公示 SINGLE_SOURCE = "single_source" # 单一来源公示 ELECTRONIC_MARKET = "electronic_market" # 电子卖场公示 ACCEPTANCE = "acceptance" # 履约验收公示 ENGINEERING = "engineering" # 工程类公告 FRAMEWORK_AGREEMENT = "framework_agreement" # 框架协议征集公告 FRAMEWORK_RESULT = "framework_result" # 框架协议入围结果公告 FRAMEWORK_SUMMARY = "framework_summary" # 框架协议成交结果汇总公告 INTENTION = "intention" # 采购意向公开 class CrawlStatus(Enum): """爬取状态枚举""" PENDING = "pending" # 待爬取 RUNNING = "running" # 爬取中 SUCCESS = "success" # 成功 FAILED = "failed" # 失败 PARTIAL = "partial" # 部分成功 @dataclass class AnnouncementSource: """公告来源""" code: str # 来源代码,如 "ZcyAnnouncement1" category_id: int # 分类ID name: str # 显示名称,如 "采购公告" type: AnnouncementType # 公告类型 @dataclass class Announcement: """公告数据模型""" id: Optional[int] = None # 数据库ID title: str = "" # 公告标题 publish_date: datetime = field(default_factory=datetime.now) # 发布时间 purchase_name: str = "" # 发布单位 content_url: str = "" # 内容链接 source_code: str = "" # 来源代码 source_name: str = "" # 来源名称 announcement_type: AnnouncementType = AnnouncementType.PURCHASE # 公告类型 # 爬取相关字段 crawled_at: Optional[datetime] = None # 爬取时间 created_at: Optional[datetime] = None # 创建时间 updated_at: Optional[datetime] = None # 更新时间 crawl_mode: str = "auto" # 爬取模式:auto(自动)/manual(手动) # 去重字段 content_hash: Optional[str] = None # 内容哈希,用于去重 # 筛选相关 keyword_matched: bool = False # 是否匹配关键词 date_filtered: bool = True # 是否在日期范围内 # 业务字段 is_new: bool = True # 是否为新公告 is_today: bool = False # 是否为今日公告 def __post_init__(self): """后初始化处理""" if isinstance(self.announcement_type, str): self.announcement_type = AnnouncementType(self.announcement_type) if self.publish_date and isinstance(self.publish_date, str): try: self.publish_date = datetime.fromisoformat(self.publish_date.replace('Z', '+00:00')) except ValueError: # 如果解析失败,使用当前时间 self.publish_date = datetime.now() # 判断是否为今日公告 today = datetime.now().date() if self.publish_date: self.is_today = self.publish_date.date() == today @property def publish_date_str(self) -> str: """获取发布日期字符串""" return self.publish_date.strftime("%Y-%m-%d") if self.publish_date else "" @property def crawled_at_str(self) -> str: """获取爬取时间字符串""" return self.crawled_at.strftime("%Y-%m-%d %H:%M:%S") if self.crawled_at else "" def to_dict(self) -> Dict[str, Any]: """转换为字典""" return { "id": self.id, "title": self.title, "publish_date": self.publish_date.isoformat() if self.publish_date else None, "purchase_name": self.purchase_name, "content_url": self.content_url, "source_code": self.source_code, "source_name": self.source_name, "announcement_type": self.announcement_type.value, "crawled_at": self.crawled_at.isoformat() if self.crawled_at else None, "created_at": self.created_at.isoformat() if self.created_at else None, "updated_at": self.updated_at.isoformat() if self.updated_at else None, "content_hash": self.content_hash, "keyword_matched": self.keyword_matched, "date_filtered": self.date_filtered, "is_new": self.is_new, "is_today": self.is_today } @classmethod def from_dict(cls, data: Dict[str, Any]) -> 'Announcement': """从字典创建实例""" # 处理枚举类型 if 'announcement_type' in data and isinstance(data['announcement_type'], str): data['announcement_type'] = AnnouncementType(data['announcement_type']) # 处理日期时间 for date_field in ['publish_date', 'crawled_at', 'created_at', 'updated_at']: if date_field in data and data[date_field] and isinstance(data[date_field], str): try: data[date_field] = datetime.fromisoformat(data[date_field].replace('Z', '+00:00')) except ValueError: data[date_field] = None return cls(**data) def generate_content_hash(self) -> str: """生成内容哈希用于去重""" import hashlib content = f"{self.title}|{self.publish_date_str}|{self.purchase_name}|{self.content_url}|{self.source_code}" self.content_hash = hashlib.md5(content.encode('utf-8')).hexdigest() return self.content_hash def matches_keywords(self, keywords: List[str]) -> bool: """检查是否匹配关键词""" if not keywords: return True search_text = f"{self.title} {self.purchase_name}".lower() for keyword in keywords: if keyword.lower() in search_text: self.keyword_matched = True return True self.keyword_matched = False return False def in_date_range(self, start_date: Optional[str], end_date: Optional[str]) -> bool: """检查是否在日期范围内""" if not self.publish_date: self.date_filtered = False return False publish_date = self.publish_date.date() try: if start_date: start = datetime.fromisoformat(start_date).date() if publish_date < start: self.date_filtered = False return False if end_date: end = datetime.fromisoformat(end_date).date() if publish_date > end: self.date_filtered = False return False self.date_filtered = True return True except ValueError: # 日期格式错误时,默认通过 self.date_filtered = True return True @dataclass class CrawlResult: """爬取结果""" source: AnnouncementSource # 公告来源 status: CrawlStatus # 爬取状态 total_count: int = 0 # 总公告数 new_count: int = 0 # 新增公告数 error_message: Optional[str] = None # 错误信息 announcements: List[Announcement] = field(default_factory=list) # 公告列表 crawled_at: datetime = field(default_factory=datetime.now) # 爬取时间 duration: float = 0.0 # 爬取耗时(秒) @dataclass class CrawlSession: """爬取会话""" session_id: str # 会话ID start_time: datetime # 开始时间 end_time: Optional[datetime] = None # 结束时间 status: CrawlStatus = CrawlStatus.PENDING # 会话状态 total_sources: int = 0 # 总来源数 completed_sources: int = 0 # 已完成来源数 total_announcements: int = 0 # 总公告数 new_announcements: int = 0 # 新增公告数 results: List[CrawlResult] = field(default_factory=list) # 各来源结果 @property def duration(self) -> float: """获取会话持续时间""" if self.end_time and self.start_time: return (self.end_time - self.start_time).total_seconds() elif self.start_time: return (datetime.now() - self.start_time).total_seconds() return 0.0 @property def progress(self) -> float: """获取完成进度(0-1)""" if self.total_sources == 0: return 0.0 return self.completed_sources / self.total_sources @dataclass class NotificationMessage: """通知消息""" title: str # 消息标题 content: str # 消息内容 message_type: str = "text" # 消息类型:text, markdown, card recipients: List[str] = field(default_factory=lambda: ["@all"]) # 接收者列表 attachments: Optional[Dict[str, Any]] = None # 附件信息 created_at: datetime = field(default_factory=datetime.now) # 创建时间 @dataclass class SystemMetrics: """系统指标""" timestamp: datetime = field(default_factory=datetime.now) # 时间戳 total_announcements: int = 0 # 总公告数 today_announcements: int = 0 # 今日公告数 new_announcements_today: int = 0 # 今日新增公告数 crawl_sessions_today: int = 0 # 今日爬取会话数 last_crawl_duration: float = 0.0 # 最后一次爬取耗时 database_size: int = 0 # 数据库大小(字节) memory_usage: float = 0.0 # 内存使用率 disk_usage: float = 0.0 # 磁盘使用率