手动模式
This commit is contained in:
@@ -0,0 +1,255 @@
|
||||
"""
|
||||
数据模型定义
|
||||
定义系统使用的数据结构和模型
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Optional, List, Dict, Any
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class AnnouncementType(Enum):
|
||||
"""公告类型枚举"""
|
||||
PURCHASE = "purchase" # 采购公告
|
||||
RESULT = "result" # 结果公告
|
||||
CONTRACT = "contract" # 合同公告
|
||||
CORRECTION = "correction" # 更正公告
|
||||
PRE_ANNOUNCEMENT = "pre_announcement" # 招标文件预公示
|
||||
SINGLE_SOURCE = "single_source" # 单一来源公示
|
||||
ELECTRONIC_MARKET = "electronic_market" # 电子卖场公示
|
||||
ACCEPTANCE = "acceptance" # 履约验收公示
|
||||
ENGINEERING = "engineering" # 工程类公告
|
||||
FRAMEWORK_AGREEMENT = "framework_agreement" # 框架协议征集公告
|
||||
FRAMEWORK_RESULT = "framework_result" # 框架协议入围结果公告
|
||||
FRAMEWORK_SUMMARY = "framework_summary" # 框架协议成交结果汇总公告
|
||||
INTENTION = "intention" # 采购意向公开
|
||||
|
||||
|
||||
class CrawlStatus(Enum):
|
||||
"""爬取状态枚举"""
|
||||
PENDING = "pending" # 待爬取
|
||||
RUNNING = "running" # 爬取中
|
||||
SUCCESS = "success" # 成功
|
||||
FAILED = "failed" # 失败
|
||||
PARTIAL = "partial" # 部分成功
|
||||
|
||||
|
||||
@dataclass
|
||||
class AnnouncementSource:
|
||||
"""公告来源"""
|
||||
code: str # 来源代码，如 "ZcyAnnouncement1"
|
||||
category_id: int # 分类ID
|
||||
name: str # 显示名称，如 "采购公告"
|
||||
type: AnnouncementType # 公告类型
|
||||
|
||||
|
||||
@dataclass
|
||||
class Announcement:
|
||||
"""公告数据模型"""
|
||||
id: Optional[int] = None # 数据库ID
|
||||
title: str = "" # 公告标题
|
||||
publish_date: datetime = field(default_factory=datetime.now) # 发布时间
|
||||
purchase_name: str = "" # 发布单位
|
||||
content_url: str = "" # 内容链接
|
||||
source_code: str = "" # 来源代码
|
||||
source_name: str = "" # 来源名称
|
||||
announcement_type: AnnouncementType = AnnouncementType.PURCHASE # 公告类型
|
||||
|
||||
# 爬取相关字段
|
||||
crawled_at: Optional[datetime] = None # 爬取时间
|
||||
created_at: Optional[datetime] = None # 创建时间
|
||||
updated_at: Optional[datetime] = None # 更新时间
|
||||
|
||||
# 去重字段
|
||||
content_hash: Optional[str] = None # 内容哈希，用于去重
|
||||
|
||||
# 筛选相关
|
||||
keyword_matched: bool = False # 是否匹配关键词
|
||||
date_filtered: bool = True # 是否在日期范围内
|
||||
|
||||
# 业务字段
|
||||
is_new: bool = True # 是否为新公告
|
||||
is_today: bool = False # 是否为今日公告
|
||||
|
||||
def __post_init__(self):
|
||||
"""后初始化处理"""
|
||||
if isinstance(self.announcement_type, str):
|
||||
self.announcement_type = AnnouncementType(self.announcement_type)
|
||||
|
||||
if self.publish_date and isinstance(self.publish_date, str):
|
||||
try:
|
||||
self.publish_date = datetime.fromisoformat(self.publish_date.replace('Z', '+00:00'))
|
||||
except ValueError:
|
||||
# 如果解析失败，使用当前时间
|
||||
self.publish_date = datetime.now()
|
||||
|
||||
# 判断是否为今日公告
|
||||
today = datetime.now().date()
|
||||
if self.publish_date:
|
||||
self.is_today = self.publish_date.date() == today
|
||||
|
||||
@property
|
||||
def publish_date_str(self) -> str:
|
||||
"""获取发布日期字符串"""
|
||||
return self.publish_date.strftime("%Y-%m-%d") if self.publish_date else ""
|
||||
|
||||
@property
|
||||
def crawled_at_str(self) -> str:
|
||||
"""获取爬取时间字符串"""
|
||||
return self.crawled_at.strftime("%Y-%m-%d %H:%M:%S") if self.crawled_at else ""
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""转换为字典"""
|
||||
return {
|
||||
"id": self.id,
|
||||
"title": self.title,
|
||||
"publish_date": self.publish_date.isoformat() if self.publish_date else None,
|
||||
"purchase_name": self.purchase_name,
|
||||
"content_url": self.content_url,
|
||||
"source_code": self.source_code,
|
||||
"source_name": self.source_name,
|
||||
"announcement_type": self.announcement_type.value,
|
||||
"crawled_at": self.crawled_at.isoformat() if self.crawled_at else None,
|
||||
"created_at": self.created_at.isoformat() if self.created_at else None,
|
||||
"updated_at": self.updated_at.isoformat() if self.updated_at else None,
|
||||
"content_hash": self.content_hash,
|
||||
"keyword_matched": self.keyword_matched,
|
||||
"date_filtered": self.date_filtered,
|
||||
"is_new": self.is_new,
|
||||
"is_today": self.is_today
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> 'Announcement':
|
||||
"""从字典创建实例"""
|
||||
# 处理枚举类型
|
||||
if 'announcement_type' in data and isinstance(data['announcement_type'], str):
|
||||
data['announcement_type'] = AnnouncementType(data['announcement_type'])
|
||||
|
||||
# 处理日期时间
|
||||
for date_field in ['publish_date', 'crawled_at', 'created_at', 'updated_at']:
|
||||
if date_field in data and data[date_field] and isinstance(data[date_field], str):
|
||||
try:
|
||||
data[date_field] = datetime.fromisoformat(data[date_field].replace('Z', '+00:00'))
|
||||
except ValueError:
|
||||
data[date_field] = None
|
||||
|
||||
return cls(**data)
|
||||
|
||||
def generate_content_hash(self) -> str:
|
||||
"""生成内容哈希用于去重"""
|
||||
import hashlib
|
||||
content = f"{self.title}|{self.publish_date_str}|{self.purchase_name}|{self.content_url}|{self.source_code}"
|
||||
self.content_hash = hashlib.md5(content.encode('utf-8')).hexdigest()
|
||||
return self.content_hash
|
||||
|
||||
def matches_keywords(self, keywords: List[str]) -> bool:
|
||||
"""检查是否匹配关键词"""
|
||||
if not keywords:
|
||||
return True
|
||||
|
||||
search_text = f"{self.title} {self.purchase_name}".lower()
|
||||
for keyword in keywords:
|
||||
if keyword.lower() in search_text:
|
||||
self.keyword_matched = True
|
||||
return True
|
||||
|
||||
self.keyword_matched = False
|
||||
return False
|
||||
|
||||
def in_date_range(self, start_date: Optional[str], end_date: Optional[str]) -> bool:
|
||||
"""检查是否在日期范围内"""
|
||||
if not self.publish_date:
|
||||
self.date_filtered = False
|
||||
return False
|
||||
|
||||
publish_date = self.publish_date.date()
|
||||
|
||||
try:
|
||||
if start_date:
|
||||
start = datetime.fromisoformat(start_date).date()
|
||||
if publish_date < start:
|
||||
self.date_filtered = False
|
||||
return False
|
||||
|
||||
if end_date:
|
||||
end = datetime.fromisoformat(end_date).date()
|
||||
if publish_date > end:
|
||||
self.date_filtered = False
|
||||
return False
|
||||
|
||||
self.date_filtered = True
|
||||
return True
|
||||
except ValueError:
|
||||
# 日期格式错误时，默认通过
|
||||
self.date_filtered = True
|
||||
return True
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlResult:
|
||||
"""爬取结果"""
|
||||
source: AnnouncementSource # 公告来源
|
||||
status: CrawlStatus # 爬取状态
|
||||
total_count: int = 0 # 总公告数
|
||||
new_count: int = 0 # 新增公告数
|
||||
error_message: Optional[str] = None # 错误信息
|
||||
announcements: List[Announcement] = field(default_factory=list) # 公告列表
|
||||
crawled_at: datetime = field(default_factory=datetime.now) # 爬取时间
|
||||
duration: float = 0.0 # 爬取耗时（秒）
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlSession:
|
||||
"""爬取会话"""
|
||||
session_id: str # 会话ID
|
||||
start_time: datetime # 开始时间
|
||||
end_time: Optional[datetime] = None # 结束时间
|
||||
status: CrawlStatus = CrawlStatus.PENDING # 会话状态
|
||||
total_sources: int = 0 # 总来源数
|
||||
completed_sources: int = 0 # 已完成来源数
|
||||
total_announcements: int = 0 # 总公告数
|
||||
new_announcements: int = 0 # 新增公告数
|
||||
results: List[CrawlResult] = field(default_factory=list) # 各来源结果
|
||||
|
||||
@property
|
||||
def duration(self) -> float:
|
||||
"""获取会话持续时间"""
|
||||
if self.end_time and self.start_time:
|
||||
return (self.end_time - self.start_time).total_seconds()
|
||||
elif self.start_time:
|
||||
return (datetime.now() - self.start_time).total_seconds()
|
||||
return 0.0
|
||||
|
||||
@property
|
||||
def progress(self) -> float:
|
||||
"""获取完成进度（0-1）"""
|
||||
if self.total_sources == 0:
|
||||
return 0.0
|
||||
return self.completed_sources / self.total_sources
|
||||
|
||||
|
||||
@dataclass
|
||||
class NotificationMessage:
|
||||
"""通知消息"""
|
||||
title: str # 消息标题
|
||||
content: str # 消息内容
|
||||
message_type: str = "text" # 消息类型：text, markdown, card
|
||||
recipients: List[str] = field(default_factory=lambda: ["@all"]) # 接收者列表
|
||||
attachments: Optional[Dict[str, Any]] = None # 附件信息
|
||||
created_at: datetime = field(default_factory=datetime.now) # 创建时间
|
||||
|
||||
|
||||
@dataclass
|
||||
class SystemMetrics:
|
||||
"""系统指标"""
|
||||
timestamp: datetime = field(default_factory=datetime.now) # 时间戳
|
||||
total_announcements: int = 0 # 总公告数
|
||||
today_announcements: int = 0 # 今日公告数
|
||||
new_announcements_today: int = 0 # 今日新增公告数
|
||||
crawl_sessions_today: int = 0 # 今日爬取会话数
|
||||
last_crawl_duration: float = 0.0 # 最后一次爬取耗时
|
||||
database_size: int = 0 # 数据库大小（字节）
|
||||
memory_usage: float = 0.0 # 内存使用率
|
||||
disk_usage: float = 0.0 # 磁盘使用率
|
||||
Reference in New Issue
Block a user