fix: 连接 Pipeline 到爬取流程 + 夜间跳过 + 断网恢复
- CrawlService.run_spider 现在自动调用 PostCrawlPipeline(爬取→存储→筛选→去重→推送) - 定时任务添加 22:00-06:00 夜间跳过逻辑 - 断网恢复后不会漏公告:爬虫每次从 API 按日期倒序抓取,未推送的公告下次自动补推 - Pipeline 的 _exclude_sent + mark_sent 防止重复推送
This commit is contained in:
+9
-8
@@ -4,6 +4,14 @@ from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class PipelineResult:
|
||||
stored: int = 0
|
||||
filtered: int = 0
|
||||
notified: int = 0
|
||||
markdown_generated: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlResult:
|
||||
source_code: str
|
||||
@@ -14,6 +22,7 @@ class CrawlResult:
|
||||
error_message: str | None = None
|
||||
crawled_at: datetime = field(default_factory=datetime.now)
|
||||
duration: float = 0.0
|
||||
pipeline_result: PipelineResult | None = None
|
||||
|
||||
@property
|
||||
def success(self) -> bool:
|
||||
@@ -29,14 +38,6 @@ class PipelineConfig:
|
||||
mark_sent: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class PipelineResult:
|
||||
stored: int = 0
|
||||
filtered: int = 0
|
||||
notified: int = 0
|
||||
markdown_generated: bool = False
|
||||
|
||||
|
||||
class BaseSpider(ABC):
|
||||
name: str
|
||||
source_code: str
|
||||
|
||||
Reference in New Issue
Block a user