fix: 连接 Pipeline 到爬取流程 + 夜间跳过 + 断网恢复

- CrawlService.run_spider 现在自动调用 PostCrawlPipeline(爬取→存储→筛选→去重→推送)
- 定时任务添加 22:00-06:00 夜间跳过逻辑
- 断网恢复后不会漏公告:爬虫每次从 API 按日期倒序抓取,未推送的公告下次自动补推
- Pipeline 的 _exclude_sent + mark_sent 防止重复推送
This commit is contained in:
2026-05-09 16:10:30 +08:00
parent d851dafea9
commit cbeb3c3504
6 changed files with 106 additions and 25 deletions
+9 -8
View File
@@ -4,6 +4,14 @@ from dataclasses import dataclass, field
from datetime import datetime
@dataclass
class PipelineResult:
stored: int = 0
filtered: int = 0
notified: int = 0
markdown_generated: bool = False
@dataclass
class CrawlResult:
source_code: str
@@ -14,6 +22,7 @@ class CrawlResult:
error_message: str | None = None
crawled_at: datetime = field(default_factory=datetime.now)
duration: float = 0.0
pipeline_result: PipelineResult | None = None
@property
def success(self) -> bool:
@@ -29,14 +38,6 @@ class PipelineConfig:
mark_sent: bool = False
@dataclass
class PipelineResult:
stored: int = 0
filtered: int = 0
notified: int = 0
markdown_generated: bool = False
class BaseSpider(ABC):
name: str
source_code: str