feat: 添加 Spider 基类 + Pipeline 配置 + 测试

app/crawler/base.py

@@ -0,0 +1,57 @@
+import hashlib
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import List, Optional
+
+
+@dataclass
+class CrawlResult:
+    source_code: str
+    source_name: str
+    total_count: int = 0
+    new_count: int = 0
+    announcements: list = field(default_factory=list)
+    error_message: Optional[str] = None
+    crawled_at: datetime = field(default_factory=datetime.now)
+    duration: float = 0.0
+
+    @property
+    def success(self) -> bool:
+        return self.error_message is None
+
+
+@dataclass
+class PipelineConfig:
+    filter_enabled: bool = True
+    keywords: List[str] = field(default_factory=list)
+    dedup_enabled: bool = True
+    notify_mode: str = "filtered"
+    mark_sent: bool = False
+
+
+@dataclass
+class PipelineResult:
+    stored: int = 0
+    filtered: int = 0
+    notified: int = 0
+    markdown_generated: bool = False
+
+
+class BaseSpider(ABC):
+    name: str
+    source_code: str
+    source_name: str
+
+    @abstractmethod
+    async def crawl(self) -> CrawlResult:
+        ...
+
+    def get_pipeline_config(self) -> PipelineConfig:
+        return PipelineConfig()
+
+    @staticmethod
+    def generate_content_hash(title: str, publish_date: str, purchase_name: str,
+                              content_url: str, source_code: str) -> str:
+        content = f"{title}|{publish_date}|{purchase_name}|{content_url}|{source_code}"
+        return hashlib.sha256(content.encode("utf-8")).hexdigest()