From d9af819d2ef43f97ebb3ffb222f2586c56bd1299 Mon Sep 17 00:00:00 2001 From: v6ole Date: Sat, 9 May 2026 13:37:40 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=20Spider=20=E5=9F=BA?= =?UTF-8?q?=E7=B1=BB=20+=20Pipeline=20=E9=85=8D=E7=BD=AE=20+=20=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/crawler/__init__.py | 0 app/crawler/base.py | 57 +++++++++++++++++++++++++ tests/__init__.py | 0 tests/conftest.py | 13 ++++++ tests/test_crawler/__init__.py | 0 tests/test_crawler/test_base.py | 76 +++++++++++++++++++++++++++++++++ 6 files changed, 146 insertions(+) create mode 100644 app/crawler/__init__.py create mode 100644 app/crawler/base.py create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/test_crawler/__init__.py create mode 100644 tests/test_crawler/test_base.py diff --git a/app/crawler/__init__.py b/app/crawler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/crawler/base.py b/app/crawler/base.py new file mode 100644 index 0000000..3e06f6c --- /dev/null +++ b/app/crawler/base.py @@ -0,0 +1,57 @@ +import hashlib +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import datetime +from typing import List, Optional + + +@dataclass +class CrawlResult: + source_code: str + source_name: str + total_count: int = 0 + new_count: int = 0 + announcements: list = field(default_factory=list) + error_message: Optional[str] = None + crawled_at: datetime = field(default_factory=datetime.now) + duration: float = 0.0 + + @property + def success(self) -> bool: + return self.error_message is None + + +@dataclass +class PipelineConfig: + filter_enabled: bool = True + keywords: List[str] = field(default_factory=list) + dedup_enabled: bool = True + notify_mode: str = "filtered" + mark_sent: bool = False + + +@dataclass +class PipelineResult: + stored: int = 0 + filtered: int = 0 + notified: int = 0 + markdown_generated: bool = False + + +class BaseSpider(ABC): + name: str + source_code: str + source_name: str + + @abstractmethod + async def crawl(self) -> CrawlResult: + ... + + def get_pipeline_config(self) -> PipelineConfig: + return PipelineConfig() + + @staticmethod + def generate_content_hash(title: str, publish_date: str, purchase_name: str, + content_url: str, source_code: str) -> str: + content = f"{title}|{publish_date}|{purchase_name}|{content_url}|{source_code}" + return hashlib.sha256(content.encode("utf-8")).hexdigest() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..22bdfc1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,13 @@ +import pytest + + +@pytest.fixture +def sample_announcement_data(): + return { + "title": "测试公告标题", + "publish_date": "2026-05-09", + "purchase_name": "测试单位", + "content_url": "https://example.com/detail/123", + "source_code": "ZcyAnnouncement1", + "source_name": "采购公告", + } diff --git a/tests/test_crawler/__init__.py b/tests/test_crawler/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_crawler/test_base.py b/tests/test_crawler/test_base.py new file mode 100644 index 0000000..880deee --- /dev/null +++ b/tests/test_crawler/test_base.py @@ -0,0 +1,76 @@ +import pytest +from datetime import datetime +from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig, PipelineResult + + +class FakeSpider(BaseSpider): + name = "test_spider" + source_code = "test_source" + source_name = "测试来源" + + async def crawl(self) -> CrawlResult: + return CrawlResult( + source_code=self.source_code, + source_name=self.source_name, + total_count=5, + new_count=3, + crawled_at=datetime.now(), + ) + + def get_pipeline_config(self) -> PipelineConfig: + return PipelineConfig( + filter_enabled=True, + keywords=["测试"], + dedup_enabled=True, + notify_mode="filtered", + mark_sent=False, + ) + + +@pytest.mark.asyncio +async def test_base_spider_crawl(): + spider = FakeSpider() + result = await spider.crawl() + assert result.source_code == "test_source" + assert result.total_count == 5 + assert result.new_count == 3 + assert result.success is True + + +def test_pipeline_config_defaults(): + config = PipelineConfig() + assert config.filter_enabled is True + assert config.keywords == [] + assert config.dedup_enabled is True + assert config.notify_mode == "filtered" + assert config.mark_sent is False + + +def test_pipeline_result_defaults(): + result = PipelineResult() + assert result.stored == 0 + assert result.filtered == 0 + assert result.notified == 0 + assert result.markdown_generated is False + + +def test_generate_content_hash(): + h = BaseSpider.generate_content_hash( + "title", "2026-05-09", "unit", "https://x.com", "source" + ) + assert len(h) == 64 + assert all(c in "0123456789abcdef" for c in h) + + +def test_generate_content_hash_deterministic(): + h1 = BaseSpider.generate_content_hash("t", "d", "p", "u", "s") + h2 = BaseSpider.generate_content_hash("t", "d", "p", "u", "s") + assert h1 == h2 + + +def test_crawl_result_failure(): + result = CrawlResult( + source_code="s", source_name="n", + error_message="连接超时" + ) + assert result.success is False