chore: ruff 代码检查与修复
130 issues auto-fixed (import ordering, UP045/UP006 type annotations), 33 issues manually fixed (E712/E501/E402/E722 + N818 rename + per-file wechat ignore for N8xx naming conventions). All 33 tests pass.
This commit is contained in:
@@ -1,22 +1,22 @@
|
||||
from typing import Dict, List
|
||||
|
||||
from app.crawler.base import BaseSpider, CrawlResult
|
||||
|
||||
|
||||
class CrawlService:
|
||||
def __init__(self):
|
||||
self.spiders: Dict[str, BaseSpider] = {}
|
||||
self.spiders: dict[str, BaseSpider] = {}
|
||||
|
||||
def register(self, spider: BaseSpider):
|
||||
self.spiders[spider.name] = spider
|
||||
|
||||
async def run_all(self) -> List[CrawlResult]:
|
||||
async def run_all(self) -> list[CrawlResult]:
|
||||
results = []
|
||||
for name, spider in self.spiders.items():
|
||||
result = await spider.crawl()
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
async def run_spider(self, name: str, **kwargs) -> List[CrawlResult]:
|
||||
async def run_spider(self, name: str, **kwargs) -> list[CrawlResult]:
|
||||
spider = self.spiders.get(name)
|
||||
if spider is None:
|
||||
return [CrawlResult(
|
||||
@@ -26,7 +26,7 @@ class CrawlService:
|
||||
result = await spider.crawl(**kwargs)
|
||||
return [result]
|
||||
|
||||
def get_spider_names(self) -> List[str]:
|
||||
def get_spider_names(self) -> list[str]:
|
||||
return list(self.spiders.keys())
|
||||
|
||||
def get_pipeline_config(self, name: str):
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
from datetime import date
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any
|
||||
|
||||
|
||||
def filter_by_keywords(announcements: List[Dict[str, Any]],
|
||||
keywords: List[str]) -> List[Dict[str, Any]]:
|
||||
def filter_by_keywords(announcements: list[dict[str, Any]],
|
||||
keywords: list[str]) -> list[dict[str, Any]]:
|
||||
if not keywords:
|
||||
return announcements
|
||||
|
||||
@@ -17,9 +17,9 @@ def filter_by_keywords(announcements: List[Dict[str, Any]],
|
||||
return filtered
|
||||
|
||||
|
||||
def filter_by_date(announcements: List[Dict[str, Any]],
|
||||
start_date: Optional[date] = None,
|
||||
end_date: Optional[date] = None) -> List[Dict[str, Any]]:
|
||||
def filter_by_date(announcements: list[dict[str, Any]],
|
||||
start_date: date | None = None,
|
||||
end_date: date | None = None) -> list[dict[str, Any]]:
|
||||
if not start_date and not end_date:
|
||||
return announcements
|
||||
|
||||
@@ -42,7 +42,7 @@ def filter_by_date(announcements: List[Dict[str, Any]],
|
||||
return filtered
|
||||
|
||||
|
||||
def dedup_by_hash(announcements: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
def dedup_by_hash(announcements: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
seen = set()
|
||||
result = []
|
||||
for ann in announcements:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any
|
||||
|
||||
from app.config import settings
|
||||
from app.wechat.client import WeChatClient
|
||||
|
||||
@@ -8,7 +8,7 @@ class NotificationService:
|
||||
def __init__(self):
|
||||
self.client = WeChatClient()
|
||||
|
||||
async def send(self, announcements: List[Dict[str, Any]]) -> int:
|
||||
async def send(self, announcements: list[dict[str, Any]]) -> int:
|
||||
if not settings.wechat_enabled:
|
||||
return 0
|
||||
if not announcements:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any
|
||||
|
||||
from app.crawler.base import PipelineConfig, PipelineResult
|
||||
from app.services.filter_service import dedup_by_hash
|
||||
|
||||
@@ -8,7 +9,7 @@ class PostCrawlPipeline:
|
||||
self.db = db_session
|
||||
self.notify = notification_service
|
||||
|
||||
async def process(self, announcements: List[Dict[str, Any]],
|
||||
async def process(self, announcements: list[dict[str, Any]],
|
||||
config: PipelineConfig) -> PipelineResult:
|
||||
result = PipelineResult()
|
||||
|
||||
@@ -47,8 +48,9 @@ class PostCrawlPipeline:
|
||||
|
||||
return result
|
||||
|
||||
async def _save_to_db(self, announcements: List[Dict[str, Any]]) -> int:
|
||||
async def _save_to_db(self, announcements: list[dict[str, Any]]) -> int:
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
|
||||
from app.models.announcement import Announcement
|
||||
|
||||
if not announcements:
|
||||
@@ -76,13 +78,14 @@ class PostCrawlPipeline:
|
||||
await self.db.commit()
|
||||
return result_proxy.rowcount if result_proxy.rowcount >= 0 else len(values)
|
||||
|
||||
async def _send_notifications(self, announcements: List[Dict[str, Any]]) -> int:
|
||||
async def _send_notifications(self, announcements: list[dict[str, Any]]) -> int:
|
||||
return await self.notify.send(announcements)
|
||||
|
||||
async def _mark_sent(self, announcements: List[Dict[str, Any]]) -> int:
|
||||
from app.models.announcement import Announcement
|
||||
async def _mark_sent(self, announcements: list[dict[str, Any]]) -> int:
|
||||
from sqlalchemy import update
|
||||
|
||||
from app.models.announcement import Announcement
|
||||
|
||||
hashes = [a["content_hash"] for a in announcements if a.get("content_hash")]
|
||||
if not hashes:
|
||||
return 0
|
||||
@@ -97,6 +100,6 @@ class PostCrawlPipeline:
|
||||
return result.rowcount
|
||||
|
||||
@staticmethod
|
||||
def _match_keywords(announcement: Dict[str, Any], keywords: List[str]) -> bool:
|
||||
def _match_keywords(announcement: dict[str, Any], keywords: list[str]) -> bool:
|
||||
text = f"{announcement.get('title', '')} {announcement.get('purchase_name', '')}"
|
||||
return any(kw in text for kw in keywords)
|
||||
|
||||
Reference in New Issue
Block a user