chore: ruff 代码检查与修复

130 issues auto-fixed (import ordering, UP045/UP006 type annotations),
33 issues manually fixed (E712/E501/E402/E722 + N818 rename + per-file
wechat ignore for N8xx naming conventions). All 33 tests pass.
This commit is contained in:
2026-05-09 14:28:09 +08:00
parent 02ea794015
commit 7455d7e426
33 changed files with 147 additions and 114 deletions
+12 -11
View File
@@ -1,11 +1,12 @@
from datetime import datetime from datetime import datetime
from typing import Optional
from fastapi import APIRouter, Depends, Query, HTTPException from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func
from app.api.deps import get_db from app.api.deps import get_db
from app.models.announcement import Announcement from app.models.announcement import Announcement
from app.models.schemas import AnnouncementResponse, AnnouncementListResponse from app.models.schemas import AnnouncementListResponse, AnnouncementResponse
router = APIRouter() router = APIRouter()
@@ -14,11 +15,11 @@ router = APIRouter()
async def list_announcements( async def list_announcements(
page: int = Query(1, ge=1), page: int = Query(1, ge=1),
page_size: int = Query(20, ge=1, le=100), page_size: int = Query(20, ge=1, le=100),
source_code: Optional[str] = None, source_code: str | None = None,
keyword: Optional[str] = None, keyword: str | None = None,
start_date: Optional[str] = None, start_date: str | None = None,
end_date: Optional[str] = None, end_date: str | None = None,
crawl_mode: Optional[str] = None, crawl_mode: str | None = None,
db: AsyncSession = Depends(get_db), db: AsyncSession = Depends(get_db),
): ):
conditions = [] conditions = []
@@ -91,12 +92,12 @@ async def get_stats(db: AsyncSession = Depends(get_db)):
).select_from(Announcement) ).select_from(Announcement)
) )
new_count = await db.execute( new_count = await db.execute(
select(func.count()).where(Announcement.is_new == True) select(func.count()).where(Announcement.is_new == True) # noqa: E712
.select_from(Announcement) .select_from(Announcement)
) )
unsent = await db.execute( unsent = await db.execute(
select(func.count()).where( select(func.count()).where(
Announcement.is_sent == False, Announcement.is_new == True Announcement.is_sent == False, Announcement.is_new == True # noqa: E712
).select_from(Announcement) ).select_from(Announcement)
) )
return { return {
+2
View File
@@ -1,4 +1,5 @@
from fastapi import APIRouter from fastapi import APIRouter
from app.api.deps import get_crawl_service from app.api.deps import get_crawl_service
from app.models.schemas import CrawlTriggerRequest from app.models.schemas import CrawlTriggerRequest
@@ -34,6 +35,7 @@ async def crawl_status():
@router.get("/crawl/sources") @router.get("/crawl/sources")
async def crawl_sources(): async def crawl_sources():
import json import json
from app.config import settings from app.config import settings
sources = json.loads(settings.announcement_sources) sources = json.loads(settings.announcement_sources)
return { return {
+3 -2
View File
@@ -1,7 +1,8 @@
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.services.crawl_service import CrawlService
from app.crawler.gxgp_spider import GXGPSpider
from app.crawler.dahuagov_spider import DahuagovSpider from app.crawler.dahuagov_spider import DahuagovSpider
from app.crawler.gxgp_spider import GXGPSpider
from app.services.crawl_service import CrawlService
async def get_db() -> AsyncSession: async def get_db() -> AsyncSession:
+3 -1
View File
@@ -1,5 +1,7 @@
from fastapi import APIRouter from fastapi import APIRouter
from app.api import announcements, crawl, wechat, scheduler as scheduler_module
from app.api import announcements, crawl, wechat
from app.api import scheduler as scheduler_module
api_router = APIRouter(prefix="/api/v1") api_router = APIRouter(prefix="/api/v1")
api_router.include_router(announcements.router, tags=["announcements"]) api_router.include_router(announcements.router, tags=["announcements"])
+2 -1
View File
@@ -1,6 +1,7 @@
from fastapi import APIRouter from fastapi import APIRouter
from app.scheduler.jobs import scheduler
from app.models.schemas import JobResponse from app.models.schemas import JobResponse
from app.scheduler.jobs import scheduler
router = APIRouter(prefix="/scheduler", tags=["scheduler"]) router = APIRouter(prefix="/scheduler", tags=["scheduler"])
+1
View File
@@ -1,4 +1,5 @@
from fastapi import APIRouter, Request, Response from fastapi import APIRouter, Request, Response
from app.wechat.handler import WeChatMessageHandler from app.wechat.handler import WeChatMessageHandler
router = APIRouter() router = APIRouter()
+3 -3
View File
@@ -1,5 +1,5 @@
from pydantic_settings import BaseSettings, SettingsConfigDict from pydantic_settings import BaseSettings, SettingsConfigDict
from typing import List
class Settings(BaseSettings): class Settings(BaseSettings):
@@ -14,7 +14,7 @@ class Settings(BaseSettings):
# 爬虫 # 爬虫
crawler_base_url: str = "https://zfcg.gxzf.gov.cn" crawler_base_url: str = "https://zfcg.gxzf.gov.cn"
crawler_keywords: List[str] = ["大化"] crawler_keywords: list[str] = ["大化"]
crawler_max_pages: int = 10 crawler_max_pages: int = 10
crawler_timeout: int = 30 crawler_timeout: int = 30
crawler_page_size: int = 100 crawler_page_size: int = 100
@@ -38,7 +38,7 @@ class Settings(BaseSettings):
markdown_output_file: str = "onu.md" markdown_output_file: str = "onu.md"
# 公告来源(JSON 字符串,从环境变量读取) # 公告来源(JSON 字符串,从环境变量读取)
announcement_sources: str = '{"ZcyAnnouncement1":{"category_id":66485,"name":"采购公告","type":"purchase"},"ZcyAnnouncement2":{"category_id":66485,"name":"结果公告","type":"result"},"ZcyAnnouncement3":{"category_id":66485,"name":"合同公告","type":"contract"},"ZcyAnnouncement4":{"category_id":66485,"name":"更正公告","type":"correction"},"ZcyAnnouncement5":{"category_id":66485,"name":"招标文件预公示","type":"pre_announcement"},"ZcyAnnouncement6":{"category_id":66485,"name":"单一来源公示","type":"single_source"},"ZcyAnnouncement7":{"category_id":66485,"name":"电子卖场公示","type":"electronic_market"},"ZcyAnnouncement10":{"category_id":66485,"name":"履约验收公示","type":"acceptance"},"ZcyAnnouncement11":{"category_id":66485,"name":"工程类公告","type":"engineering"},"ZcyAnnouncement20":{"category_id":66485,"name":"框架协议征集公告","type":"framework_agreement"},"ZcyAnnouncement21":{"category_id":66485,"name":"框架协议入围结果公告","type":"framework_result"},"ZcyAnnouncement23":{"category_id":66485,"name":"框架协议成交结果汇总公告","type":"framework_summary"},"61-266648":{"category_id":66485,"name":"采购意向公开","type":"intention"}}' announcement_sources: str = '{"ZcyAnnouncement1":{"category_id":66485,"name":"采购公告","type":"purchase"},"ZcyAnnouncement2":{"category_id":66485,"name":"结果公告","type":"result"},"ZcyAnnouncement3":{"category_id":66485,"name":"合同公告","type":"contract"},"ZcyAnnouncement4":{"category_id":66485,"name":"更正公告","type":"correction"},"ZcyAnnouncement5":{"category_id":66485,"name":"招标文件预公示","type":"pre_announcement"},"ZcyAnnouncement6":{"category_id":66485,"name":"单一来源公示","type":"single_source"},"ZcyAnnouncement7":{"category_id":66485,"name":"电子卖场公示","type":"electronic_market"},"ZcyAnnouncement10":{"category_id":66485,"name":"履约验收公示","type":"acceptance"},"ZcyAnnouncement11":{"category_id":66485,"name":"工程类公告","type":"engineering"},"ZcyAnnouncement20":{"category_id":66485,"name":"框架协议征集公告","type":"framework_agreement"},"ZcyAnnouncement21":{"category_id":66485,"name":"框架协议入围结果公告","type":"framework_result"},"ZcyAnnouncement23":{"category_id":66485,"name":"框架协议成交结果汇总公告","type":"framework_summary"},"61-266648":{"category_id":66485,"name":"采购意向公开","type":"intention"}}' # noqa: E501
settings = Settings() settings = Settings()
+2 -3
View File
@@ -2,7 +2,6 @@ import hashlib
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from dataclasses import dataclass, field from dataclasses import dataclass, field
from datetime import datetime from datetime import datetime
from typing import List, Optional
@dataclass @dataclass
@@ -12,7 +11,7 @@ class CrawlResult:
total_count: int = 0 total_count: int = 0
new_count: int = 0 new_count: int = 0
announcements: list = field(default_factory=list) announcements: list = field(default_factory=list)
error_message: Optional[str] = None error_message: str | None = None
crawled_at: datetime = field(default_factory=datetime.now) crawled_at: datetime = field(default_factory=datetime.now)
duration: float = 0.0 duration: float = 0.0
@@ -24,7 +23,7 @@ class CrawlResult:
@dataclass @dataclass
class PipelineConfig: class PipelineConfig:
filter_enabled: bool = True filter_enabled: bool = True
keywords: List[str] = field(default_factory=list) keywords: list[str] = field(default_factory=list)
dedup_enabled: bool = True dedup_enabled: bool = True
notify_mode: str = "filtered" notify_mode: str = "filtered"
mark_sent: bool = False mark_sent: bool = False
+2
View File
@@ -1,7 +1,9 @@
import asyncio import asyncio
import random import random
from datetime import datetime from datetime import datetime
import httpx import httpx
from app.config import settings from app.config import settings
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
from app.crawler.parsers import parse_dahuagov_html from app.crawler.parsers import parse_dahuagov_html
+7 -6
View File
@@ -2,11 +2,12 @@ import json
import random import random
import time import time
from datetime import datetime from datetime import datetime
from typing import List, Optional
import httpx import httpx
from app.config import settings from app.config import settings
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
from app.crawler.parsers import parse_gxgp_api_response, extract_pagination from app.crawler.parsers import extract_pagination, parse_gxgp_api_response
class GXGPSpider(BaseSpider): class GXGPSpider(BaseSpider):
@@ -27,8 +28,8 @@ class GXGPSpider(BaseSpider):
mark_sent=False, mark_sent=False,
) )
async def crawl(self, sources: Optional[List[str]] = None, async def crawl(self, sources: list[str] | None = None,
max_pages: Optional[int] = None) -> CrawlResult: max_pages: int | None = None) -> CrawlResult:
if max_pages is None: if max_pages is None:
max_pages = settings.crawler_max_pages max_pages = settings.crawler_max_pages
if sources is None: if sources is None:
@@ -88,7 +89,7 @@ class GXGPSpider(BaseSpider):
) )
async def _fetch_page(self, client: httpx.AsyncClient, source_code: str, async def _fetch_page(self, client: httpx.AsyncClient, source_code: str,
category_id: int, page_no: int) -> Optional[dict]: category_id: int, page_no: int) -> dict | None:
payload = { payload = {
"keyword": "", "keyword": "",
"publishDateBegin": "", "publishDateBegin": "",
@@ -102,7 +103,7 @@ class GXGPSpider(BaseSpider):
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Content-Type": "application/json;charset=UTF-8", "Content-Type": "application/json;charset=UTF-8",
"Origin": self.base_url, "Origin": self.base_url,
"Referer": f"{self.base_url}/site/category?parentId={category_id}&childrenCode={source_code}", "Referer": f"{self.base_url}/site/category?parentId={category_id}&childrenCode={source_code}", # noqa: E501
} }
response = await client.post( response = await client.post(
self.announcement_api, json=payload, headers=headers self.announcement_api, json=payload, headers=headers
+6 -6
View File
@@ -1,18 +1,18 @@
import hashlib import hashlib
from datetime import datetime from datetime import datetime
from typing import Any, Dict, List from typing import Any
from urllib.parse import urljoin from urllib.parse import urljoin
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
def parse_gxgp_api_response( def parse_gxgp_api_response(
response_data: Dict[str, Any], response_data: dict[str, Any],
source_code: str, source_code: str,
source_name: str, source_name: str,
crawled_at: datetime, crawled_at: datetime,
category_id: int, category_id: int,
) -> List[Dict[str, Any]]: ) -> list[dict[str, Any]]:
if not response_data.get("success"): if not response_data.get("success"):
return [] return []
data = response_data.get("result", {}).get("data", {}) data = response_data.get("result", {}).get("data", {})
@@ -62,7 +62,7 @@ def parse_gxgp_api_response(
return results return results
def extract_pagination(response_data: Dict[str, Any]) -> Dict[str, Any]: def extract_pagination(response_data: dict[str, Any]) -> dict[str, Any]:
data = response_data.get("result", {}).get("data", {}) data = response_data.get("result", {}).get("data", {})
return { return {
"total": data.get("total", 0), "total": data.get("total", 0),
@@ -75,7 +75,7 @@ def extract_pagination(response_data: Dict[str, Any]) -> Dict[str, Any]:
} }
def parse_dahuagov_html(html: str, crawled_at: datetime) -> List[Dict[str, Any]]: def parse_dahuagov_html(html: str, crawled_at: datetime) -> list[dict[str, Any]]:
soup = BeautifulSoup(html, "html.parser") soup = BeautifulSoup(html, "html.parser")
lists = soup.find_all("ul", class_="more-list") lists = soup.find_all("ul", class_="more-list")
if not lists: if not lists:
@@ -131,7 +131,7 @@ def parse_dahuagov_html(html: str, crawled_at: datetime) -> List[Dict[str, Any]]
return results return results
def _generate_hash(ann: Dict[str, Any]) -> str: def _generate_hash(ann: dict[str, Any]) -> str:
content = ( content = (
f"{ann['title']}|{ann['publish_date'].strftime('%Y-%m-%d')}" f"{ann['title']}|{ann['publish_date'].strftime('%Y-%m-%d')}"
f"|{ann['purchase_name']}|{ann['content_url']}|{ann['source_code']}" f"|{ann['purchase_name']}|{ann['content_url']}|{ann['source_code']}"
+5 -4
View File
@@ -1,9 +1,11 @@
import logging import logging
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from fastapi import FastAPI from fastapi import FastAPI
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from app.api.router import api_router
from app.config import settings from app.config import settings
from app.models.announcement import Base
engine = create_async_engine(settings.database_url, echo=settings.debug) engine = create_async_engine(settings.database_url, echo=settings.debug)
async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
@@ -20,7 +22,7 @@ async def lifespan(app: FastAPI):
level=getattr(logging, settings.log_level), level=getattr(logging, settings.log_level),
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
) )
from app.scheduler.jobs import start_scheduler, shutdown_scheduler from app.scheduler.jobs import shutdown_scheduler, start_scheduler
start_scheduler() start_scheduler()
yield yield
shutdown_scheduler() shutdown_scheduler()
@@ -35,7 +37,6 @@ app = FastAPI(
redoc_url=None, redoc_url=None,
) )
from app.api.router import api_router
app.include_router(api_router) app.include_router(api_router)
+5 -3
View File
@@ -1,6 +1,7 @@
import hashlib import hashlib
from datetime import datetime, date from datetime import datetime
from sqlalchemy import String, Boolean, DateTime, Integer, Text, func
from sqlalchemy import Boolean, DateTime, Integer, String, Text, func
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
@@ -25,7 +26,7 @@ class Announcement(Base):
is_sent: Mapped[bool] = mapped_column(Boolean, default=False) is_sent: Mapped[bool] = mapped_column(Boolean, default=False)
keyword_matched: Mapped[bool] = mapped_column(Boolean, default=False) keyword_matched: Mapped[bool] = mapped_column(Boolean, default=False)
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now()) created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), onupdate=func.now()) updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), onupdate=func.now()) # noqa: E501
@staticmethod @staticmethod
def generate_hash(title: str, publish_date: str, purchase_name: str, def generate_hash(title: str, publish_date: str, purchase_name: str,
@@ -36,5 +37,6 @@ class Announcement(Base):
@staticmethod @staticmethod
def source_map() -> dict: def source_map() -> dict:
import json import json
from app.config import settings from app.config import settings
return json.loads(settings.announcement_sources) return json.loads(settings.announcement_sources)
+5 -5
View File
@@ -1,5 +1,5 @@
from datetime import datetime from datetime import datetime
from typing import Optional
from pydantic import BaseModel from pydantic import BaseModel
@@ -29,14 +29,14 @@ class AnnouncementListResponse(BaseModel):
class CrawlTriggerRequest(BaseModel): class CrawlTriggerRequest(BaseModel):
keywords: Optional[list[str]] = None keywords: list[str] | None = None
sources: Optional[list[str]] = None sources: list[str] | None = None
manual: bool = False manual: bool = False
class CrawlStatusResponse(BaseModel): class CrawlStatusResponse(BaseModel):
running: bool running: bool
last_crawl_time: Optional[datetime] = None last_crawl_time: datetime | None = None
total_sources: int total_sources: int
@@ -53,4 +53,4 @@ class SourcesResponse(BaseModel):
class JobResponse(BaseModel): class JobResponse(BaseModel):
id: str id: str
name: str name: str
next_run_time: Optional[str] = None next_run_time: str | None = None
+3 -1
View File
@@ -1,7 +1,9 @@
import logging import logging
from apscheduler.schedulers.asyncio import AsyncIOScheduler from apscheduler.schedulers.asyncio import AsyncIOScheduler
from app.config import settings
from app.api.deps import get_crawl_service from app.api.deps import get_crawl_service
from app.config import settings
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
scheduler = AsyncIOScheduler() scheduler = AsyncIOScheduler()
+5 -5
View File
@@ -1,22 +1,22 @@
from typing import Dict, List
from app.crawler.base import BaseSpider, CrawlResult from app.crawler.base import BaseSpider, CrawlResult
class CrawlService: class CrawlService:
def __init__(self): def __init__(self):
self.spiders: Dict[str, BaseSpider] = {} self.spiders: dict[str, BaseSpider] = {}
def register(self, spider: BaseSpider): def register(self, spider: BaseSpider):
self.spiders[spider.name] = spider self.spiders[spider.name] = spider
async def run_all(self) -> List[CrawlResult]: async def run_all(self) -> list[CrawlResult]:
results = [] results = []
for name, spider in self.spiders.items(): for name, spider in self.spiders.items():
result = await spider.crawl() result = await spider.crawl()
results.append(result) results.append(result)
return results return results
async def run_spider(self, name: str, **kwargs) -> List[CrawlResult]: async def run_spider(self, name: str, **kwargs) -> list[CrawlResult]:
spider = self.spiders.get(name) spider = self.spiders.get(name)
if spider is None: if spider is None:
return [CrawlResult( return [CrawlResult(
@@ -26,7 +26,7 @@ class CrawlService:
result = await spider.crawl(**kwargs) result = await spider.crawl(**kwargs)
return [result] return [result]
def get_spider_names(self) -> List[str]: def get_spider_names(self) -> list[str]:
return list(self.spiders.keys()) return list(self.spiders.keys())
def get_pipeline_config(self, name: str): def get_pipeline_config(self, name: str):
+7 -7
View File
@@ -1,9 +1,9 @@
from datetime import date from datetime import date
from typing import Any, Dict, List, Optional from typing import Any
def filter_by_keywords(announcements: List[Dict[str, Any]], def filter_by_keywords(announcements: list[dict[str, Any]],
keywords: List[str]) -> List[Dict[str, Any]]: keywords: list[str]) -> list[dict[str, Any]]:
if not keywords: if not keywords:
return announcements return announcements
@@ -17,9 +17,9 @@ def filter_by_keywords(announcements: List[Dict[str, Any]],
return filtered return filtered
def filter_by_date(announcements: List[Dict[str, Any]], def filter_by_date(announcements: list[dict[str, Any]],
start_date: Optional[date] = None, start_date: date | None = None,
end_date: Optional[date] = None) -> List[Dict[str, Any]]: end_date: date | None = None) -> list[dict[str, Any]]:
if not start_date and not end_date: if not start_date and not end_date:
return announcements return announcements
@@ -42,7 +42,7 @@ def filter_by_date(announcements: List[Dict[str, Any]],
return filtered return filtered
def dedup_by_hash(announcements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: def dedup_by_hash(announcements: list[dict[str, Any]]) -> list[dict[str, Any]]:
seen = set() seen = set()
result = [] result = []
for ann in announcements: for ann in announcements:
+3 -3
View File
@@ -1,5 +1,5 @@
from datetime import datetime from typing import Any
from typing import Any, Dict, List
from app.config import settings from app.config import settings
from app.wechat.client import WeChatClient from app.wechat.client import WeChatClient
@@ -8,7 +8,7 @@ class NotificationService:
def __init__(self): def __init__(self):
self.client = WeChatClient() self.client = WeChatClient()
async def send(self, announcements: List[Dict[str, Any]]) -> int: async def send(self, announcements: list[dict[str, Any]]) -> int:
if not settings.wechat_enabled: if not settings.wechat_enabled:
return 0 return 0
if not announcements: if not announcements:
+10 -7
View File
@@ -1,4 +1,5 @@
from typing import Any, Dict, List from typing import Any
from app.crawler.base import PipelineConfig, PipelineResult from app.crawler.base import PipelineConfig, PipelineResult
from app.services.filter_service import dedup_by_hash from app.services.filter_service import dedup_by_hash
@@ -8,7 +9,7 @@ class PostCrawlPipeline:
self.db = db_session self.db = db_session
self.notify = notification_service self.notify = notification_service
async def process(self, announcements: List[Dict[str, Any]], async def process(self, announcements: list[dict[str, Any]],
config: PipelineConfig) -> PipelineResult: config: PipelineConfig) -> PipelineResult:
result = PipelineResult() result = PipelineResult()
@@ -47,8 +48,9 @@ class PostCrawlPipeline:
return result return result
async def _save_to_db(self, announcements: List[Dict[str, Any]]) -> int: async def _save_to_db(self, announcements: list[dict[str, Any]]) -> int:
from sqlalchemy.dialects.postgresql import insert from sqlalchemy.dialects.postgresql import insert
from app.models.announcement import Announcement from app.models.announcement import Announcement
if not announcements: if not announcements:
@@ -76,13 +78,14 @@ class PostCrawlPipeline:
await self.db.commit() await self.db.commit()
return result_proxy.rowcount if result_proxy.rowcount >= 0 else len(values) return result_proxy.rowcount if result_proxy.rowcount >= 0 else len(values)
async def _send_notifications(self, announcements: List[Dict[str, Any]]) -> int: async def _send_notifications(self, announcements: list[dict[str, Any]]) -> int:
return await self.notify.send(announcements) return await self.notify.send(announcements)
async def _mark_sent(self, announcements: List[Dict[str, Any]]) -> int: async def _mark_sent(self, announcements: list[dict[str, Any]]) -> int:
from app.models.announcement import Announcement
from sqlalchemy import update from sqlalchemy import update
from app.models.announcement import Announcement
hashes = [a["content_hash"] for a in announcements if a.get("content_hash")] hashes = [a["content_hash"] for a in announcements if a.get("content_hash")]
if not hashes: if not hashes:
return 0 return 0
@@ -97,6 +100,6 @@ class PostCrawlPipeline:
return result.rowcount return result.rowcount
@staticmethod @staticmethod
def _match_keywords(announcement: Dict[str, Any], keywords: List[str]) -> bool: def _match_keywords(announcement: dict[str, Any], keywords: list[str]) -> bool:
text = f"{announcement.get('title', '')} {announcement.get('purchase_name', '')}" text = f"{announcement.get('title', '')} {announcement.get('purchase_name', '')}"
return any(kw in text for kw in keywords) return any(kw in text for kw in keywords)
+3 -3
View File
@@ -1,16 +1,16 @@
import time import time
import httpx import httpx
from typing import Optional
from app.config import settings from app.config import settings
class WeChatClient: class WeChatClient:
def __init__(self): def __init__(self):
self._access_token: Optional[str] = None self._access_token: str | None = None
self._token_expires_at: float = 0 self._token_expires_at: float = 0
async def _get_access_token(self) -> Optional[str]: async def _get_access_token(self) -> str | None:
now = time.time() now = time.time()
if self._access_token and now < self._token_expires_at: if self._access_token and now < self._token_expires_at:
return self._access_token return self._access_token
+18 -18
View File
@@ -1,20 +1,20 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- encoding:utf-8 -*-
""" 对企业微信发送给企业后台的消息加解密示例代码. """ 对企业微信发送给企业后台的消息加解密示例代码.
@copyright: Copyright (c) 1998-2014 Tencent Inc. @copyright: Copyright (c) 1998-2014 Tencent Inc.
""" """
# ------------------------------------------------------------------------ # ------------------------------------------------------------------------
import logging
import base64 import base64
import random
import hashlib import hashlib
import time import logging
import struct import random
from Crypto.Cipher import AES
import xml.etree.cElementTree as ET
import socket import socket
import struct
import time
import xml.etree.ElementTree as ET
from Crypto.Cipher import AES
try: try:
import ierror import ierror
@@ -29,11 +29,11 @@ except ImportError:
""" """
class FormatException(Exception): class FormatError(Exception):
pass pass
def throw_exception(message, exception_class=FormatException): def throw_exception(message, exception_class=FormatError):
"""my define raise exception function""" """my define raise exception function"""
raise exception_class(message) raise exception_class(message)
@@ -104,7 +104,7 @@ class XMLParse:
return resp_xml return resp_xml
class PKCS7Encoder(): class PKCS7Encoder:
"""提供基于PKCS7算法的加解密接口""" """提供基于PKCS7算法的加解密接口"""
block_size = 32 block_size = 32
@@ -134,7 +134,7 @@ class PKCS7Encoder():
return decrypted[:-pad] return decrypted[:-pad]
class Prpcrypt(object): class Prpcrypt:
"""提供接收和推送给企业微信消息的加解密接口""" """提供接收和推送给企业微信消息的加解密接口"""
def __init__(self, key): def __init__(self, key):
@@ -151,7 +151,7 @@ class Prpcrypt(object):
""" """
# 16位随机字符串添加到明文开头 # 16位随机字符串添加到明文开头
text = text.encode() text = text.encode()
text = self.get_random_str() + struct.pack("I", socket.htonl(len(text))) + text + receiveid.encode() text = self.get_random_str() + struct.pack("I", socket.htonl(len(text))) + text + receiveid.encode() # noqa: E501
# 使用自定义的填充方式对明文进行补位填充 # 使用自定义的填充方式对明文进行补位填充
pkcs7 = PKCS7Encoder() pkcs7 = PKCS7Encoder()
@@ -206,14 +206,14 @@ class Prpcrypt(object):
return str(random.randint(1000000000000000, 9999999999999999)).encode() return str(random.randint(1000000000000000, 9999999999999999)).encode()
class WXBizMsgCrypt(object): class WXBizMsgCrypt:
# 构造函数 # 构造函数
def __init__(self, sToken, sEncodingAESKey, sReceiveId): def __init__(self, sToken, sEncodingAESKey, sReceiveId):
try: try:
self.key = base64.b64decode(sEncodingAESKey + "=") self.key = base64.b64decode(sEncodingAESKey + "=")
assert len(self.key) == 32 assert len(self.key) == 32
except: except Exception:
throw_exception("[error]: EncodingAESKey unvalid !", FormatException) throw_exception("[error]: EncodingAESKey unvalid !", FormatError)
# return ierror.WXBizMsgCrypt_IllegalAesKey,None # return ierror.WXBizMsgCrypt_IllegalAesKey,None
self.m_sToken = sToken self.m_sToken = sToken
self.m_sReceiveId = sReceiveId self.m_sReceiveId = sReceiveId
@@ -240,9 +240,9 @@ class WXBizMsgCrypt(object):
def EncryptMsg(self, sReplyMsg, sNonce, timestamp=None): def EncryptMsg(self, sReplyMsg, sNonce, timestamp=None):
# 将企业回复用户的消息加密打包 # 将企业回复用户的消息加密打包
# @param sReplyMsg: 企业号待回复用户的消息,xml格式的字符串 # @param sReplyMsg: 企业号待回复用户的消息,xml格式的字符串
# @param sTimeStamp: 时间戳,可以自己生成,也可以用URL参数的timestamp,如为None则自动用当前时间 # @param sTimeStamp: 时间戳,可以自己生成,也可以用URL参数的timestamp,如为None则自动用当前时间 # noqa: E501
# @param sNonce: 随机串,可以自己生成,也可以用URL参数的nonce # @param sNonce: 随机串,可以自己生成,也可以用URL参数的nonce # noqa: E501
# sEncryptMsg: 加密后的可以直接回复用户的密文,包括msg_signature, timestamp, nonce, encrypt的xml格式的字符串, # sEncryptMsg: 加密后的可以直接回复用户的密文,包括msg_signature, timestamp, nonce, encrypt的xml格式的字符串, # noqa: E501
# return:成功0sEncryptMsg,失败返回对应的错误码None # return:成功0sEncryptMsg,失败返回对应的错误码None
pc = Prpcrypt(self.key) pc = Prpcrypt(self.key)
ret, encrypt = pc.encrypt(sReplyMsg, self.m_sReceiveId) ret, encrypt = pc.encrypt(sReplyMsg, self.m_sReceiveId)
+8 -9
View File
@@ -1,8 +1,7 @@
import xml.etree.cElementTree as ET import xml.etree.ElementTree as ET
from typing import Optional
from app.wechat.crypto import WXBizMsgCrypt
from app.config import settings from app.config import settings
from app.wechat.crypto import WXBizMsgCrypt
class WeChatMessageHandler: class WeChatMessageHandler:
@@ -15,7 +14,7 @@ class WeChatMessageHandler:
def verify_url( def verify_url(
self, msg_signature: str, timestamp: str, nonce: str, echostr: str self, msg_signature: str, timestamp: str, nonce: str, echostr: str
) -> Optional[str]: ) -> str | None:
ret, sEchoStr = self.wxcpt.VerifyURL( ret, sEchoStr = self.wxcpt.VerifyURL(
msg_signature, timestamp, nonce, echostr msg_signature, timestamp, nonce, echostr
) )
@@ -33,7 +32,7 @@ class WeChatMessageHandler:
msg_signature: str, msg_signature: str,
timestamp: str, timestamp: str,
nonce: str, nonce: str,
) -> Optional[ET.Element]: ) -> ET.Element | None:
ret, xml_content = self.wxcpt.DecryptMsg( ret, xml_content = self.wxcpt.DecryptMsg(
post_data, msg_signature, timestamp, nonce post_data, msg_signature, timestamp, nonce
) )
@@ -43,16 +42,16 @@ class WeChatMessageHandler:
def encrypt_response( def encrypt_response(
self, response_xml: str, nonce: str, timestamp: str self, response_xml: str, nonce: str, timestamp: str
) -> Optional[str]: ) -> str | None:
ret, encrypted = self.wxcpt.EncryptMsg(response_xml, nonce, timestamp) ret, encrypted = self.wxcpt.EncryptMsg(response_xml, nonce, timestamp)
if ret == 0: if ret == 0:
return encrypted return encrypted
return None return None
def handle_event( def handle_event(
self, event: str, event_key: Optional[str], from_user: str self, event: str, event_key: str | None, from_user: str
) -> Optional[str]: ) -> str | None:
return None return None
def handle_text(self, content: str, from_user: str) -> Optional[str]: def handle_text(self, content: str, from_user: str) -> str | None:
return None return None
-1
View File
@@ -1,5 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python
# -*- coding: utf-8 -*-
######################################################################### #########################################################################
# Author: jonyqin # Author: jonyqin
# Created Time: Thu 11 Sep 2014 01:53:58 PM CST # Created Time: Thu 11 Sep 2014 01:53:58 PM CST
+3
View File
@@ -36,6 +36,9 @@ line-length = 100
[tool.ruff.lint] [tool.ruff.lint]
select = ["E", "F", "I", "N", "W", "UP"] select = ["E", "F", "I", "N", "W", "UP"]
[tool.ruff.lint.per-file-ignores]
"app/wechat/*.py" = ["N802", "N803", "N806"]
[tool.pytest.ini_options] [tool.pytest.ini_options]
asyncio_mode = "auto" asyncio_mode = "auto"
testpaths = ["tests"] testpaths = ["tests"]
+2 -1
View File
@@ -1,5 +1,6 @@
import pytest import pytest
from httpx import AsyncClient, ASGITransport from httpx import ASGITransport, AsyncClient
from app.main import app from app.main import app
+3 -1
View File
@@ -1,5 +1,7 @@
import pytest
from datetime import datetime from datetime import datetime
import pytest
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig, PipelineResult from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig, PipelineResult
+4 -2
View File
@@ -1,7 +1,9 @@
from unittest.mock import AsyncMock, MagicMock, patch
import pytest import pytest
from unittest.mock import AsyncMock, patch, MagicMock
from app.crawler.dahuagov_spider import DahuagovSpider
from app.crawler.base import PipelineConfig from app.crawler.base import PipelineConfig
from app.crawler.dahuagov_spider import DahuagovSpider
@pytest.mark.asyncio @pytest.mark.asyncio
+4 -2
View File
@@ -1,7 +1,9 @@
from unittest.mock import AsyncMock, MagicMock, patch
import pytest import pytest
from unittest.mock import AsyncMock, patch, MagicMock
from app.crawler.gxgp_spider import GXGPSpider
from app.crawler.base import PipelineConfig from app.crawler.base import PipelineConfig
from app.crawler.gxgp_spider import GXGPSpider
@pytest.mark.asyncio @pytest.mark.asyncio
+3 -2
View File
@@ -1,8 +1,9 @@
from datetime import datetime from datetime import datetime
from app.crawler.parsers import ( from app.crawler.parsers import (
parse_gxgp_api_response,
parse_dahuagov_html,
extract_pagination, extract_pagination,
parse_dahuagov_html,
parse_gxgp_api_response,
) )
+2 -1
View File
@@ -1,5 +1,6 @@
import pytest import pytest
from datetime import datetime
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
from app.services.crawl_service import CrawlService from app.services.crawl_service import CrawlService
+2 -1
View File
@@ -1,5 +1,6 @@
from datetime import datetime from datetime import datetime
from app.services.filter_service import filter_by_keywords, filter_by_date
from app.services.filter_service import filter_by_date, filter_by_keywords
def test_filter_by_keywords_match(): def test_filter_by_keywords_match():
@@ -1,5 +1,7 @@
import pytest
from unittest.mock import AsyncMock, patch from unittest.mock import AsyncMock, patch
import pytest
from app.services.notification_service import NotificationService from app.services.notification_service import NotificationService
+5 -3
View File
@@ -1,8 +1,10 @@
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
from datetime import datetime from datetime import datetime
from app.services.pipeline import PostCrawlPipeline from unittest.mock import AsyncMock, patch
import pytest
from app.crawler.base import CrawlResult, PipelineConfig from app.crawler.base import CrawlResult, PipelineConfig
from app.services.pipeline import PostCrawlPipeline
@pytest.mark.asyncio @pytest.mark.asyncio