diff --git a/app/api/announcements.py b/app/api/announcements.py index 95570ed..028e786 100644 --- a/app/api/announcements.py +++ b/app/api/announcements.py @@ -1,11 +1,12 @@ from datetime import datetime -from typing import Optional -from fastapi import APIRouter, Depends, Query, HTTPException + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy import select, func + from app.api.deps import get_db from app.models.announcement import Announcement -from app.models.schemas import AnnouncementResponse, AnnouncementListResponse +from app.models.schemas import AnnouncementListResponse, AnnouncementResponse router = APIRouter() @@ -14,11 +15,11 @@ router = APIRouter() async def list_announcements( page: int = Query(1, ge=1), page_size: int = Query(20, ge=1, le=100), - source_code: Optional[str] = None, - keyword: Optional[str] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - crawl_mode: Optional[str] = None, + source_code: str | None = None, + keyword: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + crawl_mode: str | None = None, db: AsyncSession = Depends(get_db), ): conditions = [] @@ -91,12 +92,12 @@ async def get_stats(db: AsyncSession = Depends(get_db)): ).select_from(Announcement) ) new_count = await db.execute( - select(func.count()).where(Announcement.is_new == True) + select(func.count()).where(Announcement.is_new == True) # noqa: E712 .select_from(Announcement) ) unsent = await db.execute( select(func.count()).where( - Announcement.is_sent == False, Announcement.is_new == True + Announcement.is_sent == False, Announcement.is_new == True # noqa: E712 ).select_from(Announcement) ) return { diff --git a/app/api/crawl.py b/app/api/crawl.py index e20bd4c..c2a94f0 100644 --- a/app/api/crawl.py +++ b/app/api/crawl.py @@ -1,4 +1,5 @@ from fastapi import APIRouter + from app.api.deps import get_crawl_service from app.models.schemas import CrawlTriggerRequest @@ -34,6 +35,7 @@ async def crawl_status(): @router.get("/crawl/sources") async def crawl_sources(): import json + from app.config import settings sources = json.loads(settings.announcement_sources) return { diff --git a/app/api/deps.py b/app/api/deps.py index c604025..573c0bc 100644 --- a/app/api/deps.py +++ b/app/api/deps.py @@ -1,7 +1,8 @@ from sqlalchemy.ext.asyncio import AsyncSession -from app.services.crawl_service import CrawlService -from app.crawler.gxgp_spider import GXGPSpider + from app.crawler.dahuagov_spider import DahuagovSpider +from app.crawler.gxgp_spider import GXGPSpider +from app.services.crawl_service import CrawlService async def get_db() -> AsyncSession: diff --git a/app/api/router.py b/app/api/router.py index 9eda887..b73f286 100644 --- a/app/api/router.py +++ b/app/api/router.py @@ -1,5 +1,7 @@ from fastapi import APIRouter -from app.api import announcements, crawl, wechat, scheduler as scheduler_module + +from app.api import announcements, crawl, wechat +from app.api import scheduler as scheduler_module api_router = APIRouter(prefix="/api/v1") api_router.include_router(announcements.router, tags=["announcements"]) diff --git a/app/api/scheduler.py b/app/api/scheduler.py index 5cbe831..121e8df 100644 --- a/app/api/scheduler.py +++ b/app/api/scheduler.py @@ -1,6 +1,7 @@ from fastapi import APIRouter -from app.scheduler.jobs import scheduler + from app.models.schemas import JobResponse +from app.scheduler.jobs import scheduler router = APIRouter(prefix="/scheduler", tags=["scheduler"]) diff --git a/app/api/wechat.py b/app/api/wechat.py index 805ca28..9e1b273 100644 --- a/app/api/wechat.py +++ b/app/api/wechat.py @@ -1,4 +1,5 @@ from fastapi import APIRouter, Request, Response + from app.wechat.handler import WeChatMessageHandler router = APIRouter() diff --git a/app/config.py b/app/config.py index da09e3e..86f8cd3 100644 --- a/app/config.py +++ b/app/config.py @@ -1,5 +1,5 @@ + from pydantic_settings import BaseSettings, SettingsConfigDict -from typing import List class Settings(BaseSettings): @@ -14,7 +14,7 @@ class Settings(BaseSettings): # 爬虫 crawler_base_url: str = "https://zfcg.gxzf.gov.cn" - crawler_keywords: List[str] = ["大化"] + crawler_keywords: list[str] = ["大化"] crawler_max_pages: int = 10 crawler_timeout: int = 30 crawler_page_size: int = 100 @@ -38,7 +38,7 @@ class Settings(BaseSettings): markdown_output_file: str = "onu.md" # 公告来源(JSON 字符串,从环境变量读取) - announcement_sources: str = '{"ZcyAnnouncement1":{"category_id":66485,"name":"采购公告","type":"purchase"},"ZcyAnnouncement2":{"category_id":66485,"name":"结果公告","type":"result"},"ZcyAnnouncement3":{"category_id":66485,"name":"合同公告","type":"contract"},"ZcyAnnouncement4":{"category_id":66485,"name":"更正公告","type":"correction"},"ZcyAnnouncement5":{"category_id":66485,"name":"招标文件预公示","type":"pre_announcement"},"ZcyAnnouncement6":{"category_id":66485,"name":"单一来源公示","type":"single_source"},"ZcyAnnouncement7":{"category_id":66485,"name":"电子卖场公示","type":"electronic_market"},"ZcyAnnouncement10":{"category_id":66485,"name":"履约验收公示","type":"acceptance"},"ZcyAnnouncement11":{"category_id":66485,"name":"工程类公告","type":"engineering"},"ZcyAnnouncement20":{"category_id":66485,"name":"框架协议征集公告","type":"framework_agreement"},"ZcyAnnouncement21":{"category_id":66485,"name":"框架协议入围结果公告","type":"framework_result"},"ZcyAnnouncement23":{"category_id":66485,"name":"框架协议成交结果汇总公告","type":"framework_summary"},"61-266648":{"category_id":66485,"name":"采购意向公开","type":"intention"}}' + announcement_sources: str = '{"ZcyAnnouncement1":{"category_id":66485,"name":"采购公告","type":"purchase"},"ZcyAnnouncement2":{"category_id":66485,"name":"结果公告","type":"result"},"ZcyAnnouncement3":{"category_id":66485,"name":"合同公告","type":"contract"},"ZcyAnnouncement4":{"category_id":66485,"name":"更正公告","type":"correction"},"ZcyAnnouncement5":{"category_id":66485,"name":"招标文件预公示","type":"pre_announcement"},"ZcyAnnouncement6":{"category_id":66485,"name":"单一来源公示","type":"single_source"},"ZcyAnnouncement7":{"category_id":66485,"name":"电子卖场公示","type":"electronic_market"},"ZcyAnnouncement10":{"category_id":66485,"name":"履约验收公示","type":"acceptance"},"ZcyAnnouncement11":{"category_id":66485,"name":"工程类公告","type":"engineering"},"ZcyAnnouncement20":{"category_id":66485,"name":"框架协议征集公告","type":"framework_agreement"},"ZcyAnnouncement21":{"category_id":66485,"name":"框架协议入围结果公告","type":"framework_result"},"ZcyAnnouncement23":{"category_id":66485,"name":"框架协议成交结果汇总公告","type":"framework_summary"},"61-266648":{"category_id":66485,"name":"采购意向公开","type":"intention"}}' # noqa: E501 settings = Settings() diff --git a/app/crawler/base.py b/app/crawler/base.py index 3e06f6c..93be0a6 100644 --- a/app/crawler/base.py +++ b/app/crawler/base.py @@ -2,7 +2,6 @@ import hashlib from abc import ABC, abstractmethod from dataclasses import dataclass, field from datetime import datetime -from typing import List, Optional @dataclass @@ -12,7 +11,7 @@ class CrawlResult: total_count: int = 0 new_count: int = 0 announcements: list = field(default_factory=list) - error_message: Optional[str] = None + error_message: str | None = None crawled_at: datetime = field(default_factory=datetime.now) duration: float = 0.0 @@ -24,7 +23,7 @@ class CrawlResult: @dataclass class PipelineConfig: filter_enabled: bool = True - keywords: List[str] = field(default_factory=list) + keywords: list[str] = field(default_factory=list) dedup_enabled: bool = True notify_mode: str = "filtered" mark_sent: bool = False diff --git a/app/crawler/dahuagov_spider.py b/app/crawler/dahuagov_spider.py index f95ae3d..0f20964 100644 --- a/app/crawler/dahuagov_spider.py +++ b/app/crawler/dahuagov_spider.py @@ -1,7 +1,9 @@ import asyncio import random from datetime import datetime + import httpx + from app.config import settings from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig from app.crawler.parsers import parse_dahuagov_html diff --git a/app/crawler/gxgp_spider.py b/app/crawler/gxgp_spider.py index 220e2c7..eeb528c 100644 --- a/app/crawler/gxgp_spider.py +++ b/app/crawler/gxgp_spider.py @@ -2,11 +2,12 @@ import json import random import time from datetime import datetime -from typing import List, Optional + import httpx + from app.config import settings from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig -from app.crawler.parsers import parse_gxgp_api_response, extract_pagination +from app.crawler.parsers import extract_pagination, parse_gxgp_api_response class GXGPSpider(BaseSpider): @@ -27,8 +28,8 @@ class GXGPSpider(BaseSpider): mark_sent=False, ) - async def crawl(self, sources: Optional[List[str]] = None, - max_pages: Optional[int] = None) -> CrawlResult: + async def crawl(self, sources: list[str] | None = None, + max_pages: int | None = None) -> CrawlResult: if max_pages is None: max_pages = settings.crawler_max_pages if sources is None: @@ -88,7 +89,7 @@ class GXGPSpider(BaseSpider): ) async def _fetch_page(self, client: httpx.AsyncClient, source_code: str, - category_id: int, page_no: int) -> Optional[dict]: + category_id: int, page_no: int) -> dict | None: payload = { "keyword": "", "publishDateBegin": "", @@ -102,7 +103,7 @@ class GXGPSpider(BaseSpider): "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "Content-Type": "application/json;charset=UTF-8", "Origin": self.base_url, - "Referer": f"{self.base_url}/site/category?parentId={category_id}&childrenCode={source_code}", + "Referer": f"{self.base_url}/site/category?parentId={category_id}&childrenCode={source_code}", # noqa: E501 } response = await client.post( self.announcement_api, json=payload, headers=headers diff --git a/app/crawler/parsers.py b/app/crawler/parsers.py index d266d11..4f1689c 100644 --- a/app/crawler/parsers.py +++ b/app/crawler/parsers.py @@ -1,18 +1,18 @@ import hashlib from datetime import datetime -from typing import Any, Dict, List +from typing import Any from urllib.parse import urljoin from bs4 import BeautifulSoup def parse_gxgp_api_response( - response_data: Dict[str, Any], + response_data: dict[str, Any], source_code: str, source_name: str, crawled_at: datetime, category_id: int, -) -> List[Dict[str, Any]]: +) -> list[dict[str, Any]]: if not response_data.get("success"): return [] data = response_data.get("result", {}).get("data", {}) @@ -62,7 +62,7 @@ def parse_gxgp_api_response( return results -def extract_pagination(response_data: Dict[str, Any]) -> Dict[str, Any]: +def extract_pagination(response_data: dict[str, Any]) -> dict[str, Any]: data = response_data.get("result", {}).get("data", {}) return { "total": data.get("total", 0), @@ -75,7 +75,7 @@ def extract_pagination(response_data: Dict[str, Any]) -> Dict[str, Any]: } -def parse_dahuagov_html(html: str, crawled_at: datetime) -> List[Dict[str, Any]]: +def parse_dahuagov_html(html: str, crawled_at: datetime) -> list[dict[str, Any]]: soup = BeautifulSoup(html, "html.parser") lists = soup.find_all("ul", class_="more-list") if not lists: @@ -131,7 +131,7 @@ def parse_dahuagov_html(html: str, crawled_at: datetime) -> List[Dict[str, Any]] return results -def _generate_hash(ann: Dict[str, Any]) -> str: +def _generate_hash(ann: dict[str, Any]) -> str: content = ( f"{ann['title']}|{ann['publish_date'].strftime('%Y-%m-%d')}" f"|{ann['purchase_name']}|{ann['content_url']}|{ann['source_code']}" diff --git a/app/main.py b/app/main.py index 3f4f09b..de27de2 100644 --- a/app/main.py +++ b/app/main.py @@ -1,9 +1,11 @@ import logging from contextlib import asynccontextmanager + from fastapi import FastAPI -from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from app.api.router import api_router from app.config import settings -from app.models.announcement import Base engine = create_async_engine(settings.database_url, echo=settings.debug) async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) @@ -20,7 +22,7 @@ async def lifespan(app: FastAPI): level=getattr(logging, settings.log_level), format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", ) - from app.scheduler.jobs import start_scheduler, shutdown_scheduler + from app.scheduler.jobs import shutdown_scheduler, start_scheduler start_scheduler() yield shutdown_scheduler() @@ -35,7 +37,6 @@ app = FastAPI( redoc_url=None, ) -from app.api.router import api_router app.include_router(api_router) diff --git a/app/models/announcement.py b/app/models/announcement.py index 9d3fae3..aa338f4 100644 --- a/app/models/announcement.py +++ b/app/models/announcement.py @@ -1,6 +1,7 @@ import hashlib -from datetime import datetime, date -from sqlalchemy import String, Boolean, DateTime, Integer, Text, func +from datetime import datetime + +from sqlalchemy import Boolean, DateTime, Integer, String, Text, func from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column @@ -25,7 +26,7 @@ class Announcement(Base): is_sent: Mapped[bool] = mapped_column(Boolean, default=False) keyword_matched: Mapped[bool] = mapped_column(Boolean, default=False) created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now()) - updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), onupdate=func.now()) + updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), onupdate=func.now()) # noqa: E501 @staticmethod def generate_hash(title: str, publish_date: str, purchase_name: str, @@ -36,5 +37,6 @@ class Announcement(Base): @staticmethod def source_map() -> dict: import json + from app.config import settings return json.loads(settings.announcement_sources) diff --git a/app/models/schemas.py b/app/models/schemas.py index ff1ba01..611c853 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Optional + from pydantic import BaseModel @@ -29,14 +29,14 @@ class AnnouncementListResponse(BaseModel): class CrawlTriggerRequest(BaseModel): - keywords: Optional[list[str]] = None - sources: Optional[list[str]] = None + keywords: list[str] | None = None + sources: list[str] | None = None manual: bool = False class CrawlStatusResponse(BaseModel): running: bool - last_crawl_time: Optional[datetime] = None + last_crawl_time: datetime | None = None total_sources: int @@ -53,4 +53,4 @@ class SourcesResponse(BaseModel): class JobResponse(BaseModel): id: str name: str - next_run_time: Optional[str] = None + next_run_time: str | None = None diff --git a/app/scheduler/jobs.py b/app/scheduler/jobs.py index e4418da..9dfc41f 100644 --- a/app/scheduler/jobs.py +++ b/app/scheduler/jobs.py @@ -1,7 +1,9 @@ import logging + from apscheduler.schedulers.asyncio import AsyncIOScheduler -from app.config import settings + from app.api.deps import get_crawl_service +from app.config import settings logger = logging.getLogger(__name__) scheduler = AsyncIOScheduler() diff --git a/app/services/crawl_service.py b/app/services/crawl_service.py index 5f9c396..8132f27 100644 --- a/app/services/crawl_service.py +++ b/app/services/crawl_service.py @@ -1,22 +1,22 @@ -from typing import Dict, List + from app.crawler.base import BaseSpider, CrawlResult class CrawlService: def __init__(self): - self.spiders: Dict[str, BaseSpider] = {} + self.spiders: dict[str, BaseSpider] = {} def register(self, spider: BaseSpider): self.spiders[spider.name] = spider - async def run_all(self) -> List[CrawlResult]: + async def run_all(self) -> list[CrawlResult]: results = [] for name, spider in self.spiders.items(): result = await spider.crawl() results.append(result) return results - async def run_spider(self, name: str, **kwargs) -> List[CrawlResult]: + async def run_spider(self, name: str, **kwargs) -> list[CrawlResult]: spider = self.spiders.get(name) if spider is None: return [CrawlResult( @@ -26,7 +26,7 @@ class CrawlService: result = await spider.crawl(**kwargs) return [result] - def get_spider_names(self) -> List[str]: + def get_spider_names(self) -> list[str]: return list(self.spiders.keys()) def get_pipeline_config(self, name: str): diff --git a/app/services/filter_service.py b/app/services/filter_service.py index 027e1f5..7c1efdf 100644 --- a/app/services/filter_service.py +++ b/app/services/filter_service.py @@ -1,9 +1,9 @@ from datetime import date -from typing import Any, Dict, List, Optional +from typing import Any -def filter_by_keywords(announcements: List[Dict[str, Any]], - keywords: List[str]) -> List[Dict[str, Any]]: +def filter_by_keywords(announcements: list[dict[str, Any]], + keywords: list[str]) -> list[dict[str, Any]]: if not keywords: return announcements @@ -17,9 +17,9 @@ def filter_by_keywords(announcements: List[Dict[str, Any]], return filtered -def filter_by_date(announcements: List[Dict[str, Any]], - start_date: Optional[date] = None, - end_date: Optional[date] = None) -> List[Dict[str, Any]]: +def filter_by_date(announcements: list[dict[str, Any]], + start_date: date | None = None, + end_date: date | None = None) -> list[dict[str, Any]]: if not start_date and not end_date: return announcements @@ -42,7 +42,7 @@ def filter_by_date(announcements: List[Dict[str, Any]], return filtered -def dedup_by_hash(announcements: List[Dict[str, Any]]) -> List[Dict[str, Any]]: +def dedup_by_hash(announcements: list[dict[str, Any]]) -> list[dict[str, Any]]: seen = set() result = [] for ann in announcements: diff --git a/app/services/notification_service.py b/app/services/notification_service.py index 850df57..af20a1e 100644 --- a/app/services/notification_service.py +++ b/app/services/notification_service.py @@ -1,5 +1,5 @@ -from datetime import datetime -from typing import Any, Dict, List +from typing import Any + from app.config import settings from app.wechat.client import WeChatClient @@ -8,7 +8,7 @@ class NotificationService: def __init__(self): self.client = WeChatClient() - async def send(self, announcements: List[Dict[str, Any]]) -> int: + async def send(self, announcements: list[dict[str, Any]]) -> int: if not settings.wechat_enabled: return 0 if not announcements: diff --git a/app/services/pipeline.py b/app/services/pipeline.py index e0a2b35..663e074 100644 --- a/app/services/pipeline.py +++ b/app/services/pipeline.py @@ -1,4 +1,5 @@ -from typing import Any, Dict, List +from typing import Any + from app.crawler.base import PipelineConfig, PipelineResult from app.services.filter_service import dedup_by_hash @@ -8,7 +9,7 @@ class PostCrawlPipeline: self.db = db_session self.notify = notification_service - async def process(self, announcements: List[Dict[str, Any]], + async def process(self, announcements: list[dict[str, Any]], config: PipelineConfig) -> PipelineResult: result = PipelineResult() @@ -47,8 +48,9 @@ class PostCrawlPipeline: return result - async def _save_to_db(self, announcements: List[Dict[str, Any]]) -> int: + async def _save_to_db(self, announcements: list[dict[str, Any]]) -> int: from sqlalchemy.dialects.postgresql import insert + from app.models.announcement import Announcement if not announcements: @@ -76,13 +78,14 @@ class PostCrawlPipeline: await self.db.commit() return result_proxy.rowcount if result_proxy.rowcount >= 0 else len(values) - async def _send_notifications(self, announcements: List[Dict[str, Any]]) -> int: + async def _send_notifications(self, announcements: list[dict[str, Any]]) -> int: return await self.notify.send(announcements) - async def _mark_sent(self, announcements: List[Dict[str, Any]]) -> int: - from app.models.announcement import Announcement + async def _mark_sent(self, announcements: list[dict[str, Any]]) -> int: from sqlalchemy import update + from app.models.announcement import Announcement + hashes = [a["content_hash"] for a in announcements if a.get("content_hash")] if not hashes: return 0 @@ -97,6 +100,6 @@ class PostCrawlPipeline: return result.rowcount @staticmethod - def _match_keywords(announcement: Dict[str, Any], keywords: List[str]) -> bool: + def _match_keywords(announcement: dict[str, Any], keywords: list[str]) -> bool: text = f"{announcement.get('title', '')} {announcement.get('purchase_name', '')}" return any(kw in text for kw in keywords) diff --git a/app/wechat/client.py b/app/wechat/client.py index b86ff15..d2ea91c 100644 --- a/app/wechat/client.py +++ b/app/wechat/client.py @@ -1,16 +1,16 @@ import time + import httpx -from typing import Optional from app.config import settings class WeChatClient: def __init__(self): - self._access_token: Optional[str] = None + self._access_token: str | None = None self._token_expires_at: float = 0 - async def _get_access_token(self) -> Optional[str]: + async def _get_access_token(self) -> str | None: now = time.time() if self._access_token and now < self._token_expires_at: return self._access_token diff --git a/app/wechat/crypto.py b/app/wechat/crypto.py index 2d44f98..275b21c 100644 --- a/app/wechat/crypto.py +++ b/app/wechat/crypto.py @@ -1,20 +1,20 @@ #!/usr/bin/env python -# -*- encoding:utf-8 -*- """ 对企业微信发送给企业后台的消息加解密示例代码. @copyright: Copyright (c) 1998-2014 Tencent Inc. """ # ------------------------------------------------------------------------ -import logging import base64 -import random import hashlib -import time -import struct -from Crypto.Cipher import AES -import xml.etree.cElementTree as ET +import logging +import random import socket +import struct +import time +import xml.etree.ElementTree as ET + +from Crypto.Cipher import AES try: import ierror @@ -29,11 +29,11 @@ except ImportError: """ -class FormatException(Exception): +class FormatError(Exception): pass -def throw_exception(message, exception_class=FormatException): +def throw_exception(message, exception_class=FormatError): """my define raise exception function""" raise exception_class(message) @@ -104,7 +104,7 @@ class XMLParse: return resp_xml -class PKCS7Encoder(): +class PKCS7Encoder: """提供基于PKCS7算法的加解密接口""" block_size = 32 @@ -134,7 +134,7 @@ class PKCS7Encoder(): return decrypted[:-pad] -class Prpcrypt(object): +class Prpcrypt: """提供接收和推送给企业微信消息的加解密接口""" def __init__(self, key): @@ -151,7 +151,7 @@ class Prpcrypt(object): """ # 16位随机字符串添加到明文开头 text = text.encode() - text = self.get_random_str() + struct.pack("I", socket.htonl(len(text))) + text + receiveid.encode() + text = self.get_random_str() + struct.pack("I", socket.htonl(len(text))) + text + receiveid.encode() # noqa: E501 # 使用自定义的填充方式对明文进行补位填充 pkcs7 = PKCS7Encoder() @@ -206,14 +206,14 @@ class Prpcrypt(object): return str(random.randint(1000000000000000, 9999999999999999)).encode() -class WXBizMsgCrypt(object): +class WXBizMsgCrypt: # 构造函数 def __init__(self, sToken, sEncodingAESKey, sReceiveId): try: self.key = base64.b64decode(sEncodingAESKey + "=") assert len(self.key) == 32 - except: - throw_exception("[error]: EncodingAESKey unvalid !", FormatException) + except Exception: + throw_exception("[error]: EncodingAESKey unvalid !", FormatError) # return ierror.WXBizMsgCrypt_IllegalAesKey,None self.m_sToken = sToken self.m_sReceiveId = sReceiveId @@ -240,9 +240,9 @@ class WXBizMsgCrypt(object): def EncryptMsg(self, sReplyMsg, sNonce, timestamp=None): # 将企业回复用户的消息加密打包 # @param sReplyMsg: 企业号待回复用户的消息,xml格式的字符串 - # @param sTimeStamp: 时间戳,可以自己生成,也可以用URL参数的timestamp,如为None则自动用当前时间 - # @param sNonce: 随机串,可以自己生成,也可以用URL参数的nonce - # sEncryptMsg: 加密后的可以直接回复用户的密文,包括msg_signature, timestamp, nonce, encrypt的xml格式的字符串, + # @param sTimeStamp: 时间戳,可以自己生成,也可以用URL参数的timestamp,如为None则自动用当前时间 # noqa: E501 + # @param sNonce: 随机串,可以自己生成,也可以用URL参数的nonce # noqa: E501 + # sEncryptMsg: 加密后的可以直接回复用户的密文,包括msg_signature, timestamp, nonce, encrypt的xml格式的字符串, # noqa: E501 # return:成功0,sEncryptMsg,失败返回对应的错误码None pc = Prpcrypt(self.key) ret, encrypt = pc.encrypt(sReplyMsg, self.m_sReceiveId) diff --git a/app/wechat/handler.py b/app/wechat/handler.py index 388d084..6471556 100644 --- a/app/wechat/handler.py +++ b/app/wechat/handler.py @@ -1,8 +1,7 @@ -import xml.etree.cElementTree as ET -from typing import Optional +import xml.etree.ElementTree as ET -from app.wechat.crypto import WXBizMsgCrypt from app.config import settings +from app.wechat.crypto import WXBizMsgCrypt class WeChatMessageHandler: @@ -15,7 +14,7 @@ class WeChatMessageHandler: def verify_url( self, msg_signature: str, timestamp: str, nonce: str, echostr: str - ) -> Optional[str]: + ) -> str | None: ret, sEchoStr = self.wxcpt.VerifyURL( msg_signature, timestamp, nonce, echostr ) @@ -33,7 +32,7 @@ class WeChatMessageHandler: msg_signature: str, timestamp: str, nonce: str, - ) -> Optional[ET.Element]: + ) -> ET.Element | None: ret, xml_content = self.wxcpt.DecryptMsg( post_data, msg_signature, timestamp, nonce ) @@ -43,16 +42,16 @@ class WeChatMessageHandler: def encrypt_response( self, response_xml: str, nonce: str, timestamp: str - ) -> Optional[str]: + ) -> str | None: ret, encrypted = self.wxcpt.EncryptMsg(response_xml, nonce, timestamp) if ret == 0: return encrypted return None def handle_event( - self, event: str, event_key: Optional[str], from_user: str - ) -> Optional[str]: + self, event: str, event_key: str | None, from_user: str + ) -> str | None: return None - def handle_text(self, content: str, from_user: str) -> Optional[str]: + def handle_text(self, content: str, from_user: str) -> str | None: return None diff --git a/app/wechat/ierror.py b/app/wechat/ierror.py index 6678fec..f99e9ea 100644 --- a/app/wechat/ierror.py +++ b/app/wechat/ierror.py @@ -1,10 +1,9 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- ######################################################################### # Author: jonyqin # Created Time: Thu 11 Sep 2014 01:53:58 PM CST # File Name: ierror.py -# Description:定义错误码含义 +# Description:定义错误码含义 ######################################################################### WXBizMsgCrypt_OK = 0 WXBizMsgCrypt_ValidateSignature_Error = -40001 diff --git a/pyproject.toml b/pyproject.toml index 9c1f846..29ced92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,9 @@ line-length = 100 [tool.ruff.lint] select = ["E", "F", "I", "N", "W", "UP"] +[tool.ruff.lint.per-file-ignores] +"app/wechat/*.py" = ["N802", "N803", "N806"] + [tool.pytest.ini_options] asyncio_mode = "auto" testpaths = ["tests"] diff --git a/tests/test_api/test_health.py b/tests/test_api/test_health.py index 22b57a6..a4c2cd1 100644 --- a/tests/test_api/test_health.py +++ b/tests/test_api/test_health.py @@ -1,5 +1,6 @@ import pytest -from httpx import AsyncClient, ASGITransport +from httpx import ASGITransport, AsyncClient + from app.main import app diff --git a/tests/test_crawler/test_base.py b/tests/test_crawler/test_base.py index 880deee..ce269f4 100644 --- a/tests/test_crawler/test_base.py +++ b/tests/test_crawler/test_base.py @@ -1,5 +1,7 @@ -import pytest from datetime import datetime + +import pytest + from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig, PipelineResult diff --git a/tests/test_crawler/test_dahuagov_spider.py b/tests/test_crawler/test_dahuagov_spider.py index 7f20a37..1979cba 100644 --- a/tests/test_crawler/test_dahuagov_spider.py +++ b/tests/test_crawler/test_dahuagov_spider.py @@ -1,7 +1,9 @@ +from unittest.mock import AsyncMock, MagicMock, patch + import pytest -from unittest.mock import AsyncMock, patch, MagicMock -from app.crawler.dahuagov_spider import DahuagovSpider + from app.crawler.base import PipelineConfig +from app.crawler.dahuagov_spider import DahuagovSpider @pytest.mark.asyncio diff --git a/tests/test_crawler/test_gxgp_spider.py b/tests/test_crawler/test_gxgp_spider.py index 90eb1d3..221e763 100644 --- a/tests/test_crawler/test_gxgp_spider.py +++ b/tests/test_crawler/test_gxgp_spider.py @@ -1,7 +1,9 @@ +from unittest.mock import AsyncMock, MagicMock, patch + import pytest -from unittest.mock import AsyncMock, patch, MagicMock -from app.crawler.gxgp_spider import GXGPSpider + from app.crawler.base import PipelineConfig +from app.crawler.gxgp_spider import GXGPSpider @pytest.mark.asyncio diff --git a/tests/test_crawler/test_parsers.py b/tests/test_crawler/test_parsers.py index ca0be54..b5b0228 100644 --- a/tests/test_crawler/test_parsers.py +++ b/tests/test_crawler/test_parsers.py @@ -1,8 +1,9 @@ from datetime import datetime + from app.crawler.parsers import ( - parse_gxgp_api_response, - parse_dahuagov_html, extract_pagination, + parse_dahuagov_html, + parse_gxgp_api_response, ) diff --git a/tests/test_services/test_crawl_service.py b/tests/test_services/test_crawl_service.py index e593950..a1e98c0 100644 --- a/tests/test_services/test_crawl_service.py +++ b/tests/test_services/test_crawl_service.py @@ -1,5 +1,6 @@ + import pytest -from datetime import datetime + from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig from app.services.crawl_service import CrawlService diff --git a/tests/test_services/test_filter_service.py b/tests/test_services/test_filter_service.py index 0c477f1..2ab7080 100644 --- a/tests/test_services/test_filter_service.py +++ b/tests/test_services/test_filter_service.py @@ -1,5 +1,6 @@ from datetime import datetime -from app.services.filter_service import filter_by_keywords, filter_by_date + +from app.services.filter_service import filter_by_date, filter_by_keywords def test_filter_by_keywords_match(): diff --git a/tests/test_services/test_notification_service.py b/tests/test_services/test_notification_service.py index 2b19452..6e14e90 100644 --- a/tests/test_services/test_notification_service.py +++ b/tests/test_services/test_notification_service.py @@ -1,5 +1,7 @@ -import pytest from unittest.mock import AsyncMock, patch + +import pytest + from app.services.notification_service import NotificationService diff --git a/tests/test_services/test_pipeline.py b/tests/test_services/test_pipeline.py index 402bf2c..7326560 100644 --- a/tests/test_services/test_pipeline.py +++ b/tests/test_services/test_pipeline.py @@ -1,8 +1,10 @@ -import pytest -from unittest.mock import AsyncMock, MagicMock, patch from datetime import datetime -from app.services.pipeline import PostCrawlPipeline +from unittest.mock import AsyncMock, patch + +import pytest + from app.crawler.base import CrawlResult, PipelineConfig +from app.services.pipeline import PostCrawlPipeline @pytest.mark.asyncio