chore: ruff 代码检查与修复
130 issues auto-fixed (import ordering, UP045/UP006 type annotations), 33 issues manually fixed (E712/E501/E402/E722 + N818 rename + per-file wechat ignore for N8xx naming conventions). All 33 tests pass.
This commit is contained in:
+12
-11
@@ -1,11 +1,12 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from fastapi import APIRouter, Depends, Query, HTTPException
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, func
|
||||
|
||||
from app.api.deps import get_db
|
||||
from app.models.announcement import Announcement
|
||||
from app.models.schemas import AnnouncementResponse, AnnouncementListResponse
|
||||
from app.models.schemas import AnnouncementListResponse, AnnouncementResponse
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -14,11 +15,11 @@ router = APIRouter()
|
||||
async def list_announcements(
|
||||
page: int = Query(1, ge=1),
|
||||
page_size: int = Query(20, ge=1, le=100),
|
||||
source_code: Optional[str] = None,
|
||||
keyword: Optional[str] = None,
|
||||
start_date: Optional[str] = None,
|
||||
end_date: Optional[str] = None,
|
||||
crawl_mode: Optional[str] = None,
|
||||
source_code: str | None = None,
|
||||
keyword: str | None = None,
|
||||
start_date: str | None = None,
|
||||
end_date: str | None = None,
|
||||
crawl_mode: str | None = None,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
conditions = []
|
||||
@@ -91,12 +92,12 @@ async def get_stats(db: AsyncSession = Depends(get_db)):
|
||||
).select_from(Announcement)
|
||||
)
|
||||
new_count = await db.execute(
|
||||
select(func.count()).where(Announcement.is_new == True)
|
||||
select(func.count()).where(Announcement.is_new == True) # noqa: E712
|
||||
.select_from(Announcement)
|
||||
)
|
||||
unsent = await db.execute(
|
||||
select(func.count()).where(
|
||||
Announcement.is_sent == False, Announcement.is_new == True
|
||||
Announcement.is_sent == False, Announcement.is_new == True # noqa: E712
|
||||
).select_from(Announcement)
|
||||
)
|
||||
return {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.api.deps import get_crawl_service
|
||||
from app.models.schemas import CrawlTriggerRequest
|
||||
|
||||
@@ -34,6 +35,7 @@ async def crawl_status():
|
||||
@router.get("/crawl/sources")
|
||||
async def crawl_sources():
|
||||
import json
|
||||
|
||||
from app.config import settings
|
||||
sources = json.loads(settings.announcement_sources)
|
||||
return {
|
||||
|
||||
+3
-2
@@ -1,7 +1,8 @@
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.services.crawl_service import CrawlService
|
||||
from app.crawler.gxgp_spider import GXGPSpider
|
||||
|
||||
from app.crawler.dahuagov_spider import DahuagovSpider
|
||||
from app.crawler.gxgp_spider import GXGPSpider
|
||||
from app.services.crawl_service import CrawlService
|
||||
|
||||
|
||||
async def get_db() -> AsyncSession:
|
||||
|
||||
+3
-1
@@ -1,5 +1,7 @@
|
||||
from fastapi import APIRouter
|
||||
from app.api import announcements, crawl, wechat, scheduler as scheduler_module
|
||||
|
||||
from app.api import announcements, crawl, wechat
|
||||
from app.api import scheduler as scheduler_module
|
||||
|
||||
api_router = APIRouter(prefix="/api/v1")
|
||||
api_router.include_router(announcements.router, tags=["announcements"])
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from fastapi import APIRouter
|
||||
from app.scheduler.jobs import scheduler
|
||||
|
||||
from app.models.schemas import JobResponse
|
||||
from app.scheduler.jobs import scheduler
|
||||
|
||||
router = APIRouter(prefix="/scheduler", tags=["scheduler"])
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from fastapi import APIRouter, Request, Response
|
||||
|
||||
from app.wechat.handler import WeChatMessageHandler
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
from typing import List
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
@@ -14,7 +14,7 @@ class Settings(BaseSettings):
|
||||
|
||||
# 爬虫
|
||||
crawler_base_url: str = "https://zfcg.gxzf.gov.cn"
|
||||
crawler_keywords: List[str] = ["大化"]
|
||||
crawler_keywords: list[str] = ["大化"]
|
||||
crawler_max_pages: int = 10
|
||||
crawler_timeout: int = 30
|
||||
crawler_page_size: int = 100
|
||||
@@ -38,7 +38,7 @@ class Settings(BaseSettings):
|
||||
markdown_output_file: str = "onu.md"
|
||||
|
||||
# 公告来源(JSON 字符串,从环境变量读取)
|
||||
announcement_sources: str = '{"ZcyAnnouncement1":{"category_id":66485,"name":"采购公告","type":"purchase"},"ZcyAnnouncement2":{"category_id":66485,"name":"结果公告","type":"result"},"ZcyAnnouncement3":{"category_id":66485,"name":"合同公告","type":"contract"},"ZcyAnnouncement4":{"category_id":66485,"name":"更正公告","type":"correction"},"ZcyAnnouncement5":{"category_id":66485,"name":"招标文件预公示","type":"pre_announcement"},"ZcyAnnouncement6":{"category_id":66485,"name":"单一来源公示","type":"single_source"},"ZcyAnnouncement7":{"category_id":66485,"name":"电子卖场公示","type":"electronic_market"},"ZcyAnnouncement10":{"category_id":66485,"name":"履约验收公示","type":"acceptance"},"ZcyAnnouncement11":{"category_id":66485,"name":"工程类公告","type":"engineering"},"ZcyAnnouncement20":{"category_id":66485,"name":"框架协议征集公告","type":"framework_agreement"},"ZcyAnnouncement21":{"category_id":66485,"name":"框架协议入围结果公告","type":"framework_result"},"ZcyAnnouncement23":{"category_id":66485,"name":"框架协议成交结果汇总公告","type":"framework_summary"},"61-266648":{"category_id":66485,"name":"采购意向公开","type":"intention"}}'
|
||||
announcement_sources: str = '{"ZcyAnnouncement1":{"category_id":66485,"name":"采购公告","type":"purchase"},"ZcyAnnouncement2":{"category_id":66485,"name":"结果公告","type":"result"},"ZcyAnnouncement3":{"category_id":66485,"name":"合同公告","type":"contract"},"ZcyAnnouncement4":{"category_id":66485,"name":"更正公告","type":"correction"},"ZcyAnnouncement5":{"category_id":66485,"name":"招标文件预公示","type":"pre_announcement"},"ZcyAnnouncement6":{"category_id":66485,"name":"单一来源公示","type":"single_source"},"ZcyAnnouncement7":{"category_id":66485,"name":"电子卖场公示","type":"electronic_market"},"ZcyAnnouncement10":{"category_id":66485,"name":"履约验收公示","type":"acceptance"},"ZcyAnnouncement11":{"category_id":66485,"name":"工程类公告","type":"engineering"},"ZcyAnnouncement20":{"category_id":66485,"name":"框架协议征集公告","type":"framework_agreement"},"ZcyAnnouncement21":{"category_id":66485,"name":"框架协议入围结果公告","type":"framework_result"},"ZcyAnnouncement23":{"category_id":66485,"name":"框架协议成交结果汇总公告","type":"framework_summary"},"61-266648":{"category_id":66485,"name":"采购意向公开","type":"intention"}}' # noqa: E501
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
||||
+2
-3
@@ -2,7 +2,6 @@ import hashlib
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -12,7 +11,7 @@ class CrawlResult:
|
||||
total_count: int = 0
|
||||
new_count: int = 0
|
||||
announcements: list = field(default_factory=list)
|
||||
error_message: Optional[str] = None
|
||||
error_message: str | None = None
|
||||
crawled_at: datetime = field(default_factory=datetime.now)
|
||||
duration: float = 0.0
|
||||
|
||||
@@ -24,7 +23,7 @@ class CrawlResult:
|
||||
@dataclass
|
||||
class PipelineConfig:
|
||||
filter_enabled: bool = True
|
||||
keywords: List[str] = field(default_factory=list)
|
||||
keywords: list[str] = field(default_factory=list)
|
||||
dedup_enabled: bool = True
|
||||
notify_mode: str = "filtered"
|
||||
mark_sent: bool = False
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import asyncio
|
||||
import random
|
||||
from datetime import datetime
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
||||
from app.crawler.parsers import parse_dahuagov_html
|
||||
|
||||
@@ -2,11 +2,12 @@ import json
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import List, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
from app.config import settings
|
||||
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
||||
from app.crawler.parsers import parse_gxgp_api_response, extract_pagination
|
||||
from app.crawler.parsers import extract_pagination, parse_gxgp_api_response
|
||||
|
||||
|
||||
class GXGPSpider(BaseSpider):
|
||||
@@ -27,8 +28,8 @@ class GXGPSpider(BaseSpider):
|
||||
mark_sent=False,
|
||||
)
|
||||
|
||||
async def crawl(self, sources: Optional[List[str]] = None,
|
||||
max_pages: Optional[int] = None) -> CrawlResult:
|
||||
async def crawl(self, sources: list[str] | None = None,
|
||||
max_pages: int | None = None) -> CrawlResult:
|
||||
if max_pages is None:
|
||||
max_pages = settings.crawler_max_pages
|
||||
if sources is None:
|
||||
@@ -88,7 +89,7 @@ class GXGPSpider(BaseSpider):
|
||||
)
|
||||
|
||||
async def _fetch_page(self, client: httpx.AsyncClient, source_code: str,
|
||||
category_id: int, page_no: int) -> Optional[dict]:
|
||||
category_id: int, page_no: int) -> dict | None:
|
||||
payload = {
|
||||
"keyword": "",
|
||||
"publishDateBegin": "",
|
||||
@@ -102,7 +103,7 @@ class GXGPSpider(BaseSpider):
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||
"Content-Type": "application/json;charset=UTF-8",
|
||||
"Origin": self.base_url,
|
||||
"Referer": f"{self.base_url}/site/category?parentId={category_id}&childrenCode={source_code}",
|
||||
"Referer": f"{self.base_url}/site/category?parentId={category_id}&childrenCode={source_code}", # noqa: E501
|
||||
}
|
||||
response = await client.post(
|
||||
self.announcement_api, json=payload, headers=headers
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
import hashlib
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
def parse_gxgp_api_response(
|
||||
response_data: Dict[str, Any],
|
||||
response_data: dict[str, Any],
|
||||
source_code: str,
|
||||
source_name: str,
|
||||
crawled_at: datetime,
|
||||
category_id: int,
|
||||
) -> List[Dict[str, Any]]:
|
||||
) -> list[dict[str, Any]]:
|
||||
if not response_data.get("success"):
|
||||
return []
|
||||
data = response_data.get("result", {}).get("data", {})
|
||||
@@ -62,7 +62,7 @@ def parse_gxgp_api_response(
|
||||
return results
|
||||
|
||||
|
||||
def extract_pagination(response_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
def extract_pagination(response_data: dict[str, Any]) -> dict[str, Any]:
|
||||
data = response_data.get("result", {}).get("data", {})
|
||||
return {
|
||||
"total": data.get("total", 0),
|
||||
@@ -75,7 +75,7 @@ def extract_pagination(response_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def parse_dahuagov_html(html: str, crawled_at: datetime) -> List[Dict[str, Any]]:
|
||||
def parse_dahuagov_html(html: str, crawled_at: datetime) -> list[dict[str, Any]]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
lists = soup.find_all("ul", class_="more-list")
|
||||
if not lists:
|
||||
@@ -131,7 +131,7 @@ def parse_dahuagov_html(html: str, crawled_at: datetime) -> List[Dict[str, Any]]
|
||||
return results
|
||||
|
||||
|
||||
def _generate_hash(ann: Dict[str, Any]) -> str:
|
||||
def _generate_hash(ann: dict[str, Any]) -> str:
|
||||
content = (
|
||||
f"{ann['title']}|{ann['publish_date'].strftime('%Y-%m-%d')}"
|
||||
f"|{ann['purchase_name']}|{ann['content_url']}|{ann['source_code']}"
|
||||
|
||||
+5
-4
@@ -1,9 +1,11 @@
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
from app.api.router import api_router
|
||||
from app.config import settings
|
||||
from app.models.announcement import Base
|
||||
|
||||
engine = create_async_engine(settings.database_url, echo=settings.debug)
|
||||
async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
@@ -20,7 +22,7 @@ async def lifespan(app: FastAPI):
|
||||
level=getattr(logging, settings.log_level),
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
)
|
||||
from app.scheduler.jobs import start_scheduler, shutdown_scheduler
|
||||
from app.scheduler.jobs import shutdown_scheduler, start_scheduler
|
||||
start_scheduler()
|
||||
yield
|
||||
shutdown_scheduler()
|
||||
@@ -35,7 +37,6 @@ app = FastAPI(
|
||||
redoc_url=None,
|
||||
)
|
||||
|
||||
from app.api.router import api_router
|
||||
app.include_router(api_router)
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import hashlib
|
||||
from datetime import datetime, date
|
||||
from sqlalchemy import String, Boolean, DateTime, Integer, Text, func
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Boolean, DateTime, Integer, String, Text, func
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
|
||||
|
||||
@@ -25,7 +26,7 @@ class Announcement(Base):
|
||||
is_sent: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
keyword_matched: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), onupdate=func.now()) # noqa: E501
|
||||
|
||||
@staticmethod
|
||||
def generate_hash(title: str, publish_date: str, purchase_name: str,
|
||||
@@ -36,5 +37,6 @@ class Announcement(Base):
|
||||
@staticmethod
|
||||
def source_map() -> dict:
|
||||
import json
|
||||
|
||||
from app.config import settings
|
||||
return json.loads(settings.announcement_sources)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
@@ -29,14 +29,14 @@ class AnnouncementListResponse(BaseModel):
|
||||
|
||||
|
||||
class CrawlTriggerRequest(BaseModel):
|
||||
keywords: Optional[list[str]] = None
|
||||
sources: Optional[list[str]] = None
|
||||
keywords: list[str] | None = None
|
||||
sources: list[str] | None = None
|
||||
manual: bool = False
|
||||
|
||||
|
||||
class CrawlStatusResponse(BaseModel):
|
||||
running: bool
|
||||
last_crawl_time: Optional[datetime] = None
|
||||
last_crawl_time: datetime | None = None
|
||||
total_sources: int
|
||||
|
||||
|
||||
@@ -53,4 +53,4 @@ class SourcesResponse(BaseModel):
|
||||
class JobResponse(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
next_run_time: Optional[str] = None
|
||||
next_run_time: str | None = None
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import logging
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from app.config import settings
|
||||
|
||||
from app.api.deps import get_crawl_service
|
||||
from app.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
scheduler = AsyncIOScheduler()
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
from typing import Dict, List
|
||||
|
||||
from app.crawler.base import BaseSpider, CrawlResult
|
||||
|
||||
|
||||
class CrawlService:
|
||||
def __init__(self):
|
||||
self.spiders: Dict[str, BaseSpider] = {}
|
||||
self.spiders: dict[str, BaseSpider] = {}
|
||||
|
||||
def register(self, spider: BaseSpider):
|
||||
self.spiders[spider.name] = spider
|
||||
|
||||
async def run_all(self) -> List[CrawlResult]:
|
||||
async def run_all(self) -> list[CrawlResult]:
|
||||
results = []
|
||||
for name, spider in self.spiders.items():
|
||||
result = await spider.crawl()
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
async def run_spider(self, name: str, **kwargs) -> List[CrawlResult]:
|
||||
async def run_spider(self, name: str, **kwargs) -> list[CrawlResult]:
|
||||
spider = self.spiders.get(name)
|
||||
if spider is None:
|
||||
return [CrawlResult(
|
||||
@@ -26,7 +26,7 @@ class CrawlService:
|
||||
result = await spider.crawl(**kwargs)
|
||||
return [result]
|
||||
|
||||
def get_spider_names(self) -> List[str]:
|
||||
def get_spider_names(self) -> list[str]:
|
||||
return list(self.spiders.keys())
|
||||
|
||||
def get_pipeline_config(self, name: str):
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
from datetime import date
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any
|
||||
|
||||
|
||||
def filter_by_keywords(announcements: List[Dict[str, Any]],
|
||||
keywords: List[str]) -> List[Dict[str, Any]]:
|
||||
def filter_by_keywords(announcements: list[dict[str, Any]],
|
||||
keywords: list[str]) -> list[dict[str, Any]]:
|
||||
if not keywords:
|
||||
return announcements
|
||||
|
||||
@@ -17,9 +17,9 @@ def filter_by_keywords(announcements: List[Dict[str, Any]],
|
||||
return filtered
|
||||
|
||||
|
||||
def filter_by_date(announcements: List[Dict[str, Any]],
|
||||
start_date: Optional[date] = None,
|
||||
end_date: Optional[date] = None) -> List[Dict[str, Any]]:
|
||||
def filter_by_date(announcements: list[dict[str, Any]],
|
||||
start_date: date | None = None,
|
||||
end_date: date | None = None) -> list[dict[str, Any]]:
|
||||
if not start_date and not end_date:
|
||||
return announcements
|
||||
|
||||
@@ -42,7 +42,7 @@ def filter_by_date(announcements: List[Dict[str, Any]],
|
||||
return filtered
|
||||
|
||||
|
||||
def dedup_by_hash(announcements: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
def dedup_by_hash(announcements: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
seen = set()
|
||||
result = []
|
||||
for ann in announcements:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any
|
||||
|
||||
from app.config import settings
|
||||
from app.wechat.client import WeChatClient
|
||||
|
||||
@@ -8,7 +8,7 @@ class NotificationService:
|
||||
def __init__(self):
|
||||
self.client = WeChatClient()
|
||||
|
||||
async def send(self, announcements: List[Dict[str, Any]]) -> int:
|
||||
async def send(self, announcements: list[dict[str, Any]]) -> int:
|
||||
if not settings.wechat_enabled:
|
||||
return 0
|
||||
if not announcements:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any
|
||||
|
||||
from app.crawler.base import PipelineConfig, PipelineResult
|
||||
from app.services.filter_service import dedup_by_hash
|
||||
|
||||
@@ -8,7 +9,7 @@ class PostCrawlPipeline:
|
||||
self.db = db_session
|
||||
self.notify = notification_service
|
||||
|
||||
async def process(self, announcements: List[Dict[str, Any]],
|
||||
async def process(self, announcements: list[dict[str, Any]],
|
||||
config: PipelineConfig) -> PipelineResult:
|
||||
result = PipelineResult()
|
||||
|
||||
@@ -47,8 +48,9 @@ class PostCrawlPipeline:
|
||||
|
||||
return result
|
||||
|
||||
async def _save_to_db(self, announcements: List[Dict[str, Any]]) -> int:
|
||||
async def _save_to_db(self, announcements: list[dict[str, Any]]) -> int:
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
|
||||
from app.models.announcement import Announcement
|
||||
|
||||
if not announcements:
|
||||
@@ -76,13 +78,14 @@ class PostCrawlPipeline:
|
||||
await self.db.commit()
|
||||
return result_proxy.rowcount if result_proxy.rowcount >= 0 else len(values)
|
||||
|
||||
async def _send_notifications(self, announcements: List[Dict[str, Any]]) -> int:
|
||||
async def _send_notifications(self, announcements: list[dict[str, Any]]) -> int:
|
||||
return await self.notify.send(announcements)
|
||||
|
||||
async def _mark_sent(self, announcements: List[Dict[str, Any]]) -> int:
|
||||
from app.models.announcement import Announcement
|
||||
async def _mark_sent(self, announcements: list[dict[str, Any]]) -> int:
|
||||
from sqlalchemy import update
|
||||
|
||||
from app.models.announcement import Announcement
|
||||
|
||||
hashes = [a["content_hash"] for a in announcements if a.get("content_hash")]
|
||||
if not hashes:
|
||||
return 0
|
||||
@@ -97,6 +100,6 @@ class PostCrawlPipeline:
|
||||
return result.rowcount
|
||||
|
||||
@staticmethod
|
||||
def _match_keywords(announcement: Dict[str, Any], keywords: List[str]) -> bool:
|
||||
def _match_keywords(announcement: dict[str, Any], keywords: list[str]) -> bool:
|
||||
text = f"{announcement.get('title', '')} {announcement.get('purchase_name', '')}"
|
||||
return any(kw in text for kw in keywords)
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
import time
|
||||
|
||||
import httpx
|
||||
from typing import Optional
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
class WeChatClient:
|
||||
def __init__(self):
|
||||
self._access_token: Optional[str] = None
|
||||
self._access_token: str | None = None
|
||||
self._token_expires_at: float = 0
|
||||
|
||||
async def _get_access_token(self) -> Optional[str]:
|
||||
async def _get_access_token(self) -> str | None:
|
||||
now = time.time()
|
||||
if self._access_token and now < self._token_expires_at:
|
||||
return self._access_token
|
||||
|
||||
+18
-18
@@ -1,20 +1,20 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- encoding:utf-8 -*-
|
||||
|
||||
""" 对企业微信发送给企业后台的消息加解密示例代码.
|
||||
@copyright: Copyright (c) 1998-2014 Tencent Inc.
|
||||
|
||||
"""
|
||||
# ------------------------------------------------------------------------
|
||||
import logging
|
||||
import base64
|
||||
import random
|
||||
import hashlib
|
||||
import time
|
||||
import struct
|
||||
from Crypto.Cipher import AES
|
||||
import xml.etree.cElementTree as ET
|
||||
import logging
|
||||
import random
|
||||
import socket
|
||||
import struct
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from Crypto.Cipher import AES
|
||||
|
||||
try:
|
||||
import ierror
|
||||
@@ -29,11 +29,11 @@ except ImportError:
|
||||
"""
|
||||
|
||||
|
||||
class FormatException(Exception):
|
||||
class FormatError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def throw_exception(message, exception_class=FormatException):
|
||||
def throw_exception(message, exception_class=FormatError):
|
||||
"""my define raise exception function"""
|
||||
raise exception_class(message)
|
||||
|
||||
@@ -104,7 +104,7 @@ class XMLParse:
|
||||
return resp_xml
|
||||
|
||||
|
||||
class PKCS7Encoder():
|
||||
class PKCS7Encoder:
|
||||
"""提供基于PKCS7算法的加解密接口"""
|
||||
|
||||
block_size = 32
|
||||
@@ -134,7 +134,7 @@ class PKCS7Encoder():
|
||||
return decrypted[:-pad]
|
||||
|
||||
|
||||
class Prpcrypt(object):
|
||||
class Prpcrypt:
|
||||
"""提供接收和推送给企业微信消息的加解密接口"""
|
||||
|
||||
def __init__(self, key):
|
||||
@@ -151,7 +151,7 @@ class Prpcrypt(object):
|
||||
"""
|
||||
# 16位随机字符串添加到明文开头
|
||||
text = text.encode()
|
||||
text = self.get_random_str() + struct.pack("I", socket.htonl(len(text))) + text + receiveid.encode()
|
||||
text = self.get_random_str() + struct.pack("I", socket.htonl(len(text))) + text + receiveid.encode() # noqa: E501
|
||||
|
||||
# 使用自定义的填充方式对明文进行补位填充
|
||||
pkcs7 = PKCS7Encoder()
|
||||
@@ -206,14 +206,14 @@ class Prpcrypt(object):
|
||||
return str(random.randint(1000000000000000, 9999999999999999)).encode()
|
||||
|
||||
|
||||
class WXBizMsgCrypt(object):
|
||||
class WXBizMsgCrypt:
|
||||
# 构造函数
|
||||
def __init__(self, sToken, sEncodingAESKey, sReceiveId):
|
||||
try:
|
||||
self.key = base64.b64decode(sEncodingAESKey + "=")
|
||||
assert len(self.key) == 32
|
||||
except:
|
||||
throw_exception("[error]: EncodingAESKey unvalid !", FormatException)
|
||||
except Exception:
|
||||
throw_exception("[error]: EncodingAESKey unvalid !", FormatError)
|
||||
# return ierror.WXBizMsgCrypt_IllegalAesKey,None
|
||||
self.m_sToken = sToken
|
||||
self.m_sReceiveId = sReceiveId
|
||||
@@ -240,9 +240,9 @@ class WXBizMsgCrypt(object):
|
||||
def EncryptMsg(self, sReplyMsg, sNonce, timestamp=None):
|
||||
# 将企业回复用户的消息加密打包
|
||||
# @param sReplyMsg: 企业号待回复用户的消息,xml格式的字符串
|
||||
# @param sTimeStamp: 时间戳,可以自己生成,也可以用URL参数的timestamp,如为None则自动用当前时间
|
||||
# @param sNonce: 随机串,可以自己生成,也可以用URL参数的nonce
|
||||
# sEncryptMsg: 加密后的可以直接回复用户的密文,包括msg_signature, timestamp, nonce, encrypt的xml格式的字符串,
|
||||
# @param sTimeStamp: 时间戳,可以自己生成,也可以用URL参数的timestamp,如为None则自动用当前时间 # noqa: E501
|
||||
# @param sNonce: 随机串,可以自己生成,也可以用URL参数的nonce # noqa: E501
|
||||
# sEncryptMsg: 加密后的可以直接回复用户的密文,包括msg_signature, timestamp, nonce, encrypt的xml格式的字符串, # noqa: E501
|
||||
# return:成功0,sEncryptMsg,失败返回对应的错误码None
|
||||
pc = Prpcrypt(self.key)
|
||||
ret, encrypt = pc.encrypt(sReplyMsg, self.m_sReceiveId)
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
import xml.etree.cElementTree as ET
|
||||
from typing import Optional
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from app.wechat.crypto import WXBizMsgCrypt
|
||||
from app.config import settings
|
||||
from app.wechat.crypto import WXBizMsgCrypt
|
||||
|
||||
|
||||
class WeChatMessageHandler:
|
||||
@@ -15,7 +14,7 @@ class WeChatMessageHandler:
|
||||
|
||||
def verify_url(
|
||||
self, msg_signature: str, timestamp: str, nonce: str, echostr: str
|
||||
) -> Optional[str]:
|
||||
) -> str | None:
|
||||
ret, sEchoStr = self.wxcpt.VerifyURL(
|
||||
msg_signature, timestamp, nonce, echostr
|
||||
)
|
||||
@@ -33,7 +32,7 @@ class WeChatMessageHandler:
|
||||
msg_signature: str,
|
||||
timestamp: str,
|
||||
nonce: str,
|
||||
) -> Optional[ET.Element]:
|
||||
) -> ET.Element | None:
|
||||
ret, xml_content = self.wxcpt.DecryptMsg(
|
||||
post_data, msg_signature, timestamp, nonce
|
||||
)
|
||||
@@ -43,16 +42,16 @@ class WeChatMessageHandler:
|
||||
|
||||
def encrypt_response(
|
||||
self, response_xml: str, nonce: str, timestamp: str
|
||||
) -> Optional[str]:
|
||||
) -> str | None:
|
||||
ret, encrypted = self.wxcpt.EncryptMsg(response_xml, nonce, timestamp)
|
||||
if ret == 0:
|
||||
return encrypted
|
||||
return None
|
||||
|
||||
def handle_event(
|
||||
self, event: str, event_key: Optional[str], from_user: str
|
||||
) -> Optional[str]:
|
||||
self, event: str, event_key: str | None, from_user: str
|
||||
) -> str | None:
|
||||
return None
|
||||
|
||||
def handle_text(self, content: str, from_user: str) -> Optional[str]:
|
||||
def handle_text(self, content: str, from_user: str) -> str | None:
|
||||
return None
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
#########################################################################
|
||||
# Author: jonyqin
|
||||
# Created Time: Thu 11 Sep 2014 01:53:58 PM CST
|
||||
# File Name: ierror.py
|
||||
# Description:定义错误码含义
|
||||
# Description:定义错误码含义
|
||||
#########################################################################
|
||||
WXBizMsgCrypt_OK = 0
|
||||
WXBizMsgCrypt_ValidateSignature_Error = -40001
|
||||
|
||||
@@ -36,6 +36,9 @@ line-length = 100
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I", "N", "W", "UP"]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"app/wechat/*.py" = ["N802", "N803", "N806"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
asyncio_mode = "auto"
|
||||
testpaths = ["tests"]
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import pytest
|
||||
from httpx import AsyncClient, ASGITransport
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.main import app
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig, PipelineResult
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, patch, MagicMock
|
||||
from app.crawler.dahuagov_spider import DahuagovSpider
|
||||
|
||||
from app.crawler.base import PipelineConfig
|
||||
from app.crawler.dahuagov_spider import DahuagovSpider
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, patch, MagicMock
|
||||
from app.crawler.gxgp_spider import GXGPSpider
|
||||
|
||||
from app.crawler.base import PipelineConfig
|
||||
from app.crawler.gxgp_spider import GXGPSpider
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
from datetime import datetime
|
||||
|
||||
from app.crawler.parsers import (
|
||||
parse_gxgp_api_response,
|
||||
parse_dahuagov_html,
|
||||
extract_pagination,
|
||||
parse_dahuagov_html,
|
||||
parse_gxgp_api_response,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
|
||||
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
||||
from app.services.crawl_service import CrawlService
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from datetime import datetime
|
||||
from app.services.filter_service import filter_by_keywords, filter_by_date
|
||||
|
||||
from app.services.filter_service import filter_by_date, filter_by_keywords
|
||||
|
||||
|
||||
def test_filter_by_keywords_match():
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.notification_service import NotificationService
|
||||
|
||||
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from datetime import datetime
|
||||
from app.services.pipeline import PostCrawlPipeline
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.crawler.base import CrawlResult, PipelineConfig
|
||||
from app.services.pipeline import PostCrawlPipeline
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
Reference in New Issue
Block a user