chore: ruff 代码检查与修复
130 issues auto-fixed (import ordering, UP045/UP006 type annotations), 33 issues manually fixed (E712/E501/E402/E722 + N818 rename + per-file wechat ignore for N8xx naming conventions). All 33 tests pass.
This commit is contained in:
+12
-11
@@ -1,11 +1,12 @@
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Optional
|
|
||||||
from fastapi import APIRouter, Depends, Query, HTTPException
|
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||||
|
from sqlalchemy import func, select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from sqlalchemy import select, func
|
|
||||||
from app.api.deps import get_db
|
from app.api.deps import get_db
|
||||||
from app.models.announcement import Announcement
|
from app.models.announcement import Announcement
|
||||||
from app.models.schemas import AnnouncementResponse, AnnouncementListResponse
|
from app.models.schemas import AnnouncementListResponse, AnnouncementResponse
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
@@ -14,11 +15,11 @@ router = APIRouter()
|
|||||||
async def list_announcements(
|
async def list_announcements(
|
||||||
page: int = Query(1, ge=1),
|
page: int = Query(1, ge=1),
|
||||||
page_size: int = Query(20, ge=1, le=100),
|
page_size: int = Query(20, ge=1, le=100),
|
||||||
source_code: Optional[str] = None,
|
source_code: str | None = None,
|
||||||
keyword: Optional[str] = None,
|
keyword: str | None = None,
|
||||||
start_date: Optional[str] = None,
|
start_date: str | None = None,
|
||||||
end_date: Optional[str] = None,
|
end_date: str | None = None,
|
||||||
crawl_mode: Optional[str] = None,
|
crawl_mode: str | None = None,
|
||||||
db: AsyncSession = Depends(get_db),
|
db: AsyncSession = Depends(get_db),
|
||||||
):
|
):
|
||||||
conditions = []
|
conditions = []
|
||||||
@@ -91,12 +92,12 @@ async def get_stats(db: AsyncSession = Depends(get_db)):
|
|||||||
).select_from(Announcement)
|
).select_from(Announcement)
|
||||||
)
|
)
|
||||||
new_count = await db.execute(
|
new_count = await db.execute(
|
||||||
select(func.count()).where(Announcement.is_new == True)
|
select(func.count()).where(Announcement.is_new == True) # noqa: E712
|
||||||
.select_from(Announcement)
|
.select_from(Announcement)
|
||||||
)
|
)
|
||||||
unsent = await db.execute(
|
unsent = await db.execute(
|
||||||
select(func.count()).where(
|
select(func.count()).where(
|
||||||
Announcement.is_sent == False, Announcement.is_new == True
|
Announcement.is_sent == False, Announcement.is_new == True # noqa: E712
|
||||||
).select_from(Announcement)
|
).select_from(Announcement)
|
||||||
)
|
)
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
from fastapi import APIRouter
|
from fastapi import APIRouter
|
||||||
|
|
||||||
from app.api.deps import get_crawl_service
|
from app.api.deps import get_crawl_service
|
||||||
from app.models.schemas import CrawlTriggerRequest
|
from app.models.schemas import CrawlTriggerRequest
|
||||||
|
|
||||||
@@ -34,6 +35,7 @@ async def crawl_status():
|
|||||||
@router.get("/crawl/sources")
|
@router.get("/crawl/sources")
|
||||||
async def crawl_sources():
|
async def crawl_sources():
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
sources = json.loads(settings.announcement_sources)
|
sources = json.loads(settings.announcement_sources)
|
||||||
return {
|
return {
|
||||||
|
|||||||
+3
-2
@@ -1,7 +1,8 @@
|
|||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from app.services.crawl_service import CrawlService
|
|
||||||
from app.crawler.gxgp_spider import GXGPSpider
|
|
||||||
from app.crawler.dahuagov_spider import DahuagovSpider
|
from app.crawler.dahuagov_spider import DahuagovSpider
|
||||||
|
from app.crawler.gxgp_spider import GXGPSpider
|
||||||
|
from app.services.crawl_service import CrawlService
|
||||||
|
|
||||||
|
|
||||||
async def get_db() -> AsyncSession:
|
async def get_db() -> AsyncSession:
|
||||||
|
|||||||
+3
-1
@@ -1,5 +1,7 @@
|
|||||||
from fastapi import APIRouter
|
from fastapi import APIRouter
|
||||||
from app.api import announcements, crawl, wechat, scheduler as scheduler_module
|
|
||||||
|
from app.api import announcements, crawl, wechat
|
||||||
|
from app.api import scheduler as scheduler_module
|
||||||
|
|
||||||
api_router = APIRouter(prefix="/api/v1")
|
api_router = APIRouter(prefix="/api/v1")
|
||||||
api_router.include_router(announcements.router, tags=["announcements"])
|
api_router.include_router(announcements.router, tags=["announcements"])
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
from fastapi import APIRouter
|
from fastapi import APIRouter
|
||||||
from app.scheduler.jobs import scheduler
|
|
||||||
from app.models.schemas import JobResponse
|
from app.models.schemas import JobResponse
|
||||||
|
from app.scheduler.jobs import scheduler
|
||||||
|
|
||||||
router = APIRouter(prefix="/scheduler", tags=["scheduler"])
|
router = APIRouter(prefix="/scheduler", tags=["scheduler"])
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
from fastapi import APIRouter, Request, Response
|
from fastapi import APIRouter, Request, Response
|
||||||
|
|
||||||
from app.wechat.handler import WeChatMessageHandler
|
from app.wechat.handler import WeChatMessageHandler
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|||||||
+3
-3
@@ -1,5 +1,5 @@
|
|||||||
|
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
from typing import List
|
|
||||||
|
|
||||||
|
|
||||||
class Settings(BaseSettings):
|
class Settings(BaseSettings):
|
||||||
@@ -14,7 +14,7 @@ class Settings(BaseSettings):
|
|||||||
|
|
||||||
# 爬虫
|
# 爬虫
|
||||||
crawler_base_url: str = "https://zfcg.gxzf.gov.cn"
|
crawler_base_url: str = "https://zfcg.gxzf.gov.cn"
|
||||||
crawler_keywords: List[str] = ["大化"]
|
crawler_keywords: list[str] = ["大化"]
|
||||||
crawler_max_pages: int = 10
|
crawler_max_pages: int = 10
|
||||||
crawler_timeout: int = 30
|
crawler_timeout: int = 30
|
||||||
crawler_page_size: int = 100
|
crawler_page_size: int = 100
|
||||||
@@ -38,7 +38,7 @@ class Settings(BaseSettings):
|
|||||||
markdown_output_file: str = "onu.md"
|
markdown_output_file: str = "onu.md"
|
||||||
|
|
||||||
# 公告来源(JSON 字符串,从环境变量读取)
|
# 公告来源(JSON 字符串,从环境变量读取)
|
||||||
announcement_sources: str = '{"ZcyAnnouncement1":{"category_id":66485,"name":"采购公告","type":"purchase"},"ZcyAnnouncement2":{"category_id":66485,"name":"结果公告","type":"result"},"ZcyAnnouncement3":{"category_id":66485,"name":"合同公告","type":"contract"},"ZcyAnnouncement4":{"category_id":66485,"name":"更正公告","type":"correction"},"ZcyAnnouncement5":{"category_id":66485,"name":"招标文件预公示","type":"pre_announcement"},"ZcyAnnouncement6":{"category_id":66485,"name":"单一来源公示","type":"single_source"},"ZcyAnnouncement7":{"category_id":66485,"name":"电子卖场公示","type":"electronic_market"},"ZcyAnnouncement10":{"category_id":66485,"name":"履约验收公示","type":"acceptance"},"ZcyAnnouncement11":{"category_id":66485,"name":"工程类公告","type":"engineering"},"ZcyAnnouncement20":{"category_id":66485,"name":"框架协议征集公告","type":"framework_agreement"},"ZcyAnnouncement21":{"category_id":66485,"name":"框架协议入围结果公告","type":"framework_result"},"ZcyAnnouncement23":{"category_id":66485,"name":"框架协议成交结果汇总公告","type":"framework_summary"},"61-266648":{"category_id":66485,"name":"采购意向公开","type":"intention"}}'
|
announcement_sources: str = '{"ZcyAnnouncement1":{"category_id":66485,"name":"采购公告","type":"purchase"},"ZcyAnnouncement2":{"category_id":66485,"name":"结果公告","type":"result"},"ZcyAnnouncement3":{"category_id":66485,"name":"合同公告","type":"contract"},"ZcyAnnouncement4":{"category_id":66485,"name":"更正公告","type":"correction"},"ZcyAnnouncement5":{"category_id":66485,"name":"招标文件预公示","type":"pre_announcement"},"ZcyAnnouncement6":{"category_id":66485,"name":"单一来源公示","type":"single_source"},"ZcyAnnouncement7":{"category_id":66485,"name":"电子卖场公示","type":"electronic_market"},"ZcyAnnouncement10":{"category_id":66485,"name":"履约验收公示","type":"acceptance"},"ZcyAnnouncement11":{"category_id":66485,"name":"工程类公告","type":"engineering"},"ZcyAnnouncement20":{"category_id":66485,"name":"框架协议征集公告","type":"framework_agreement"},"ZcyAnnouncement21":{"category_id":66485,"name":"框架协议入围结果公告","type":"framework_result"},"ZcyAnnouncement23":{"category_id":66485,"name":"框架协议成交结果汇总公告","type":"framework_summary"},"61-266648":{"category_id":66485,"name":"采购意向公开","type":"intention"}}' # noqa: E501
|
||||||
|
|
||||||
|
|
||||||
settings = Settings()
|
settings = Settings()
|
||||||
|
|||||||
+2
-3
@@ -2,7 +2,6 @@ import hashlib
|
|||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import List, Optional
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -12,7 +11,7 @@ class CrawlResult:
|
|||||||
total_count: int = 0
|
total_count: int = 0
|
||||||
new_count: int = 0
|
new_count: int = 0
|
||||||
announcements: list = field(default_factory=list)
|
announcements: list = field(default_factory=list)
|
||||||
error_message: Optional[str] = None
|
error_message: str | None = None
|
||||||
crawled_at: datetime = field(default_factory=datetime.now)
|
crawled_at: datetime = field(default_factory=datetime.now)
|
||||||
duration: float = 0.0
|
duration: float = 0.0
|
||||||
|
|
||||||
@@ -24,7 +23,7 @@ class CrawlResult:
|
|||||||
@dataclass
|
@dataclass
|
||||||
class PipelineConfig:
|
class PipelineConfig:
|
||||||
filter_enabled: bool = True
|
filter_enabled: bool = True
|
||||||
keywords: List[str] = field(default_factory=list)
|
keywords: list[str] = field(default_factory=list)
|
||||||
dedup_enabled: bool = True
|
dedup_enabled: bool = True
|
||||||
notify_mode: str = "filtered"
|
notify_mode: str = "filtered"
|
||||||
mark_sent: bool = False
|
mark_sent: bool = False
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import random
|
import random
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
||||||
from app.crawler.parsers import parse_dahuagov_html
|
from app.crawler.parsers import parse_dahuagov_html
|
||||||
|
|||||||
@@ -2,11 +2,12 @@ import json
|
|||||||
import random
|
import random
|
||||||
import time
|
import time
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import List, Optional
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
||||||
from app.crawler.parsers import parse_gxgp_api_response, extract_pagination
|
from app.crawler.parsers import extract_pagination, parse_gxgp_api_response
|
||||||
|
|
||||||
|
|
||||||
class GXGPSpider(BaseSpider):
|
class GXGPSpider(BaseSpider):
|
||||||
@@ -27,8 +28,8 @@ class GXGPSpider(BaseSpider):
|
|||||||
mark_sent=False,
|
mark_sent=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def crawl(self, sources: Optional[List[str]] = None,
|
async def crawl(self, sources: list[str] | None = None,
|
||||||
max_pages: Optional[int] = None) -> CrawlResult:
|
max_pages: int | None = None) -> CrawlResult:
|
||||||
if max_pages is None:
|
if max_pages is None:
|
||||||
max_pages = settings.crawler_max_pages
|
max_pages = settings.crawler_max_pages
|
||||||
if sources is None:
|
if sources is None:
|
||||||
@@ -88,7 +89,7 @@ class GXGPSpider(BaseSpider):
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def _fetch_page(self, client: httpx.AsyncClient, source_code: str,
|
async def _fetch_page(self, client: httpx.AsyncClient, source_code: str,
|
||||||
category_id: int, page_no: int) -> Optional[dict]:
|
category_id: int, page_no: int) -> dict | None:
|
||||||
payload = {
|
payload = {
|
||||||
"keyword": "",
|
"keyword": "",
|
||||||
"publishDateBegin": "",
|
"publishDateBegin": "",
|
||||||
@@ -102,7 +103,7 @@ class GXGPSpider(BaseSpider):
|
|||||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||||||
"Content-Type": "application/json;charset=UTF-8",
|
"Content-Type": "application/json;charset=UTF-8",
|
||||||
"Origin": self.base_url,
|
"Origin": self.base_url,
|
||||||
"Referer": f"{self.base_url}/site/category?parentId={category_id}&childrenCode={source_code}",
|
"Referer": f"{self.base_url}/site/category?parentId={category_id}&childrenCode={source_code}", # noqa: E501
|
||||||
}
|
}
|
||||||
response = await client.post(
|
response = await client.post(
|
||||||
self.announcement_api, json=payload, headers=headers
|
self.announcement_api, json=payload, headers=headers
|
||||||
|
|||||||
@@ -1,18 +1,18 @@
|
|||||||
import hashlib
|
import hashlib
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any, Dict, List
|
from typing import Any
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
|
||||||
def parse_gxgp_api_response(
|
def parse_gxgp_api_response(
|
||||||
response_data: Dict[str, Any],
|
response_data: dict[str, Any],
|
||||||
source_code: str,
|
source_code: str,
|
||||||
source_name: str,
|
source_name: str,
|
||||||
crawled_at: datetime,
|
crawled_at: datetime,
|
||||||
category_id: int,
|
category_id: int,
|
||||||
) -> List[Dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
if not response_data.get("success"):
|
if not response_data.get("success"):
|
||||||
return []
|
return []
|
||||||
data = response_data.get("result", {}).get("data", {})
|
data = response_data.get("result", {}).get("data", {})
|
||||||
@@ -62,7 +62,7 @@ def parse_gxgp_api_response(
|
|||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
def extract_pagination(response_data: Dict[str, Any]) -> Dict[str, Any]:
|
def extract_pagination(response_data: dict[str, Any]) -> dict[str, Any]:
|
||||||
data = response_data.get("result", {}).get("data", {})
|
data = response_data.get("result", {}).get("data", {})
|
||||||
return {
|
return {
|
||||||
"total": data.get("total", 0),
|
"total": data.get("total", 0),
|
||||||
@@ -75,7 +75,7 @@ def extract_pagination(response_data: Dict[str, Any]) -> Dict[str, Any]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def parse_dahuagov_html(html: str, crawled_at: datetime) -> List[Dict[str, Any]]:
|
def parse_dahuagov_html(html: str, crawled_at: datetime) -> list[dict[str, Any]]:
|
||||||
soup = BeautifulSoup(html, "html.parser")
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
lists = soup.find_all("ul", class_="more-list")
|
lists = soup.find_all("ul", class_="more-list")
|
||||||
if not lists:
|
if not lists:
|
||||||
@@ -131,7 +131,7 @@ def parse_dahuagov_html(html: str, crawled_at: datetime) -> List[Dict[str, Any]]
|
|||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
def _generate_hash(ann: Dict[str, Any]) -> str:
|
def _generate_hash(ann: dict[str, Any]) -> str:
|
||||||
content = (
|
content = (
|
||||||
f"{ann['title']}|{ann['publish_date'].strftime('%Y-%m-%d')}"
|
f"{ann['title']}|{ann['publish_date'].strftime('%Y-%m-%d')}"
|
||||||
f"|{ann['purchase_name']}|{ann['content_url']}|{ann['source_code']}"
|
f"|{ann['purchase_name']}|{ann['content_url']}|{ann['source_code']}"
|
||||||
|
|||||||
+5
-4
@@ -1,9 +1,11 @@
|
|||||||
import logging
|
import logging
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
|
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker, AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||||
|
|
||||||
|
from app.api.router import api_router
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.models.announcement import Base
|
|
||||||
|
|
||||||
engine = create_async_engine(settings.database_url, echo=settings.debug)
|
engine = create_async_engine(settings.database_url, echo=settings.debug)
|
||||||
async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||||
@@ -20,7 +22,7 @@ async def lifespan(app: FastAPI):
|
|||||||
level=getattr(logging, settings.log_level),
|
level=getattr(logging, settings.log_level),
|
||||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||||
)
|
)
|
||||||
from app.scheduler.jobs import start_scheduler, shutdown_scheduler
|
from app.scheduler.jobs import shutdown_scheduler, start_scheduler
|
||||||
start_scheduler()
|
start_scheduler()
|
||||||
yield
|
yield
|
||||||
shutdown_scheduler()
|
shutdown_scheduler()
|
||||||
@@ -35,7 +37,6 @@ app = FastAPI(
|
|||||||
redoc_url=None,
|
redoc_url=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
from app.api.router import api_router
|
|
||||||
app.include_router(api_router)
|
app.include_router(api_router)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import hashlib
|
import hashlib
|
||||||
from datetime import datetime, date
|
from datetime import datetime
|
||||||
from sqlalchemy import String, Boolean, DateTime, Integer, Text, func
|
|
||||||
|
from sqlalchemy import Boolean, DateTime, Integer, String, Text, func
|
||||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||||
|
|
||||||
|
|
||||||
@@ -25,7 +26,7 @@ class Announcement(Base):
|
|||||||
is_sent: Mapped[bool] = mapped_column(Boolean, default=False)
|
is_sent: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||||
keyword_matched: Mapped[bool] = mapped_column(Boolean, default=False)
|
keyword_matched: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
|
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
|
||||||
updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), onupdate=func.now())
|
updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), onupdate=func.now()) # noqa: E501
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def generate_hash(title: str, publish_date: str, purchase_name: str,
|
def generate_hash(title: str, publish_date: str, purchase_name: str,
|
||||||
@@ -36,5 +37,6 @@ class Announcement(Base):
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def source_map() -> dict:
|
def source_map() -> dict:
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
return json.loads(settings.announcement_sources)
|
return json.loads(settings.announcement_sources)
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Optional
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
|
||||||
@@ -29,14 +29,14 @@ class AnnouncementListResponse(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class CrawlTriggerRequest(BaseModel):
|
class CrawlTriggerRequest(BaseModel):
|
||||||
keywords: Optional[list[str]] = None
|
keywords: list[str] | None = None
|
||||||
sources: Optional[list[str]] = None
|
sources: list[str] | None = None
|
||||||
manual: bool = False
|
manual: bool = False
|
||||||
|
|
||||||
|
|
||||||
class CrawlStatusResponse(BaseModel):
|
class CrawlStatusResponse(BaseModel):
|
||||||
running: bool
|
running: bool
|
||||||
last_crawl_time: Optional[datetime] = None
|
last_crawl_time: datetime | None = None
|
||||||
total_sources: int
|
total_sources: int
|
||||||
|
|
||||||
|
|
||||||
@@ -53,4 +53,4 @@ class SourcesResponse(BaseModel):
|
|||||||
class JobResponse(BaseModel):
|
class JobResponse(BaseModel):
|
||||||
id: str
|
id: str
|
||||||
name: str
|
name: str
|
||||||
next_run_time: Optional[str] = None
|
next_run_time: str | None = None
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
import logging
|
import logging
|
||||||
|
|
||||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||||
from app.config import settings
|
|
||||||
from app.api.deps import get_crawl_service
|
from app.api.deps import get_crawl_service
|
||||||
|
from app.config import settings
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
scheduler = AsyncIOScheduler()
|
scheduler = AsyncIOScheduler()
|
||||||
|
|||||||
@@ -1,22 +1,22 @@
|
|||||||
from typing import Dict, List
|
|
||||||
from app.crawler.base import BaseSpider, CrawlResult
|
from app.crawler.base import BaseSpider, CrawlResult
|
||||||
|
|
||||||
|
|
||||||
class CrawlService:
|
class CrawlService:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.spiders: Dict[str, BaseSpider] = {}
|
self.spiders: dict[str, BaseSpider] = {}
|
||||||
|
|
||||||
def register(self, spider: BaseSpider):
|
def register(self, spider: BaseSpider):
|
||||||
self.spiders[spider.name] = spider
|
self.spiders[spider.name] = spider
|
||||||
|
|
||||||
async def run_all(self) -> List[CrawlResult]:
|
async def run_all(self) -> list[CrawlResult]:
|
||||||
results = []
|
results = []
|
||||||
for name, spider in self.spiders.items():
|
for name, spider in self.spiders.items():
|
||||||
result = await spider.crawl()
|
result = await spider.crawl()
|
||||||
results.append(result)
|
results.append(result)
|
||||||
return results
|
return results
|
||||||
|
|
||||||
async def run_spider(self, name: str, **kwargs) -> List[CrawlResult]:
|
async def run_spider(self, name: str, **kwargs) -> list[CrawlResult]:
|
||||||
spider = self.spiders.get(name)
|
spider = self.spiders.get(name)
|
||||||
if spider is None:
|
if spider is None:
|
||||||
return [CrawlResult(
|
return [CrawlResult(
|
||||||
@@ -26,7 +26,7 @@ class CrawlService:
|
|||||||
result = await spider.crawl(**kwargs)
|
result = await spider.crawl(**kwargs)
|
||||||
return [result]
|
return [result]
|
||||||
|
|
||||||
def get_spider_names(self) -> List[str]:
|
def get_spider_names(self) -> list[str]:
|
||||||
return list(self.spiders.keys())
|
return list(self.spiders.keys())
|
||||||
|
|
||||||
def get_pipeline_config(self, name: str):
|
def get_pipeline_config(self, name: str):
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
from datetime import date
|
from datetime import date
|
||||||
from typing import Any, Dict, List, Optional
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
def filter_by_keywords(announcements: List[Dict[str, Any]],
|
def filter_by_keywords(announcements: list[dict[str, Any]],
|
||||||
keywords: List[str]) -> List[Dict[str, Any]]:
|
keywords: list[str]) -> list[dict[str, Any]]:
|
||||||
if not keywords:
|
if not keywords:
|
||||||
return announcements
|
return announcements
|
||||||
|
|
||||||
@@ -17,9 +17,9 @@ def filter_by_keywords(announcements: List[Dict[str, Any]],
|
|||||||
return filtered
|
return filtered
|
||||||
|
|
||||||
|
|
||||||
def filter_by_date(announcements: List[Dict[str, Any]],
|
def filter_by_date(announcements: list[dict[str, Any]],
|
||||||
start_date: Optional[date] = None,
|
start_date: date | None = None,
|
||||||
end_date: Optional[date] = None) -> List[Dict[str, Any]]:
|
end_date: date | None = None) -> list[dict[str, Any]]:
|
||||||
if not start_date and not end_date:
|
if not start_date and not end_date:
|
||||||
return announcements
|
return announcements
|
||||||
|
|
||||||
@@ -42,7 +42,7 @@ def filter_by_date(announcements: List[Dict[str, Any]],
|
|||||||
return filtered
|
return filtered
|
||||||
|
|
||||||
|
|
||||||
def dedup_by_hash(announcements: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
def dedup_by_hash(announcements: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||||
seen = set()
|
seen = set()
|
||||||
result = []
|
result = []
|
||||||
for ann in announcements:
|
for ann in announcements:
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from datetime import datetime
|
from typing import Any
|
||||||
from typing import Any, Dict, List
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.wechat.client import WeChatClient
|
from app.wechat.client import WeChatClient
|
||||||
|
|
||||||
@@ -8,7 +8,7 @@ class NotificationService:
|
|||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.client = WeChatClient()
|
self.client = WeChatClient()
|
||||||
|
|
||||||
async def send(self, announcements: List[Dict[str, Any]]) -> int:
|
async def send(self, announcements: list[dict[str, Any]]) -> int:
|
||||||
if not settings.wechat_enabled:
|
if not settings.wechat_enabled:
|
||||||
return 0
|
return 0
|
||||||
if not announcements:
|
if not announcements:
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
from typing import Any, Dict, List
|
from typing import Any
|
||||||
|
|
||||||
from app.crawler.base import PipelineConfig, PipelineResult
|
from app.crawler.base import PipelineConfig, PipelineResult
|
||||||
from app.services.filter_service import dedup_by_hash
|
from app.services.filter_service import dedup_by_hash
|
||||||
|
|
||||||
@@ -8,7 +9,7 @@ class PostCrawlPipeline:
|
|||||||
self.db = db_session
|
self.db = db_session
|
||||||
self.notify = notification_service
|
self.notify = notification_service
|
||||||
|
|
||||||
async def process(self, announcements: List[Dict[str, Any]],
|
async def process(self, announcements: list[dict[str, Any]],
|
||||||
config: PipelineConfig) -> PipelineResult:
|
config: PipelineConfig) -> PipelineResult:
|
||||||
result = PipelineResult()
|
result = PipelineResult()
|
||||||
|
|
||||||
@@ -47,8 +48,9 @@ class PostCrawlPipeline:
|
|||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
async def _save_to_db(self, announcements: List[Dict[str, Any]]) -> int:
|
async def _save_to_db(self, announcements: list[dict[str, Any]]) -> int:
|
||||||
from sqlalchemy.dialects.postgresql import insert
|
from sqlalchemy.dialects.postgresql import insert
|
||||||
|
|
||||||
from app.models.announcement import Announcement
|
from app.models.announcement import Announcement
|
||||||
|
|
||||||
if not announcements:
|
if not announcements:
|
||||||
@@ -76,13 +78,14 @@ class PostCrawlPipeline:
|
|||||||
await self.db.commit()
|
await self.db.commit()
|
||||||
return result_proxy.rowcount if result_proxy.rowcount >= 0 else len(values)
|
return result_proxy.rowcount if result_proxy.rowcount >= 0 else len(values)
|
||||||
|
|
||||||
async def _send_notifications(self, announcements: List[Dict[str, Any]]) -> int:
|
async def _send_notifications(self, announcements: list[dict[str, Any]]) -> int:
|
||||||
return await self.notify.send(announcements)
|
return await self.notify.send(announcements)
|
||||||
|
|
||||||
async def _mark_sent(self, announcements: List[Dict[str, Any]]) -> int:
|
async def _mark_sent(self, announcements: list[dict[str, Any]]) -> int:
|
||||||
from app.models.announcement import Announcement
|
|
||||||
from sqlalchemy import update
|
from sqlalchemy import update
|
||||||
|
|
||||||
|
from app.models.announcement import Announcement
|
||||||
|
|
||||||
hashes = [a["content_hash"] for a in announcements if a.get("content_hash")]
|
hashes = [a["content_hash"] for a in announcements if a.get("content_hash")]
|
||||||
if not hashes:
|
if not hashes:
|
||||||
return 0
|
return 0
|
||||||
@@ -97,6 +100,6 @@ class PostCrawlPipeline:
|
|||||||
return result.rowcount
|
return result.rowcount
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _match_keywords(announcement: Dict[str, Any], keywords: List[str]) -> bool:
|
def _match_keywords(announcement: dict[str, Any], keywords: list[str]) -> bool:
|
||||||
text = f"{announcement.get('title', '')} {announcement.get('purchase_name', '')}"
|
text = f"{announcement.get('title', '')} {announcement.get('purchase_name', '')}"
|
||||||
return any(kw in text for kw in keywords)
|
return any(kw in text for kw in keywords)
|
||||||
|
|||||||
@@ -1,16 +1,16 @@
|
|||||||
import time
|
import time
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
|
||||||
|
|
||||||
class WeChatClient:
|
class WeChatClient:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self._access_token: Optional[str] = None
|
self._access_token: str | None = None
|
||||||
self._token_expires_at: float = 0
|
self._token_expires_at: float = 0
|
||||||
|
|
||||||
async def _get_access_token(self) -> Optional[str]:
|
async def _get_access_token(self) -> str | None:
|
||||||
now = time.time()
|
now = time.time()
|
||||||
if self._access_token and now < self._token_expires_at:
|
if self._access_token and now < self._token_expires_at:
|
||||||
return self._access_token
|
return self._access_token
|
||||||
|
|||||||
+18
-18
@@ -1,20 +1,20 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
# -*- encoding:utf-8 -*-
|
|
||||||
|
|
||||||
""" 对企业微信发送给企业后台的消息加解密示例代码.
|
""" 对企业微信发送给企业后台的消息加解密示例代码.
|
||||||
@copyright: Copyright (c) 1998-2014 Tencent Inc.
|
@copyright: Copyright (c) 1998-2014 Tencent Inc.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
# ------------------------------------------------------------------------
|
# ------------------------------------------------------------------------
|
||||||
import logging
|
|
||||||
import base64
|
import base64
|
||||||
import random
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import time
|
import logging
|
||||||
import struct
|
import random
|
||||||
from Crypto.Cipher import AES
|
|
||||||
import xml.etree.cElementTree as ET
|
|
||||||
import socket
|
import socket
|
||||||
|
import struct
|
||||||
|
import time
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
|
||||||
|
from Crypto.Cipher import AES
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import ierror
|
import ierror
|
||||||
@@ -29,11 +29,11 @@ except ImportError:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
class FormatException(Exception):
|
class FormatError(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
def throw_exception(message, exception_class=FormatException):
|
def throw_exception(message, exception_class=FormatError):
|
||||||
"""my define raise exception function"""
|
"""my define raise exception function"""
|
||||||
raise exception_class(message)
|
raise exception_class(message)
|
||||||
|
|
||||||
@@ -104,7 +104,7 @@ class XMLParse:
|
|||||||
return resp_xml
|
return resp_xml
|
||||||
|
|
||||||
|
|
||||||
class PKCS7Encoder():
|
class PKCS7Encoder:
|
||||||
"""提供基于PKCS7算法的加解密接口"""
|
"""提供基于PKCS7算法的加解密接口"""
|
||||||
|
|
||||||
block_size = 32
|
block_size = 32
|
||||||
@@ -134,7 +134,7 @@ class PKCS7Encoder():
|
|||||||
return decrypted[:-pad]
|
return decrypted[:-pad]
|
||||||
|
|
||||||
|
|
||||||
class Prpcrypt(object):
|
class Prpcrypt:
|
||||||
"""提供接收和推送给企业微信消息的加解密接口"""
|
"""提供接收和推送给企业微信消息的加解密接口"""
|
||||||
|
|
||||||
def __init__(self, key):
|
def __init__(self, key):
|
||||||
@@ -151,7 +151,7 @@ class Prpcrypt(object):
|
|||||||
"""
|
"""
|
||||||
# 16位随机字符串添加到明文开头
|
# 16位随机字符串添加到明文开头
|
||||||
text = text.encode()
|
text = text.encode()
|
||||||
text = self.get_random_str() + struct.pack("I", socket.htonl(len(text))) + text + receiveid.encode()
|
text = self.get_random_str() + struct.pack("I", socket.htonl(len(text))) + text + receiveid.encode() # noqa: E501
|
||||||
|
|
||||||
# 使用自定义的填充方式对明文进行补位填充
|
# 使用自定义的填充方式对明文进行补位填充
|
||||||
pkcs7 = PKCS7Encoder()
|
pkcs7 = PKCS7Encoder()
|
||||||
@@ -206,14 +206,14 @@ class Prpcrypt(object):
|
|||||||
return str(random.randint(1000000000000000, 9999999999999999)).encode()
|
return str(random.randint(1000000000000000, 9999999999999999)).encode()
|
||||||
|
|
||||||
|
|
||||||
class WXBizMsgCrypt(object):
|
class WXBizMsgCrypt:
|
||||||
# 构造函数
|
# 构造函数
|
||||||
def __init__(self, sToken, sEncodingAESKey, sReceiveId):
|
def __init__(self, sToken, sEncodingAESKey, sReceiveId):
|
||||||
try:
|
try:
|
||||||
self.key = base64.b64decode(sEncodingAESKey + "=")
|
self.key = base64.b64decode(sEncodingAESKey + "=")
|
||||||
assert len(self.key) == 32
|
assert len(self.key) == 32
|
||||||
except:
|
except Exception:
|
||||||
throw_exception("[error]: EncodingAESKey unvalid !", FormatException)
|
throw_exception("[error]: EncodingAESKey unvalid !", FormatError)
|
||||||
# return ierror.WXBizMsgCrypt_IllegalAesKey,None
|
# return ierror.WXBizMsgCrypt_IllegalAesKey,None
|
||||||
self.m_sToken = sToken
|
self.m_sToken = sToken
|
||||||
self.m_sReceiveId = sReceiveId
|
self.m_sReceiveId = sReceiveId
|
||||||
@@ -240,9 +240,9 @@ class WXBizMsgCrypt(object):
|
|||||||
def EncryptMsg(self, sReplyMsg, sNonce, timestamp=None):
|
def EncryptMsg(self, sReplyMsg, sNonce, timestamp=None):
|
||||||
# 将企业回复用户的消息加密打包
|
# 将企业回复用户的消息加密打包
|
||||||
# @param sReplyMsg: 企业号待回复用户的消息,xml格式的字符串
|
# @param sReplyMsg: 企业号待回复用户的消息,xml格式的字符串
|
||||||
# @param sTimeStamp: 时间戳,可以自己生成,也可以用URL参数的timestamp,如为None则自动用当前时间
|
# @param sTimeStamp: 时间戳,可以自己生成,也可以用URL参数的timestamp,如为None则自动用当前时间 # noqa: E501
|
||||||
# @param sNonce: 随机串,可以自己生成,也可以用URL参数的nonce
|
# @param sNonce: 随机串,可以自己生成,也可以用URL参数的nonce # noqa: E501
|
||||||
# sEncryptMsg: 加密后的可以直接回复用户的密文,包括msg_signature, timestamp, nonce, encrypt的xml格式的字符串,
|
# sEncryptMsg: 加密后的可以直接回复用户的密文,包括msg_signature, timestamp, nonce, encrypt的xml格式的字符串, # noqa: E501
|
||||||
# return:成功0,sEncryptMsg,失败返回对应的错误码None
|
# return:成功0,sEncryptMsg,失败返回对应的错误码None
|
||||||
pc = Prpcrypt(self.key)
|
pc = Prpcrypt(self.key)
|
||||||
ret, encrypt = pc.encrypt(sReplyMsg, self.m_sReceiveId)
|
ret, encrypt = pc.encrypt(sReplyMsg, self.m_sReceiveId)
|
||||||
|
|||||||
@@ -1,8 +1,7 @@
|
|||||||
import xml.etree.cElementTree as ET
|
import xml.etree.ElementTree as ET
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
from app.wechat.crypto import WXBizMsgCrypt
|
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
|
from app.wechat.crypto import WXBizMsgCrypt
|
||||||
|
|
||||||
|
|
||||||
class WeChatMessageHandler:
|
class WeChatMessageHandler:
|
||||||
@@ -15,7 +14,7 @@ class WeChatMessageHandler:
|
|||||||
|
|
||||||
def verify_url(
|
def verify_url(
|
||||||
self, msg_signature: str, timestamp: str, nonce: str, echostr: str
|
self, msg_signature: str, timestamp: str, nonce: str, echostr: str
|
||||||
) -> Optional[str]:
|
) -> str | None:
|
||||||
ret, sEchoStr = self.wxcpt.VerifyURL(
|
ret, sEchoStr = self.wxcpt.VerifyURL(
|
||||||
msg_signature, timestamp, nonce, echostr
|
msg_signature, timestamp, nonce, echostr
|
||||||
)
|
)
|
||||||
@@ -33,7 +32,7 @@ class WeChatMessageHandler:
|
|||||||
msg_signature: str,
|
msg_signature: str,
|
||||||
timestamp: str,
|
timestamp: str,
|
||||||
nonce: str,
|
nonce: str,
|
||||||
) -> Optional[ET.Element]:
|
) -> ET.Element | None:
|
||||||
ret, xml_content = self.wxcpt.DecryptMsg(
|
ret, xml_content = self.wxcpt.DecryptMsg(
|
||||||
post_data, msg_signature, timestamp, nonce
|
post_data, msg_signature, timestamp, nonce
|
||||||
)
|
)
|
||||||
@@ -43,16 +42,16 @@ class WeChatMessageHandler:
|
|||||||
|
|
||||||
def encrypt_response(
|
def encrypt_response(
|
||||||
self, response_xml: str, nonce: str, timestamp: str
|
self, response_xml: str, nonce: str, timestamp: str
|
||||||
) -> Optional[str]:
|
) -> str | None:
|
||||||
ret, encrypted = self.wxcpt.EncryptMsg(response_xml, nonce, timestamp)
|
ret, encrypted = self.wxcpt.EncryptMsg(response_xml, nonce, timestamp)
|
||||||
if ret == 0:
|
if ret == 0:
|
||||||
return encrypted
|
return encrypted
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def handle_event(
|
def handle_event(
|
||||||
self, event: str, event_key: Optional[str], from_user: str
|
self, event: str, event_key: str | None, from_user: str
|
||||||
) -> Optional[str]:
|
) -> str | None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def handle_text(self, content: str, from_user: str) -> Optional[str]:
|
def handle_text(self, content: str, from_user: str) -> str | None:
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -1,5 +1,4 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
#########################################################################
|
#########################################################################
|
||||||
# Author: jonyqin
|
# Author: jonyqin
|
||||||
# Created Time: Thu 11 Sep 2014 01:53:58 PM CST
|
# Created Time: Thu 11 Sep 2014 01:53:58 PM CST
|
||||||
|
|||||||
@@ -36,6 +36,9 @@ line-length = 100
|
|||||||
[tool.ruff.lint]
|
[tool.ruff.lint]
|
||||||
select = ["E", "F", "I", "N", "W", "UP"]
|
select = ["E", "F", "I", "N", "W", "UP"]
|
||||||
|
|
||||||
|
[tool.ruff.lint.per-file-ignores]
|
||||||
|
"app/wechat/*.py" = ["N802", "N803", "N806"]
|
||||||
|
|
||||||
[tool.pytest.ini_options]
|
[tool.pytest.ini_options]
|
||||||
asyncio_mode = "auto"
|
asyncio_mode = "auto"
|
||||||
testpaths = ["tests"]
|
testpaths = ["tests"]
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import pytest
|
import pytest
|
||||||
from httpx import AsyncClient, ASGITransport
|
from httpx import ASGITransport, AsyncClient
|
||||||
|
|
||||||
from app.main import app
|
from app.main import app
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
import pytest
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig, PipelineResult
|
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig, PipelineResult
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from unittest.mock import AsyncMock, patch, MagicMock
|
|
||||||
from app.crawler.dahuagov_spider import DahuagovSpider
|
|
||||||
from app.crawler.base import PipelineConfig
|
from app.crawler.base import PipelineConfig
|
||||||
|
from app.crawler.dahuagov_spider import DahuagovSpider
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from unittest.mock import AsyncMock, patch, MagicMock
|
|
||||||
from app.crawler.gxgp_spider import GXGPSpider
|
|
||||||
from app.crawler.base import PipelineConfig
|
from app.crawler.base import PipelineConfig
|
||||||
|
from app.crawler.gxgp_spider import GXGPSpider
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from app.crawler.parsers import (
|
from app.crawler.parsers import (
|
||||||
parse_gxgp_api_response,
|
|
||||||
parse_dahuagov_html,
|
|
||||||
extract_pagination,
|
extract_pagination,
|
||||||
|
parse_dahuagov_html,
|
||||||
|
parse_gxgp_api_response,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from datetime import datetime
|
|
||||||
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
from app.crawler.base import BaseSpider, CrawlResult, PipelineConfig
|
||||||
from app.services.crawl_service import CrawlService
|
from app.services.crawl_service import CrawlService
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from app.services.filter_service import filter_by_keywords, filter_by_date
|
|
||||||
|
from app.services.filter_service import filter_by_date, filter_by_keywords
|
||||||
|
|
||||||
|
|
||||||
def test_filter_by_keywords_match():
|
def test_filter_by_keywords_match():
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
import pytest
|
|
||||||
from unittest.mock import AsyncMock, patch
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
from app.services.notification_service import NotificationService
|
from app.services.notification_service import NotificationService
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
import pytest
|
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from app.services.pipeline import PostCrawlPipeline
|
from unittest.mock import AsyncMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
from app.crawler.base import CrawlResult, PipelineConfig
|
from app.crawler.base import CrawlResult, PipelineConfig
|
||||||
|
from app.services.pipeline import PostCrawlPipeline
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
Reference in New Issue
Block a user