v6ole
1f18d2ec87
- 新增 chinese_holidays 表，通过 timor.tech API 同步节假日数据 - 修正 holiday 字段解读：holiday=true → 休息日，holiday=false → 调休工作日 - 工作日 8:00-22:00 每小时爬取，周末/节假日/夜间自动跳过 - 新增 /api/v1/holidays/sync 和 /api/v1/holidays/today 接口 - 企微菜单新增「同步节假日」按钮，支持手动触发同步
82 lines
2.4 KiB
Python
82 lines
2.4 KiB
Python
import logging
|
|
from datetime import datetime
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
|
from apscheduler.triggers.cron import CronTrigger
|
|
|
|
from app.api.deps import get_db, get_crawl_service
|
|
from app.config import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
scheduler = AsyncIOScheduler()
|
|
|
|
NIGHT_START = 22 # 22:00
|
|
NIGHT_END = 8 # 08:00
|
|
TZ = ZoneInfo("Asia/Shanghai")
|
|
|
|
|
|
def _is_night_time() -> bool:
|
|
"""22:00 ~ 次日 08:00 夜间时段"""
|
|
current = datetime.now(TZ).hour
|
|
return current >= NIGHT_START or current < NIGHT_END
|
|
|
|
|
|
async def _should_skip() -> bool:
|
|
"""检查是否应该跳过爬取"""
|
|
if _is_night_time():
|
|
logger.info("夜间时段 (22:00-08:00)，跳过爬取")
|
|
return True
|
|
|
|
from app.services.holiday_service import is_workday
|
|
async for db in get_db():
|
|
if not await is_workday(db):
|
|
logger.info("非工作日，跳过爬取")
|
|
return True
|
|
return False
|
|
|
|
|
|
async def scheduled_crawl():
|
|
if await _should_skip():
|
|
return
|
|
|
|
logger.info("开始定时爬取任务")
|
|
service = get_crawl_service()
|
|
for name in service.get_spider_names():
|
|
try:
|
|
results = await service.run_spider(name)
|
|
for r in results:
|
|
if not r.success:
|
|
logger.error(f"Spider {name} 失败: {r.error_message}")
|
|
elif r.pipeline_result:
|
|
logger.info(
|
|
f"Spider {name}: 抓取{r.total_count}条, "
|
|
f"新增{r.pipeline_result.stored}条, "
|
|
f"通知{r.pipeline_result.notified}条"
|
|
)
|
|
else:
|
|
logger.info(f"Spider {name}: 抓取{r.total_count}条 (无新增)")
|
|
except Exception as e:
|
|
logger.error(f"Spider {name} 异常: {e}")
|
|
logger.info("定时爬取任务完成")
|
|
|
|
|
|
def start_scheduler():
|
|
if not settings.scheduler_enabled:
|
|
return
|
|
trigger = CronTrigger.from_crontab(settings.scheduler_cron, timezone=TZ)
|
|
scheduler.add_job(
|
|
scheduled_crawl,
|
|
trigger=trigger,
|
|
id="scheduled_crawl",
|
|
name="定时爬取",
|
|
)
|
|
scheduler.start()
|
|
logger.info(f"APScheduler 已启动 (cron: {settings.scheduler_cron})")
|
|
|
|
|
|
def shutdown_scheduler():
|
|
if scheduler.running:
|
|
scheduler.shutdown(wait=False)
|
|
logger.info("APScheduler 已停止")
|