v6ole/GX-gp-notify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

手动模式

Browse Source
This commit is contained in:
v6ole
2026-01-07 17:37:09 +08:00
commit 28c57a040a
57 changed files with 7921 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
old/WeChatService.py
+625
View File
@@ -0,0 +1,625 @@
import requests
import json
import xml.etree.ElementTree as ET
import hashlib
import time
import random
import string
from config import Config
import logging
from Crypto.Cipher import AES
import base64
import socket
import struct
import urllib.parse
import redis
logger = logging.getLogger(__name__)
class WeChatService:
def __init__(self):
self.corpid = Config.WECHAT_CORPID
self.corpsecret = Config.WECHAT_CORPSECRET
self.agentid = Config.WECHAT_AGENTID
self.token = Config.WECHAT_TOKEN
self.encoding_aes_key = Config.WECHAT_ENCODING_AES_KEY
self.access_token = None
self.token_expires_at = 0
# 添加代理API配置
self.use_proxy = getattr(Config, 'USE_WECHAT_PROXY', False)
self.proxy_api_url = getattr(Config, 'WECHAT_PROXY_API_URL', 'https://api.v6ole.top')
logger.info("WeChatService初始化完成")
def _get_redis_connection(self):
"""获取Redis连接"""
try:
redis_conn = redis.Redis(
host=Config.REDIS_HOST,
port=Config.REDIS_PORT,
db=Config.REDIS_DB,
password=Config.REDIS_PASSWORD,
decode_responses=True, # 自动将响应解码为字符串
socket_timeout=5, # 设置超时时间
socket_connect_timeout=5
)
# 测试连接
redis_conn.ping()
return redis_conn
except Exception as e:
logger.error(f"Redis连接失败: {str(e)}")
raise
def get_access_token(self):
"""获取access_token"""
current_time = time.time()
# 如果access_token未过期，直接返回
if self.access_token and current_time < self.token_expires_at:
return self.access_token
try:
# 使用代理API获取access_token
if self.use_proxy:
url = f"{self.proxy_api_url}/cgi-bin/gettoken?corpid={self.corpid}&corpsecret={self.corpsecret}"
logger.info(f"使用代理API获取access_token: {url}")
else:
url = f"https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={self.corpid}&corpsecret={self.corpsecret}"
response = requests.get(url)
result = response.json()
if result.get("errcode") == 0:
self.access_token = result.get("access_token")
# 设置过期时间，提前5分钟过期
expires_in = result.get("expires_in", 7200) - 300
self.token_expires_at = current_time + expires_in
# 转换过期时间为可读格式
expiry_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self.token_expires_at))
logger.info(f"成功获取access_token，过期时间: {expiry_time}")
return self.access_token
elif result.get("errcode") == 60020 and not self.use_proxy:
# 如果是IP限制错误并且未使用代理，尝试使用代理
logger.info("IP受限，尝试使用代理API获取access_token")
self.use_proxy = True
return self.get_access_token()
else:
logger.error(f"获取access_token失败: {result}")
return None
except Exception as e:
logger.error(f"获取access_token异常: {str(e)}")
return None
def verify_url(self, signature, timestamp, nonce, echostr):
"""验证URL有效性"""
try:
# URL解码echostr
echostr = urllib.parse.unquote(echostr)
# 1. 将token、timestamp、nonce、echostr四个参数进行字典序排序
temp_list = [self.token, timestamp, nonce, echostr]
temp_list.sort()
# 2. 将四个参数字符串拼接成一个字符串进行sha1加密
temp_str = ''.join(temp_list)
hash_obj = hashlib.sha1(temp_str.encode('utf-8'))
hash_str = hash_obj.hexdigest()
# 3. 开发者获得加密后的字符串可与signature对比，标识该请求来源于微信
if hash_str == signature:
# 如果验证成功，需要解密echostr
if self.encoding_aes_key:
return self.decrypt_echostr(echostr)
return echostr
else:
logger.error(f"URL验证失败: signature={signature}, hash_str={hash_str}")
return "URL验证失败"
except Exception as e:
logger.error(f"URL验证异常: {str(e)}")
return "URL验证异常"
def decrypt_echostr(self, echostr):
"""解密echostr"""
try:
# 1. 对密文进行base64解码
aes_key = base64.b64decode(self.encoding_aes_key + '=')
encrypted = base64.b64decode(echostr)
# 2. 使用AES解密
cipher = AES.new(aes_key, AES.MODE_CBC, aes_key[:16])
decrypted = cipher.decrypt(encrypted)
# 3. 去除补位字符
unpad = lambda s: s[:-ord(s[len(s)-1:])]
decrypted = unpad(decrypted)
# 4. 去除16位随机字符串
content = decrypted[16:]
xml_len = socket.ntohl(struct.unpack("I", content[:4])[0])
xml_content = content[4:xml_len+4]
# 5. 验证企业ID
received_id = content[xml_len+4:].decode('utf-8')
if received_id != self.corpid:
logger.error(f"企业ID验证失败: received={received_id}, expected={self.corpid}")
return "企业ID验证失败"
return xml_content.decode('utf-8')
except Exception as e:
logger.error(f"解密echostr失败: {str(e)}")
return "解密失败"
def parse_message(self, xml_data):
"""解析接收到的XML消息"""
try:
root = ET.fromstring(xml_data)
msg = {}
for child in root:
msg[child.tag] = child.text
# 如果消息是加密的，需要解密
if 'Encrypt' in msg:
logger.info("消息已加密，开始解密")
decrypted = self.decrypt_message(msg['Encrypt'])
logger.info(f"解密后的消息: {decrypted}")
# 解析解密后的XML
decrypted_root = ET.fromstring(decrypted)
msg = {}
for child in decrypted_root:
msg[child.tag] = child.text
logger.info(f"最终解析的消息: {msg}")
return msg
except Exception as e:
logger.error(f"解析消息失败: {str(e)}")
return None
def decrypt_message(self, encrypted_msg):
"""解密消息"""
try:
# 1. 对密文进行base64解码
aes_key = base64.b64decode(self.encoding_aes_key + '=')
encrypted = base64.b64decode(encrypted_msg)
# 2. 使用AES解密
cipher = AES.new(aes_key, AES.MODE_CBC, aes_key[:16])
decrypted = cipher.decrypt(encrypted)
# 3. 去除补位字符
unpad = lambda s: s[:-ord(s[len(s)-1:])]
decrypted = unpad(decrypted)
# 4. 去除16位随机字符串
content = decrypted[16:]
xml_len = socket.ntohl(struct.unpack("I", content[:4])[0])
xml_content = content[4:xml_len+4]
# 5. 验证企业ID
received_id = content[xml_len+4:].decode('utf-8')
if received_id != self.corpid:
logger.error(f"企业ID验证失败: received={received_id}, expected={self.corpid}")
raise Exception("企业ID验证失败")
return xml_content.decode('utf-8')
except Exception as e:
logger.error(f"解密消息失败: {str(e)}")
raise
def send_text_message(self, content, to_user='@all', to_party='', to_tag=''):
"""发送文本消息"""
max_retries = 3
retry_count = 0
while retry_count < max_retries:
try:
access_token = self.get_access_token()
# 构建消息数据
data = {
"touser": to_user,
"toparty": to_party,
"totag": to_tag,
"msgtype": "text",
"agentid": self.agentid,
"text": {
"content": content
}
}
logger.info(f"发送文本消息: {data}")
# 使用代理API发送消息
if self.use_proxy:
url = f"{self.proxy_api_url}/cgi-bin/message/send?access_token={access_token}"
logger.info(f"使用代理API发送消息: {url}")
else:
url = f"https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={access_token}"
response = requests.post(url, json=data)
result = response.json()
logger.info(f"发送文本消息响应: {result}")
# 如果token过期，重新获取并重试
if result.get('errcode') == 40014:
logger.info("access_token过期，重新获取")
self.access_token = None
self.token_expires_at = 0
retry_count += 1
continue
# 如果是IP限制错误并且未使用代理，尝试使用代理
if result.get('errcode') == 60020 and not self.use_proxy:
logger.info("IP受限，尝试使用代理API发送")
self.use_proxy = True
retry_count += 1
continue
# 如果是其他错误，记录并返回
if result.get('errcode') != 0:
logger.error(f"发送消息失败: {result.get('errmsg')}")
if result.get('errcode') == 301002: # 应用ID不匹配
logger.error("应用ID不匹配，请检查配置")
break
retry_count += 1
continue
return result
except Exception as e:
logger.error(f"发送文本消息失败: {str(e)}")
retry_count += 1
if retry_count >= max_retries:
raise
time.sleep(1) # 等待1秒后重试
return {"errcode": -1, "errmsg": "发送消息失败，已达到最大重试次数"}
def send_markdown_message(self, content, to_user='@all', to_party='', to_tag=''):
"""发送markdown消息"""
try:
access_token = self.get_access_token()
data = {
"touser": to_user,
"toparty": to_party,
"totag": to_tag,
"msgtype": "markdown",
"agentid": self.agentid,
"markdown": {
"content": content
}
}
logger.info(f"发送markdown消息: {data}")
# 使用代理API发送消息
if self.use_proxy:
url = f"{self.proxy_api_url}/cgi-bin/message/send?access_token={access_token}"
logger.info(f"使用代理API发送消息: {url}")
else:
url = f"https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={access_token}"
response = requests.post(url, json=data)
result = response.json()
logger.info(f"发送markdown消息响应: {result}")
# 如果token过期，重新获取并重试
if result.get('errcode') == 40014:
logger.info("access_token过期，重新获取")
self.access_token = None
self.token_expires_at = 0
return self.send_markdown_message(content, to_user, to_party, to_tag)
# 如果是IP限制错误并且未使用代理，尝试使用代理
if result.get('errcode') == 60020 and not self.use_proxy:
logger.info("IP受限，尝试使用代理API发送")
self.use_proxy = True
return self.send_markdown_message(content, to_user, to_party, to_tag)
return result
except Exception as e:
logger.error(f"发送markdown消息失败: {str(e)}")
raise
def get_user_info(self, userid):
"""获取用户信息"""
try:
access_token = self.get_access_token()
# 使用代理API获取用户信息
if self.use_proxy:
url = f"{self.proxy_api_url}/cgi-bin/user/get?access_token={access_token}&userid={userid}"
logger.info(f"使用代理API获取用户信息: {url}")
else:
url = f"https://qyapi.weixin.qq.com/cgi-bin/user/get?access_token={access_token}&userid={userid}"
logger.info(f"获取用户信息: {url}")
response = requests.get(url)
result = response.json()
# 如果是IP限制错误并且未使用代理，尝试使用代理
if result.get('errcode') == 60020 and not self.use_proxy:
logger.info("IP受限，尝试使用代理API获取用户信息")
self.use_proxy = True
return self.get_user_info(userid)
logger.info(f"获取用户信息响应: {result}")
return result
except Exception as e:
logger.error(f"获取用户信息失败: {str(e)}")
return {"errcode": -1, "errmsg": str(e)}
def generate_temp_token(self, user_id, user_name=None):
"""生成临时访问令牌"""
try:
# 生成随机令牌
token = ''.join(random.choices(string.ascii_letters + string.digits, k=32))
# 存储令牌信息到Redis
token_data = {
'user_id': user_id,
'name': user_name or '微信用户',
'created_at': int(time.time())
}
# 使用Redis存储令牌，设置过期时间
key = f"temp_token:{token}"
redis_conn = self._get_redis_connection()
redis_conn.setex(
key,
Config.REDIS_TEMP_TOKEN_EXPIRE,
json.dumps(token_data)
)
# 验证令牌是否成功存储
stored_data = redis_conn.get(key)
if not stored_data:
logger.error("令牌存储失败")
return None
logger.info(f"成功生成临时令牌: {token}, 存储数据: {stored_data}")
return token
except Exception as e:
logger.error(f"生成临时令牌失败: {str(e)}")
return None
def verify_temp_token(self, token):
"""验证临时访问令牌"""
try:
# 从Redis中获取令牌信息
key = f"temp_token:{token}"
redis_conn = self._get_redis_connection()
token_info = redis_conn.get(key)
if not token_info:
logger.error(f"临时令牌不存在或已过期: {token}")
return None
# 解析令牌信息
token_data = json.loads(token_info)
user_id = token_data.get('user_id')
if not user_id:
logger.error(f"临时令牌中未找到用户ID: {token_info}")
return None
logger.info(f"临时令牌验证成功: {token}, 用户信息: {token_data}")
return token_data
except Exception as e:
logger.error(f"验证临时令牌失败: {str(e)}")
return None
def send_card_message(self, title, description, url, to_user):
"""发送卡片消息"""
try:
# 获取access_token
access_token = self.get_access_token()
if not access_token:
logger.error("获取access_token失败")
return None
# 构建消息内容
data = {
"touser": to_user,
"toparty": "",
"totag": "",
"msgtype": "textcard",
"agentid": self.agentid,
"textcard": {
"title": title,
"description": description,
"url": url,
"btntxt": "查看详情"
}
}
logger.info(f"发送卡片消息: {data}")
# 使用代理API发送消息
if self.use_proxy:
api_url = f"{self.proxy_api_url}/cgi-bin/message/send?access_token={access_token}"
logger.info(f"使用代理API发送消息: {api_url}")
else:
api_url = f"https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={access_token}"
# 发送消息
response = requests.post(api_url, json=data)
result = response.json()
# 如果是IP限制错误并且未使用代理，尝试使用代理
if result.get('errcode') == 60020 and not self.use_proxy:
logger.info("IP受限，尝试使用代理API发送")
self.use_proxy = True
return self.send_card_message(title, description, url, to_user)
if result.get('errcode') == 0:
logger.info(f"发送卡片消息成功: {result}")
return result
else:
logger.error(f"发送卡片消息失败: {result}")
return None
except Exception as e:
logger.error(f"发送卡片消息异常: {str(e)}")
return None
def create_menu(self):
"""创建应用菜单"""
try:
# 获取access_token
access_token = self.get_access_token()
if not access_token:
logger.error("获取access_token失败")
return None
# 菜单配置
menu_data = {
"button": [
{
"name": "设备查询",
"sub_button": [
{
"type": "click",
"name": "在线统计",
"key": "online"
},
{
"type": "click",
"name": "设备状态",
"key": "status"
}
]
},
{
"name": "设备管理",
"sub_button": [
{
"type": "click",
"name": "业务下发",
"key": "deploy"
},
{
"type": "view",
"name": "设备管理",
"url": f"{Config.BASE_URL}/devices"
},
{
"type": "click",
"name": "序列修复",
"key": "sequence_fix"
}
]
},
{
"type": "click",
"name": "帮助",
"key": "help"
}
]
}
logger.info(f"创建菜单: {menu_data}")
# 使用代理API创建菜单
if self.use_proxy:
api_url = f"{self.proxy_api_url}/cgi-bin/menu/create?access_token={access_token}&agentid={self.agentid}"
logger.info(f"使用代理API创建菜单: {api_url}")
else:
api_url = f"https://qyapi.weixin.qq.com/cgi-bin/menu/create?access_token={access_token}&agentid={self.agentid}"
# 发送请求
response = requests.post(api_url, json=menu_data)
result = response.json()
# 如果是IP限制错误并且未使用代理，尝试使用代理
if result.get('errcode') == 60020 and not self.use_proxy:
logger.info("IP受限，尝试使用代理API创建菜单")
self.use_proxy = True
return self.create_menu()
if result.get('errcode') == 0:
logger.info(f"创建菜单成功: {result}")
return result
else:
logger.error(f"创建菜单失败: {result}")
return None
except Exception as e:
logger.error(f"创建菜单异常: {str(e)}")
return None
def delete_menu(self):
"""删除应用菜单"""
try:
# 获取access_token
access_token = self.get_access_token()
if not access_token:
logger.error("获取access_token失败")
return None
# 使用代理API删除菜单
if self.use_proxy:
api_url = f"{self.proxy_api_url}/cgi-bin/menu/delete?access_token={access_token}&agentid={self.agentid}"
logger.info(f"使用代理API删除菜单: {api_url}")
else:
api_url = f"https://qyapi.weixin.qq.com/cgi-bin/menu/delete?access_token={access_token}&agentid={self.agentid}"
# 发送请求
response = requests.get(api_url)
result = response.json()
# 如果是IP限制错误并且未使用代理，尝试使用代理
if result.get('errcode') == 60020 and not self.use_proxy:
logger.info("IP受限，尝试使用代理API删除菜单")
self.use_proxy = True
return self.delete_menu()
if result.get('errcode') == 0:
logger.info(f"删除菜单成功: {result}")
return result
else:
logger.error(f"删除菜单失败: {result}")
return None
except Exception as e:
logger.error(f"删除菜单异常: {str(e)}")
return None
def get_menu(self):
"""获取应用菜单"""
try:
# 获取access_token
access_token = self.get_access_token()
if not access_token:
logger.error("获取access_token失败")
return None
# 使用代理API获取菜单
if self.use_proxy:
api_url = f"{self.proxy_api_url}/cgi-bin/menu/get?access_token={access_token}&agentid={self.agentid}"
logger.info(f"使用代理API获取菜单: {api_url}")
else:
api_url = f"https://qyapi.weixin.qq.com/cgi-bin/menu/get?access_token={access_token}&agentid={self.agentid}"
# 发送请求
response = requests.get(api_url)
result = response.json()
# 如果是IP限制错误并且未使用代理，尝试使用代理
if result.get('errcode') == 60020 and not self.use_proxy:
logger.info("IP受限，尝试使用代理API获取菜单")
self.use_proxy = True
return self.get_menu()
if result.get('errcode') == 0:
logger.info(f"获取菜单成功: {result}")
return result
else:
logger.error(f"获取菜单失败: {result}")
return None
except Exception as e:
logger.error(f"获取菜单异常: {str(e)}")
return None
old/config.yaml
+104
View File
@@ -0,0 +1,104 @@
# 广西政府采购网公告监控系统配置文件
# 复制此文件为 config.yaml 并修改相应配置
# 调试模式
debug: false
# 日志配置
log_level: INFO
log_file: logs/gx_gp_monitor.log
# 爬虫配置
crawler:
base_url: "https://zfcg.gxzf.gov.cn"
timeout: 30 # 请求超时时间（秒）
max_retries: 3 # 最大重试次数
retry_delay: 1.0 # 重试初始延迟
max_retry_delay: 60.0 # 重试最大延迟
backoff_factor: 2.0 # 退避因子
user_agents: # User-Agent列表
- "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"
proxies: [] # 代理列表
request_delay: 1.0 # 请求间延迟
request_delay_max: 3.0 # 请求间最大延迟
keyword: ["大化", "信息化"] # 关键词筛选（支持多个关键词）
start_date: "" # 开始日期 (YYYY-MM-DD)
end_date: "" # 结束日期 (YYYY-MM-DD)
max_pages: 10 # 最大页数
page_size: 100 # 每页大小
# 企业微信通知配置
wechat_app:
enabled: true # 是否启用企业微信通知
corp_id: "ww69e8e44636f47780" # 企业ID
agent_id: "1000007" # 应用ID
secret: "SmelCwKFoL0E9ATWFzr-w7gsfXBTN72lT1UqnNd0HpI" # 应用Secret
token: "DmvL98cAF6x9CFtQZwqD2emGL8S7HxA" # Token
encoding_aes_key: "yAc4OoSCP92YTefHXYfw27WeG9oF11W9d6nw6QYlU3D" # 消息加密Key
port: 18001 # 服务端口
host: "0.0.0.0" # 服务主机
debug: false # 调试模式
# 数据库配置
DB_NAME=gx-gp-notify
DB_USER=gx-gp-notify
DB_PASSWORD=MA6RBX4F6Bd5DGmw
DB_HOST=10.10.10.14
DB_PORT=5432
# 公告来源配置
sources:
ZcyAnnouncement1:
category_id: 66485
name: "采购公告"
type: "purchase"
ZcyAnnouncement2:
category_id: 66485
name: "结果公告"
type: "result"
ZcyAnnouncement3:
category_id: 66485
name: "合同公告"
type: "contract"
ZcyAnnouncement4:
category_id: 66485
name: "更正公告"
type: "correction"
ZcyAnnouncement5:
category_id: 66485
name: "招标文件预公示"
type: "pre_announcement"
ZcyAnnouncement6:
category_id: 66485
name: "单一来源公示"
type: "single_source"
ZcyAnnouncement7:
category_id: 66485
name: "电子卖场公示"
type: "electronic_market"
ZcyAnnouncement10:
category_id: 66485
name: "履约验收公示"
type: "acceptance"
ZcyAnnouncement11:
category_id: 66485
name: "工程类公告"
type: "engineering"
ZcyAnnouncement20:
category_id: 66485
name: "框架协议征集公告"
type: "framework_agreement"
ZcyAnnouncement21:
category_id: 66485
name: "框架协议入围结果公告"
type: "framework_result"
ZcyAnnouncement23:
category_id: 66485
name: "框架协议成交结果汇总公告"
type: "framework_summary"
"61-266648":
category_id: 66485
name: "采购意向公开"
type: "intention"
old/main/Main_script.py
Executable
+44
View File
@@ -0,0 +1,44 @@
import web_crawler as webc
"""脚本运行指南"""
# 整个程序的启动需点击右上角的绿色三角按钮(或使用快捷组合键Shift+F10快捷启动)，注意！在启动前，请先在按钮的左侧选择“当前文件”再执行。
# 注意！！！请不要在打开公告数据导入的目标excel文件时启动该爬虫程序，程序无法对正在运行的进程文件进行修改。务必在爬虫程序启动并将数据成功导入目标excel文件后再打开并查看目标excel文件
"""脚本结果读取指南"""
# 这个脚本旨在爬取广西政府采购网的多个公告栏目的公告信息。
#该脚本在运行结束之后会返回以下结果：<某专栏> 新增 xxx 条公告，已保存至excel文件中，建议确认是否有新增公告后再查看excel文件
#每次爬取之后会把数据导出至指定excel表格，表格将会把数据按时间倒序的方式排列公告数据，同时会把属于今天的公告数据标红。excel文件默认为桌面的政府采购公告.xlsx('D:/Document/政府采购公告.xlsx')
"""脚本参数修改指南"""
# 若没有创建相应的excel表格文件，无需担心，程序会先检测是否有对应的excel文件，若不存在该文件，程序便会自动生成。
# excel表格的存取路径可以在utils.py中修改。请点击左侧的"项目"(或使用快捷组合键Alt+1打开项目栏)，选择utils.py并打开。可以在这个文件中修改某些参数。
# 若需要修改公告信息的筛选条件以及启动爬虫程序的代理列表，请点击左侧的"项目"(或使用快捷组合键Alt+1打开项目栏)，选择utils.py并打开。可以在这个文件中修改这些参数。
# 我已经设置了一个定时器，在每天的8:40、14:40、18:00这三个时间段，定时器会启动爬虫程序开始爬取网站。如果需要修改时间段，可以在代码的下半部分修改
# 修改参数以及代码片段后请重新启动程序。
# """脚本定时启动"""
# def job():
# print(f"任务执行于 {datetime.now()}")
# #调用爬取函数
# webc.web_crawler()
# print(f"任务完成于 {datetime.now()}")
#
# # 安排任务在每天的8:40、14:40、18:00执行
# # 此处可以修改时间段，只需修改括号内的时间即可，格式参照原先括号里的数即可，如需额外增加时间段，请复制代码：schedule.every().day.at("时间段").do(job)并粘贴到下方，可增加任意数量的时间段
# schedule.every().day.at("08:37").do(job)
# schedule.every().day.at("11:55").do(job)
# schedule.every().day.at("14:45").do(job)
# schedule.every().day.at("18:00").do(job)
#
# print("脚本定时任务已启动...")
# while True:
# schedule.run_pending()
# time.sleep(1)
"""脚本单次启动"""
def main():
print("脚本单次任务已启动...")
webc.web_crawler()
if __name__ == "__main__":
main()
old/main/utils.py
Executable
+50
View File
@@ -0,0 +1,50 @@
# 定义公告导出的目标excel文件路径
FILE_PATH = "D:/Document/政府采购公告.xlsx"
# 初始化excel工作表头
HEADER = ["标题", "发布时间", "发布单位", "内容链接", "来源栏目"]
# 初始化工作表数量以及名称
sheet_names = [
"采购公告", "招标文件预公示", "采购意向公开", "结果公告", "合同公告",
"更正公告", "单一来源公示", "电子卖场公示", "履约验收公示",
"工程类公告", "框架协议征集公告", "框架协议入围结果公告",
"框架协议成交结果汇总公告", "其他"
]
# === 初始化筛选条件 ===
KEYWORD ="大化" #筛选关键词
START_DATE ="2025-12-01" #筛选最早发布日期
END_DATE ="2026-01-31" #筛选最晚发布日期
# 代理列表
PROXIES = [
#若要添加代理，请先测试代理是否能正常连接，否则请将代理置空
#"http://10.10.1.10:3218", #此为无效代理,仅作示例
#"http://user:pass@10.10.1.10:8080", #此为无效代理,仅作示例
# 带认证的代理
# 添加更多代理...
]
USER_AGENTS = [
#此为用户代理,根据实际情况修改
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15",
# 添加更多 User-Agent...
]
# 公告来源链接
SOURCE_URL = {
"ZcyAnnouncement2":[66485,"结果公告"],# 结果公告
"ZcyAnnouncement3":[66485,"合同公告"],# 合同公告
"ZcyAnnouncement4":[66485,"更正公告"],# 更正公告
"ZcyAnnouncement6":[66485,"单一来源公示"],# 单一来源公示
"ZcyAnnouncement7":[66485,"电子卖场公示"],# 电子卖场公示
"ZcyAnnouncement10":[66485,"履约验收公示"],# 履约验收公示
"ZcyAnnouncement11":[66485,"工程类公告"],# 工程类公告
"ZcyAnnouncement20":[66485,"框架协议征集公告"],# 框架协议征集公告
"ZcyAnnouncement21":[66485,"框架协议入围结果公告"],# 框架协议入围结果公告
"ZcyAnnouncement23":[66485,"框架协议成交结果汇总公告"],# 框架协议成交结果汇总公告
"ZcyAnnouncement1":[66485,"采购公告"],# 采购公告
"ZcyAnnouncement5":[66485,"招标文件预公示"],# 招标文件预公示
"61-266648":[66485,"采购意向公开"]# 采购意向公开
}
old/main/web_crawler.py
Executable
+191
View File
@@ -0,0 +1,191 @@
import requests
import time
from datetime import datetime
import random
from fake_useragent import UserAgent
import logging
import write_to_excel as wte
import utils as Utils
# === 获取筛选条件 ===
KEYWORD =Utils.KEYWORD #关键词筛选
START_DATE =Utils.START_DATE
END_DATE =Utils.END_DATE
#获取公告来源链接
SOURCE_URL = Utils.SOURCE_URL
# === 反扒配置 ===
#获取代理列表
PROXIES = Utils.PROXIES
USER_AGENTS = Utils.USER_AGENTS
MAX_RETRIES = 3 # 最大重连次数
DELAY_MIN, DELAY_MAX = 1, 3 # 随机延迟范围（秒）
# === 日志配置 ===
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
def get_random_user_agent():
"""随机选择 User-Agent"""
return random.choice(USER_AGENTS)
def get_random_proxy():
"""随机选择代理"""
if not PROXIES:
return None
proxy = random.choice(PROXIES)
return {"http": proxy, "https": proxy}
def check_sensitive_words(user_agent,payload,category_code,childrencode):
"""敏感词检查请求"""
# === 接口地址 ===
url = "https://zfcg.gxzf.gov.cn/portal/sensitiveWords/check"
headers = {
"User-Agent": user_agent,
"Content-Type": "application/json;charset=UTF-8",
"Origin": "https://zfcg.gxzf.gov.cn",
"Referer": f"https://zfcg.gxzf.gov.cn/site/category?parentId={category_code}&childrenCode={childrencode}",
"Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64"
}
response = requests.post(url, json=payload, headers=headers)
# print("这是敏感词检查请求返回的响应预览",response.json())
return response.json()
def get_announcements(category_code, childrencode,page_no=1, session=None):
"""请求接口获取公告列表函数(带反扒机制)"""
if session is None:
session = requests.Session()
payload = {
# 接口的请求参数
"keyword": KEYWORD, #关键词筛选，此参数即为搜索框中的关键词输入值
"publishDateBegin": START_DATE, #最早日期筛选
"publishDateEnd": END_DATE, #最晚日期筛选
"pageNo": page_no, #页码数
"pageSize": 15, #页容量
"categoryCode": category_code, # 该参数指定当前查找的公告栏目
"_t": int(time.time() * 1000) # 动态时间戳
}
# 先执行敏感词检查
user_agent=get_random_user_agent()
check_response = check_sensitive_words(user_agent,payload, category_code, childrencode)
if not check_response.get("success", False):
logging.warning("敏感词检查失败")
return None
# 再执行公告数据请求
# === 接口地址 ===
api_url = "https://zfcg.gxzf.gov.cn/portal/category"
headers = {
"User-Agent": user_agent,
"Content-Type": "application/json;charset=UTF-8",
"Origin": "https://zfcg.gxzf.gov.cn",
"Referer": f"https://zfcg.gxzf.gov.cn/site/category?parentId={category_code}&childrenCode={childrencode}",
"Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64" # 如果需要登录
}
#若添加了至少1条代理，则尝试使用代理
if PROXIES:
proxy = 'Default_value'
else:
proxy=None
for retry in range(MAX_RETRIES):
try:
# 尝试使用代理，失败则切换为无代理
if proxy :
proxy = get_random_proxy()
if proxy:
logging.info(f"使用代理: {proxy['http']}")
else:
logging.info("未使用代理")
response = session.post(
api_url,
json=payload,
headers=headers,
proxies=proxy,
timeout=10
)
if response.status_code == 200:
data = response.json()
if data.get("success",True):
return data
else:
logging.warning(f"接口返回失败: {data.get('error', '未知错误')}")
else:
logging.warning(f"请求失败，状态码: {response.status_code}")
except requests.exceptions.ProxyError as pe:
# 代理错误时记录并跳过，继续无代理请求
logging.error(f"代理错误: {pe}. 切换为无代理模式")
proxy = None # 下次请求不使用代理
except Exception as e:
logging.error(f"请求异常: {e}")
# 重试前等待
delay = random.uniform(DELAY_MIN, DELAY_MAX)
logging.info(f"第 {retry + 1}/{MAX_RETRIES} 次重试，等待 {delay:.1f} 秒...")
time.sleep(delay)
return None # 所有重试失败
def parse_data(data,category_code,source_name):
"""解析公告数据函数"""
results = []
for item in data["result"]["data"]["data"]:
results.append({
"标题": item["title"],
"发布时间": datetime.fromtimestamp(int(item["publishDate"]) / 1000).strftime("%Y-%m-%d"),
"发布单位": item["purchaseName"],
"内容链接": f"https://zfcg.gxzf.gov.cn/site/detail?parentId={category_code}&articleId={item['articleId']}",
"来源栏目":source_name
})
return results
#爬虫主函数
def web_crawler():
session = requests.Session()
# print("这是本地会话存储：",session)
# 初始化引用值为信息公告,先爬取信息公告栏目的公告数据
childrencode = "ZcyAnnouncement"
# 爬取代码主体，外循环为来源栏目的循环，即遍历需要爬取的所有来源栏目
for key in SOURCE_URL:
page = 1
all_results = []
# print("这是正在爬取的栏目", SOURCE_URL[key][1])
# print("这是正在爬取的栏目的目录码", str(key))
# 判断正确的引用值
# 内循环主体即为在符合筛选条件的公告列表中遍历所有公告，爬取每个公告需要的指定数据。
while True:
# 这是分割线
# logging.info("爬虫日志分割线————————————————————————————————————————————————爬虫日志分割线")
# logging.info("爬虫日志分割线————————————————————————————————————————————————爬虫日志分割线")
# logging.info("爬虫日志分割线————————————————————————————————————————————————爬虫日志分割线")
# logging.info(f"正在爬取第 {page} 页...")
response = get_announcements(str(key), childrencode, page,session)
if response["result"]["data"]["empty"] or not response["result"]["data"]["data"]:
break
# logging.info(f"请求后共返回 {response["result"]["data"]["total"]} 条公告")
current_data = parse_data(response, SOURCE_URL[key][0], SOURCE_URL[key][1])
if not current_data:
break
# logging.info(f"解析后共爬取 {len(current_data)} 条公告")
all_results.extend(current_data)
page += 1
# 随机延迟
delay = random.uniform(DELAY_MIN, DELAY_MAX)
time.sleep(delay)
# print(f"从 {SOURCE_URL[key][1]} 共爬取 {len(all_results)} 条公告")
# 调用excel读写函数，将公告数据筛选后保存到Excel中
wte.write_to_excel(all_results,SOURCE_URL[key][1])
# 这是分割线
print("公告栏目分割线————————————————————————————————————————————————公告栏目分割线")
print("公告栏目分割线————————————————————————————————————————————————公告栏目分割线")
print("公告栏目分割线————————————————————————————————————————————————公告栏目分割线")
old/main/write_to_excel.py
Executable
+84
View File
@@ -0,0 +1,84 @@
import pandas as pd
from datetime import datetime
from openpyxl.styles import Font, Alignment, PatternFill, Border, Side
import schedule
import time
import utils as Utils
# 获取excel文件路径和excel的工作表头
FILE_PATH = Utils.FILE_PATH
HEADER = Utils.HEADER
# 获取工作表数量以及名称
sheet_names = Utils.sheet_names
def write_to_excel(results,source=""):
# 检查文件是否存在，如果不存在则创建一个新的Excel文件
try:
df_existing = pd.read_excel(FILE_PATH, sheet_name=None)
except FileNotFoundError:
df_existing = {name: pd.DataFrame(columns=HEADER) for name in sheet_names}
today_str = datetime.now().strftime('%Y-%m-%d')
aditional_item=0 # 记录本次爬取新增的公告数量
# 将爬取的公告对号入座填充至excel文件的工作表中
for result in results:
sheet_name = result["来源栏目"] if result["来源栏目"] in sheet_names else "其他"
df_sheet = df_existing[sheet_name]
# 检查是否已存在相同内容链接的记录
if not df_sheet[df_sheet["内容链接"] == result["内容链接"]].empty:
continue
# 添加新记录
new_row = pd.DataFrame([result])
df_sheet = pd.concat([df_sheet, new_row], ignore_index=True)
aditional_item+=1 # 若有新增公告，则加1
# 按时间降序排列
df_sheet.sort_values(by='发布时间', ascending=False, inplace=True)
# 更新现有数据
df_existing[sheet_name] = df_sheet
# 写入Excel文件
with pd.ExcelWriter(FILE_PATH, engine='openpyxl') as writer:
for sheet_name, df in df_existing.items():
df.to_excel(writer, sheet_name=sheet_name, index=False)
# 获取工作表对象
worksheet = writer.sheets[sheet_name]
# 设置列宽
for col_idx, column_width in enumerate([40, 15, 20, 30, 20]):
worksheet.column_dimensions[chr(65 + col_idx)].width = column_width
# 创建红色填充
red_fill = PatternFill(start_color="FF0000", end_color="FF0000", fill_type="solid")
# 创建边框样式
thin_border = Border(left=Side(style='thin'),
right=Side(style='thin'),
top=Side(style='thin'),
bottom=Side(style='thin'))
# 应用样式
for row in worksheet.iter_rows(min_row=1, max_col=len(HEADER), max_row=worksheet.max_row):
publish_date = row[1].value if len(row) > 1 else None
# print("这是publish_date",publish_date)
# 若公告发布日期为今天，则将其单元格背景填充至红色
if publish_date == today_str:
for cell in row:
cell.fill = red_fill
for idx, cell in enumerate(row):
alignment = Alignment(horizontal='left' if idx < 4 else 'general', vertical='top', wrap_text=True)
cell.alignment = alignment
cell.border = thin_border
print(f"<{source}> 新增 {aditional_item} 条公告，已保存至excel文件中")