手动模式
This commit is contained in:
Executable
+44
@@ -0,0 +1,44 @@
|
||||
import web_crawler as webc
|
||||
|
||||
"""脚本运行指南"""
|
||||
# 整个程序的启动需点击右上角的绿色三角按钮(或使用快捷组合键Shift+F10快捷启动)，注意！在启动前，请先在按钮的左侧选择“当前文件”再执行。
|
||||
# 注意！！！请不要在打开公告数据导入的目标excel文件时启动该爬虫程序，程序无法对正在运行的进程文件进行修改。务必在爬虫程序启动并将数据成功导入目标excel文件后再打开并查看目标excel文件
|
||||
|
||||
"""脚本结果读取指南"""
|
||||
# 这个脚本旨在爬取广西政府采购网的多个公告栏目的公告信息。
|
||||
#该脚本在运行结束之后会返回以下结果：<某专栏> 新增 xxx 条公告，已保存至excel文件中，建议确认是否有新增公告后再查看excel文件
|
||||
#每次爬取之后会把数据导出至指定excel表格，表格将会把数据按时间倒序的方式排列公告数据，同时会把属于今天的公告数据标红。excel文件默认为桌面的政府采购公告.xlsx('D:/Document/政府采购公告.xlsx')
|
||||
|
||||
"""脚本参数修改指南"""
|
||||
# 若没有创建相应的excel表格文件，无需担心，程序会先检测是否有对应的excel文件，若不存在该文件，程序便会自动生成。
|
||||
# excel表格的存取路径可以在utils.py中修改。请点击左侧的"项目"(或使用快捷组合键Alt+1打开项目栏)，选择utils.py并打开。可以在这个文件中修改某些参数。
|
||||
# 若需要修改公告信息的筛选条件以及启动爬虫程序的代理列表，请点击左侧的"项目"(或使用快捷组合键Alt+1打开项目栏)，选择utils.py并打开。可以在这个文件中修改这些参数。
|
||||
# 我已经设置了一个定时器，在每天的8:40、14:40、18:00这三个时间段，定时器会启动爬虫程序开始爬取网站。如果需要修改时间段，可以在代码的下半部分修改
|
||||
# 修改参数以及代码片段后请重新启动程序。
|
||||
|
||||
# """脚本定时启动"""
|
||||
# def job():
|
||||
# print(f"任务执行于 {datetime.now()}")
|
||||
# #调用爬取函数
|
||||
# webc.web_crawler()
|
||||
# print(f"任务完成于 {datetime.now()}")
|
||||
#
|
||||
# # 安排任务在每天的8:40、14:40、18:00执行
|
||||
# # 此处可以修改时间段，只需修改括号内的时间即可，格式参照原先括号里的数即可，如需额外增加时间段，请复制代码：schedule.every().day.at("时间段").do(job)并粘贴到下方，可增加任意数量的时间段
|
||||
# schedule.every().day.at("08:37").do(job)
|
||||
# schedule.every().day.at("11:55").do(job)
|
||||
# schedule.every().day.at("14:45").do(job)
|
||||
# schedule.every().day.at("18:00").do(job)
|
||||
#
|
||||
# print("脚本定时任务已启动...")
|
||||
# while True:
|
||||
# schedule.run_pending()
|
||||
# time.sleep(1)
|
||||
|
||||
"""脚本单次启动"""
|
||||
def main():
|
||||
print("脚本单次任务已启动...")
|
||||
webc.web_crawler()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
# 定义公告导出的目标excel文件路径
|
||||
FILE_PATH = "D:/Document/政府采购公告.xlsx"
|
||||
# 初始化excel工作表头
|
||||
HEADER = ["标题", "发布时间", "发布单位", "内容链接", "来源栏目"]
|
||||
|
||||
# 初始化工作表数量以及名称
|
||||
sheet_names = [
|
||||
"采购公告", "招标文件预公示", "采购意向公开", "结果公告", "合同公告",
|
||||
"更正公告", "单一来源公示", "电子卖场公示", "履约验收公示",
|
||||
"工程类公告", "框架协议征集公告", "框架协议入围结果公告",
|
||||
"框架协议成交结果汇总公告", "其他"
|
||||
]
|
||||
|
||||
# === 初始化筛选条件 ===
|
||||
KEYWORD ="大化" #筛选关键词
|
||||
START_DATE ="2025-12-01" #筛选最早发布日期
|
||||
END_DATE ="2026-01-31" #筛选最晚发布日期
|
||||
|
||||
# 代理列表
|
||||
PROXIES = [
|
||||
#若要添加代理，请先测试代理是否能正常连接，否则请将代理置空
|
||||
#"http://10.10.1.10:3218", #此为无效代理,仅作示例
|
||||
#"http://user:pass@10.10.1.10:8080", #此为无效代理,仅作示例
|
||||
# 带认证的代理
|
||||
# 添加更多代理...
|
||||
]
|
||||
|
||||
USER_AGENTS = [
|
||||
#此为用户代理,根据实际情况修改
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15",
|
||||
# 添加更多 User-Agent...
|
||||
]
|
||||
|
||||
# 公告来源链接
|
||||
SOURCE_URL = {
|
||||
"ZcyAnnouncement2":[66485,"结果公告"],# 结果公告
|
||||
"ZcyAnnouncement3":[66485,"合同公告"],# 合同公告
|
||||
"ZcyAnnouncement4":[66485,"更正公告"],# 更正公告
|
||||
"ZcyAnnouncement6":[66485,"单一来源公示"],# 单一来源公示
|
||||
"ZcyAnnouncement7":[66485,"电子卖场公示"],# 电子卖场公示
|
||||
"ZcyAnnouncement10":[66485,"履约验收公示"],# 履约验收公示
|
||||
"ZcyAnnouncement11":[66485,"工程类公告"],# 工程类公告
|
||||
"ZcyAnnouncement20":[66485,"框架协议征集公告"],# 框架协议征集公告
|
||||
"ZcyAnnouncement21":[66485,"框架协议入围结果公告"],# 框架协议入围结果公告
|
||||
"ZcyAnnouncement23":[66485,"框架协议成交结果汇总公告"],# 框架协议成交结果汇总公告
|
||||
"ZcyAnnouncement1":[66485,"采购公告"],# 采购公告
|
||||
"ZcyAnnouncement5":[66485,"招标文件预公示"],# 招标文件预公示
|
||||
"61-266648":[66485,"采购意向公开"]# 采购意向公开
|
||||
}
|
||||
Executable
+191
@@ -0,0 +1,191 @@
|
||||
import requests
|
||||
import time
|
||||
from datetime import datetime
|
||||
import random
|
||||
from fake_useragent import UserAgent
|
||||
import logging
|
||||
import write_to_excel as wte
|
||||
import utils as Utils
|
||||
|
||||
# === 获取筛选条件 ===
|
||||
KEYWORD =Utils.KEYWORD #关键词筛选
|
||||
START_DATE =Utils.START_DATE
|
||||
END_DATE =Utils.END_DATE
|
||||
|
||||
#获取公告来源链接
|
||||
SOURCE_URL = Utils.SOURCE_URL
|
||||
|
||||
# === 反扒配置 ===
|
||||
|
||||
#获取代理列表
|
||||
PROXIES = Utils.PROXIES
|
||||
USER_AGENTS = Utils.USER_AGENTS
|
||||
|
||||
MAX_RETRIES = 3 # 最大重连次数
|
||||
DELAY_MIN, DELAY_MAX = 1, 3 # 随机延迟范围（秒）
|
||||
|
||||
# === 日志配置 ===
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
||||
|
||||
def get_random_user_agent():
|
||||
"""随机选择 User-Agent"""
|
||||
return random.choice(USER_AGENTS)
|
||||
|
||||
def get_random_proxy():
|
||||
"""随机选择代理"""
|
||||
if not PROXIES:
|
||||
return None
|
||||
proxy = random.choice(PROXIES)
|
||||
return {"http": proxy, "https": proxy}
|
||||
|
||||
def check_sensitive_words(user_agent,payload,category_code,childrencode):
|
||||
"""敏感词检查请求"""
|
||||
# === 接口地址 ===
|
||||
url = "https://zfcg.gxzf.gov.cn/portal/sensitiveWords/check"
|
||||
headers = {
|
||||
"User-Agent": user_agent,
|
||||
"Content-Type": "application/json;charset=UTF-8",
|
||||
"Origin": "https://zfcg.gxzf.gov.cn",
|
||||
"Referer": f"https://zfcg.gxzf.gov.cn/site/category?parentId={category_code}&childrenCode={childrencode}",
|
||||
"Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64"
|
||||
}
|
||||
response = requests.post(url, json=payload, headers=headers)
|
||||
# print("这是敏感词检查请求返回的响应预览",response.json())
|
||||
return response.json()
|
||||
|
||||
def get_announcements(category_code, childrencode,page_no=1, session=None):
|
||||
"""请求接口获取公告列表函数(带反扒机制)"""
|
||||
if session is None:
|
||||
session = requests.Session()
|
||||
payload = {
|
||||
# 接口的请求参数
|
||||
"keyword": KEYWORD, #关键词筛选，此参数即为搜索框中的关键词输入值
|
||||
"publishDateBegin": START_DATE, #最早日期筛选
|
||||
"publishDateEnd": END_DATE, #最晚日期筛选
|
||||
"pageNo": page_no, #页码数
|
||||
"pageSize": 15, #页容量
|
||||
"categoryCode": category_code, # 该参数指定当前查找的公告栏目
|
||||
"_t": int(time.time() * 1000) # 动态时间戳
|
||||
}
|
||||
|
||||
# 先执行敏感词检查
|
||||
user_agent=get_random_user_agent()
|
||||
check_response = check_sensitive_words(user_agent,payload, category_code, childrencode)
|
||||
if not check_response.get("success", False):
|
||||
logging.warning("敏感词检查失败")
|
||||
return None
|
||||
|
||||
# 再执行公告数据请求
|
||||
# === 接口地址 ===
|
||||
api_url = "https://zfcg.gxzf.gov.cn/portal/category"
|
||||
headers = {
|
||||
"User-Agent": user_agent,
|
||||
"Content-Type": "application/json;charset=UTF-8",
|
||||
"Origin": "https://zfcg.gxzf.gov.cn",
|
||||
"Referer": f"https://zfcg.gxzf.gov.cn/site/category?parentId={category_code}&childrenCode={childrencode}",
|
||||
"Cookie": "_zcy_log_client_uuid=71e283e0-23d2-11f0-844a-eb67dfa7ab64" # 如果需要登录
|
||||
}
|
||||
#若添加了至少1条代理，则尝试使用代理
|
||||
if PROXIES:
|
||||
proxy = 'Default_value'
|
||||
else:
|
||||
proxy=None
|
||||
for retry in range(MAX_RETRIES):
|
||||
try:
|
||||
# 尝试使用代理，失败则切换为无代理
|
||||
if proxy :
|
||||
proxy = get_random_proxy()
|
||||
|
||||
if proxy:
|
||||
logging.info(f"使用代理: {proxy['http']}")
|
||||
else:
|
||||
logging.info("未使用代理")
|
||||
|
||||
response = session.post(
|
||||
api_url,
|
||||
json=payload,
|
||||
headers=headers,
|
||||
proxies=proxy,
|
||||
timeout=10
|
||||
)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get("success",True):
|
||||
return data
|
||||
else:
|
||||
logging.warning(f"接口返回失败: {data.get('error', '未知错误')}")
|
||||
else:
|
||||
logging.warning(f"请求失败，状态码: {response.status_code}")
|
||||
except requests.exceptions.ProxyError as pe:
|
||||
# 代理错误时记录并跳过，继续无代理请求
|
||||
logging.error(f"代理错误: {pe}. 切换为无代理模式")
|
||||
proxy = None # 下次请求不使用代理
|
||||
except Exception as e:
|
||||
logging.error(f"请求异常: {e}")
|
||||
|
||||
# 重试前等待
|
||||
delay = random.uniform(DELAY_MIN, DELAY_MAX)
|
||||
logging.info(f"第 {retry + 1}/{MAX_RETRIES} 次重试，等待 {delay:.1f} 秒...")
|
||||
time.sleep(delay)
|
||||
|
||||
return None # 所有重试失败
|
||||
|
||||
def parse_data(data,category_code,source_name):
|
||||
"""解析公告数据函数"""
|
||||
results = []
|
||||
for item in data["result"]["data"]["data"]:
|
||||
results.append({
|
||||
"标题": item["title"],
|
||||
"发布时间": datetime.fromtimestamp(int(item["publishDate"]) / 1000).strftime("%Y-%m-%d"),
|
||||
"发布单位": item["purchaseName"],
|
||||
"内容链接": f"https://zfcg.gxzf.gov.cn/site/detail?parentId={category_code}&articleId={item['articleId']}",
|
||||
"来源栏目":source_name
|
||||
})
|
||||
return results
|
||||
|
||||
#爬虫主函数
|
||||
def web_crawler():
|
||||
session = requests.Session()
|
||||
# print("这是本地会话存储：",session)
|
||||
# 初始化引用值为信息公告,先爬取信息公告栏目的公告数据
|
||||
childrencode = "ZcyAnnouncement"
|
||||
# 爬取代码主体，外循环为来源栏目的循环，即遍历需要爬取的所有来源栏目
|
||||
for key in SOURCE_URL:
|
||||
page = 1
|
||||
all_results = []
|
||||
# print("这是正在爬取的栏目", SOURCE_URL[key][1])
|
||||
# print("这是正在爬取的栏目的目录码", str(key))
|
||||
# 判断正确的引用值
|
||||
# 内循环主体即为在符合筛选条件的公告列表中遍历所有公告，爬取每个公告需要的指定数据。
|
||||
while True:
|
||||
# 这是分割线
|
||||
# logging.info("爬虫日志分割线————————————————————————————————————————————————爬虫日志分割线")
|
||||
# logging.info("爬虫日志分割线————————————————————————————————————————————————爬虫日志分割线")
|
||||
# logging.info("爬虫日志分割线————————————————————————————————————————————————爬虫日志分割线")
|
||||
# logging.info(f"正在爬取第 {page} 页...")
|
||||
response = get_announcements(str(key), childrencode, page,session)
|
||||
if response["result"]["data"]["empty"] or not response["result"]["data"]["data"]:
|
||||
break
|
||||
# logging.info(f"请求后共返回 {response["result"]["data"]["total"]} 条公告")
|
||||
current_data = parse_data(response, SOURCE_URL[key][0], SOURCE_URL[key][1])
|
||||
if not current_data:
|
||||
break
|
||||
# logging.info(f"解析后共爬取 {len(current_data)} 条公告")
|
||||
all_results.extend(current_data)
|
||||
page += 1
|
||||
# 随机延迟
|
||||
delay = random.uniform(DELAY_MIN, DELAY_MAX)
|
||||
time.sleep(delay)
|
||||
|
||||
# print(f"从 {SOURCE_URL[key][1]} 共爬取 {len(all_results)} 条公告")
|
||||
# 调用excel读写函数，将公告数据筛选后保存到Excel中
|
||||
wte.write_to_excel(all_results,SOURCE_URL[key][1])
|
||||
# 这是分割线
|
||||
print("公告栏目分割线————————————————————————————————————————————————公告栏目分割线")
|
||||
print("公告栏目分割线————————————————————————————————————————————————公告栏目分割线")
|
||||
print("公告栏目分割线————————————————————————————————————————————————公告栏目分割线")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
from openpyxl.styles import Font, Alignment, PatternFill, Border, Side
|
||||
import schedule
|
||||
import time
|
||||
import utils as Utils
|
||||
|
||||
# 获取excel文件路径和excel的工作表头
|
||||
FILE_PATH = Utils.FILE_PATH
|
||||
HEADER = Utils.HEADER
|
||||
|
||||
# 获取工作表数量以及名称
|
||||
sheet_names = Utils.sheet_names
|
||||
|
||||
|
||||
def write_to_excel(results,source=""):
|
||||
# 检查文件是否存在，如果不存在则创建一个新的Excel文件
|
||||
try:
|
||||
df_existing = pd.read_excel(FILE_PATH, sheet_name=None)
|
||||
except FileNotFoundError:
|
||||
df_existing = {name: pd.DataFrame(columns=HEADER) for name in sheet_names}
|
||||
|
||||
today_str = datetime.now().strftime('%Y-%m-%d')
|
||||
aditional_item=0 # 记录本次爬取新增的公告数量
|
||||
# 将爬取的公告对号入座填充至excel文件的工作表中
|
||||
for result in results:
|
||||
sheet_name = result["来源栏目"] if result["来源栏目"] in sheet_names else "其他"
|
||||
df_sheet = df_existing[sheet_name]
|
||||
|
||||
# 检查是否已存在相同内容链接的记录
|
||||
if not df_sheet[df_sheet["内容链接"] == result["内容链接"]].empty:
|
||||
continue
|
||||
|
||||
# 添加新记录
|
||||
new_row = pd.DataFrame([result])
|
||||
df_sheet = pd.concat([df_sheet, new_row], ignore_index=True)
|
||||
aditional_item+=1 # 若有新增公告，则加1
|
||||
# 按时间降序排列
|
||||
df_sheet.sort_values(by='发布时间', ascending=False, inplace=True)
|
||||
|
||||
# 更新现有数据
|
||||
df_existing[sheet_name] = df_sheet
|
||||
|
||||
# 写入Excel文件
|
||||
with pd.ExcelWriter(FILE_PATH, engine='openpyxl') as writer:
|
||||
for sheet_name, df in df_existing.items():
|
||||
df.to_excel(writer, sheet_name=sheet_name, index=False)
|
||||
|
||||
# 获取工作表对象
|
||||
worksheet = writer.sheets[sheet_name]
|
||||
|
||||
# 设置列宽
|
||||
for col_idx, column_width in enumerate([40, 15, 20, 30, 20]):
|
||||
worksheet.column_dimensions[chr(65 + col_idx)].width = column_width
|
||||
|
||||
# 创建红色填充
|
||||
red_fill = PatternFill(start_color="FF0000", end_color="FF0000", fill_type="solid")
|
||||
|
||||
# 创建边框样式
|
||||
thin_border = Border(left=Side(style='thin'),
|
||||
right=Side(style='thin'),
|
||||
top=Side(style='thin'),
|
||||
bottom=Side(style='thin'))
|
||||
|
||||
# 应用样式
|
||||
for row in worksheet.iter_rows(min_row=1, max_col=len(HEADER), max_row=worksheet.max_row):
|
||||
publish_date = row[1].value if len(row) > 1 else None
|
||||
# print("这是publish_date",publish_date)
|
||||
# 若公告发布日期为今天，则将其单元格背景填充至红色
|
||||
if publish_date == today_str:
|
||||
for cell in row:
|
||||
cell.fill = red_fill
|
||||
|
||||
for idx, cell in enumerate(row):
|
||||
alignment = Alignment(horizontal='left' if idx < 4 else 'general', vertical='top', wrap_text=True)
|
||||
cell.alignment = alignment
|
||||
cell.border = thin_border
|
||||
print(f"<{source}> 新增 {aditional_item} 条公告，已保存至excel文件中")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user