Enhance announcement storage functionality by adding a method to save all announcements by source, retaining the latest 100 per source. Update related logging and comments for clarity. Adjusted the main script to utilize the new storage method and log the saving process. Updated the markdown file timestamp.
This commit is contained in:
@@ -797,3 +797,62 @@
|
||||
2026-01-07 17:36:12 - __main__ - [32mINFO[0m - [32m筛选后剩余 0 条公告[0m
|
||||
2026-01-07 17:36:12 - storage.md_generator - [32mINFO[0m - [32mMarkdown文件已保存到: onu.md (共 0 条公告)[0m
|
||||
2026-01-07 17:36:12 - __main__ - [32mINFO[0m - [32m爬取任务完成: {'success': True, 'total_crawled': 1300, 'filtered': 0, 'saved': 0, 'markdown_generated': True, 'notification_sent': False, 'filter_stats': {'keyword_filtered': 1282, 'date_filtered': 0, 'duplicate_filtered': 18, 'source_filtered': 0}}[0m
|
||||
2026-01-08 08:17:30 - __main__ - [32mINFO[0m - [32m=== 广西政府采购网公告监控系统启动 ===[0m
|
||||
2026-01-08 08:17:30 - __main__ - [32mINFO[0m - [32m版本: 1.0.0[0m
|
||||
2026-01-08 08:17:30 - __main__ - [32mINFO[0m - [32m配置文件: 默认配置[0m
|
||||
2026-01-08 08:17:30 - core.reliability - [32mINFO[0m - [32m健康检查通过[0m
|
||||
2026-01-08 08:17:30 - core.database - [32mINFO[0m - [32m数据库连接池初始化成功[0m
|
||||
2026-01-08 08:17:30 - core.database - [32mINFO[0m - [32m开始初始化数据库表结构[0m
|
||||
2026-01-08 08:17:30 - core.database - [32mINFO[0m - [32m数据库表结构初始化完成[0m
|
||||
2026-01-08 08:17:30 - storage.postgresql - [32mINFO[0m - [32m存储初始化完成[0m
|
||||
2026-01-08 08:17:30 - __main__ - [32mINFO[0m - [32m应用初始化完成[0m
|
||||
2026-01-08 08:17:30 - __main__ - [32mINFO[0m - [32m开始执行爬取任务[0m
|
||||
2026-01-08 08:17:30 - crawler.spider - [32mINFO[0m - [32m开始爬取 13 个公告来源[0m
|
||||
2026-01-08 08:17:30 - gx_gp_monitor - [32mINFO[0m - [32m开始爬取 采购公告[0m
|
||||
2026-01-08 08:17:31 - crawler.parsers - [32mINFO[0m - [32m成功解析 100/100 条公告记录[0m
|
||||
2026-01-08 08:17:31 - gx_gp_monitor - [32mINFO[0m - [32m采购公告 爬取完成，共获取 100 条公告，耗时 0.53秒[0m
|
||||
2026-01-08 08:17:31 - gx_gp_monitor - [32mINFO[0m - [32m开始爬取 结果公告[0m
|
||||
2026-01-08 08:17:31 - crawler.parsers - [32mINFO[0m - [32m成功解析 100/100 条公告记录[0m
|
||||
2026-01-08 08:17:31 - gx_gp_monitor - [32mINFO[0m - [32m结果公告 爬取完成，共获取 100 条公告，耗时 0.19秒[0m
|
||||
2026-01-08 08:17:31 - gx_gp_monitor - [32mINFO[0m - [32m开始爬取 合同公告[0m
|
||||
2026-01-08 08:17:31 - crawler.parsers - [32mINFO[0m - [32m成功解析 100/100 条公告记录[0m
|
||||
2026-01-08 08:17:31 - gx_gp_monitor - [32mINFO[0m - [32m合同公告 爬取完成，共获取 100 条公告，耗时 0.20秒[0m
|
||||
2026-01-08 08:17:31 - gx_gp_monitor - [32mINFO[0m - [32m开始爬取 更正公告[0m
|
||||
2026-01-08 08:17:31 - crawler.parsers - [32mINFO[0m - [32m成功解析 100/100 条公告记录[0m
|
||||
2026-01-08 08:17:31 - gx_gp_monitor - [32mINFO[0m - [32m更正公告 爬取完成，共获取 100 条公告，耗时 0.28秒[0m
|
||||
2026-01-08 08:17:31 - gx_gp_monitor - [32mINFO[0m - [32m开始爬取 招标文件预公示[0m
|
||||
2026-01-08 08:17:31 - crawler.parsers - [32mINFO[0m - [32m成功解析 100/100 条公告记录[0m
|
||||
2026-01-08 08:17:31 - gx_gp_monitor - [32mINFO[0m - [32m招标文件预公示 爬取完成，共获取 100 条公告，耗时 0.18秒[0m
|
||||
2026-01-08 08:17:31 - gx_gp_monitor - [32mINFO[0m - [32m开始爬取 单一来源公示[0m
|
||||
2026-01-08 08:17:32 - crawler.parsers - [32mINFO[0m - [32m成功解析 100/100 条公告记录[0m
|
||||
2026-01-08 08:17:32 - gx_gp_monitor - [32mINFO[0m - [32m单一来源公示 爬取完成，共获取 100 条公告，耗时 0.16秒[0m
|
||||
2026-01-08 08:17:32 - gx_gp_monitor - [32mINFO[0m - [32m开始爬取 电子卖场公示[0m
|
||||
2026-01-08 08:17:32 - crawler.parsers - [32mINFO[0m - [32m成功解析 100/100 条公告记录[0m
|
||||
2026-01-08 08:17:32 - gx_gp_monitor - [32mINFO[0m - [32m电子卖场公示 爬取完成，共获取 100 条公告，耗时 0.34秒[0m
|
||||
2026-01-08 08:17:32 - gx_gp_monitor - [32mINFO[0m - [32m开始爬取 履约验收公示[0m
|
||||
2026-01-08 08:17:32 - crawler.parsers - [32mINFO[0m - [32m成功解析 100/100 条公告记录[0m
|
||||
2026-01-08 08:17:32 - gx_gp_monitor - [32mINFO[0m - [32m履约验收公示 爬取完成，共获取 100 条公告，耗时 0.48秒[0m
|
||||
2026-01-08 08:17:32 - gx_gp_monitor - [32mINFO[0m - [32m开始爬取 工程类公告[0m
|
||||
2026-01-08 08:17:33 - crawler.parsers - [32mINFO[0m - [32m成功解析 100/100 条公告记录[0m
|
||||
2026-01-08 08:17:33 - gx_gp_monitor - [32mINFO[0m - [32m工程类公告 爬取完成，共获取 100 条公告，耗时 0.23秒[0m
|
||||
2026-01-08 08:17:33 - gx_gp_monitor - [32mINFO[0m - [32m开始爬取 框架协议征集公告[0m
|
||||
2026-01-08 08:17:33 - crawler.parsers - [32mINFO[0m - [32m成功解析 100/100 条公告记录[0m
|
||||
2026-01-08 08:17:33 - gx_gp_monitor - [32mINFO[0m - [32m框架协议征集公告 爬取完成，共获取 100 条公告，耗时 0.18秒[0m
|
||||
2026-01-08 08:17:33 - gx_gp_monitor - [32mINFO[0m - [32m开始爬取 框架协议入围结果公告[0m
|
||||
2026-01-08 08:17:33 - crawler.parsers - [32mINFO[0m - [32m成功解析 100/100 条公告记录[0m
|
||||
2026-01-08 08:17:33 - gx_gp_monitor - [32mINFO[0m - [32m框架协议入围结果公告 爬取完成，共获取 100 条公告，耗时 0.23秒[0m
|
||||
2026-01-08 08:17:33 - gx_gp_monitor - [32mINFO[0m - [32m开始爬取 框架协议成交结果汇总公告[0m
|
||||
2026-01-08 08:17:33 - crawler.parsers - [32mINFO[0m - [32m成功解析 100/100 条公告记录[0m
|
||||
2026-01-08 08:17:33 - gx_gp_monitor - [32mINFO[0m - [32m框架协议成交结果汇总公告 爬取完成，共获取 100 条公告，耗时 0.16秒[0m
|
||||
2026-01-08 08:17:33 - gx_gp_monitor - [32mINFO[0m - [32m开始爬取 采购意向公开[0m
|
||||
2026-01-08 08:17:33 - crawler.parsers - [32mINFO[0m - [32m成功解析 100/100 条公告记录[0m
|
||||
2026-01-08 08:17:33 - gx_gp_monitor - [32mINFO[0m - [32m采购意向公开 爬取完成，共获取 100 条公告，耗时 0.19秒[0m
|
||||
2026-01-08 08:17:33 - crawler.spider - [32mINFO[0m - [32m爬取完成: 共处理 13 个来源，成功 13 个，失败 0 个，获取 1300 条公告[0m
|
||||
2026-01-08 08:17:33 - __main__ - [32mINFO[0m - [32m爬取到 1300 条原始公告[0m
|
||||
2026-01-08 08:17:33 - filters.filters - [32mINFO[0m - [32m来源筛选: 1300 -> 1300 条公告[0m
|
||||
2026-01-08 08:17:34 - filters.filters - [32mINFO[0m - [32m去重筛选: 移除了 17 条重复公告[0m
|
||||
2026-01-08 08:17:34 - filters.filters - [32mINFO[0m - [32m关键词筛选: 1283 -> 0 条公告[0m
|
||||
2026-01-08 08:17:34 - filters.filters - [32mINFO[0m - [32m筛选完成: 总数 1300 -> 筛选后 0 (关键词: 1283, 日期: 0, 去重: 17, 来源: 0)[0m
|
||||
2026-01-08 08:17:34 - __main__ - [32mINFO[0m - [32m筛选后剩余 0 条公告[0m
|
||||
2026-01-08 08:17:34 - storage.md_generator - [32mINFO[0m - [32mMarkdown文件已保存到: onu.md (共 0 条公告)[0m
|
||||
2026-01-08 08:17:34 - __main__ - [32mINFO[0m - [32m爬取任务完成: {'success': True, 'total_crawled': 1300, 'filtered': 0, 'saved': 0, 'markdown_generated': True, 'notification_sent': False, 'filter_stats': {'keyword_filtered': 1283, 'date_filtered': 0, 'duplicate_filtered': 17, 'source_filtered': 0}}[0m
|
||||
|
||||
Reference in New Issue
Block a user