Implement manual crawling feature in GXGPMonitorApp, allowing users to filter announcements by date and keywords. Update Markdown generation to include time period and keyword details. Remove outdated onu.md file. Enhance WeChat message handling to support manual crawl results and improve logging for better traceability.

2026-01-08 18:24:00 +08:00
parent b52d9c069d
commit 04d254cfef
11 changed files with 5741 additions and 555 deletions
@@ -89,10 +89,17 @@ class GXGPMonitorApp:
            print(f"应用初始化失败: {str(e)}", file=sys.stderr)
            return False

-    def run_crawl(self, keywords: list = None, sources: list = None, max_pages: int = None):
-        """执行爬取任务"""
+    def run_crawl(self, keywords: list = None, sources: list = None, max_pages: int = None, manual_crawl: bool = False):
+        """执行爬取任务
+
+        Args:
+            keywords: 关键词列表
+            sources: 来源列表
+            max_pages: 最大页数
+            manual_crawl: 是否为手动爬取（只筛选今天的数据）
+        """
        try:
-            logger.info("开始执行爬取任务")
+            logger.info(f"开始执行爬取任务 (手动爬取: {manual_crawl})")

            # 执行爬取
            crawl_results = crawl_announcements(keywords, sources)
@@ -115,21 +122,69 @@ class GXGPMonitorApp:
            logger.info(f"保存所有公告完成：共保存 {all_saved_count} 条，按来源统计: {all_saved_stats}")

            # 筛选公告
-            filter_obj = filter_from_config()
-            filtered_announcements, filter_stats = filter_obj.filter(all_announcements)
+            if manual_crawl:
+                # 手动爬取：只筛选今天的公告和用户指定的关键词，不进行去重
+                from .filters.filters import KeywordFilter, DateFilter, SourceFilter
+                from datetime import date
+
+                # 1. 日期筛选：只保留今天的公告
+                date_filter = DateFilter()
+                date_filtered = date_filter.filter_announcements(all_announcements, start_date=date.today(), end_date=date.today())
+
+                # 2. 关键词筛选
+                keyword_filter = KeywordFilter()
+                keyword_filtered = keyword_filter.filter_announcements(date_filtered, keywords=keywords or [])
+
+                # 3. 来源筛选
+                source_filter = SourceFilter()
+                filtered_announcements = source_filter.filter_announcements(keyword_filtered, sources or list(self.config.sources.keys()))
+
+                # 计算统计信息
+                filter_stats = type('FilterResult', (), {
+                    "keyword_filtered": len(date_filtered) - len(keyword_filtered),
+                    "date_filtered": len(all_announcements) - len(date_filtered),
+                    "duplicate_filtered": 0,  # 手动爬取不进行去重
+                    "source_filtered": len(keyword_filtered) - len(filtered_announcements)
+                })()
+            else:
+                # 自动爬取：筛选出新公告并应用关键词筛选
+                # 1. 筛选出数据库中没有的新公告
+                new_announcements = [ann for ann in all_announcements if ann.is_new]
+                logger.info(f"筛选出 {len(new_announcements)} 条新公告")
+
+                # 2. 对新公告应用关键词筛选等
+                if new_announcements:
+                    filter_obj = filter_from_config()
+                    filtered_announcements, filter_stats = filter_obj.filter(new_announcements)
+                    # 更新统计信息，加上未筛选的新公告数量
+                    filter_stats.keyword_filtered += len(new_announcements) - len(filtered_announcements)
+                else:
+                    filtered_announcements = []
+                    filter_stats = type('FilterResult', (), {
+                        "keyword_filtered": 0,
+                        "date_filtered": 0,
+                        "duplicate_filtered": len(all_announcements),
+                        "source_filtered": 0
+                    })()

            logger.info(f"筛选后剩余 {len(filtered_announcements)} 条公告")

            # 保存筛选后的公告（用于标记关键词匹配等）
            saved_count = save_announcements_to_storage(filtered_announcements)

-            # 生成Markdown文件
-            md_success = generate_onu_md(filtered_announcements)
+            # 生成Markdown文件（只在自动爬取时生成）
+            md_success = False
+            if not manual_crawl:
+                md_success = generate_onu_md(filtered_announcements)

            # 发送通知
            notify_success = False
-            if filtered_announcements and self.config.wechat_app.enabled:
+            if not manual_crawl and filtered_announcements and self.config.wechat_app.enabled:
+                # 自动爬取时发送卡片消息
                notify_success = send_announcements_notification(filtered_announcements)
+            elif manual_crawl and filtered_announcements and self.config.wechat_app.enabled:
+                # 手动爬取时不在这里发送消息，由消息处理器负责发送markdown消息
+                notify_success = True  # 标记为成功，因为消息会通过其他方式发送

            result = {
                "success": True,
@@ -146,6 +201,10 @@ class GXGPMonitorApp:
                }
            }

+            # 对于手动爬取，额外返回筛选后的公告列表
+            if manual_crawl:
+                result["filtered_announcements"] = filtered_announcements
+
            logger.info(f"爬取任务完成: {result}")
            return result

@@ -42,7 +42,8 @@ class MarkdownGenerator:
        output_path.parent.mkdir(parents=True, exist_ok=True)

    def generate_markdown(self, announcements: List[Announcement],
-                         title: str = "广西政府采购网公告监控") -> str:
+                         title: str = "广西政府采购网公告监控",
+                         time_period: str = None) -> str:
        """
        生成Markdown内容

@@ -54,22 +55,31 @@ class MarkdownGenerator:
            str: Markdown格式的文本
        """
        if not announcements:
-            return self._generate_empty_markdown(title)
+            return self._generate_empty_markdown(title, time_period)

        # 按来源分组
        grouped_announcements = self._group_announcements_by_source(announcements)

        # 生成Markdown
        lines = []
-        lines.append(f"# {title}")
-        lines.append("")
-        lines.append(f"**更新时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
-        lines.append(f"**总公告数**: {len(announcements)}")
+        lines.append("# 搜索完成")
        lines.append("")

-        # 生成目录
-        lines.extend(self._generate_toc(grouped_announcements))
-        lines.append("---")
+        # 解析标题中的关键词
+        keyword = "未知"
+        if "关键词:" in title:
+            keyword_part = title.split("关键词:")[-1].strip()
+            keyword = keyword_part.split()[0] if keyword_part else "未知"
+
+        lines.append(f"📋 关键词搜索: `{keyword}` - 总公告数: `{len(announcements)}`")
+        lines.append("")
+
+        # 使用传入的时间段或默认的更新时间
+        if time_period:
+            lines.append(f"**时间段: {time_period}**")
+        else:
+            lines.append(f"**更新时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}**")
+        lines.append("")
        lines.append("")

        # 生成各来源的公告
@@ -107,11 +117,8 @@ class MarkdownGenerator:
    def _generate_source_section(self, source_name: str, announcements: List[Announcement]) -> List[str]:
        """生成来源章节"""
        lines = []
-        anchor = self._create_anchor(source_name)

-        lines.append(f"## {source_name}")
-        lines.append("")
-        lines.append(f"**共 {len(announcements)} 条公告**")
+        lines.append(f"## {source_name} - **共 {len(announcements)} 条**")
        lines.append("")

        # 生成公告列表
@@ -124,59 +131,49 @@ class MarkdownGenerator:
        """生成单个公告项"""
        lines = []

-        # 公告标题
-        title_prefix = ""
-        if self.include_today_highlight and announcement.is_today:
-            title_prefix = "🆕 **【今日】**"
-
-        title_line = f"### {index}. {title_prefix}[{announcement.title}]({announcement.content_url})"
+        # 公告标题（包含超链接）
+        title_line = f"### {index}. [{announcement.title}]({announcement.content_url})"
        lines.append(title_line)
        lines.append("")

-        # 公告信息
-        info_items = []
+        # 公告信息 - 简洁格式
+        info_parts = []

        if announcement.publish_date:
            publish_date = announcement.publish_date.strftime("%Y-%m-%d")
-            info_items.append(f"📅 发布时间: {publish_date}")
+            info_parts.append(publish_date)

        if announcement.purchase_name:
-            info_items.append(f"🏢 发布单位: {announcement.purchase_name}")
+            info_parts.append(announcement.purchase_name)

-        info_items.append(f"📄 来源: {announcement.source_name}")
+        info_parts.append(announcement.source_name)

-        if announcement.crawled_at:
-            crawled_time = announcement.crawled_at.strftime("%m-%d %H:%M")
-            info_items.append(f"🤖 爬取时间: {crawled_time}")
-
-        if info_items:
-            lines.append(" | ".join(info_items))
+        if info_parts:
+            info_line = " | ".join(info_parts)
+            lines.append(info_line)
            lines.append("")
-
-        # 如果是新公告，添加标记
-        if announcement.is_new:
-            lines.append("*🚀 新公告*")
            lines.append("")

-        lines.append("---")
-        lines.append("")
-
        return lines

-    def _generate_empty_markdown(self, title: str) -> str:
+    def _generate_empty_markdown(self, title: str, time_period: str = None) -> str:
        """生成空内容的Markdown"""
+        # 解析标题中的关键词
+        keyword = "未知"
+        if "关键词:" in title:
+            keyword_part = title.split("关键词:")[-1].strip()
+            keyword = keyword_part.split()[0] if keyword_part else "未知"
+
        lines = [
-            f"# {title}",
+            f"📋 关键词搜索: `{keyword}` - 总公告数: `0`",
            "",
-            f"**更新时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
+            f"**时间段: {time_period}**" if time_period else f"**更新时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}**",
            "",
-            "## 无新公告",
            "",
-            "当前时间范围内没有找到符合条件的公告。",
+            "## 无匹配公告",
            "",
-            "---",
-            "",
-            f"*由广西政府采购网公告监控系统生成*"
+            "在指定时间范围内没有找到符合条件的公告。",
+            ""
        ]

        return "\n".join(lines)
@@ -276,31 +276,118 @@ class WeChatMessageHandler:
            if not app:
                return self._create_text_response("系统初始化失败，请稍后重试", from_user)

-            # 执行爬取
-            result = app.run_crawl(keywords=keywords)
+            # 执行爬取（手动爬取，只筛选今天的公告）
+            result = app.run_crawl(keywords=keywords, manual_crawl=True)

            if result.get("success"):
                total = result.get("total_crawled", 0)
                filtered = result.get("filtered", 0)
+                filtered_announcements = result.get("filtered_announcements", [])
+
+                # 获取今天的日期范围
+                from datetime import datetime, date
+                today = date.today()
+                time_period = f"{today.strftime('%Y-%m-%d')} 00:00 至 {datetime.now().strftime('%Y-%m-%d %H:%M')}"

                if filtered > 0:
-                    response = f"""✅ 爬取完成！
+                    # 生成markdown汇总消息并发送
+                    try:
+                        from ..storage.md_generator import MarkdownGenerator
+                        from ..notification.wechat import send_system_notification

-关键词: {' '.join(keywords)}
-📊 统计信息：
+                        # 生成markdown内容
+                        md_generator = MarkdownGenerator()
+                        title = f"手动爬取结果 - 关键词: {' '.join(keywords)}"
+                        markdown_content = md_generator.generate_markdown(filtered_announcements, title, time_period)
+
+                        # 发送markdown消息
+                        notify_success = send_system_notification(
+                            title="🔍 搜索完成",
+                            content=markdown_content
+                        )
+
+                        if notify_success:
+                            # 成功发送markdown消息，返回空响应（不发送额外文本消息）
+                            response = ""
+                        else:
+                            response = f"""✅ 爬取完成！
+
+🔍 搜索条件：
+• 关键词: {' '.join(keywords)}
+• 时间段: {time_period}
+
+📊 统计结果：
 • 总共发现: {total} 条公告
 • 匹配筛选: {filtered} 条

-相关公告已推送，请查收。"""
-                else:
-                    response = f"""✅ 爬取完成！
+⚠️ 公告汇总推送失败，但数据已生成。"""
+                    except Exception as notify_error:
+                        logger.error(f"生成或发送公告汇总失败: {notify_error}")
+                        # 降级处理：手动构建简单的文本响应
+                        announcement_list = []
+                        for i, ann in enumerate(filtered_announcements[:10], 1):  # 最多显示10条
+                            announcement_list.append(f"{i}. {ann.title[:50]}...")

-关键词: {' '.join(keywords)}
-📊 统计信息：
+                        remaining = len(filtered_announcements) - 10
+                        if remaining > 0:
+                            announcement_list.append(f"... 还有 {remaining} 条公告")
+
+                        response = f"""✅ 爬取完成！
+
+🔍 搜索条件：
+• 关键词: {' '.join(keywords)}
+• 时间段: {time_period}
+
+📊 统计结果：
+• 总共发现: {total} 条公告
+• 匹配筛选: {filtered} 条
+
+📋 匹配公告：
+{chr(10).join(announcement_list)}
+
+💡 公告详情已保存，可通过其他方式查看。"""
+                else:
+                    # 没有找到匹配的公告，发送markdown格式的空结果
+                    try:
+                        from ..storage.md_generator import MarkdownGenerator
+                        from ..notification.wechat import send_system_notification
+
+                        md_generator = MarkdownGenerator()
+                        title = f"手动爬取结果 - 关键词: {' '.join(keywords)}"
+                        markdown_content = md_generator.generate_markdown([], title, time_period)
+
+                        notify_success = send_system_notification(
+                            title="🔍 搜索完成",
+                            content=markdown_content
+                        )
+
+                        if notify_success:
+                            response = ""
+                        else:
+                            response = f"""✅ 爬取完成！
+
+🔍 搜索条件：
+• 关键词: {' '.join(keywords)}
+• 时间段: {time_period}
+
+📊 统计结果：
 • 总共发现: {total} 条公告
 • 匹配筛选: 0 条

-没有找到匹配的公告。"""
+❌ 在指定时间段内没有找到匹配的公告。"""
+                    except Exception as notify_error:
+                        logger.error(f"生成或发送公告汇总失败: {notify_error}")
+                        response = f"""✅ 爬取完成！
+
+🔍 搜索条件：
+• 关键词: {' '.join(keywords)}
+• 时间段: {time_period}
+
+📊 统计结果：
+• 总共发现: {total} 条公告
+• 匹配筛选: 0 条
+
+❌ 在指定时间段内没有找到匹配的公告。"""
            else:
                error = result.get("error", "未知错误")
                response = f"❌ 爬取失败: {error}"