from typing import Any from app.crawler.base import PipelineConfig, PipelineResult from app.services.filter_service import dedup_by_hash class PostCrawlPipeline: def __init__(self, db_session, notification_service): self.db = db_session self.notify = notification_service async def process(self, announcements: list[dict[str, Any]], config: PipelineConfig) -> PipelineResult: result = PipelineResult() if not announcements: return result # 1. Dedup if config.dedup_enabled: announcements = dedup_by_hash(announcements) # 2. Store to database stored = await self._save_to_db(announcements) result.stored = stored to_notify = announcements # 3. Filter if config.filter_enabled and config.keywords: before = len(to_notify) to_notify = [a for a in to_notify if self._match_keywords(a, config.keywords)] result.filtered = before - len(to_notify) # 4. Skip already-notified if to_notify: to_notify = await self._exclude_sent(to_notify) # 5. Notify if config.notify_mode == "all": result.notified = await self._send_notifications(to_notify) elif config.notify_mode == "filtered": if config.filter_enabled and config.keywords: result.notified = await self._send_notifications(to_notify) elif not config.filter_enabled: result.notified = await self._send_notifications(to_notify) # 6. Mark sent if config.mark_sent and result.notified > 0: await self._mark_sent(to_notify) return result async def _save_to_db(self, announcements: list[dict[str, Any]]) -> int: from sqlalchemy.dialects.postgresql import insert from app.models.announcement import Announcement if not announcements: return 0 values = [{ "title": a["title"], "publish_date": a["publish_date"], "purchase_name": a.get("purchase_name", ""), "content_url": a.get("content_url", ""), "source_code": a["source_code"], "source_name": a["source_name"], "announcement_type": a.get("announcement_type", "purchase"), "content_hash": a["content_hash"], "crawl_mode": a.get("crawl_mode", "auto"), "is_new": a.get("is_new", True), "is_sent": False, "keyword_matched": a.get("keyword_matched", False), } for a in announcements] stmt = insert(Announcement).values(values) stmt = stmt.on_conflict_do_nothing(index_elements=["content_hash"]) result_proxy = await self.db.execute(stmt) await self.db.commit() return result_proxy.rowcount if result_proxy.rowcount >= 0 else len(values) async def _send_notifications(self, announcements: list[dict[str, Any]]) -> int: return await self.notify.send(announcements) async def _mark_sent(self, announcements: list[dict[str, Any]]) -> int: from sqlalchemy import update from app.models.announcement import Announcement hashes = [a["content_hash"] for a in announcements if a.get("content_hash")] if not hashes: return 0 stmt = ( update(Announcement) .where(Announcement.content_hash.in_(hashes)) .values(is_sent=True) ) result = await self.db.execute(stmt) await self.db.commit() return result.rowcount async def _exclude_sent(self, announcements: list[dict[str, Any]]) -> list[dict[str, Any]]: from sqlalchemy import select from app.models.announcement import Announcement hashes = [a["content_hash"] for a in announcements if a.get("content_hash")] if not hashes: return announcements stmt = select(Announcement.content_hash).where( Announcement.content_hash.in_(hashes), Announcement.is_sent == True, # noqa: E712 ) result = await self.db.execute(stmt) sent_hashes = {row[0] for row in result.fetchall()} return [a for a in announcements if a.get("content_hash") not in sent_hashes] @staticmethod def _match_keywords(announcement: dict[str, Any], keywords: list[str]) -> bool: text = f"{announcement.get('title', '')} {announcement.get('purchase_name', '')}" return any(kw in text for kw in keywords)