from __future__ import annotations import json import logging from datetime import date, datetime, timedelta from typing import Any, Optional from urllib.error import HTTPError, URLError from urllib.parse import urlencode from urllib.request import Request, urlopen from app.core.config import Settings from app.schemas.agent import SignalItem logger = logging.getLogger(__name__) class NewsSearchClient: """通过公开新闻搜索接口采集动态AI新闻候选。""" endpoint = "https://api.gdeltproject.org/api/v2/doc/doc" def __init__(self, settings: Settings) -> None: self.settings = settings def collect_ai_news(self, run_date: date) -> list[SignalItem]: # GDELT按日期窗口返回全球新闻候选,模型后续只从这些证据里挑选日报条目。 since = run_date - timedelta(days=self.settings.news_recent_days) articles: dict[str, dict[str, Any]] = {} payload = self._search(query=self._query(), since=since, until=run_date) for article in payload.get("articles", []): url = article.get("url") if isinstance(url, str) and url.startswith("http"): articles[url] = article candidates = [self._article_to_signal(article) for article in articles.values()] candidates = [candidate for candidate in candidates if candidate is not None] candidates.sort(key=lambda item: item.published_at or datetime.min, reverse=True) logger.info("news_candidates_collected days=%s items=%s", self.settings.news_recent_days, len(candidates)) return candidates[: self.settings.news_search_max_records] def _query(self) -> str: # GDELT限制请求频率,单次组合查询覆盖模型、Agent、开源和融资四类情报。 return ( '("AI agent" OR "agent framework" OR "LLM agent" OR ' '"large language model" OR LLM OR "AI model" OR ' '"open source AI" OR "open-source AI" OR "AI startup funding")' ) def _search(self, query: str, since: date, until: date) -> dict[str, Any]: # ArtList模式返回标题、URL、域名和发现时间,正好满足候选证据采集。 params = { "query": query, "mode": "ArtList", "format": "json", "maxrecords": str(self.settings.news_search_max_records), "sort": "HybridRel", "startdatetime": since.strftime("%Y%m%d000000"), "enddatetime": until.strftime("%Y%m%d235959"), } url = f"{self.endpoint}?{urlencode(params)}" request = Request(url, headers={"User-Agent": "SignalScout"}) try: with urlopen(request, timeout=30) as response: body = response.read().decode("utf-8") except HTTPError as exc: message = exc.read().decode("utf-8", errors="replace") raise RuntimeError(f"News search failed: HTTP {exc.code} {message}") from exc except URLError as exc: raise RuntimeError(f"News search failed: {exc.reason}") from exc return json.loads(body) def _article_to_signal(self, article: dict[str, Any]) -> Optional[SignalItem]: # 新闻候选先保留标题、链接和来源,重要性与摘要由模型最终确定。 title = article.get("title") url = article.get("url") if not isinstance(title, str) or not isinstance(url, str): return None domain = str(article.get("domain") or "News") source_country = str(article.get("sourcecountry") or "") published_at = self._parse_gdelt_date(article.get("seendate")) return SignalItem( topic="AI 新闻候选", title=title[:300], summary=f"来自 {domain} 的新闻候选,GDELT发现时间为 {article.get('seendate') or '未知'}。", source_url=url, source_name=domain, published_at=published_at, importance=3, entities=[value for value in [domain, source_country] if value], ) def _parse_gdelt_date(self, value: Any) -> Optional[datetime]: # GDELT使用类似20260708T120000Z的时间格式。 if not isinstance(value, str): return None try: return datetime.strptime(value, "%Y%m%dT%H%M%SZ") except ValueError: return None