Codex
100 lines
4.3 KiB
Python
100 lines
4.3 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from datetime import date, datetime, timedelta
|
|
from typing import Any, Optional
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urlencode
|
|
from urllib.request import Request, urlopen
|
|
|
|
from app.core.config import Settings
|
|
from app.schemas.agent import SignalItem
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class NewsSearchClient:
|
|
"""通过公开新闻搜索接口采集动态AI新闻候选。"""
|
|
|
|
endpoint = "https://api.gdeltproject.org/api/v2/doc/doc"
|
|
|
|
def __init__(self, settings: Settings) -> None:
|
|
self.settings = settings
|
|
|
|
def collect_ai_news(self, run_date: date) -> list[SignalItem]:
|
|
# GDELT按日期窗口返回全球新闻候选，模型后续只从这些证据里挑选日报条目。
|
|
since = run_date - timedelta(days=self.settings.news_recent_days)
|
|
articles: dict[str, dict[str, Any]] = {}
|
|
payload = self._search(query=self._query(), since=since, until=run_date)
|
|
for article in payload.get("articles", []):
|
|
url = article.get("url")
|
|
if isinstance(url, str) and url.startswith("http"):
|
|
articles[url] = article
|
|
|
|
candidates = [self._article_to_signal(article) for article in articles.values()]
|
|
candidates = [candidate for candidate in candidates if candidate is not None]
|
|
candidates.sort(key=lambda item: item.published_at or datetime.min, reverse=True)
|
|
logger.info("news_candidates_collected days=%s items=%s", self.settings.news_recent_days, len(candidates))
|
|
return candidates[: self.settings.news_search_max_records]
|
|
|
|
def _query(self) -> str:
|
|
# GDELT限制请求频率，单次组合查询覆盖模型、Agent、开源和融资四类情报。
|
|
return (
|
|
'("AI agent" OR "agent framework" OR "LLM agent" OR '
|
|
'"large language model" OR LLM OR "AI model" OR '
|
|
'"open source AI" OR "open-source AI" OR "AI startup funding")'
|
|
)
|
|
|
|
def _search(self, query: str, since: date, until: date) -> dict[str, Any]:
|
|
# ArtList模式返回标题、URL、域名和发现时间，正好满足候选证据采集。
|
|
params = {
|
|
"query": query,
|
|
"mode": "ArtList",
|
|
"format": "json",
|
|
"maxrecords": str(self.settings.news_search_max_records),
|
|
"sort": "HybridRel",
|
|
"startdatetime": since.strftime("%Y%m%d000000"),
|
|
"enddatetime": until.strftime("%Y%m%d235959"),
|
|
}
|
|
url = f"{self.endpoint}?{urlencode(params)}"
|
|
request = Request(url, headers={"User-Agent": "SignalScout"})
|
|
try:
|
|
with urlopen(request, timeout=30) as response:
|
|
body = response.read().decode("utf-8")
|
|
except HTTPError as exc:
|
|
message = exc.read().decode("utf-8", errors="replace")
|
|
raise RuntimeError(f"News search failed: HTTP {exc.code} {message}") from exc
|
|
except URLError as exc:
|
|
raise RuntimeError(f"News search failed: {exc.reason}") from exc
|
|
return json.loads(body)
|
|
|
|
def _article_to_signal(self, article: dict[str, Any]) -> Optional[SignalItem]:
|
|
# 新闻候选先保留标题、链接和来源，重要性与摘要由模型最终确定。
|
|
title = article.get("title")
|
|
url = article.get("url")
|
|
if not isinstance(title, str) or not isinstance(url, str):
|
|
return None
|
|
domain = str(article.get("domain") or "News")
|
|
source_country = str(article.get("sourcecountry") or "")
|
|
published_at = self._parse_gdelt_date(article.get("seendate"))
|
|
return SignalItem(
|
|
topic="AI 新闻候选",
|
|
title=title[:300],
|
|
summary=f"来自 {domain} 的新闻候选，GDELT发现时间为 {article.get('seendate') or '未知'}。",
|
|
source_url=url,
|
|
source_name=domain,
|
|
published_at=published_at,
|
|
importance=3,
|
|
entities=[value for value in [domain, source_country] if value],
|
|
)
|
|
|
|
def _parse_gdelt_date(self, value: Any) -> Optional[datetime]:
|
|
# GDELT使用类似20260708T120000Z的时间格式。
|
|
if not isinstance(value, str):
|
|
return None
|
|
try:
|
|
return datetime.strptime(value, "%Y%m%dT%H%M%SZ")
|
|
except ValueError:
|
|
return None
|