修正北京时间并更换新闻候选源

2026-07-08 23:08:55 +08:00
parent 3ece2327c8
commit 88e1cf4b7a
10 changed files with 178 additions and 55 deletions
@@ -9,6 +9,7 @@ from urllib.parse import urlencode
 from urllib.request import Request, urlopen

 from app.core.config import Settings
+from app.core.timezone import to_beijing_naive
 from app.schemas.agent import SignalItem

 logger = logging.getLogger(__name__)
@@ -110,7 +111,7 @@ class GitHubHotProjectClient:
        return stars * 3 + forks + recency_bonus

    def _parse_datetime(self, value: Any) -> Optional[datetime]:
-        # GitHub时间戳使用UTC ISO格式和Z后缀。
+        # GitHub时间戳使用UTC ISO格式和Z后缀，入库前先换算为北京时间。
        if not isinstance(value, str):
            return None
-        return datetime.fromisoformat(value.replace("Z", "+00:00")).replace(tzinfo=None)
+        return to_beijing_naive(datetime.fromisoformat(value.replace("Z", "+00:00")))
@@ -2,65 +2,69 @@ from __future__ import annotations

 import json
 import logging
-from datetime import date, datetime, timedelta
+from datetime import date, datetime, time, timedelta
 from typing import Any, Optional
 from urllib.error import HTTPError, URLError
 from urllib.parse import urlencode
 from urllib.request import Request, urlopen

 from app.core.config import Settings
+from app.core.timezone import BEIJING_TZ, to_beijing_naive
 from app.schemas.agent import SignalItem

 logger = logging.getLogger(__name__)


 class NewsSearchClient:
-    """通过公开新闻搜索接口采集动态AI新闻候选。"""
+    """通过公开技术新闻检索接口采集动态AI新闻候选。"""

-    endpoint = "https://api.gdeltproject.org/api/v2/doc/doc"
+    endpoint = "https://hn.algolia.com/api/v1/search_by_date"

    def __init__(self, settings: Settings) -> None:
        self.settings = settings

    def collect_ai_news(self, run_date: date) -> list[SignalItem]:
-        # GDELT按日期窗口返回全球新闻候选，模型后续只从这些证据里挑选日报条目。
+        # HN Algolia按时间倒序返回技术新闻候选，模型后续只从这些证据里挑选日报条目。
        since = run_date - timedelta(days=self.settings.news_recent_days)
-        articles: dict[str, dict[str, Any]] = {}
-        payload = self._search(query=self._query(), since=since, until=run_date)
-        for article in payload.get("articles", []):
-            url = article.get("url")
-            if isinstance(url, str) and url.startswith("http"):
-                articles[url] = article
+        stories: dict[str, dict[str, Any]] = {}
+        for query in self._queries():
+            payload = self._search(query=query, since=since, until=run_date)
+            for story in payload.get("hits", []):
+                url = self._story_url(story)
+                object_id = story.get("objectID")
+                key = str(object_id or url)
+                if isinstance(url, str) and url.startswith("http"):
+                    stories[key] = story

-        candidates = [self._article_to_signal(article) for article in articles.values()]
+        candidates = [self._story_to_signal(story) for story in stories.values()]
        candidates = [candidate for candidate in candidates if candidate is not None]
        candidates.sort(key=lambda item: item.published_at or datetime.min, reverse=True)
        logger.info("news_candidates_collected days=%s items=%s", self.settings.news_recent_days, len(candidates))
-        return candidates[: self.settings.news_search_max_records]
+        return candidates[: self.settings.news_max_items]

-    def _query(self) -> str:
-        # GDELT限制请求频率，单次组合查询覆盖模型、Agent、开源和融资四类情报。
-        return (
-            '("AI agent" OR "agent framework" OR "LLM agent" OR '
-            '"large language model" OR LLM OR "AI model" OR '
-            '"open source AI" OR "open-source AI" OR "AI startup funding")'
-        )
+    def _queries(self) -> list[str]:
+        # 查询词覆盖模型、Agent、开源AI和融资产品动态，避免单一关键词漏掉日报候选。
+        return [
+            "AI OR LLM OR agent",
+            "large language model",
+            "OpenAI Anthropic",
+            "AI startup funding",
+        ]

    def _search(self, query: str, since: date, until: date) -> dict[str, Any]:
-        # ArtList模式返回标题、URL、域名和发现时间，正好满足候选证据采集。
+        # search_by_date接口稳定返回最新故事，numericFilters把候选限制在本次日报窗口内。
+        start_at = int(datetime.combine(since, time.min, tzinfo=BEIJING_TZ).timestamp())
+        end_at = int(datetime.combine(until, time.max, tzinfo=BEIJING_TZ).timestamp())
        params = {
            "query": query,
-            "mode": "ArtList",
-            "format": "json",
-            "maxrecords": str(self.settings.news_search_max_records),
-            "sort": "HybridRel",
-            "startdatetime": since.strftime("%Y%m%d000000"),
-            "enddatetime": until.strftime("%Y%m%d235959"),
+            "tags": "story",
+            "hitsPerPage": str(self.settings.news_search_max_records),
+            "numericFilters": f"created_at_i>={start_at},created_at_i<={end_at}",
        }
        url = f"{self.endpoint}?{urlencode(params)}"
        request = Request(url, headers={"User-Agent": "SignalScout"})
        try:
-            with urlopen(request, timeout=30) as response:
+            with urlopen(request, timeout=20) as response:
                body = response.read().decode("utf-8")
        except HTTPError as exc:
            message = exc.read().decode("utf-8", errors="replace")
@@ -69,31 +73,42 @@ class NewsSearchClient:
            raise RuntimeError(f"News search failed: {exc.reason}") from exc
        return json.loads(body)

-    def _article_to_signal(self, article: dict[str, Any]) -> Optional[SignalItem]:
-        # 新闻候选先保留标题、链接和来源，重要性与摘要由模型最终确定。
-        title = article.get("title")
-        url = article.get("url")
+    def _story_to_signal(self, story: dict[str, Any]) -> Optional[SignalItem]:
+        # HN故事先保留标题、链接和互动指标，重要性与摘要由模型最终确定。
+        title = story.get("title")
+        url = self._story_url(story)
        if not isinstance(title, str) or not isinstance(url, str):
            return None
-        domain = str(article.get("domain") or "News")
-        source_country = str(article.get("sourcecountry") or "")
-        published_at = self._parse_gdelt_date(article.get("seendate"))
+        author = str(story.get("author") or "unknown")
+        points = int(story.get("points") or 0)
+        comments = int(story.get("num_comments") or 0)
+        published_at = self._parse_hn_date(story.get("created_at"))
        return SignalItem(
            topic="AI 新闻候选",
            title=title[:300],
-            summary=f"来自 {domain} 的新闻候选，GDELT发现时间为 {article.get('seendate') or '未知'}。",
+            summary=f"Hacker News 最新技术讨论，作者 {author}，约 {points} points、{comments} 条评论。",
            source_url=url,
-            source_name=domain,
+            source_name="Hacker News",
            published_at=published_at,
            importance=3,
-            entities=[value for value in [domain, source_country] if value],
+            entities=["Hacker News", author],
        )

-    def _parse_gdelt_date(self, value: Any) -> Optional[datetime]:
-        # GDELT使用类似20260708T120000Z的时间格式。
+    def _story_url(self, story: dict[str, Any]) -> Optional[str]:
+        # 外链优先；没有外链的Ask/Show HN故事使用HN讨论页作为证据链接。
+        url = story.get("url") or story.get("story_url")
+        if isinstance(url, str) and url.startswith("http"):
+            return url
+        object_id = story.get("objectID")
+        if object_id:
+            return f"https://news.ycombinator.com/item?id={object_id}"
+        return None
+
+    def _parse_hn_date(self, value: Any) -> Optional[datetime]:
+        # HN Algolia返回UTC ISO时间，入库前先换算为北京时间。
        if not isinstance(value, str):
            return None
        try:
-            return datetime.strptime(value, "%Y%m%dT%H%M%SZ")
+            return to_beijing_naive(datetime.fromisoformat(value.replace("Z", "+00:00")))
        except ValueError:
            return None