重写为 AI 情报 Agent 并接入 Jenkins 流水线

app/integrations/__init__.py

@@ -0,0 +1 @@
+"""External data-source clients used by the agent pipeline."""

app/integrations/github_client.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import json
+import logging
+from datetime import date, datetime, timedelta
+from typing import Any, Optional
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+from app.core.config import Settings
+from app.schemas.agent import SignalItem
+
+logger = logging.getLogger(__name__)
+
+
+class GitHubHotProjectClient:
+    """Collects recently active, high-signal GitHub AI repositories."""
+
+    endpoint = "https://api.github.com/search/repositories"
+
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def collect_hot_projects(self, run_date: date) -> list[SignalItem]:
+        # GitHub官方搜索提供开源项目热度候选，最终是否入选交给模型判断。
+        since = run_date - timedelta(days=self.settings.github_recent_days)
+        repositories: dict[str, dict[str, Any]] = {}
+        for query in self._queries(since=since):
+            payload = self._search(query)
+            for repository in payload.get("items", []):
+                html_url = repository.get("html_url")
+                if isinstance(html_url, str):
+                    repositories[html_url] = repository
+
+        ranked = sorted(repositories.values(), key=lambda item: self._heat_score(item, run_date), reverse=True)
+        projects = [self._to_signal_item(repository) for repository in ranked[: self.settings.github_max_projects]]
+        logger.info(
+            "github_hot_projects_collected days=%s min_stars=%s projects=%s",
+            self.settings.github_recent_days,
+            self.settings.github_min_stars,
+            len(projects),
+        )
+        return projects
+
+    def _queries(self, since: date) -> list[str]:
+        # 查询集聚焦新近创建的AI项目，避免长期头部仓库挤占每日发现。
+        since_text = since.isoformat()
+        min_stars = self.settings.github_min_stars
+        return [
+            f"topic:llm created:>={since_text} stars:>={min_stars}",
+            f"topic:ai-agent created:>={since_text} stars:>={min_stars}",
+            f"topic:rag created:>={since_text} stars:>={min_stars}",
+            f"llm in:name,description created:>={since_text} stars:>={min_stars}",
+            f"llm agent in:name,description created:>={since_text} stars:>={min_stars}",
+        ]
+
+    def _search(self, query: str) -> dict[str, Any]:
+        # GitHub REST搜索直接返回排序后的仓库元数据，便于后续模型研判。
+        url = f"{self.endpoint}?{urlencode({'q': query, 'sort': 'stars', 'order': 'desc', 'per_page': '10'})}"
+        request = Request(
+            url,
+            headers={
+                "Accept": "application/vnd.github+json",
+                "User-Agent": "SignalScout",
+                "X-GitHub-Api-Version": "2022-11-28",
+            },
+        )
+        try:
+            with urlopen(request, timeout=20) as response:
+                body = response.read().decode("utf-8")
+        except HTTPError as exc:
+            message = exc.read().decode("utf-8", errors="replace")
+            raise RuntimeError(f"GitHub project search failed: HTTP {exc.code} {message}") from exc
+        except URLError as exc:
+            raise RuntimeError(f"GitHub project search failed: {exc.reason}") from exc
+        return json.loads(body)
+
+    def _to_signal_item(self, repository: dict[str, Any]) -> SignalItem:
+        # 仓库元数据先标准化为信号候选，后续由模型统一筛选和改写摘要。
+        full_name = str(repository.get("full_name") or repository.get("name") or "Unknown repository")
+        description = str(repository.get("description") or "暂无项目描述")
+        stars = int(repository.get("stargazers_count") or 0)
+        forks = int(repository.get("forks_count") or 0)
+        language = repository.get("language") or "未标注语言"
+        created_at = self._parse_datetime(repository.get("created_at"))
+        pushed_at = self._parse_datetime(repository.get("pushed_at"))
+        freshness = created_at.date().isoformat() if created_at else "最近"
+        summary = (
+            f"{description} 该项目创建于 {freshness}，当前约 {stars} stars、{forks} forks，"
+            f"主要语言为 {language}。"
+        )
+        return SignalItem(
+            topic="GitHub 热门项目",
+            title=full_name,
+            summary=summary,
+            source_url=str(repository["html_url"]),
+            source_name="GitHub",
+            published_at=created_at or pushed_at,
+            importance=3,
+            entities=[full_name, "GitHub", str(language)],
+        )
+
+    def _heat_score(self, repository: dict[str, Any], run_date: date) -> int:
+        # Stars表示关注度，forks表示采用迹象，新项目获得额外新鲜度权重。
+        stars = int(repository.get("stargazers_count") or 0)
+        forks = int(repository.get("forks_count") or 0)
+        created_at = self._parse_datetime(repository.get("created_at"))
+        recency_bonus = 100 if created_at and created_at.date() >= run_date - timedelta(days=7) else 0
+        return stars * 3 + forks + recency_bonus
+
+    def _parse_datetime(self, value: Any) -> Optional[datetime]:
+        # GitHub时间戳使用UTC ISO格式和Z后缀。
+        if not isinstance(value, str):
+            return None
+        return datetime.fromisoformat(value.replace("Z", "+00:00")).replace(tzinfo=None)

app/integrations/news_search_client.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import json
+import logging
+from datetime import date, datetime, timedelta
+from typing import Any, Optional
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+
+from app.core.config import Settings
+from app.schemas.agent import SignalItem
+
+logger = logging.getLogger(__name__)
+
+
+class NewsSearchClient:
+    """通过公开新闻搜索接口采集动态AI新闻候选。"""
+
+    endpoint = "https://api.gdeltproject.org/api/v2/doc/doc"
+
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def collect_ai_news(self, run_date: date) -> list[SignalItem]:
+        # GDELT按日期窗口返回全球新闻候选，模型后续只从这些证据里挑选日报条目。
+        since = run_date - timedelta(days=self.settings.news_recent_days)
+        articles: dict[str, dict[str, Any]] = {}
+        payload = self._search(query=self._query(), since=since, until=run_date)
+        for article in payload.get("articles", []):
+            url = article.get("url")
+            if isinstance(url, str) and url.startswith("http"):
+                articles[url] = article
+
+        candidates = [self._article_to_signal(article) for article in articles.values()]
+        candidates = [candidate for candidate in candidates if candidate is not None]
+        candidates.sort(key=lambda item: item.published_at or datetime.min, reverse=True)
+        logger.info("news_candidates_collected days=%s items=%s", self.settings.news_recent_days, len(candidates))
+        return candidates[: self.settings.news_search_max_records]
+
+    def _query(self) -> str:
+        # GDELT限制请求频率，单次组合查询覆盖模型、Agent、开源和融资四类情报。
+        return (
+            '("AI agent" OR "agent framework" OR "LLM agent" OR '
+            '"large language model" OR LLM OR "AI model" OR '
+            '"open source AI" OR "open-source AI" OR "AI startup funding")'
+        )
+
+    def _search(self, query: str, since: date, until: date) -> dict[str, Any]:
+        # ArtList模式返回标题、URL、域名和发现时间，正好满足候选证据采集。
+        params = {
+            "query": query,
+            "mode": "ArtList",
+            "format": "json",
+            "maxrecords": str(self.settings.news_search_max_records),
+            "sort": "HybridRel",
+            "startdatetime": since.strftime("%Y%m%d000000"),
+            "enddatetime": until.strftime("%Y%m%d235959"),
+        }
+        url = f"{self.endpoint}?{urlencode(params)}"
+        request = Request(url, headers={"User-Agent": "SignalScout"})
+        try:
+            with urlopen(request, timeout=30) as response:
+                body = response.read().decode("utf-8")
+        except HTTPError as exc:
+            message = exc.read().decode("utf-8", errors="replace")
+            raise RuntimeError(f"News search failed: HTTP {exc.code} {message}") from exc
+        except URLError as exc:
+            raise RuntimeError(f"News search failed: {exc.reason}") from exc
+        return json.loads(body)
+
+    def _article_to_signal(self, article: dict[str, Any]) -> Optional[SignalItem]:
+        # 新闻候选先保留标题、链接和来源，重要性与摘要由模型最终确定。
+        title = article.get("title")
+        url = article.get("url")
+        if not isinstance(title, str) or not isinstance(url, str):
+            return None
+        domain = str(article.get("domain") or "News")
+        source_country = str(article.get("sourcecountry") or "")
+        published_at = self._parse_gdelt_date(article.get("seendate"))
+        return SignalItem(
+            topic="AI 新闻候选",
+            title=title[:300],
+            summary=f"来自 {domain} 的新闻候选，GDELT发现时间为 {article.get('seendate') or '未知'}。",
+            source_url=url,
+            source_name=domain,
+            published_at=published_at,
+            importance=3,
+            entities=[value for value in [domain, source_country] if value],
+        )
+
+    def _parse_gdelt_date(self, value: Any) -> Optional[datetime]:
+        # GDELT使用类似20260708T120000Z的时间格式。
+        if not isinstance(value, str):
+            return None
+        try:
+            return datetime.strptime(value, "%Y%m%dT%H%M%SZ")
+        except ValueError:
+            return None