gitadmin/signalscout-backend
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

修正北京时间并更换新闻候选源

Browse Source
This commit is contained in:
Codex
2026-07-08 23:08:55 +08:00
parent 3ece2327c8
commit 88e1cf4b7a
10 changed files with 178 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/agent/runner.py
+2 -1
View File
@@ -8,6 +8,7 @@ from sqlalchemy.orm import Session
from app.agent.report_writer import ReportWriter
from app.core.config import Settings
from app.core.timezone import beijing_today
from app.db.repository import AgentRepository
from app.integrations.github_client import GitHubHotProjectClient
from app.integrations.news_search_client import NewsSearchClient
@@ -30,7 +31,7 @@ class AgentRunner:
def run_daily(self, run_date: Optional[date] = None) -> RunResponse:
# Runner统一编排候选采集、模型研判和持久化，保证一次运行只有一种业务路径。
current_date = run_date or date.today()
current_date = run_date or beijing_today()
request_payload = {
"run_date": current_date.isoformat(),
"mode": "grok_agent",
app/core/timezone.py
+22
View File
@@ -0,0 +1,22 @@
from datetime import datetime
from datetime import date
from zoneinfo import ZoneInfo
BEIJING_TZ = ZoneInfo("Asia/Shanghai")
def beijing_now() -> datetime:
# 数据库字段使用无时区DateTime，写入前统一转换成北京时间，避免UTC时间直接显示到业务界面。
return datetime.now(BEIJING_TZ).replace(tzinfo=None)
def beijing_today() -> date:
# Agent日报的业务日期按北京时间计算，避免UTC服务器在清晨生成前一天日报。
return datetime.now(BEIJING_TZ).date()
def to_beijing_naive(value: datetime) -> datetime:
# 外部接口常返回带时区时间，入库前统一换算为北京时间的无时区值。
if value.tzinfo is None:
return value
return value.astimezone(BEIJING_TZ).replace(tzinfo=None)
app/db/repository.py
+12 -5
View File
@@ -1,12 +1,12 @@
from __future__ import annotations
from datetime import datetime
from hashlib import sha256
from typing import Optional
from sqlalchemy import select
from sqlalchemy.orm import Session
from app.core.timezone import beijing_now
from app.db.models import AgentRun, Report, Signal, Topic
from app.schemas.agent import SignalItem
@@ -23,20 +23,20 @@ class AgentRepository:
def create_run(self, request_payload: dict) -> AgentRun:
# 运行先进入running状态，情报和日报保存完成后再标记完成。
run = AgentRun(status="running", request_payload=request_payload)
run = AgentRun(status="running", started_at=beijing_now(), request_payload=request_payload)
self.db.add(run)
self.db.flush()
return run
def mark_run_complete(self, run: AgentRun, raw_response: dict) -> None:
run.status = "completed"
run.finished_at = datetime.utcnow()
run.finished_at = beijing_now()
run.raw_response = raw_response
self.db.flush()
def mark_run_failed(self, run: AgentRun, error: str) -> None:
run.status = "failed"
run.finished_at = datetime.utcnow()
run.finished_at = beijing_now()
run.error = error
self.db.flush()
@@ -77,6 +77,7 @@ class AgentRepository:
published_at=item.published_at,
importance=item.importance,
entities=item.entities,
created_at=beijing_now(),
)
self.db.add(signal)
self.db.flush()
@@ -88,7 +89,13 @@ class AgentRepository:
def save_report(self, run: AgentRun, title: str, content_md: str, file_path: str) -> Report:
# 文件路径便于人工打开Markdown产物，MySQL保存可查询正文。
report = Report(run_id=run.id, title=title, content_md=content_md, file_path=file_path)
report = Report(
run_id=run.id,
title=title,
content_md=content_md,
file_path=file_path,
created_at=beijing_now(),
)
self.db.add(report)
self.db.flush()
return report
app/integrations/github_client.py
+3 -2
View File
@@ -9,6 +9,7 @@ from urllib.parse import urlencode
from urllib.request import Request, urlopen
from app.core.config import Settings
from app.core.timezone import to_beijing_naive
from app.schemas.agent import SignalItem
logger = logging.getLogger(__name__)
@@ -110,7 +111,7 @@ class GitHubHotProjectClient:
return stars * 3 + forks + recency_bonus
def _parse_datetime(self, value: Any) -> Optional[datetime]:
# GitHub时间戳使用UTC ISO格式和Z后缀。
# GitHub时间戳使用UTC ISO格式和Z后缀，入库前先换算为北京时间。
if not isinstance(value, str):
return None
return datetime.fromisoformat(value.replace("Z", "+00:00")).replace(tzinfo=None)
return to_beijing_naive(datetime.fromisoformat(value.replace("Z", "+00:00")))
app/integrations/news_search_client.py
+54 -39
View File
@@ -2,65 +2,69 @@ from __future__ import annotations
import json
import logging
from datetime import date, datetime, timedelta
from datetime import date, datetime, time, timedelta
from typing import Any, Optional
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from app.core.config import Settings
from app.core.timezone import BEIJING_TZ, to_beijing_naive
from app.schemas.agent import SignalItem
logger = logging.getLogger(__name__)
class NewsSearchClient:
"""通过公开新闻搜索接口采集动态AI新闻候选。"""
"""通过公开技术新闻检索接口采集动态AI新闻候选。"""
endpoint = "https://api.gdeltproject.org/api/v2/doc/doc"
endpoint = "https://hn.algolia.com/api/v1/search_by_date"
def __init__(self, settings: Settings) -> None:
self.settings = settings
def collect_ai_news(self, run_date: date) -> list[SignalItem]:
# GDELT按日期窗口返回全球新闻候选，模型后续只从这些证据里挑选日报条目。
# HN Algolia按时间倒序返回技术新闻候选，模型后续只从这些证据里挑选日报条目。
since = run_date - timedelta(days=self.settings.news_recent_days)
articles: dict[str, dict[str, Any]] = {}
payload = self._search(query=self._query(), since=since, until=run_date)
for article in payload.get("articles", []):
url = article.get("url")
stories: dict[str, dict[str, Any]] = {}
for query in self._queries():
payload = self._search(query=query, since=since, until=run_date)
for story in payload.get("hits", []):
url = self._story_url(story)
object_id = story.get("objectID")
key = str(object_id or url)
if isinstance(url, str) and url.startswith("http"):
articles[url] = article
stories[key] = story
candidates = [self._article_to_signal(article) for article in articles.values()]
candidates = [self._story_to_signal(story) for story in stories.values()]
candidates = [candidate for candidate in candidates if candidate is not None]
candidates.sort(key=lambda item: item.published_at or datetime.min, reverse=True)
logger.info("news_candidates_collected days=%s items=%s", self.settings.news_recent_days, len(candidates))
return candidates[: self.settings.news_search_max_records]
return candidates[: self.settings.news_max_items]
def _query(self) -> str:
# GDELT限制请求频率，单次组合查询覆盖模型、Agent、开源和融资四类情报。
return (
'("AI agent" OR "agent framework" OR "LLM agent" OR '
'"large language model" OR LLM OR "AI model" OR '
'"open source AI" OR "open-source AI" OR "AI startup funding")'
)
def _queries(self) -> list[str]:
# 查询词覆盖模型、Agent、开源AI和融资产品动态，避免单一关键词漏掉日报候选。
return [
"AI OR LLM OR agent",
"large language model",
"OpenAI Anthropic",
"AI startup funding",
]
def _search(self, query: str, since: date, until: date) -> dict[str, Any]:
# ArtList模式返回标题、URL、域名和发现时间，正好满足候选证据采集。
# search_by_date接口稳定返回最新故事，numericFilters把候选限制在本次日报窗口内。
start_at = int(datetime.combine(since, time.min, tzinfo=BEIJING_TZ).timestamp())
end_at = int(datetime.combine(until, time.max, tzinfo=BEIJING_TZ).timestamp())
params = {
"query": query,
"mode": "ArtList",
"format": "json",
"maxrecords": str(self.settings.news_search_max_records),
"sort": "HybridRel",
"startdatetime": since.strftime("%Y%m%d000000"),
"enddatetime": until.strftime("%Y%m%d235959"),
"tags": "story",
"hitsPerPage": str(self.settings.news_search_max_records),
"numericFilters": f"created_at_i>={start_at},created_at_i<={end_at}",
}
url = f"{self.endpoint}?{urlencode(params)}"
request = Request(url, headers={"User-Agent": "SignalScout"})
try:
with urlopen(request, timeout=30) as response:
with urlopen(request, timeout=20) as response:
body = response.read().decode("utf-8")
except HTTPError as exc:
message = exc.read().decode("utf-8", errors="replace")
@@ -69,31 +73,42 @@ class NewsSearchClient:
raise RuntimeError(f"News search failed: {exc.reason}") from exc
return json.loads(body)
def _article_to_signal(self, article: dict[str, Any]) -> Optional[SignalItem]:
# 新闻候选先保留标题、链接和来源，重要性与摘要由模型最终确定。
title = article.get("title")
url = article.get("url")
def _story_to_signal(self, story: dict[str, Any]) -> Optional[SignalItem]:
# HN故事先保留标题、链接和互动指标，重要性与摘要由模型最终确定。
title = story.get("title")
url = self._story_url(story)
if not isinstance(title, str) or not isinstance(url, str):
return None
domain = str(article.get("domain") or "News")
source_country = str(article.get("sourcecountry") or "")
published_at = self._parse_gdelt_date(article.get("seendate"))
author = str(story.get("author") or "unknown")
points = int(story.get("points") or 0)
comments = int(story.get("num_comments") or 0)
published_at = self._parse_hn_date(story.get("created_at"))
return SignalItem(
topic="AI 新闻候选",
title=title[:300],
summary=f"来自 {domain} 的新闻候选，GDELT发现时间为 {article.get('seendate') or '未知'}。",
summary=f"Hacker News 最新技术讨论，作者 {author}，约 {points} points、{comments} 条评论。",
source_url=url,
source_name=domain,
source_name="Hacker News",
published_at=published_at,
importance=3,
entities=[value for value in [domain, source_country] if value],
entities=["Hacker News", author],
)
def _parse_gdelt_date(self, value: Any) -> Optional[datetime]:
# GDELT使用类似20260708T120000Z的时间格式。
def _story_url(self, story: dict[str, Any]) -> Optional[str]:
# 外链优先；没有外链的Ask/Show HN故事使用HN讨论页作为证据链接。
url = story.get("url") or story.get("story_url")
if isinstance(url, str) and url.startswith("http"):
return url
object_id = story.get("objectID")
if object_id:
return f"https://news.ycombinator.com/item?id={object_id}"
return None
def _parse_hn_date(self, value: Any) -> Optional[datetime]:
# HN Algolia返回UTC ISO时间，入库前先换算为北京时间。
if not isinstance(value, str):
return None
try:
return datetime.strptime(value, "%Y%m%dT%H%M%SZ")
return to_beijing_naive(datetime.fromisoformat(value.replace("Z", "+00:00")))
except ValueError:
return None
app/ui/home.py
+4 -1
View File
@@ -454,9 +454,12 @@ HOME_PAGE_HTML = """
function formatDate(value) {
if (!value) return "未标注时间";
const date = new Date(value);
const text = String(value);
const normalized = /(?:Z|[+-]\\d{2}:\\d{2})$/.test(text) ? text : `${text}+08:00`;
const date = new Date(normalized);
if (Number.isNaN(date.getTime())) return "未标注时间";
return date.toLocaleString("zh-CN", {
timeZone: "Asia/Shanghai",
month: "2-digit",
day: "2-digit",
hour: "2-digit",
migrations/versions/202607081530_store_business_time.py
+45
View File
@@ -0,0 +1,45 @@
"""store business time in beijing timezone
Revision ID: 202607081530
Revises: 202607080900
Create Date: 2026-07-08 15:30:00
"""
from typing import Sequence, Union
from alembic import op
revision: str = "202607081530"
down_revision: Union[str, None] = "202607080900"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# 历史数据由UTC换算为北京时间，后续新数据由业务层直接写入北京时间。
op.execute("UPDATE agent_runs SET started_at = DATE_ADD(started_at, INTERVAL 8 HOUR)")
op.execute(
"UPDATE agent_runs SET finished_at = DATE_ADD(finished_at, INTERVAL 8 HOUR) "
"WHERE finished_at IS NOT NULL"
)
op.execute("UPDATE reports SET created_at = DATE_ADD(created_at, INTERVAL 8 HOUR)")
op.execute("UPDATE signals SET created_at = DATE_ADD(created_at, INTERVAL 8 HOUR)")
op.execute(
"UPDATE signals SET published_at = DATE_ADD(published_at, INTERVAL 8 HOUR) "
"WHERE published_at IS NOT NULL"
)
def downgrade() -> None:
# 回滚时恢复迁移前的UTC记录值，便于迁移链保持可逆。
op.execute("UPDATE agent_runs SET started_at = DATE_SUB(started_at, INTERVAL 8 HOUR)")
op.execute(
"UPDATE agent_runs SET finished_at = DATE_SUB(finished_at, INTERVAL 8 HOUR) "
"WHERE finished_at IS NOT NULL"
)
op.execute("UPDATE reports SET created_at = DATE_SUB(created_at, INTERVAL 8 HOUR)")
op.execute("UPDATE signals SET created_at = DATE_SUB(created_at, INTERVAL 8 HOUR)")
op.execute(
"UPDATE signals SET published_at = DATE_SUB(published_at, INTERVAL 8 HOUR) "
"WHERE published_at IS NOT NULL"
)
tests/test_github_client.py
+2
View File
@@ -23,3 +23,5 @@ def test_github_client_maps_repository_to_signal() -> None:
assert signal.source_url == "https://github.com/example/agent-kit"
assert "2400 stars" in signal.summary
assert "2026-05-01" in signal.summary
assert signal.published_at is not None
assert signal.published_at.hour == 17
tests/test_news_search_client.py
+10 -6
View File
@@ -5,18 +5,22 @@ from app.integrations.news_search_client import NewsSearchClient
def test_news_search_client_maps_article_to_candidate() -> None:
# 新闻搜索结果会先转为候选信号，再交给模型筛选成最终情报。
client = NewsSearchClient(Settings())
signal = client._article_to_signal(
signal = client._story_to_signal(
{
"title": "AI agent platform launches",
"title": "Show HN: AI agent platform launches",
"url": "https://example.com/ai-agent-platform",
"domain": "example.com",
"sourcecountry": "US",
"seendate": "20260708T120000Z",
"author": "builder",
"points": 128,
"num_comments": 24,
"created_at": "2026-07-08T12:00:00Z",
"objectID": "123",
}
)
assert signal is not None
assert signal.topic == "AI 新闻候选"
assert signal.source_name == "example.com"
assert signal.source_name == "Hacker News"
assert signal.published_at is not None
assert signal.published_at.year == 2026
assert signal.published_at.hour == 20
assert "128 points" in signal.summary
tests/test_timezone.py
+23
View File
@@ -0,0 +1,23 @@
from datetime import datetime, timezone
from app.core.timezone import beijing_now, beijing_today, to_beijing_naive
def test_beijing_now_returns_naive_business_time() -> None:
# 数据库存储无时区时间，业务层写入前统一生成北京时间。
current = beijing_now()
assert current.tzinfo is None
def test_beijing_today_returns_date() -> None:
# 日报业务日期只需要日期部分，由北京时间统一决定。
assert beijing_today().isoformat()
def test_to_beijing_naive_converts_utc_time() -> None:
# 外部UTC时间进入数据库前要先换算成北京时间，避免前端显示少8小时。
current = to_beijing_naive(datetime(2026, 7, 8, 14, 46, tzinfo=timezone.utc))
assert current.tzinfo is None
assert current.hour == 22