暂停情报采集任务
This commit is contained in:
+2
-1
@@ -3,7 +3,7 @@ SIGNALSCOUT_ENV=dev
|
|||||||
SIGNALSCOUT_DATABASE_URL=mysql+pymysql://signalscout:signalscout@127.0.0.1:3306/signalscout?charset=utf8mb4
|
SIGNALSCOUT_DATABASE_URL=mysql+pymysql://signalscout:signalscout@127.0.0.1:3306/signalscout?charset=utf8mb4
|
||||||
SIGNALSCOUT_LLM_BASE_URL=https://api.zayuapi.com/v1
|
SIGNALSCOUT_LLM_BASE_URL=https://api.zayuapi.com/v1
|
||||||
SIGNALSCOUT_LLM_API_KEY=
|
SIGNALSCOUT_LLM_API_KEY=
|
||||||
SIGNALSCOUT_LLM_MODEL=grok-4.3
|
SIGNALSCOUT_LLM_MODEL=grok-4.5
|
||||||
SIGNALSCOUT_LLM_TIMEOUT_SECONDS=180
|
SIGNALSCOUT_LLM_TIMEOUT_SECONDS=180
|
||||||
SIGNALSCOUT_LLM_MAX_TOKENS=8000
|
SIGNALSCOUT_LLM_MAX_TOKENS=8000
|
||||||
SIGNALSCOUT_NEWS_RECENT_DAYS=3
|
SIGNALSCOUT_NEWS_RECENT_DAYS=3
|
||||||
@@ -13,6 +13,7 @@ SIGNALSCOUT_GITHUB_RECENT_DAYS=30
|
|||||||
SIGNALSCOUT_GITHUB_MIN_STARS=20
|
SIGNALSCOUT_GITHUB_MIN_STARS=20
|
||||||
SIGNALSCOUT_GITHUB_MAX_PROJECTS=40
|
SIGNALSCOUT_GITHUB_MAX_PROJECTS=40
|
||||||
SIGNALSCOUT_AGENT_TIMEZONE=Asia/Shanghai
|
SIGNALSCOUT_AGENT_TIMEZONE=Asia/Shanghai
|
||||||
|
SIGNALSCOUT_AGENT_ENABLED=false
|
||||||
SIGNALSCOUT_AGENT_INTERVAL_MINUTES=30
|
SIGNALSCOUT_AGENT_INTERVAL_MINUTES=30
|
||||||
SIGNALSCOUT_REPORT_DIR=reports
|
SIGNALSCOUT_REPORT_DIR=reports
|
||||||
SIGNALSCOUT_LOG_FILE=logs/app.log
|
SIGNALSCOUT_LOG_FILE=logs/app.log
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ cp .env.example .env
|
|||||||
SIGNALSCOUT_DATABASE_URL=mysql+pymysql://user:password@host:3306/signalscout?charset=utf8mb4
|
SIGNALSCOUT_DATABASE_URL=mysql+pymysql://user:password@host:3306/signalscout?charset=utf8mb4
|
||||||
SIGNALSCOUT_LLM_BASE_URL=https://api.zayuapi.com/v1
|
SIGNALSCOUT_LLM_BASE_URL=https://api.zayuapi.com/v1
|
||||||
SIGNALSCOUT_LLM_API_KEY=你的中转站 key
|
SIGNALSCOUT_LLM_API_KEY=你的中转站 key
|
||||||
SIGNALSCOUT_LLM_MODEL=grok-4.3
|
SIGNALSCOUT_LLM_MODEL=grok-4.5
|
||||||
SIGNALSCOUT_LLM_TIMEOUT_SECONDS=180
|
SIGNALSCOUT_LLM_TIMEOUT_SECONDS=180
|
||||||
SIGNALSCOUT_LLM_MAX_TOKENS=8000
|
SIGNALSCOUT_LLM_MAX_TOKENS=8000
|
||||||
SIGNALSCOUT_NEWS_RECENT_DAYS=3
|
SIGNALSCOUT_NEWS_RECENT_DAYS=3
|
||||||
@@ -37,11 +37,14 @@ SIGNALSCOUT_NEWS_SEARCH_MAX_RECORDS=100
|
|||||||
SIGNALSCOUT_GITHUB_RECENT_DAYS=30
|
SIGNALSCOUT_GITHUB_RECENT_DAYS=30
|
||||||
SIGNALSCOUT_GITHUB_MIN_STARS=20
|
SIGNALSCOUT_GITHUB_MIN_STARS=20
|
||||||
SIGNALSCOUT_GITHUB_MAX_PROJECTS=40
|
SIGNALSCOUT_GITHUB_MAX_PROJECTS=40
|
||||||
|
SIGNALSCOUT_AGENT_ENABLED=false
|
||||||
SIGNALSCOUT_AGENT_INTERVAL_MINUTES=30
|
SIGNALSCOUT_AGENT_INTERVAL_MINUTES=30
|
||||||
```
|
```
|
||||||
|
|
||||||
单次运行的目标上限是新闻 60 条、GitHub 项目 40 条;实际入库数量会根据候选质量、链接去重和模型筛选结果浮动。
|
单次运行的目标上限是新闻 60 条、GitHub 项目 40 条;实际入库数量会根据候选质量、链接去重和模型筛选结果浮动。
|
||||||
|
|
||||||
|
设置 `SIGNALSCOUT_AGENT_ENABLED=true` 后,服务才会启用定时和手动情报采集。
|
||||||
|
|
||||||
## 本机运行
|
## 本机运行
|
||||||
|
|
||||||
按项目规则,本机 Python 使用:
|
按项目规则,本机 Python 使用:
|
||||||
|
|||||||
@@ -16,8 +16,12 @@ def run_scheduled_job(settings: Settings) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def build_scheduler(settings: Settings) -> BackgroundScheduler:
|
def build_scheduler(settings: Settings) -> BackgroundScheduler:
|
||||||
# APScheduler负责按固定间隔自动运行,避免服务重启时连续触发外部采集限流。
|
# 调度器始终可供手动任务复用,自动采集关闭时不注册任何周期任务。
|
||||||
scheduler = BackgroundScheduler(timezone=settings.agent_timezone)
|
scheduler = BackgroundScheduler(timezone=settings.agent_timezone)
|
||||||
|
if not settings.agent_enabled:
|
||||||
|
return scheduler
|
||||||
|
|
||||||
|
# APScheduler负责按固定间隔自动运行,避免服务重启时连续触发外部采集限流。
|
||||||
scheduler.add_job(
|
scheduler.add_job(
|
||||||
run_scheduled_job,
|
run_scheduled_job,
|
||||||
trigger="interval",
|
trigger="interval",
|
||||||
|
|||||||
+4
-1
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
from typing import Literal, Optional
|
from typing import Literal, Optional
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Request
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||||
from fastapi.responses import HTMLResponse
|
from fastapi.responses import HTMLResponse
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
@@ -50,6 +50,9 @@ def trigger_daily_run(
|
|||||||
settings: Settings = Depends(get_settings),
|
settings: Settings = Depends(get_settings),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> RunTriggerRead:
|
) -> RunTriggerRead:
|
||||||
|
# 暂停期间拒绝所有入口的采集请求,保证不会产生外部模型调用。
|
||||||
|
if not settings.agent_enabled:
|
||||||
|
raise HTTPException(status_code=503, detail="情报采集当前已暂停")
|
||||||
# 手动搜索投递到调度器线程,避免HTTP请求承载长时间Grok搜索任务。
|
# 手动搜索投递到调度器线程,避免HTTP请求承载长时间Grok搜索任务。
|
||||||
running = db.scalar(
|
running = db.scalar(
|
||||||
select(AgentRun).where(AgentRun.status == "running").order_by(AgentRun.id.desc()).limit(1)
|
select(AgentRun).where(AgentRun.status == "running").order_by(AgentRun.id.desc()).limit(1)
|
||||||
|
|||||||
+3
-1
@@ -14,7 +14,7 @@ class Settings(BaseSettings):
|
|||||||
)
|
)
|
||||||
llm_base_url: str = "https://api.zayuapi.com/v1"
|
llm_base_url: str = "https://api.zayuapi.com/v1"
|
||||||
llm_api_key: str = ""
|
llm_api_key: str = ""
|
||||||
llm_model: str = "grok-4.3"
|
llm_model: str = "grok-4.5"
|
||||||
llm_timeout_seconds: int = 180
|
llm_timeout_seconds: int = 180
|
||||||
llm_max_tokens: int = 8000
|
llm_max_tokens: int = 8000
|
||||||
news_recent_days: int = 3
|
news_recent_days: int = 3
|
||||||
@@ -24,6 +24,8 @@ class Settings(BaseSettings):
|
|||||||
github_min_stars: int = 20
|
github_min_stars: int = 20
|
||||||
github_max_projects: int = 40
|
github_max_projects: int = 40
|
||||||
agent_timezone: str = "Asia/Shanghai"
|
agent_timezone: str = "Asia/Shanghai"
|
||||||
|
# 采集开关统一控制定时任务和手动触发,暂停期默认不调用外部模型。
|
||||||
|
agent_enabled: bool = False
|
||||||
agent_interval_minutes: int = 30
|
agent_interval_minutes: int = 30
|
||||||
report_dir: Path = Path("reports")
|
report_dir: Path = Path("reports")
|
||||||
log_file: Path = Path("logs/app.log")
|
log_file: Path = Path("logs/app.log")
|
||||||
|
|||||||
@@ -162,30 +162,6 @@ HOME_PAGE_HTML = """
|
|||||||
margin-top: 5px;
|
margin-top: 5px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.actions {
|
|
||||||
display: flex;
|
|
||||||
justify-content: flex-end;
|
|
||||||
margin-top: 12px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.action-button {
|
|
||||||
border: 1px solid oklch(25% 0.02 210);
|
|
||||||
border-radius: 8px;
|
|
||||||
padding: 9px 13px;
|
|
||||||
background: var(--ink);
|
|
||||||
color: white;
|
|
||||||
font: inherit;
|
|
||||||
font-size: 13px;
|
|
||||||
font-weight: 700;
|
|
||||||
cursor: pointer;
|
|
||||||
min-width: 96px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.action-button:disabled {
|
|
||||||
cursor: wait;
|
|
||||||
opacity: 0.66;
|
|
||||||
}
|
|
||||||
|
|
||||||
.layout {
|
.layout {
|
||||||
display: grid;
|
display: grid;
|
||||||
grid-template-columns: minmax(0, 1.2fr) minmax(340px, 0.8fr);
|
grid-template-columns: minmax(0, 1.2fr) minmax(340px, 0.8fr);
|
||||||
@@ -434,9 +410,6 @@ HOME_PAGE_HTML = """
|
|||||||
<div class="pulse">
|
<div class="pulse">
|
||||||
<strong id="todayCountText">读取中</strong>
|
<strong id="todayCountText">读取中</strong>
|
||||||
<span id="todayCountDetail">正在整理新闻与项目。</span>
|
<span id="todayCountDetail">正在整理新闻与项目。</span>
|
||||||
<div class="actions">
|
|
||||||
<button class="action-button" id="manualRunButton" type="button">手动搜索</button>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
@@ -511,7 +484,6 @@ HOME_PAGE_HTML = """
|
|||||||
const runCountFull = document.querySelector("#runCountFull");
|
const runCountFull = document.querySelector("#runCountFull");
|
||||||
const todayCountText = document.querySelector("#todayCountText");
|
const todayCountText = document.querySelector("#todayCountText");
|
||||||
const todayCountDetail = document.querySelector("#todayCountDetail");
|
const todayCountDetail = document.querySelector("#todayCountDetail");
|
||||||
const manualRunButton = document.querySelector("#manualRunButton");
|
|
||||||
|
|
||||||
const stateText = {
|
const stateText = {
|
||||||
completed: "完成",
|
completed: "完成",
|
||||||
@@ -695,34 +667,9 @@ HOME_PAGE_HTML = """
|
|||||||
renderRuns(data.runs || []);
|
renderRuns(data.runs || []);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function triggerManualRun() {
|
|
||||||
manualRunButton.disabled = true;
|
|
||||||
manualRunButton.textContent = "搜索中";
|
|
||||||
try {
|
|
||||||
const response = await fetch("/agent/runs/daily", {
|
|
||||||
method: "POST",
|
|
||||||
headers: { Accept: "application/json" }
|
|
||||||
});
|
|
||||||
if (!response.ok) throw new Error("手动搜索失败");
|
|
||||||
await loadDashboard();
|
|
||||||
} finally {
|
|
||||||
window.setTimeout(() => {
|
|
||||||
manualRunButton.disabled = false;
|
|
||||||
manualRunButton.textContent = "手动搜索";
|
|
||||||
loadDashboard().catch(() => {});
|
|
||||||
}, 3000);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
navButtons.forEach((button) => {
|
navButtons.forEach((button) => {
|
||||||
button.addEventListener("click", () => activateView(button.dataset.viewTarget));
|
button.addEventListener("click", () => activateView(button.dataset.viewTarget));
|
||||||
});
|
});
|
||||||
manualRunButton.addEventListener("click", () => {
|
|
||||||
triggerManualRun().catch(() => {
|
|
||||||
manualRunButton.disabled = false;
|
|
||||||
manualRunButton.textContent = "手动搜索";
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
const initialView = window.location.hash.replace("#", "");
|
const initialView = window.location.hash.replace("#", "");
|
||||||
if (["signals", "report", "runs"].includes(initialView)) {
|
if (["signals", "report", "runs"].includes(initialView)) {
|
||||||
|
|||||||
@@ -36,15 +36,16 @@ def test_home_page_returns_dashboard_shell(monkeypatch) -> None:
|
|||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
assert "SignalScout" in response.text
|
assert "SignalScout" in response.text
|
||||||
assert "今日 AI 信号" in response.text
|
assert "今日 AI 信号" in response.text
|
||||||
assert "手动搜索" in response.text
|
assert "手动搜索" not in response.text
|
||||||
assert "renderMarkdownReport" in response.text
|
assert "renderMarkdownReport" in response.text
|
||||||
assert "自动记录" not in response.text
|
assert "自动记录" not in response.text
|
||||||
assert "采集今日情报" not in response.text
|
assert "采集今日情报" not in response.text
|
||||||
|
|
||||||
|
|
||||||
def test_manual_agent_run_endpoint_starts_background_job(monkeypatch) -> None:
|
def test_manual_agent_run_endpoint_starts_background_job_when_enabled(monkeypatch) -> None:
|
||||||
# 手动搜索入口复用调度任务,方便验证每轮候选和入库数量。
|
# 启用采集后手动入口复用调度任务,方便验证每轮候选和入库数量。
|
||||||
monkeypatch.setenv("SIGNALSCOUT_ENV", "test")
|
monkeypatch.setenv("SIGNALSCOUT_ENV", "test")
|
||||||
|
monkeypatch.setenv("SIGNALSCOUT_AGENT_ENABLED", "true")
|
||||||
get_settings.cache_clear()
|
get_settings.cache_clear()
|
||||||
started = []
|
started = []
|
||||||
engine = create_engine(
|
engine = create_engine(
|
||||||
@@ -82,6 +83,7 @@ def test_manual_agent_run_endpoint_starts_background_job(monkeypatch) -> None:
|
|||||||
def test_manual_agent_run_endpoint_uses_scheduler_when_available(monkeypatch) -> None:
|
def test_manual_agent_run_endpoint_uses_scheduler_when_available(monkeypatch) -> None:
|
||||||
# 生产环境手动搜索投递到常驻调度器,避免HTTP请求承载长时间联网搜索。
|
# 生产环境手动搜索投递到常驻调度器,避免HTTP请求承载长时间联网搜索。
|
||||||
monkeypatch.setenv("SIGNALSCOUT_ENV", "test")
|
monkeypatch.setenv("SIGNALSCOUT_ENV", "test")
|
||||||
|
monkeypatch.setenv("SIGNALSCOUT_AGENT_ENABLED", "true")
|
||||||
get_settings.cache_clear()
|
get_settings.cache_clear()
|
||||||
engine = create_engine(
|
engine = create_engine(
|
||||||
"sqlite:///:memory:",
|
"sqlite:///:memory:",
|
||||||
@@ -121,6 +123,19 @@ def test_manual_agent_run_endpoint_uses_scheduler_when_available(monkeypatch) ->
|
|||||||
assert scheduled[0]["run_date"].tzinfo is not None
|
assert scheduled[0]["run_date"].tzinfo is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_manual_agent_run_endpoint_rejects_requests_when_disabled(monkeypatch) -> None:
|
||||||
|
# 采集暂停时接口拒绝请求,避免其他调用方绕过页面入口消耗模型额度。
|
||||||
|
monkeypatch.setenv("SIGNALSCOUT_ENV", "test")
|
||||||
|
monkeypatch.setenv("SIGNALSCOUT_AGENT_ENABLED", "false")
|
||||||
|
get_settings.cache_clear()
|
||||||
|
with TestClient(create_app()) as client:
|
||||||
|
response = client.post("/agent/runs/daily")
|
||||||
|
get_settings.cache_clear()
|
||||||
|
|
||||||
|
assert response.status_code == 503
|
||||||
|
assert response.json()["detail"] == "情报采集当前已暂停"
|
||||||
|
|
||||||
|
|
||||||
def test_dashboard_returns_news_and_github_signals_separately(monkeypatch) -> None:
|
def test_dashboard_returns_news_and_github_signals_separately(monkeypatch) -> None:
|
||||||
# 仪表盘接口直接返回分组后的业务数据,前端不再用标题或来源名自行猜测分类。
|
# 仪表盘接口直接返回分组后的业务数据,前端不再用标题或来源名自行猜测分类。
|
||||||
monkeypatch.setenv("SIGNALSCOUT_ENV", "test")
|
monkeypatch.setenv("SIGNALSCOUT_ENV", "test")
|
||||||
|
|||||||
@@ -3,10 +3,17 @@ from app.core.config import Settings
|
|||||||
|
|
||||||
|
|
||||||
def test_scheduler_runs_on_interval() -> None:
|
def test_scheduler_runs_on_interval() -> None:
|
||||||
# Agent默认按固定间隔自动运行,避免产品依赖用户手动触发。
|
# 启用采集后按固定间隔自动运行,避免产品依赖用户手动触发。
|
||||||
scheduler = build_scheduler(Settings(agent_interval_minutes=30))
|
scheduler = build_scheduler(Settings(agent_enabled=True, agent_interval_minutes=30))
|
||||||
job = scheduler.get_job("continuous-intelligence")
|
job = scheduler.get_job("continuous-intelligence")
|
||||||
|
|
||||||
assert job is not None
|
assert job is not None
|
||||||
assert job.trigger.interval.total_seconds() == 1800
|
assert job.trigger.interval.total_seconds() == 1800
|
||||||
assert job.max_instances == 1
|
assert job.max_instances == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_scheduler_skips_periodic_job_when_agent_is_disabled() -> None:
|
||||||
|
# 采集暂停时不注册任务,确保调度器没有可执行的外部模型调用。
|
||||||
|
scheduler = build_scheduler(Settings(agent_enabled=False))
|
||||||
|
|
||||||
|
assert scheduler.get_job("continuous-intelligence") is None
|
||||||
|
|||||||
Reference in New Issue
Block a user