sirius/osint-dashboard
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Revert "feat: rebuild news summarizer on local Ollama"

Browse source 
This reverts commit 58ff43bdee.
This commit is contained in:
Sirius DevOps 2026-08-28 19:49:51 -04:00
parent 58ff43bdee
commit c10b617f1f
15 changed files with 338 additions and 335 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
.env.example
View file

@ -60,7 +60,7 @@ FIRMS_INTERVAL=900
INGEST_FIRES=1 INGEST_FIRES=1
# ── API keys (managed from the dashboard UI) ────────────────────────────── # ── API keys (managed from the dashboard UI) ──────────────────────────────
# Keys such as TELEGRAM_TOKEN are stored in the Postgres # Keys such as NOUS_API_KEY and TELEGRAM_TOKEN are stored in the Postgres
# `api_keys` table and managed from the dashboard's "Keys" tab # `api_keys` table and managed from the dashboard's "Keys" tab
# (GET/POST/DELETE /api/keys/{name}) — see app/keystore.py. The FIRMS ingestor # (GET/POST/DELETE /api/keys/{name}) — see app/keystore.py. The FIRMS ingestor
# currently reads FIRMS_MAP_KEY from .env (above); wiring the Keys-UI store as # currently reads FIRMS_MAP_KEY from .env (above); wiring the Keys-UI store as
@ -68,15 +68,17 @@ INGEST_FIRES=1
# ── News pipeline (scraper + summarizer, profile `ingest`) ──────────────── # ── News pipeline (scraper + summarizer, profile `ingest`) ────────────────
# Hourly: the scraper crawls 257 RSS sources at minute :00 and the summarizer # Hourly: the scraper crawls 257 RSS sources at minute :00 and the summarizer
# runs the local Ollama map-reduce at minute :05, both writing to the shared osint-db # runs the Nous map-reduce at minute :05, both writing to the shared osint-db
# (tables `articles` + `article_summaries`, created by alembic 003_news). # (tables `articles` + `article_summaries`, created by alembic 003_news).
# Consume via GET /api/news and GET /api/news/summaries. # Consume via GET /api/news and GET /api/news/summaries.
# LLM_URL is the Ollama origin (Pi container cannot use 127.0.0.1 for a laptop # NOUS_API_KEY is also (preferably) set in the Keys UI; env is an override.
# GPU). Example: LLM_URL=http://100.91.7.84:11434 # Unset in both env and api_keys = summarizer logs and idles (never crashes).
# LLM_URL= NOUS_API_KEY=
NOUS_BASE_URL=https://inference-api.nousresearch.com/v1
# Optional LLM knobs
# SUMMARY_MODEL is an optional override. Leave unset so Settings # SUMMARY_MODEL is an optional override. Leave unset so Settings
# (app_settings.SUMMARY_MODEL) can reach the summarizer. Code default # (app_settings.SUMMARY_MODEL) can reach the summarizer. Code default
# qwen3:30b-a3b remains after a Postgres miss. Env wins when set. # Hermes-4.3-36B remains after a Postgres miss. Env wins when set.
# SUMMARY_MODEL= # SUMMARY_MODEL=
NEWS_BATCH_SIZE=50 NEWS_BATCH_SIZE=50
SUMMARY_WINDOW_HOURS=1 SUMMARY_WINDOW_HOURS=1

7
app/keystore.py
View file

@ -57,6 +57,11 @@ KEY_REGISTRY: dict[str, dict] = {
"pattern": r"^[0-9a-fA-F]{32}$", "pattern": r"^[0-9a-fA-F]{32}$",
"example": "32-char hex string (e.g. 5f3c…9a02)", "example": "32-char hex string (e.g. 5f3c…9a02)",
}, },
"NOUS_API_KEY": {
"description": "Nous Portal API key — hourly news summarizer (inference-api.nousresearch.com).",
"pattern": r"^.{16,}$",
"example": "key from https://portal.nousresearch.com (API keys page)",
},
"TELEGRAM_TOKEN": { "TELEGRAM_TOKEN": {
"description": "Telegram bot token — push alert notifications to a channel.", "description": "Telegram bot token — push alert notifications to a channel.",
"pattern": r"^\d{8,10}:[0-9A-Za-z_-]{35}$", "pattern": r"^\d{8,10}:[0-9A-Za-z_-]{35}$",
@ -209,7 +214,7 @@ async def get_api_key(name: str) -> str | None:
"""Read a stored key value — used by ingest services, never by the API. """Read a stored key value — used by ingest services, never by the API.
Returns the raw value (or None when unset) so producers can pass it to Returns the raw value (or None when unset) so producers can pass it to
external APIs (FIRMS, Telegram, …). Reads live from Postgres, so a external APIs (FIRMS, Nous, Telegram, …). Reads live from Postgres, so a
key set via the dashboard is picked up on the next poll — no restart. key set via the dashboard is picked up on the next poll — no restart.
""" """
await ensure_api_keys_table() await ensure_api_keys_table()

10
app/main.py
View file

@ -605,7 +605,7 @@ async def list_documents(
async def list_api_keys(): async def list_api_keys():
"""List known API keys with set/missing status — masked, never raw. """List known API keys with set/missing status — masked, never raw.
Registered keys (FIRMS_MAP_KEY, TELEGRAM_TOKEN) Registered keys (FIRMS_MAP_KEY, NOUS_API_KEY, TELEGRAM_TOKEN)
are always included. Any extra stored keys are appended. are always included. Any extra stored keys are appended.
""" """
return await list_keys() return await list_keys()
@ -638,13 +638,13 @@ async def remove_api_key(name: str):
@app.get("/api/settings", response_model=SettingsOut) @app.get("/api/settings", response_model=SettingsOut)
async def get_settings(): async def get_settings():
"""Summarizer model + read-only Ollama URL.""" """Summarizer model + read-only Nous base URL."""
return await get_app_settings() return await get_app_settings()
@app.put("/api/settings", response_model=SettingsOut) @app.put("/api/settings", response_model=SettingsOut)
async def put_settings(payload: SettingsIn): async def put_settings(payload: SettingsIn):
"""Persist SUMMARY_MODEL. ``llm_url`` is ignored even if sent.""" """Persist SUMMARY_MODEL. ``nous_base_url`` is ignored even if sent."""
try: try:
return await set_summary_model(payload.summary_model) return await set_summary_model(payload.summary_model)
except SettingsError as exc: except SettingsError as exc:
@ -1084,7 +1084,7 @@ async def camera_hls_segment(camera_id: UUID, u: str = Query(..., min_length=8))
# ── News pipeline (scraper + summarizer) ────────────────────────────────── # ── News pipeline (scraper + summarizer) ──────────────────────────────────
# Backing data for the frontend news panel. Written by the vendored # Backing data for the frontend news panel. Written by the vendored
# news-scraper (hourly Scrapy crawl) and news-summarizer (hourly Ollama # news-scraper (hourly Scrapy crawl) and news-summarizer (hourly Gemini
# map-reduce) services into the shared osint-db. # map-reduce) services into the shared osint-db.
@app.get("/api/news", response_model=list[NewsArticleOut]) @app.get("/api/news", response_model=list[NewsArticleOut])
@ -1249,7 +1249,7 @@ async def list_news_map(
@app.get("/api/news/models", response_model=NewsModelsOut) @app.get("/api/news/models", response_model=NewsModelsOut)
async def list_news_models(): async def list_news_models():
"""Ollama model catalog for the summarizer selector. Never 502s.""" """Nous model catalog for the summarizer selector. Never 502s."""
return await list_models() return await list_models()

6
app/schemas.py
View file

@ -316,7 +316,7 @@ class NewsModelsOut(BaseModel):
class SettingsIn(BaseModel): class SettingsIn(BaseModel):
"""Body for PUT /api/settings. ``llm_url`` is not writable.""" """Body for PUT /api/settings. ``nous_base_url`` is not writable."""
summary_model: str = Field(..., min_length=1, max_length=128) summary_model: str = Field(..., min_length=1, max_length=128)
@ -332,10 +332,10 @@ class SettingsIn(BaseModel):
class SettingsOut(BaseModel): class SettingsOut(BaseModel):
"""Current summarizer settings. ``llm_url`` is read-only.""" """Current summarizer settings. ``nous_base_url`` is read-only."""
summary_model: str summary_model: str
llm_url: str nous_base_url: str
# ─── Aggregations ──────────────────────────────────────────────────────── # ─── Aggregations ────────────────────────────────────────────────────────

54
app/settings_store.py
View file

@ -17,10 +17,11 @@ from datetime import datetime, timezone
import httpx import httpx
from sqlalchemy import Column, DateTime, String, Table, Text, func, select, text from sqlalchemy import Column, DateTime, String, Table, Text, func, select, text
import keystore
from database import async_session, engine, metadata from database import async_session, engine, metadata
DEFAULT_LLM_URL = "http://127.0.0.1:11434" DEFAULT_NOUS_BASE_URL = "https://inference-api.nousresearch.com/v1"
DEFAULT_SUMMARY_MODEL = "qwen3:30b-a3b" DEFAULT_SUMMARY_MODEL = "Hermes-4.3-36B"
SETTING_SUMMARY_MODEL = "SUMMARY_MODEL" SETTING_SUMMARY_MODEL = "SUMMARY_MODEL"
ALLOWED_SETTINGS = frozenset({SETTING_SUMMARY_MODEL}) ALLOWED_SETTINGS = frozenset({SETTING_SUMMARY_MODEL})
MODELS_CACHE_TTL_S = 600.0 MODELS_CACHE_TTL_S = 600.0
@ -28,7 +29,12 @@ MODELS_TIMEOUT_S = 8.0
DEFAULT_MODELS_USER_AGENT = "osint-dashboard-news-summarizer" DEFAULT_MODELS_USER_AGENT = "osint-dashboard-news-summarizer"
FALLBACK_MODELS = [ FALLBACK_MODELS = [
"qwen3:30b-a3b", "Hermes-4.3-36B",
"Hermes-4-70B",
"google/gemini-2.5-flash",
"anthropic/claude-haiku-4.5",
"openai/gpt-4.1-mini",
"x-ai/grok-4",
] ]
app_settings = Table( app_settings = Table(
@ -71,14 +77,10 @@ async def ensure_app_settings_table() -> None:
_ensured = True _ensured = True
def llm_url() -> str: def nous_base_url() -> str:
"""Read-only Ollama origin (env, never writable from the UI).""" """Read-only Nous inference base URL (env, never writable from the UI)."""
raw = (os.getenv("LLM_URL") or "").strip().rstrip("/") raw = (os.getenv("NOUS_BASE_URL") or "").strip().rstrip("/")
for suffix in ("/api/generate", "/api/chat", "/api", "/v1"): return raw or DEFAULT_NOUS_BASE_URL
if raw.endswith(suffix):
raw = raw[: -len(suffix)].rstrip("/")
break
return raw or DEFAULT_LLM_URL
def _validate_summary_model(value: str) -> str: def _validate_summary_model(value: str) -> str:
@ -89,7 +91,7 @@ def _validate_summary_model(value: str) -> str:
async def get_summary_model() -> str: async def get_summary_model() -> str:
"""Stored SUMMARY_MODEL, else env, else qwen3:30b-a3b.""" """Stored SUMMARY_MODEL, else env, else Hermes-4.3-36B."""
await ensure_app_settings_table() await ensure_app_settings_table()
async with async_session() as session: async with async_session() as session:
row = ( row = (
@ -133,7 +135,7 @@ async def set_summary_model(value: str) -> dict:
async def get_app_settings() -> dict: async def get_app_settings() -> dict:
return { return {
"summary_model": await get_summary_model(), "summary_model": await get_summary_model(),
"llm_url": llm_url(), "nous_base_url": nous_base_url(),
} }
@ -144,6 +146,18 @@ def _fallback_payload() -> dict:
} }
async def _nous_api_key() -> str | None:
"""Keystore first, then env. Any lookup failure is treated as missing."""
try:
stored = await keystore.get_api_key("NOUS_API_KEY")
except Exception:
stored = None
if stored and str(stored).strip():
return str(stored).strip()
env = (os.getenv("NOUS_API_KEY") or "").strip()
return env or None
def _models_user_agent() -> str: def _models_user_agent() -> str:
return os.getenv("OSINT_USER_AGENT") or DEFAULT_MODELS_USER_AGENT return os.getenv("OSINT_USER_AGENT") or DEFAULT_MODELS_USER_AGENT
@ -160,7 +174,7 @@ def _parse_models_payload(body: object) -> list[dict[str, str]]:
if isinstance(item, str) and item.strip(): if isinstance(item, str) and item.strip():
out.append({"id": item.strip()}) out.append({"id": item.strip()})
elif isinstance(item, dict): elif isinstance(item, dict):
mid = item.get("id") or item.get("name") or item.get("model") mid = item.get("id") or item.get("name")
if mid: if mid:
out.append({"id": str(mid)}) out.append({"id": str(mid)})
return out return out
@ -172,15 +186,23 @@ async def _http_get(url: str, *, headers: dict[str, str], timeout: float) -> htt
async def list_models() -> dict: async def list_models() -> dict:
"""Live ``GET {llm_url}/api/tags``. Never 502 — fallback on failure.""" """Live ``GET {base}/models`` when a key is present; otherwise curated fallback.
Never raises to the caller for missing key or upstream failure.
"""
global _models_cache global _models_cache
key = await _nous_api_key()
if not key:
return _fallback_payload()
now = time.monotonic() now = time.monotonic()
hit = _models_cache hit = _models_cache
if hit and now - hit[0] < MODELS_CACHE_TTL_S: if hit and now - hit[0] < MODELS_CACHE_TTL_S:
return hit[1] return hit[1]
url = f"{llm_url()}/api/tags" url = f"{nous_base_url()}/models"
headers = { headers = {
"Authorization": f"Bearer {key}",
"User-Agent": _models_user_agent(), "User-Agent": _models_user_agent(),
"Accept": "application/json", "Accept": "application/json",
} }

11
app/static/index.html
View file

@ -840,7 +840,7 @@
<div class="ns-kicker">Executive Summary <span class="ns-badge" id="ns-badge">WAITING</span></div> <div class="ns-kicker">Executive Summary <span class="ns-badge" id="ns-badge">WAITING</span></div>
<div class="ns-time" id="ns-time">—</div> <div class="ns-time" id="ns-time">—</div>
<div class="ns-body" id="ns-body"> <div class="ns-body" id="ns-body">
<div class="ns-empty">No executive summary yet — the Ollama summarizer is idle until a model is chosen in Settings and <code>LLM_URL</code> is reachable.</div> <div class="ns-empty">No executive summary yet — the Nous summarizer is idle until <code>NOUS_API_KEY</code> is set on the Pi and a model is chosen in Settings.</div>
</div> </div>
</div> </div>
@ -968,7 +968,7 @@
<section class="view" id="view-keys" aria-label="API keys"> <section class="view" id="view-keys" aria-label="API keys">
<div class="subview"> <div class="subview">
<h2><span class="tick">⚿</span> API Keys</h2> <h2><span class="tick">⚿</span> API Keys</h2>
<p class="sub">Keys used by ingest services (FIRMS, Telegram, AIS, …). Stored in Postgres, never shown again — only status + last 4 chars.</p> <p class="sub">Keys used by ingest services (FIRMS, Nous Portal, Telegram, AIS, …). Stored in Postgres, never shown again — only status + last 4 chars.</p>
<div class="key-grid" id="keys-grid"></div> <div class="key-grid" id="keys-grid"></div>
</div> </div>
</section> </section>
@ -997,11 +997,11 @@
<p class="set-note" style="margin-bottom:0.9rem">Takes effect on the next hourly run (:05). No container restart.</p> <p class="set-note" style="margin-bottom:0.9rem">Takes effect on the next hourly run (:05). No container restart.</p>
<div class="set-row"> <div class="set-row">
<label>Provider</label> <label>Provider</label>
<span>Local Ollama (LLM_URL)</span> <span>Nous Portal (inference-api.nousresearch.com)</span>
</div> </div>
<div class="set-row"> <div class="set-row">
<label>API key</label> <label>API key</label>
<span>none — Ollama on the GPU host</span> <span>set in API Keys as NOUS_API_KEY (never shown here)</span>
</div> </div>
<div class="set-row"> <div class="set-row">
<label for="set-summary-model">Model</label> <label for="set-summary-model">Model</label>
@ -1370,7 +1370,7 @@ function renderNewsSummary(summaries) {
const badgeEl = document.getElementById('ns-badge'); const badgeEl = document.getElementById('ns-badge');
if (!bodyEl) return; if (!bodyEl) return;
if (!summaries || !summaries.length) { if (!summaries || !summaries.length) {
bodyEl.innerHTML = '<div class="ns-empty">No executive summary yet — the Ollama summarizer is idle until a model is chosen in Settings and <code>LLM_URL</code> is reachable.</div>'; bodyEl.innerHTML = '<div class="ns-empty">No executive summary yet — the Nous summarizer is idle until <code>NOUS_API_KEY</code> is set on the Pi and a model is chosen in Settings.</div>';
timeEl.textContent = '—'; timeEl.textContent = '—';
badgeEl.textContent = 'WAITING'; badgeEl.textContent = 'WAITING';
return; return;
@ -1547,6 +1547,7 @@ function sentimentBadge(label) {
/* ═══════════════ API KEYS ═══════════════ */ /* ═══════════════ API KEYS ═══════════════ */
const KEY_PATTERNS = { const KEY_PATTERNS = {
'FIRMS_MAP_KEY': /^[0-9a-fA-F]{32}$/, 'FIRMS_MAP_KEY': /^[0-9a-fA-F]{32}$/,
'NOUS_API_KEY': /^.{16,}$/,
'TELEGRAM_TOKEN': /^\d{8,10}:[0-9A-Za-z_-]{35}$/, 'TELEGRAM_TOKEN': /^\d{8,10}:[0-9A-Za-z_-]{35}$/,
}; };
async function loadKeys() { async function loadKeys() {

3
docker-compose.yml
View file

@ -224,7 +224,8 @@ services:
DB_HOST: db DB_HOST: db
DB_PORT: ${DB_PORT:-5432} DB_PORT: ${DB_PORT:-5432}
DB_NAME: ${DB_NAME:-osint_data} DB_NAME: ${DB_NAME:-osint_data}
LLM_URL: ${LLM_URL:-} NOUS_API_KEY: ${NOUS_API_KEY:-}
NOUS_BASE_URL: ${NOUS_BASE_URL:-https://inference-api.nousresearch.com/v1}
SUMMARY_MODEL: ${SUMMARY_MODEL:-} SUMMARY_MODEL: ${SUMMARY_MODEL:-}
OSINT_USER_AGENT: ${OSINT_USER_AGENT:-osint-dashboard-news-summarizer} OSINT_USER_AGENT: ${OSINT_USER_AGENT:-osint-dashboard-news-summarizer}
BATCH_SIZE: ${NEWS_BATCH_SIZE:-50} BATCH_SIZE: ${NEWS_BATCH_SIZE:-50}

73
docs/news.md
View file

@ -1,12 +1,11 @@
# News pipeline — scraper + local Ollama summarizer # News pipeline — scraper + Nous Portal summarizer
The OSINT dashboard ingests ~257 global news RSS sources hourly and produces The OSINT dashboard ingests ~257 global news RSS sources hourly and produces
an English LLM brief plus flagged ticker/map rows. Both services were vendored an English LLM brief plus flagged ticker/map rows. Both services were vendored
from the upstream `~/Projects/newsPipeline` project and re-integrated here to from the upstream `~/Projects/newsPipeline` project and re-integrated here to
replace the old k8s CronJob choreography with in-compose scheduling against replace the old k8s CronJob choreography with in-compose scheduling against
the EXISTING osint-db — **no second Postgres**. The LLM is **local Ollama** the EXISTING osint-db — **no second Postgres**. The LLM is **Nous Portal**
(`LLM_URL`, default `http://127.0.0.1:11434`) — the newsPipeline `local_llm` (`inference-api.nousresearch.com`) — not Gemini.
path (`POST /api/generate`). No API key.
## Architecture ## Architecture
@ -17,7 +16,7 @@ path (`POST /api/generate`). No API key.
news-scraper (Scrapy, hourly :00) ──► articles table (osint-db) news-scraper (Scrapy, hourly :00) ──► articles table (osint-db)
│ │ │ │
│ ▼ │ ▼
news-summarizer (Ollama map-reduce, :05) ──► article_summaries + news_items news-summarizer (Nous Portal map-reduce, :05) ──► article_summaries + news_items
│ │
▼ ▼
GET /api/news · /api/news/summaries · /api/news/ticker · /api/news/map GET /api/news · /api/news/summaries · /api/news/ticker · /api/news/map
@ -47,8 +46,8 @@ feeds (`GET /api/news` exact key set is unchanged on purpose).
(`ON CONFLICT (url) DO NOTHING`). (`ON CONFLICT (url) DO NOTHING`).
2. **Summarizer** — `news/summerizer/run_news_summarizer.py` runs 2. **Summarizer** — `news/summerizer/run_news_summarizer.py` runs
`summarizer.py` at :05 past each hour. It reads articles from the last `summarizer.py` at :05 past each hour. It reads articles from the last
`SUMMARY_WINDOW_HOURS`, map-reduces them through local Ollama `SUMMARY_WINDOW_HOURS`, map-reduces them through Nous Portal
(`SUMMARY_MODEL` / Settings, default `qwen3:30b-a3b`), writes the English (`SUMMARY_MODEL` / Settings, default `Hermes-4.3-36B`), writes the English
brief to `article_summaries` (column `model` is the LLM id), and flagged brief to `article_summaries` (column `model` is the LLM id), and flagged
ticker/map rows to `news_items`. ticker/map rows to `news_items`.
@ -68,16 +67,17 @@ Container startup order doesn't matter.
## Keys and Settings ## Keys and Settings
- **No API key.** Ollama on the GPU host; the summarizer container calls - **`NOUS_API_KEY`** — paste in the dashboard **Keys** UI (`api_keys` /
`LLM_URL` (`POST /api/generate`, `GET /api/tags`). `keystore.KEY_REGISTRY`). Env / `.env` is an **override** (env wins, same
- **Idle without Ollama** — if `/api/tags` is down or the chosen model is not as FIRMS). Never returned by any API; never emitted into `index.html`;
pulled, the summarizer logs and idles (never crashes). News intel APIs never proxied from the browser.
return `[]`. - **Idle without a key** — if env is unset **and** the keystore row is empty,
the summarizer logs and idles (never crashes). News intel APIs return `[]`.
- **Model** — non-secret. Settings UI model selector `PUT /api/settings` - **Model** — non-secret. Settings UI model selector `PUT /api/settings`
`{ "summary_model": "…" }` stores `SUMMARY_MODEL` in `app_settings` (1–128 `{ "summary_model": "…" }` stores `SUMMARY_MODEL` in `app_settings` (1–128
chars). `GET /api/settings` echoes `{summary_model, llm_url}`. chars). `GET /api/settings` echoes `{summary_model, nous_base_url}`.
`llm_url` is read-only. Default `qwen3:30b-a3b`. Live catalog is `nous_base_url` is read-only. Default `Hermes-4.3-36B`. Live catalog is
best-effort `GET /api/news/models` (`GET {LLM_URL}/api/tags`). best-effort `GET /api/news/models`.
## Endpoints ## Endpoints
@ -192,15 +192,15 @@ Never 502s. `{ "source": "live"|"fallback", "models": [{"id": "…"}] }`.
### GET /api/settings · PUT /api/settings ### GET /api/settings · PUT /api/settings
```json ```json
{ "summary_model": "qwen3:30b-a3b", "llm_url": "http://127.0.0.1:11434" } { "summary_model": "Hermes-4.3-36B", "nous_base_url": "https://inference-api.nousresearch.com/v1" }
``` ```
PUT body is `{ "summary_model": "<1–128 char id>" }`. `llm_url` is PUT body is `{ "summary_model": "<1–128 char id>" }`. `nous_base_url` is
ignored even if sent. ignored even if sent.
## Reduce JSON contract ## Reduce JSON contract
Reduce phase (`format: json` on Ollama generate, English only) must be a single Reduce phase (`response_format: json_object`, English only) must be a single
object. Parser (`intel.parse_reduce_json`) strips `<think>…</think>` and object. Parser (`intel.parse_reduce_json`) strips `<think>…</think>` and
markdown json fences, then brace-slices: markdown json fences, then brace-slices:
@ -234,8 +234,9 @@ lands in `article_summaries.summary_text`.
| Var | Default | Notes | | Var | Default | Notes |
|---|---|---| |---|---|---|
| `LLM_URL` | `http://127.0.0.1:11434` | Ollama origin. Pi container must point at the GPU host (e.g. Tailscale). Compose leaves this empty so `.env` wins. | | `NOUS_API_KEY` | *(blank)* | **Required for summaries.** Prefer Keys UI; env overrides. Unset in **both** env and `api_keys` = summarizer logs and idles (never crashes); APIs return `[]`. |
| `SUMMARY_MODEL` | `qwen3:30b-a3b` | Leave compose unset so Settings → `app_settings.SUMMARY_MODEL` reaches the worker. Env wins when set. | | `NOUS_BASE_URL` | `https://inference-api.nousresearch.com/v1` | Read-only in Settings. |
| `SUMMARY_MODEL` | `Hermes-4.3-36B` | Compose default. Operator-facing choice is Settings → `app_settings.SUMMARY_MODEL`. |
| `NEWS_BATCH_SIZE` | `50` | Articles per map-phase batch (compose maps to container `BATCH_SIZE`). | | `NEWS_BATCH_SIZE` | `50` | Articles per map-phase batch (compose maps to container `BATCH_SIZE`). |
| `SUMMARY_WINDOW_HOURS` | `1` | How far back the summarizer looks for new articles. | | `SUMMARY_WINDOW_HOURS` | `1` | How far back the summarizer looks for new articles. |
| `INCLUDE_FUTURES` | `0` | Legacy futures-prices coupling (upstream pipeline). OFF for OSINT; set `1` + install `yfinance` to enable. | | `INCLUDE_FUTURES` | `0` | Legacy futures-prices coupling (upstream pipeline). OFF for OSINT; set `1` + install `yfinance` to enable. |
@ -245,14 +246,15 @@ lands in `article_summaries.summary_text`.
| `NEWS_SUMMARIZE_RUN_ON_START` | `1` | Run one summarize immediately on container start. | | `NEWS_SUMMARIZE_RUN_ON_START` | `1` | Run one summarize immediately on container start. |
| `NEWS_SUMMARIZE_FORCE` | `0` | `1` ignores the current-UTC-hour idempotency skip (double-pins on recreate). | | `NEWS_SUMMARIZE_FORCE` | `0` | `1` ignores the current-UTC-hour idempotency skip (double-pins on recreate). |
| `NEWS_LOG_LEVEL` | `INFO` | Scrapy log level. | | `NEWS_LOG_LEVEL` | `INFO` | Scrapy log level. |
| `OSINT_USER_AGENT` | `osint-dashboard-news-summarizer` | Sent on every outbound Ollama call. | | `OSINT_USER_AGENT` | `osint-dashboard-news-summarizer` | Sent on every outbound Nous call. |
| `TELEGRAM_TOKEN` / `TELEGRAM_CHAT_ID` | *(blank)* | Reserved for the (out-of-scope) Telegram delivery bot. | | `TELEGRAM_TOKEN` / `TELEGRAM_CHAT_ID` | *(blank)* | Reserved for the (out-of-scope) Telegram delivery bot. |
DB_* for both services is mapped to the shared osint-db credentials DB_* for both services is mapped to the shared osint-db credentials
(`DB_HOST=db`, same `DB_USER/DB_PASSWORD/DB_NAME` as the rest of the stack). (`DB_HOST=db`, same `DB_USER/DB_PASSWORD/DB_NAME` as the rest of the stack).
Ollama generate: `POST {LLM_URL}/api/generate` via `news/summerizer/ollama_client.py` Nous chat: `POST {NOUS_BASE_URL}/chat/completions` via `news/summerizer/nous_client.py`
(`httpx`, no `ollama` Python SDK). No API key. Reduce uses `format: json`. (`httpx`, no `openai` SDK). Auth is a Bearer token from `NOUS_API_KEY`.
No Hermes-4 reasoning system prompt. Reduce uses `json_mode=True`.
## Prompts ## Prompts
@ -266,7 +268,7 @@ language is gated behind `INCLUDE_FUTURES=1`.
```bash ```bash
PYTHONPATH=news/summerizer pytest news/summerizer/tests -v PYTHONPATH=news/summerizer pytest news/summerizer/tests -v
# intel + ollama_client tests PASS (no network) # intel + nous_client tests PASS (no network)
PYTHONPATH=app pytest tests/test_api_news.py \ PYTHONPATH=app pytest tests/test_api_news.py \
tests/test_api_settings.py tests/test_api_live_layers.py -v tests/test_api_settings.py tests/test_api_live_layers.py -v
@ -278,17 +280,16 @@ PYTHONPATH=app pytest tests/test_api_news.py \
After deploy / compose rebuild of `news-summarizer` on the Pi: After deploy / compose rebuild of `news-summarizer` on the Pi:
1. Run Ollama on the GPU host and pull the model (`ollama pull qwen3:30b-a3b`). 1. Keys UI: save `NOUS_API_KEY` → status `****last4`.
2. Set `LLM_URL` on the Pi to that host (Tailscale IP + `:11434`). 2. Settings: pick a model → Save → `GET /api/settings` echoes it.
3. Settings: pick a model → Save → `GET /api/settings` echoes it. 3. `docker compose --profile ingest logs -f news-summarizer` — next run (or
4. `docker compose --profile ingest logs -f news-summarizer` — next run (or
`NEWS_SUMMARIZE_RUN_ON_START=1` recreate) logs `Processing N articles with <model>`. `NEWS_SUMMARIZE_RUN_ON_START=1` recreate) logs `Processing N articles with <model>`.
5. `curl -s localhost:8000/api/news/summaries?limit=1` — English `summary_text`, `model` set. 4. `curl -s localhost:8000/api/news/summaries?limit=1` — English `summary_text`, `model` set.
6. `curl -s localhost:8000/api/news/ticker` — flagged headlines only. 5. `curl -s localhost:8000/api/news/ticker` — flagged headlines only.
7. `curl -s localhost:8000/api/news/map` — only rows with lat/lon. 6. `curl -s localhost:8000/api/news/map` — only rows with lat/lon.
8. HUD: NEWS ticker scrolls flagged items; map overlay pins popup with location. 7. HUD: NEWS ticker scrolls flagged items; map overlay pins popup with location.
9. Ollama down / model missing → summarizer logs idle, APIs return `[]`, no crash. 8. Unset key + empty keystore → summarizer logs idle, APIs return `[]`, no crash.
**Operator action after merge:** start Ollama on the laptop GPU, set `LLM_URL` **Operator action after merge:** paste a Nous Portal API key in API Keys; pick
in Pi `.env`, pick a model in Settings if the default `qwen3:30b-a3b` is not a model in Settings if the default `Hermes-4.3-36B` is not wanted; rebuild
wanted; rebuild `osint-news-summarizer` on the Pi (`pi-app-deploy` / compose). `osint-news-summarizer` on the Pi (`pi-app-deploy` / compose).

36
news/summerizer/nous_client.py Normal file
View file

@ -0,0 +1,36 @@
"""HTTP client for the Nous inference chat completions API."""
from __future__ import annotations
import os
import httpx
_DEFAULT_UA = "osint-dashboard-news-summarizer"
_DEFAULT_BASE = "https://inference-api.nousresearch.com/v1"
def chat(prompt, *, api_key, model, base_url, json_mode=False) -> str:
resolved = (base_url or os.environ.get("NOUS_BASE_URL", _DEFAULT_BASE)).rstrip("/")
url = f"{resolved}/chat/completions"
headers = {
"Authorization": f"Bearer {api_key}",
"User-Agent": os.environ.get("OSINT_USER_AGENT") or _DEFAULT_UA,
}
payload = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"temperature": 0.2,
"max_tokens": 4096,
}
if json_mode:
payload["response_format"] = {"type": "json_object"}
try:
with httpx.Client(timeout=60.0) as client:
resp = client.post(url, headers=headers, json=payload)
if resp.status_code == 401 or resp.status_code >= 500:
return ""
data = resp.json()
return data["choices"][0]["message"]["content"]
except Exception:
return ""

102
news/summerizer/ollama_client.py
View file

@ -1,102 +0,0 @@
"""HTTP client for a local Ollama generate API.
Matches the newsPipeline local_llm summarizer: POST /api/generate,
GET /api/tags. No API key. ``LLM_URL`` may be a host, ``/api``, or
``/api/generate`` — we normalize to the origin.
"""
from __future__ import annotations
import os
import httpx
_DEFAULT_UA = "osint-dashboard-news-summarizer"
_DEFAULT_BASE = "http://127.0.0.1:11434"
_GENERATE_TIMEOUT = 300.0
_TAGS_TIMEOUT = 8.0
_GENERATE_OPTIONS = {
"num_predict": 4096,
"temperature": 0.6,
"top_p": 0.9,
"top_k": 40,
"num_ctx": 32768,
"repeat_penalty": 1.1,
}
def normalize_base(url: str | None) -> str:
"""Strip path suffixes so LLM_URL variants share one origin."""
raw = (url or os.environ.get("LLM_URL", "") or _DEFAULT_BASE).strip()
raw = raw.rstrip("/")
for suffix in ("/api/generate", "/api/chat", "/api", "/v1"):
if raw.endswith(suffix):
raw = raw[: -len(suffix)].rstrip("/")
break
return raw or _DEFAULT_BASE
def _headers() -> dict[str, str]:
return {
"User-Agent": os.environ.get("OSINT_USER_AGENT") or _DEFAULT_UA,
"Accept": "application/json",
}
def chat(prompt: str, *, model: str, base_url: str | None = None, json_mode: bool = False) -> str:
"""POST /api/generate. Returns response text, or \"\" on any failure."""
if not (model or "").strip():
return ""
origin = normalize_base(base_url)
url = f"{origin}/api/generate"
payload: dict = {
"model": model.strip(),
"prompt": prompt,
"stream": False,
"options": dict(_GENERATE_OPTIONS),
}
if json_mode:
payload["format"] = "json"
try:
with httpx.Client(timeout=_GENERATE_TIMEOUT) as client:
resp = client.post(url, headers=_headers(), json=payload)
if resp.status_code == 401 or resp.status_code >= 500:
return ""
data = resp.json()
text = data.get("response") if isinstance(data, dict) else None
return text if isinstance(text, str) else ""
except Exception:
return ""
def list_tags(base_url: str | None = None) -> list[str]:
"""GET /api/tags model names. Empty list on failure."""
origin = normalize_base(base_url)
url = f"{origin}/api/tags"
try:
with httpx.Client(timeout=_TAGS_TIMEOUT) as client:
resp = client.get(url, headers=_headers())
if resp.status_code != 200:
return []
models = (resp.json() or {}).get("models") or []
names: list[str] = []
for item in models:
if isinstance(item, dict):
name = item.get("name") or item.get("model") or ""
if name:
names.append(str(name))
return names
except Exception:
return []
def ollama_ready(model: str, *, base_url: str | None = None) -> bool:
"""True when /api/tags is up and ``model`` is present (prefix match)."""
if not (model or "").strip():
return False
names = list_tags(base_url)
if not names:
return False
want = model.strip()
return any(n == want or n.startswith(want) for n in names)

18
news/summerizer/run_news_summarizer.py
View file

@ -1,11 +1,16 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Scheduler loop for the news summarizer — hourly summarize at minute :05. """Scheduler loop for the news summarizer — hourly summarize at minute :05.
Replaces the k8s CronJob (`5 * * * *`) with an in-compose loop. Runs once on
boot (catches up on any articles scraped since the last summary), then fires
at each :NEWS_SUMMARIZE_MINUTE wall-clock boundary.
The loop is serial, so a slow LLM pass never overlaps the next run.
Env (all optional, 12-factor): Env (all optional, 12-factor):
NEWS_SUMMARIZE_MINUTE minute of the hour to fire (default 5) NEWS_SUMMARIZE_MINUTE minute of the hour to fire (default 5)
NEWS_SUMMARIZE_RUN_ON_START "1" to summarize once immediately on boot (default 1) NEWS_SUMMARIZE_RUN_ON_START "1" to summarize once immediately on boot (default 1)
LLM_URL Ollama origin; default http://127.0.0.1:11434 NOUS_API_KEY optional in env; Keys UI / api_keys also works
SUMMARY_MODEL Ollama tag (else app_settings)
""" """
from __future__ import annotations from __future__ import annotations
@ -41,10 +46,13 @@ def run_summarize() -> None:
def main() -> None: def main() -> None:
llm_url = os.getenv("LLM_URL", "").strip() or "http://127.0.0.1:11434" if not os.getenv("NOUS_API_KEY", "").strip():
logger.warning(
"NOUS_API_KEY unset in env — will read api_keys on each run; idle if both empty"
)
logger.info( logger.info(
"news summarizer loop starting (minute=%s, run_on_start=%s, llm_url=%s)", "news summarizer loop starting (minute=%s, run_on_start=%s)",
MINUTE, RUN_ON_START, llm_url, MINUTE, RUN_ON_START,
) )
if RUN_ON_START: if RUN_ON_START:
run_summarize() run_summarize()

64
news/summerizer/summarizer.py
View file

@ -1,17 +1,19 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""News summarizer — local Ollama map-reduce into brief/ticker/map. """News summarizer — Nous map-reduce of scraped articles into brief/ticker/map.
Reads articles scraped within the last hour from the shared `articles` table, Reads articles scraped within the last hour from the shared `articles` table,
maps them with a local Ollama model (per-article English fact blocks), reduces maps them with Nous (per-article English fact blocks), reduces to one JSON
to one JSON object (summary_en + ticker + map_items), and stores the brief in object (summary_en + ticker + map_items), and stores the brief in
`article_summaries` plus flagged rows in `news_items`. Tables live in the `article_summaries` plus flagged rows in `news_items`. Tables live in the
EXISTING osint-db (alembic 003_news + 005_news_items, idempotent). EXISTING osint-db (alembic 003_news + 005_news_items, idempotent).
LLM is the newsPipeline local_llm path: Ollama POST /api/generate. No API key. Everything is env-driven (12-factor). Secrets/config are resolved at the start
of each summarize_news() — env wins, else api_keys / app_settings:
DB_HOST / DB_NAME / DB_USER / DB_PASSWORD / DB_PORT PostgreSQL (osint-db) DB_HOST / DB_NAME / DB_USER / DB_PASSWORD / DB_PORT PostgreSQL (osint-db)
LLM_URL Ollama origin (default http://127.0.0.1:11434) NOUS_API_KEY Nous Portal key (else api_keys.name='NOUS_API_KEY')
SUMMARY_MODEL Ollama tag (else app_settings; default qwen3:30b-a3b) NOUS_BASE_URL default https://inference-api.nousresearch.com/v1
SUMMARY_MODEL default Hermes-4.3-36B (else app_settings)
BATCH_SIZE articles per map-phase batch (default 50) BATCH_SIZE articles per map-phase batch (default 50)
SUMMARY_WINDOW_HOURS look-back window in hours (default 1) SUMMARY_WINDOW_HOURS look-back window in hours (default 1)
OSINT_USER_AGENT default osint-dashboard-news-summarizer OSINT_USER_AGENT default osint-dashboard-news-summarizer
@ -19,6 +21,11 @@ LLM is the newsPipeline local_llm path: Ollama POST /api/generate. No API key.
SUMMARY_PROMPT override reduce-phase prompt (uses {final_input}) SUMMARY_PROMPT override reduce-phase prompt (uses {final_input})
NEWS_SUMMARIZE_FORCE "1" to ignore the current-UTC-hour idempotency skip NEWS_SUMMARIZE_FORCE "1" to ignore the current-UTC-hour idempotency skip
INCLUDE_FUTURES "1" to prepend live futures prices (default 0) INCLUDE_FUTURES "1" to prepend live futures prices (default 0)
The futures/markets coupling from the original pipeline is gated behind
INCLUDE_FUTURES and OFF by default — it is irrelevant to the OSINT dashboard
and pulled yfinance into the image. Re-enable by installing yfinance and
setting INCLUDE_FUTURES=1.
""" """
from __future__ import annotations from __future__ import annotations
@ -30,7 +37,7 @@ from datetime import datetime
import psycopg2 import psycopg2
from intel import parse_reduce_json, select_map, select_ticker from intel import parse_reduce_json, select_map, select_ticker
from ollama_client import chat, ollama_ready from nous_client import chat
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
logger = logging.getLogger("news.summarizer") logger = logging.getLogger("news.summarizer")
@ -44,8 +51,8 @@ DB_CONFIG = {
"port": int(os.getenv("DB_PORT", "5432")), "port": int(os.getenv("DB_PORT", "5432")),
} }
DEFAULT_LLM_URL = "http://127.0.0.1:11434" DEFAULT_NOUS_BASE_URL = "https://inference-api.nousresearch.com/v1"
DEFAULT_SUMMARY_MODEL = "qwen3:30b-a3b" DEFAULT_SUMMARY_MODEL = "Hermes-4.3-36B"
BATCH_SIZE = int(os.getenv("BATCH_SIZE", "50")) BATCH_SIZE = int(os.getenv("BATCH_SIZE", "50"))
SUMMARY_WINDOW_HOURS = int(os.getenv("SUMMARY_WINDOW_HOURS", "1")) SUMMARY_WINDOW_HOURS = int(os.getenv("SUMMARY_WINDOW_HOURS", "1"))
INCLUDE_FUTURES = os.getenv("INCLUDE_FUTURES", "0").lower() in ("1", "true", "yes") INCLUDE_FUTURES = os.getenv("INCLUDE_FUTURES", "0").lower() in ("1", "true", "yes")
@ -127,6 +134,20 @@ def _kv(conn, table, name) -> str:
return (row[0] or "").strip() if row else "" return (row[0] or "").strip() if row else ""
def resolve_api_key() -> str:
env = os.getenv("NOUS_API_KEY", "").strip()
if env:
return env
try:
conn = psycopg2.connect(**DB_CONFIG)
try:
return _kv(conn, "api_keys", "NOUS_API_KEY")
finally:
conn.close()
except Exception: # noqa: BLE001
return ""
def resolve_model() -> str: def resolve_model() -> str:
env = os.getenv("SUMMARY_MODEL", "").strip() env = os.getenv("SUMMARY_MODEL", "").strip()
if env: if env:
@ -143,15 +164,15 @@ def resolve_model() -> str:
def resolve_base_url() -> str: def resolve_base_url() -> str:
return os.getenv("LLM_URL", DEFAULT_LLM_URL).strip() or DEFAULT_LLM_URL return os.getenv("NOUS_BASE_URL", DEFAULT_NOUS_BASE_URL).strip() or DEFAULT_NOUS_BASE_URL
def call_llm(prompt: str, *, model: str, base_url: str, json_mode: bool = False) -> str: def call_llm(prompt: str, *, api_key: str, model: str, base_url: str, json_mode: bool = False) -> str:
"""Send a prompt to Ollama generate and return the text (\"\" on failure).""" """Send a prompt to Nous chat completions and return the text (\"\" on failure)."""
if not model: if not api_key:
logger.warning("SUMMARY_MODEL empty — skipping LLM call") logger.warning("NOUS_API_KEY not set — skipping LLM call")
return "" return ""
return chat(prompt, model=model, base_url=base_url, json_mode=json_mode) return chat(prompt, api_key=api_key, model=model, base_url=base_url, json_mode=json_mode)
# ── Futures (legacy, gated) ──────────────────────────────────────────────── # ── Futures (legacy, gated) ────────────────────────────────────────────────
@ -405,16 +426,11 @@ def summarize_news() -> None:
) )
return return
api_key = resolve_api_key()
model = resolve_model() model = resolve_model()
base_url = resolve_base_url() base_url = resolve_base_url()
if not model: if not api_key:
logger.warning("SUMMARY_MODEL empty — idle this run") logger.warning("NOUS_API_KEY unset in env and api_keys — idle this run")
return
if not ollama_ready(model, base_url=base_url):
logger.warning(
"Ollama not ready at %s (model %s missing or /api/tags down) — idle this run",
base_url, model,
)
return return
articles = get_recent_news() articles = get_recent_news()
@ -436,6 +452,7 @@ def summarize_news() -> None:
) )
summary = call_llm( summary = call_llm(
build_map_prompt(batch), build_map_prompt(batch),
api_key=api_key,
model=model, model=model,
base_url=base_url, base_url=base_url,
json_mode=False, json_mode=False,
@ -451,6 +468,7 @@ def summarize_news() -> None:
logger.info("reduce phase over %d partial summaries", len(partial_summaries)) logger.info("reduce phase over %d partial summaries", len(partial_summaries))
master_raw = call_llm( master_raw = call_llm(
build_master_prompt(final_input), build_master_prompt(final_input),
api_key=api_key,
model=model, model=model,
base_url=base_url, base_url=base_url,
json_mode=True, json_mode=True,

97
news/summerizer/tests/test_nous_client.py Normal file
View file

@ -0,0 +1,97 @@
from unittest.mock import MagicMock
import httpx
from nous_client import chat
DEFAULT_UA = "osint-dashboard-news-summarizer"
BASE = "https://inference-api.nousresearch.com/v1"
def _ok_response(content="hello"):
resp = MagicMock()
resp.status_code = 200
resp.json.return_value = {"choices": [{"message": {"content": content}}]}
return resp
def _install_fake(monkeypatch, post_impl):
captured = {}
class FakeClient:
def __init__(self, timeout=None, **kwargs):
captured["timeout"] = timeout
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def post(self, url, *, headers=None, json=None, **kwargs):
captured["url"] = url
captured["headers"] = headers
captured["json"] = json
return post_impl(url, headers, json)
monkeypatch.setattr(httpx, "Client", FakeClient)
return captured
def test_posts_chat_completions_with_auth_body_and_returns_content(monkeypatch):
captured = _install_fake(monkeypatch, lambda *a: _ok_response("the-content"))
out = chat(
"summarize this",
api_key="secret-key",
model="hermes-3",
base_url=BASE,
)
assert out == "the-content"
assert captured["url"] == f"{BASE}/chat/completions"
assert captured["headers"]["Authorization"] == "Bearer secret-key"
assert captured["headers"]["User-Agent"] == DEFAULT_UA
assert captured["json"]["model"] == "hermes-3"
assert captured["json"]["messages"] == [{"role": "user", "content": "summarize this"}]
assert captured["json"]["temperature"] == 0.2
assert captured["json"]["max_tokens"] == 4096
assert "response_format" not in captured["json"]
def test_user_agent_equals_osint_user_agent_env(monkeypatch):
monkeypatch.setenv("OSINT_USER_AGENT", "custom-ua/2.0")
captured = _install_fake(monkeypatch, lambda *a: _ok_response("ok"))
chat("p", api_key="k", model="m", base_url=BASE)
assert captured["headers"]["User-Agent"] == "custom-ua/2.0"
def test_json_mode_sets_response_format(monkeypatch):
captured = _install_fake(monkeypatch, lambda *a: _ok_response("{}"))
chat("p", api_key="k", model="m", base_url=BASE, json_mode=True)
assert captured["json"]["response_format"] == {"type": "json_object"}
def test_401_returns_empty_string(monkeypatch):
def post_impl(*a):
resp = MagicMock()
resp.status_code = 401
return resp
_install_fake(monkeypatch, post_impl)
assert chat("p", api_key="bad", model="m", base_url=BASE) == ""
def test_5xx_returns_empty_string(monkeypatch):
def post_impl(*a):
resp = MagicMock()
resp.status_code = 503
return resp
_install_fake(monkeypatch, post_impl)
assert chat("p", api_key="k", model="m", base_url=BASE) == ""
def test_timeout_returns_empty_string(monkeypatch):
def post_impl(*a):
raise httpx.TimeoutException("timed out")
_install_fake(monkeypatch, post_impl)
assert chat("p", api_key="k", model="m", base_url=BASE) == ""

111
news/summerizer/tests/test_ollama_client.py
View file

@ -1,111 +0,0 @@
from unittest.mock import MagicMock
import httpx
from ollama_client import chat, list_tags, normalize_base, ollama_ready
DEFAULT_UA = "osint-dashboard-news-summarizer"
BASE = "http://127.0.0.1:11434"
def _ok_response(content="hello"):
resp = MagicMock()
resp.status_code = 200
resp.json.return_value = {"response": content}
return resp
def _install_fake(monkeypatch, post_impl=None, get_impl=None):
captured = {}
class FakeClient:
def __init__(self, timeout=None, **kwargs):
captured["timeout"] = timeout
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def post(self, url, *, headers=None, json=None, **kwargs):
captured["url"] = url
captured["headers"] = headers
captured["json"] = json
if post_impl is None:
raise AssertionError("post not expected")
return post_impl(url, headers, json)
def get(self, url, *, headers=None, **kwargs):
captured["url"] = url
captured["headers"] = headers
if get_impl is None:
raise AssertionError("get not expected")
return get_impl(url, headers)
monkeypatch.setattr(httpx, "Client", FakeClient)
return captured
def test_normalize_base_strips_generate_path():
assert normalize_base("http://ollama:11434/api/generate") == "http://ollama:11434"
assert normalize_base("http://ollama:11434/api/") == "http://ollama:11434"
assert normalize_base("http://ollama:11434") == "http://ollama:11434"
def test_posts_generate_and_returns_response(monkeypatch):
captured = _install_fake(monkeypatch, post_impl=lambda *a: _ok_response("the-content"))
out = chat("summarize this", model="qwen3:30b-a3b", base_url=BASE)
assert out == "the-content"
assert captured["url"] == f"{BASE}/api/generate"
assert captured["headers"]["User-Agent"] == DEFAULT_UA
assert captured["json"]["model"] == "qwen3:30b-a3b"
assert captured["json"]["prompt"] == "summarize this"
assert captured["json"]["stream"] is False
assert "format" not in captured["json"]
assert captured["json"]["options"]["num_ctx"] == 32768
assert captured["timeout"] == 300.0
def test_json_mode_sets_format_json(monkeypatch):
captured = _install_fake(monkeypatch, post_impl=lambda *a: _ok_response("{}"))
chat("p", model="m", base_url=BASE, json_mode=True)
assert captured["json"]["format"] == "json"
def test_empty_model_returns_empty(monkeypatch):
assert chat("p", model="", base_url=BASE) == ""
def test_5xx_returns_empty_string(monkeypatch):
def post_impl(*a):
resp = MagicMock()
resp.status_code = 503
return resp
_install_fake(monkeypatch, post_impl=post_impl)
assert chat("p", model="m", base_url=BASE) == ""
def test_timeout_returns_empty_string(monkeypatch):
def post_impl(*a):
raise httpx.TimeoutException("timed out")
_install_fake(monkeypatch, post_impl=post_impl)
assert chat("p", model="m", base_url=BASE) == ""
def test_list_tags_and_ready(monkeypatch):
def get_impl(*a):
resp = MagicMock()
resp.status_code = 200
resp.json.return_value = {
"models": [{"name": "qwen3:30b-a3b"}, {"name": "llama3.2:latest"}],
}
return resp
captured = _install_fake(monkeypatch, get_impl=get_impl)
names = list_tags(BASE)
assert names == ["qwen3:30b-a3b", "llama3.2:latest"]
assert captured["url"] == f"{BASE}/api/tags"
assert ollama_ready("qwen3:30b-a3b", base_url=BASE) is True
assert ollama_ready("missing", base_url=BASE) is False

67
tests/test_api_settings.py
View file

@ -64,26 +64,31 @@ def _put(path: str, json: dict) -> httpx.Response:
return _request("PUT", path, json=json) return _request("PUT", path, json=json)
def test_get_news_models_ollama_down_returns_fallback(monkeypatch): def test_get_news_models_without_key_returns_fallback(monkeypatch):
async def boom_http_get(url, *, headers, timeout): monkeypatch.delenv("NOUS_API_KEY", raising=False)
raise httpx.ConnectError("ollama down") monkeypatch.setattr("keystore.get_api_key", _missing_key)
import settings_store
monkeypatch.setattr(settings_store, "_http_get", boom_http_get)
settings_store._models_cache = None
resp = _get("/api/news/models") resp = _get("/api/news/models")
assert resp.status_code == 200 assert resp.status_code == 200
body = resp.json() body = resp.json()
assert body["source"] == "fallback" assert body["source"] == "fallback"
ids = [m["id"] for m in body["models"]] ids = [m["id"] for m in body["models"]]
assert "Hermes-4.3-36B" in ids
from settings_store import FALLBACK_MODELS from settings_store import FALLBACK_MODELS
assert ids == FALLBACK_MODELS assert ids == FALLBACK_MODELS
assert "qwen3:30b-a3b" in ids
def test_get_news_models_live_from_ollama_tags(monkeypatch): async def _missing_key(name: str):
monkeypatch.setenv("LLM_URL", "http://ollama:11434") return None
async def _present_key(name: str):
return "test-nous-api-key-1234"
def test_get_news_models_live_from_upstream(monkeypatch):
monkeypatch.setattr("keystore.get_api_key", _present_key)
monkeypatch.setenv("NOUS_BASE_URL", "https://inference-api.nousresearch.com/v1")
monkeypatch.delenv("OSINT_USER_AGENT", raising=False) monkeypatch.delenv("OSINT_USER_AGENT", raising=False)
captured: dict = {} captured: dict = {}
@ -95,7 +100,7 @@ def test_get_news_models_live_from_ollama_tags(monkeypatch):
return None return None
def json(self): def json(self):
return {"models": [{"name": "qwen3:30b-a3b"}, {"name": "llama3.2:latest"}]} return {"data": [{"id": "live-model-a"}, {"id": "Hermes-4.3-36B"}]}
async def fake_http_get(url, *, headers, timeout): async def fake_http_get(url, *, headers, timeout):
captured["url"] = url captured["url"] = url
@ -112,10 +117,30 @@ def test_get_news_models_live_from_ollama_tags(monkeypatch):
body = resp.json() body = resp.json()
assert body["source"] == "live" assert body["source"] == "live"
ids = [m["id"] for m in body["models"]] ids = [m["id"] for m in body["models"]]
assert ids == ["qwen3:30b-a3b", "llama3.2:latest"] assert ids == ["live-model-a", "Hermes-4.3-36B"]
assert captured["url"] == "http://ollama:11434/api/tags" assert captured["url"] == "https://inference-api.nousresearch.com/v1/models"
assert captured["headers"]["User-Agent"] == "osint-dashboard-news-summarizer" assert captured["headers"]["User-Agent"] == "osint-dashboard-news-summarizer"
assert "Authorization" not in captured["headers"] assert captured["headers"]["Authorization"] == "Bearer test-nous-api-key-1234"
timeout = captured["timeout"]
assert timeout == 8 or getattr(timeout, "read", timeout) == 8 or float(timeout) == 8.0
def test_get_news_models_upstream_failure_returns_fallback(monkeypatch):
monkeypatch.setattr("keystore.get_api_key", _present_key)
async def boom_http_get(url, *, headers, timeout):
raise httpx.ConnectError("upstream down")
import settings_store
monkeypatch.setattr(settings_store, "_http_get", boom_http_get)
settings_store._models_cache = None
resp = _get("/api/news/models")
assert resp.status_code == 200
body = resp.json()
assert body["source"] == "fallback"
ids = [m["id"] for m in body["models"]]
assert "Hermes-4.3-36B" in ids
def test_put_settings_empty_returns_422(): def test_put_settings_empty_returns_422():
@ -131,17 +156,17 @@ def test_put_settings_whitespace_only_returns_422():
@requires_db @requires_db
def test_put_settings_round_trip(clean_settings, monkeypatch): def test_put_settings_round_trip(clean_settings, monkeypatch):
monkeypatch.delenv("SUMMARY_MODEL", raising=False) monkeypatch.delenv("SUMMARY_MODEL", raising=False)
monkeypatch.delenv("LLM_URL", raising=False) monkeypatch.delenv("NOUS_BASE_URL", raising=False)
put_resp = _put("/api/settings", {"summary_model": "qwen3:30b-a3b"}) put_resp = _put("/api/settings", {"summary_model": "google/gemini-2.5-flash"})
assert put_resp.status_code == 200 assert put_resp.status_code == 200
put_body = put_resp.json() put_body = put_resp.json()
assert put_body["summary_model"] == "qwen3:30b-a3b" assert put_body["summary_model"] == "google/gemini-2.5-flash"
assert put_body["llm_url"] == "http://127.0.0.1:11434" assert put_body["nous_base_url"] == "https://inference-api.nousresearch.com/v1"
get_resp = _get("/api/settings") get_resp = _get("/api/settings")
assert get_resp.status_code == 200 assert get_resp.status_code == 200
get_body = get_resp.json() get_body = get_resp.json()
assert get_body["summary_model"] == "qwen3:30b-a3b" assert get_body["summary_model"] == "google/gemini-2.5-flash"
assert get_body["llm_url"] == "http://127.0.0.1:11434" assert get_body["nous_base_url"] == "https://inference-api.nousresearch.com/v1"
assert set(get_body.keys()) == {"summary_model", "llm_url"} assert set(get_body.keys()) == {"summary_model", "nous_base_url"}