From 58ff43bdee2af8f8ac792cdba88717073e74f0da Mon Sep 17 00:00:00 2001 From: Sirius DevOps Date: Fri, 28 Aug 2026 19:38:14 -0400 Subject: [PATCH] feat: rebuild news summarizer on local Ollama Swap Nous Portal for the newsPipeline local_llm generate path (POST /api/generate, GET /api/tags). No API key. Settings model selector lists Ollama tags; idle if the host is down. --- .env.example | 14 ++- app/keystore.py | 7 +- app/main.py | 10 +- app/schemas.py | 6 +- app/settings_store.py | 54 +++------- app/static/index.html | 11 +- docker-compose.yml | 3 +- docs/news.md | 73 +++++++------ news/summerizer/nous_client.py | 36 ------- news/summerizer/ollama_client.py | 102 ++++++++++++++++++ news/summerizer/run_news_summarizer.py | 18 +--- news/summerizer/summarizer.py | 64 ++++------- news/summerizer/tests/test_nous_client.py | 97 ----------------- news/summerizer/tests/test_ollama_client.py | 111 ++++++++++++++++++++ tests/test_api_settings.py | 67 ++++-------- 15 files changed, 335 insertions(+), 338 deletions(-) delete mode 100644 news/summerizer/nous_client.py create mode 100644 news/summerizer/ollama_client.py delete mode 100644 news/summerizer/tests/test_nous_client.py create mode 100644 news/summerizer/tests/test_ollama_client.py diff --git a/.env.example b/.env.example index ac898a9..2ba3877 100644 --- a/.env.example +++ b/.env.example @@ -60,7 +60,7 @@ FIRMS_INTERVAL=900 INGEST_FIRES=1 # ── API keys (managed from the dashboard UI) ────────────────────────────── -# Keys such as NOUS_API_KEY and TELEGRAM_TOKEN are stored in the Postgres +# Keys such as TELEGRAM_TOKEN are stored in the Postgres # `api_keys` table and managed from the dashboard's "Keys" tab # (GET/POST/DELETE /api/keys/{name}) — see app/keystore.py. The FIRMS ingestor # currently reads FIRMS_MAP_KEY from .env (above); wiring the Keys-UI store as @@ -68,17 +68,15 @@ INGEST_FIRES=1 # ── News pipeline (scraper + summarizer, profile `ingest`) ──────────────── # Hourly: the scraper crawls 257 RSS sources at minute :00 and the summarizer -# runs the Nous map-reduce at minute :05, both writing to the shared osint-db +# runs the local Ollama map-reduce at minute :05, both writing to the shared osint-db # (tables `articles` + `article_summaries`, created by alembic 003_news). # Consume via GET /api/news and GET /api/news/summaries. -# NOUS_API_KEY is also (preferably) set in the Keys UI; env is an override. -# Unset in both env and api_keys = summarizer logs and idles (never crashes). -NOUS_API_KEY= -NOUS_BASE_URL=https://inference-api.nousresearch.com/v1 -# Optional LLM knobs +# LLM_URL is the Ollama origin (Pi container cannot use 127.0.0.1 for a laptop +# GPU). Example: LLM_URL=http://100.91.7.84:11434 +# LLM_URL= # SUMMARY_MODEL is an optional override. Leave unset so Settings # (app_settings.SUMMARY_MODEL) can reach the summarizer. Code default -# Hermes-4.3-36B remains after a Postgres miss. Env wins when set. +# qwen3:30b-a3b remains after a Postgres miss. Env wins when set. # SUMMARY_MODEL= NEWS_BATCH_SIZE=50 SUMMARY_WINDOW_HOURS=1 diff --git a/app/keystore.py b/app/keystore.py index 0eb49a4..a701fb7 100644 --- a/app/keystore.py +++ b/app/keystore.py @@ -57,11 +57,6 @@ KEY_REGISTRY: dict[str, dict] = { "pattern": r"^[0-9a-fA-F]{32}$", "example": "32-char hex string (e.g. 5f3c…9a02)", }, - "NOUS_API_KEY": { - "description": "Nous Portal API key — hourly news summarizer (inference-api.nousresearch.com).", - "pattern": r"^.{16,}$", - "example": "key from https://portal.nousresearch.com (API keys page)", - }, "TELEGRAM_TOKEN": { "description": "Telegram bot token — push alert notifications to a channel.", "pattern": r"^\d{8,10}:[0-9A-Za-z_-]{35}$", @@ -214,7 +209,7 @@ async def get_api_key(name: str) -> str | None: """Read a stored key value — used by ingest services, never by the API. Returns the raw value (or None when unset) so producers can pass it to - external APIs (FIRMS, Nous, Telegram, …). Reads live from Postgres, so a + external APIs (FIRMS, Telegram, …). Reads live from Postgres, so a key set via the dashboard is picked up on the next poll — no restart. """ await ensure_api_keys_table() diff --git a/app/main.py b/app/main.py index a2e47ee..df90401 100644 --- a/app/main.py +++ b/app/main.py @@ -605,7 +605,7 @@ async def list_documents( async def list_api_keys(): """List known API keys with set/missing status — masked, never raw. - Registered keys (FIRMS_MAP_KEY, NOUS_API_KEY, TELEGRAM_TOKEN) + Registered keys (FIRMS_MAP_KEY, TELEGRAM_TOKEN) are always included. Any extra stored keys are appended. """ return await list_keys() @@ -638,13 +638,13 @@ async def remove_api_key(name: str): @app.get("/api/settings", response_model=SettingsOut) async def get_settings(): - """Summarizer model + read-only Nous base URL.""" + """Summarizer model + read-only Ollama URL.""" return await get_app_settings() @app.put("/api/settings", response_model=SettingsOut) async def put_settings(payload: SettingsIn): - """Persist SUMMARY_MODEL. ``nous_base_url`` is ignored even if sent.""" + """Persist SUMMARY_MODEL. ``llm_url`` is ignored even if sent.""" try: return await set_summary_model(payload.summary_model) except SettingsError as exc: @@ -1084,7 +1084,7 @@ async def camera_hls_segment(camera_id: UUID, u: str = Query(..., min_length=8)) # ── News pipeline (scraper + summarizer) ────────────────────────────────── # Backing data for the frontend news panel. Written by the vendored -# news-scraper (hourly Scrapy crawl) and news-summarizer (hourly Gemini +# news-scraper (hourly Scrapy crawl) and news-summarizer (hourly Ollama # map-reduce) services into the shared osint-db. @app.get("/api/news", response_model=list[NewsArticleOut]) @@ -1249,7 +1249,7 @@ async def list_news_map( @app.get("/api/news/models", response_model=NewsModelsOut) async def list_news_models(): - """Nous model catalog for the summarizer selector. Never 502s.""" + """Ollama model catalog for the summarizer selector. Never 502s.""" return await list_models() diff --git a/app/schemas.py b/app/schemas.py index b61e296..8fd4c68 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -316,7 +316,7 @@ class NewsModelsOut(BaseModel): class SettingsIn(BaseModel): - """Body for PUT /api/settings. ``nous_base_url`` is not writable.""" + """Body for PUT /api/settings. ``llm_url`` is not writable.""" summary_model: str = Field(..., min_length=1, max_length=128) @@ -332,10 +332,10 @@ class SettingsIn(BaseModel): class SettingsOut(BaseModel): - """Current summarizer settings. ``nous_base_url`` is read-only.""" + """Current summarizer settings. ``llm_url`` is read-only.""" summary_model: str - nous_base_url: str + llm_url: str # ─── Aggregations ──────────────────────────────────────────────────────── diff --git a/app/settings_store.py b/app/settings_store.py index 314db42..44434dc 100644 --- a/app/settings_store.py +++ b/app/settings_store.py @@ -17,11 +17,10 @@ from datetime import datetime, timezone import httpx from sqlalchemy import Column, DateTime, String, Table, Text, func, select, text -import keystore from database import async_session, engine, metadata -DEFAULT_NOUS_BASE_URL = "https://inference-api.nousresearch.com/v1" -DEFAULT_SUMMARY_MODEL = "Hermes-4.3-36B" +DEFAULT_LLM_URL = "http://127.0.0.1:11434" +DEFAULT_SUMMARY_MODEL = "qwen3:30b-a3b" SETTING_SUMMARY_MODEL = "SUMMARY_MODEL" ALLOWED_SETTINGS = frozenset({SETTING_SUMMARY_MODEL}) MODELS_CACHE_TTL_S = 600.0 @@ -29,12 +28,7 @@ MODELS_TIMEOUT_S = 8.0 DEFAULT_MODELS_USER_AGENT = "osint-dashboard-news-summarizer" FALLBACK_MODELS = [ - "Hermes-4.3-36B", - "Hermes-4-70B", - "google/gemini-2.5-flash", - "anthropic/claude-haiku-4.5", - "openai/gpt-4.1-mini", - "x-ai/grok-4", + "qwen3:30b-a3b", ] app_settings = Table( @@ -77,10 +71,14 @@ async def ensure_app_settings_table() -> None: _ensured = True -def nous_base_url() -> str: - """Read-only Nous inference base URL (env, never writable from the UI).""" - raw = (os.getenv("NOUS_BASE_URL") or "").strip().rstrip("/") - return raw or DEFAULT_NOUS_BASE_URL +def llm_url() -> str: + """Read-only Ollama origin (env, never writable from the UI).""" + raw = (os.getenv("LLM_URL") or "").strip().rstrip("/") + for suffix in ("/api/generate", "/api/chat", "/api", "/v1"): + if raw.endswith(suffix): + raw = raw[: -len(suffix)].rstrip("/") + break + return raw or DEFAULT_LLM_URL def _validate_summary_model(value: str) -> str: @@ -91,7 +89,7 @@ def _validate_summary_model(value: str) -> str: async def get_summary_model() -> str: - """Stored SUMMARY_MODEL, else env, else Hermes-4.3-36B.""" + """Stored SUMMARY_MODEL, else env, else qwen3:30b-a3b.""" await ensure_app_settings_table() async with async_session() as session: row = ( @@ -135,7 +133,7 @@ async def set_summary_model(value: str) -> dict: async def get_app_settings() -> dict: return { "summary_model": await get_summary_model(), - "nous_base_url": nous_base_url(), + "llm_url": llm_url(), } @@ -146,18 +144,6 @@ def _fallback_payload() -> dict: } -async def _nous_api_key() -> str | None: - """Keystore first, then env. Any lookup failure is treated as missing.""" - try: - stored = await keystore.get_api_key("NOUS_API_KEY") - except Exception: - stored = None - if stored and str(stored).strip(): - return str(stored).strip() - env = (os.getenv("NOUS_API_KEY") or "").strip() - return env or None - - def _models_user_agent() -> str: return os.getenv("OSINT_USER_AGENT") or DEFAULT_MODELS_USER_AGENT @@ -174,7 +160,7 @@ def _parse_models_payload(body: object) -> list[dict[str, str]]: if isinstance(item, str) and item.strip(): out.append({"id": item.strip()}) elif isinstance(item, dict): - mid = item.get("id") or item.get("name") + mid = item.get("id") or item.get("name") or item.get("model") if mid: out.append({"id": str(mid)}) return out @@ -186,23 +172,15 @@ async def _http_get(url: str, *, headers: dict[str, str], timeout: float) -> htt async def list_models() -> dict: - """Live ``GET {base}/models`` when a key is present; otherwise curated fallback. - - Never raises to the caller for missing key or upstream failure. - """ + """Live ``GET {llm_url}/api/tags``. Never 502 — fallback on failure.""" global _models_cache - key = await _nous_api_key() - if not key: - return _fallback_payload() - now = time.monotonic() hit = _models_cache if hit and now - hit[0] < MODELS_CACHE_TTL_S: return hit[1] - url = f"{nous_base_url()}/models" + url = f"{llm_url()}/api/tags" headers = { - "Authorization": f"Bearer {key}", "User-Agent": _models_user_agent(), "Accept": "application/json", } diff --git a/app/static/index.html b/app/static/index.html index db4fb01..c4f8190 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -840,7 +840,7 @@
Executive Summary WAITING
—
-
No executive summary yet — the Nous summarizer is idle until NOUS_API_KEY is set on the Pi and a model is chosen in Settings.
+
No executive summary yet — the Ollama summarizer is idle until a model is chosen in Settings and LLM_URL is reachable.
@@ -968,7 +968,7 @@

⚿ API Keys

-

Keys used by ingest services (FIRMS, Nous Portal, Telegram, AIS, …). Stored in Postgres, never shown again — only status + last 4 chars.

+

Keys used by ingest services (FIRMS, Telegram, AIS, …). Stored in Postgres, never shown again — only status + last 4 chars.

@@ -997,11 +997,11 @@

Takes effect on the next hourly run (:05). No container restart.

- Nous Portal (inference-api.nousresearch.com) + Local Ollama (LLM_URL)
- set in API Keys as NOUS_API_KEY (never shown here) + none — Ollama on the GPU host
@@ -1370,7 +1370,7 @@ function renderNewsSummary(summaries) { const badgeEl = document.getElementById('ns-badge'); if (!bodyEl) return; if (!summaries || !summaries.length) { - bodyEl.innerHTML = '
No executive summary yet — the Nous summarizer is idle until NOUS_API_KEY is set on the Pi and a model is chosen in Settings.
'; + bodyEl.innerHTML = '
No executive summary yet — the Ollama summarizer is idle until a model is chosen in Settings and LLM_URL is reachable.
'; timeEl.textContent = '—'; badgeEl.textContent = 'WAITING'; return; @@ -1547,7 +1547,6 @@ function sentimentBadge(label) { /* ═══════════════ API KEYS ═══════════════ */ const KEY_PATTERNS = { 'FIRMS_MAP_KEY': /^[0-9a-fA-F]{32}$/, - 'NOUS_API_KEY': /^.{16,}$/, 'TELEGRAM_TOKEN': /^\d{8,10}:[0-9A-Za-z_-]{35}$/, }; async function loadKeys() { diff --git a/docker-compose.yml b/docker-compose.yml index aa674f7..400c6b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -224,8 +224,7 @@ services: DB_HOST: db DB_PORT: ${DB_PORT:-5432} DB_NAME: ${DB_NAME:-osint_data} - NOUS_API_KEY: ${NOUS_API_KEY:-} - NOUS_BASE_URL: ${NOUS_BASE_URL:-https://inference-api.nousresearch.com/v1} + LLM_URL: ${LLM_URL:-} SUMMARY_MODEL: ${SUMMARY_MODEL:-} OSINT_USER_AGENT: ${OSINT_USER_AGENT:-osint-dashboard-news-summarizer} BATCH_SIZE: ${NEWS_BATCH_SIZE:-50} diff --git a/docs/news.md b/docs/news.md index 43a39c0..41e5ced 100644 --- a/docs/news.md +++ b/docs/news.md @@ -1,11 +1,12 @@ -# News pipeline — scraper + Nous Portal summarizer +# News pipeline — scraper + local Ollama summarizer The OSINT dashboard ingests ~257 global news RSS sources hourly and produces an English LLM brief plus flagged ticker/map rows. Both services were vendored from the upstream `~/Projects/newsPipeline` project and re-integrated here to replace the old k8s CronJob choreography with in-compose scheduling against -the EXISTING osint-db — **no second Postgres**. The LLM is **Nous Portal** -(`inference-api.nousresearch.com`) — not Gemini. +the EXISTING osint-db — **no second Postgres**. The LLM is **local Ollama** +(`LLM_URL`, default `http://127.0.0.1:11434`) — the newsPipeline `local_llm` +path (`POST /api/generate`). No API key. ## Architecture @@ -16,7 +17,7 @@ the EXISTING osint-db — **no second Postgres**. The LLM is **Nous Portal** news-scraper (Scrapy, hourly :00) ──► articles table (osint-db) │ │ │ ▼ -news-summarizer (Nous Portal map-reduce, :05) ──► article_summaries + news_items +news-summarizer (Ollama map-reduce, :05) ──► article_summaries + news_items │ ▼ GET /api/news · /api/news/summaries · /api/news/ticker · /api/news/map @@ -46,8 +47,8 @@ feeds (`GET /api/news` exact key set is unchanged on purpose). (`ON CONFLICT (url) DO NOTHING`). 2. **Summarizer** — `news/summerizer/run_news_summarizer.py` runs `summarizer.py` at :05 past each hour. It reads articles from the last - `SUMMARY_WINDOW_HOURS`, map-reduces them through Nous Portal - (`SUMMARY_MODEL` / Settings, default `Hermes-4.3-36B`), writes the English + `SUMMARY_WINDOW_HOURS`, map-reduces them through local Ollama + (`SUMMARY_MODEL` / Settings, default `qwen3:30b-a3b`), writes the English brief to `article_summaries` (column `model` is the LLM id), and flagged ticker/map rows to `news_items`. @@ -67,17 +68,16 @@ Container startup order doesn't matter. ## Keys and Settings -- **`NOUS_API_KEY`** — paste in the dashboard **Keys** UI (`api_keys` / - `keystore.KEY_REGISTRY`). Env / `.env` is an **override** (env wins, same - as FIRMS). Never returned by any API; never emitted into `index.html`; - never proxied from the browser. -- **Idle without a key** — if env is unset **and** the keystore row is empty, - the summarizer logs and idles (never crashes). News intel APIs return `[]`. +- **No API key.** Ollama on the GPU host; the summarizer container calls + `LLM_URL` (`POST /api/generate`, `GET /api/tags`). +- **Idle without Ollama** — if `/api/tags` is down or the chosen model is not + pulled, the summarizer logs and idles (never crashes). News intel APIs + return `[]`. - **Model** — non-secret. Settings UI model selector `PUT /api/settings` `{ "summary_model": "…" }` stores `SUMMARY_MODEL` in `app_settings` (1–128 - chars). `GET /api/settings` echoes `{summary_model, nous_base_url}`. - `nous_base_url` is read-only. Default `Hermes-4.3-36B`. Live catalog is - best-effort `GET /api/news/models`. + chars). `GET /api/settings` echoes `{summary_model, llm_url}`. + `llm_url` is read-only. Default `qwen3:30b-a3b`. Live catalog is + best-effort `GET /api/news/models` (`GET {LLM_URL}/api/tags`). ## Endpoints @@ -192,15 +192,15 @@ Never 502s. `{ "source": "live"|"fallback", "models": [{"id": "…"}] }`. ### GET /api/settings · PUT /api/settings ```json -{ "summary_model": "Hermes-4.3-36B", "nous_base_url": "https://inference-api.nousresearch.com/v1" } +{ "summary_model": "qwen3:30b-a3b", "llm_url": "http://127.0.0.1:11434" } ``` -PUT body is `{ "summary_model": "<1–128 char id>" }`. `nous_base_url` is +PUT body is `{ "summary_model": "<1–128 char id>" }`. `llm_url` is ignored even if sent. ## Reduce JSON contract -Reduce phase (`response_format: json_object`, English only) must be a single +Reduce phase (`format: json` on Ollama generate, English only) must be a single object. Parser (`intel.parse_reduce_json`) strips `…` and markdown json fences, then brace-slices: @@ -234,9 +234,8 @@ lands in `article_summaries.summary_text`. | Var | Default | Notes | |---|---|---| -| `NOUS_API_KEY` | *(blank)* | **Required for summaries.** Prefer Keys UI; env overrides. Unset in **both** env and `api_keys` = summarizer logs and idles (never crashes); APIs return `[]`. | -| `NOUS_BASE_URL` | `https://inference-api.nousresearch.com/v1` | Read-only in Settings. | -| `SUMMARY_MODEL` | `Hermes-4.3-36B` | Compose default. Operator-facing choice is Settings → `app_settings.SUMMARY_MODEL`. | +| `LLM_URL` | `http://127.0.0.1:11434` | Ollama origin. Pi container must point at the GPU host (e.g. Tailscale). Compose leaves this empty so `.env` wins. | +| `SUMMARY_MODEL` | `qwen3:30b-a3b` | Leave compose unset so Settings → `app_settings.SUMMARY_MODEL` reaches the worker. Env wins when set. | | `NEWS_BATCH_SIZE` | `50` | Articles per map-phase batch (compose maps to container `BATCH_SIZE`). | | `SUMMARY_WINDOW_HOURS` | `1` | How far back the summarizer looks for new articles. | | `INCLUDE_FUTURES` | `0` | Legacy futures-prices coupling (upstream pipeline). OFF for OSINT; set `1` + install `yfinance` to enable. | @@ -246,15 +245,14 @@ lands in `article_summaries.summary_text`. | `NEWS_SUMMARIZE_RUN_ON_START` | `1` | Run one summarize immediately on container start. | | `NEWS_SUMMARIZE_FORCE` | `0` | `1` ignores the current-UTC-hour idempotency skip (double-pins on recreate). | | `NEWS_LOG_LEVEL` | `INFO` | Scrapy log level. | -| `OSINT_USER_AGENT` | `osint-dashboard-news-summarizer` | Sent on every outbound Nous call. | +| `OSINT_USER_AGENT` | `osint-dashboard-news-summarizer` | Sent on every outbound Ollama call. | | `TELEGRAM_TOKEN` / `TELEGRAM_CHAT_ID` | *(blank)* | Reserved for the (out-of-scope) Telegram delivery bot. | DB_* for both services is mapped to the shared osint-db credentials (`DB_HOST=db`, same `DB_USER/DB_PASSWORD/DB_NAME` as the rest of the stack). -Nous chat: `POST {NOUS_BASE_URL}/chat/completions` via `news/summerizer/nous_client.py` -(`httpx`, no `openai` SDK). Auth is a Bearer token from `NOUS_API_KEY`. -No Hermes-4 reasoning system prompt. Reduce uses `json_mode=True`. +Ollama generate: `POST {LLM_URL}/api/generate` via `news/summerizer/ollama_client.py` +(`httpx`, no `ollama` Python SDK). No API key. Reduce uses `format: json`. ## Prompts @@ -268,7 +266,7 @@ language is gated behind `INCLUDE_FUTURES=1`. ```bash PYTHONPATH=news/summerizer pytest news/summerizer/tests -v -# intel + nous_client tests PASS (no network) +# intel + ollama_client tests PASS (no network) PYTHONPATH=app pytest tests/test_api_news.py \ tests/test_api_settings.py tests/test_api_live_layers.py -v @@ -280,16 +278,17 @@ PYTHONPATH=app pytest tests/test_api_news.py \ After deploy / compose rebuild of `news-summarizer` on the Pi: -1. Keys UI: save `NOUS_API_KEY` → status `****last4`. -2. Settings: pick a model → Save → `GET /api/settings` echoes it. -3. `docker compose --profile ingest logs -f news-summarizer` — next run (or +1. Run Ollama on the GPU host and pull the model (`ollama pull qwen3:30b-a3b`). +2. Set `LLM_URL` on the Pi to that host (Tailscale IP + `:11434`). +3. Settings: pick a model → Save → `GET /api/settings` echoes it. +4. `docker compose --profile ingest logs -f news-summarizer` — next run (or `NEWS_SUMMARIZE_RUN_ON_START=1` recreate) logs `Processing N articles with `. -4. `curl -s localhost:8000/api/news/summaries?limit=1` — English `summary_text`, `model` set. -5. `curl -s localhost:8000/api/news/ticker` — flagged headlines only. -6. `curl -s localhost:8000/api/news/map` — only rows with lat/lon. -7. HUD: NEWS ticker scrolls flagged items; map overlay pins popup with location. -8. Unset key + empty keystore → summarizer logs idle, APIs return `[]`, no crash. +5. `curl -s localhost:8000/api/news/summaries?limit=1` — English `summary_text`, `model` set. +6. `curl -s localhost:8000/api/news/ticker` — flagged headlines only. +7. `curl -s localhost:8000/api/news/map` — only rows with lat/lon. +8. HUD: NEWS ticker scrolls flagged items; map overlay pins popup with location. +9. Ollama down / model missing → summarizer logs idle, APIs return `[]`, no crash. -**Operator action after merge:** paste a Nous Portal API key in API Keys; pick -a model in Settings if the default `Hermes-4.3-36B` is not wanted; rebuild -`osint-news-summarizer` on the Pi (`pi-app-deploy` / compose). +**Operator action after merge:** start Ollama on the laptop GPU, set `LLM_URL` +in Pi `.env`, pick a model in Settings if the default `qwen3:30b-a3b` is not +wanted; rebuild `osint-news-summarizer` on the Pi (`pi-app-deploy` / compose). diff --git a/news/summerizer/nous_client.py b/news/summerizer/nous_client.py deleted file mode 100644 index da0b2c4..0000000 --- a/news/summerizer/nous_client.py +++ /dev/null @@ -1,36 +0,0 @@ -"""HTTP client for the Nous inference chat completions API.""" - -from __future__ import annotations - -import os - -import httpx - -_DEFAULT_UA = "osint-dashboard-news-summarizer" -_DEFAULT_BASE = "https://inference-api.nousresearch.com/v1" - - -def chat(prompt, *, api_key, model, base_url, json_mode=False) -> str: - resolved = (base_url or os.environ.get("NOUS_BASE_URL", _DEFAULT_BASE)).rstrip("/") - url = f"{resolved}/chat/completions" - headers = { - "Authorization": f"Bearer {api_key}", - "User-Agent": os.environ.get("OSINT_USER_AGENT") or _DEFAULT_UA, - } - payload = { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "temperature": 0.2, - "max_tokens": 4096, - } - if json_mode: - payload["response_format"] = {"type": "json_object"} - try: - with httpx.Client(timeout=60.0) as client: - resp = client.post(url, headers=headers, json=payload) - if resp.status_code == 401 or resp.status_code >= 500: - return "" - data = resp.json() - return data["choices"][0]["message"]["content"] - except Exception: - return "" diff --git a/news/summerizer/ollama_client.py b/news/summerizer/ollama_client.py new file mode 100644 index 0000000..057b437 --- /dev/null +++ b/news/summerizer/ollama_client.py @@ -0,0 +1,102 @@ +"""HTTP client for a local Ollama generate API. + +Matches the newsPipeline local_llm summarizer: POST /api/generate, +GET /api/tags. No API key. ``LLM_URL`` may be a host, ``/api``, or +``/api/generate`` — we normalize to the origin. +""" + +from __future__ import annotations + +import os + +import httpx + +_DEFAULT_UA = "osint-dashboard-news-summarizer" +_DEFAULT_BASE = "http://127.0.0.1:11434" +_GENERATE_TIMEOUT = 300.0 +_TAGS_TIMEOUT = 8.0 + +_GENERATE_OPTIONS = { + "num_predict": 4096, + "temperature": 0.6, + "top_p": 0.9, + "top_k": 40, + "num_ctx": 32768, + "repeat_penalty": 1.1, +} + + +def normalize_base(url: str | None) -> str: + """Strip path suffixes so LLM_URL variants share one origin.""" + raw = (url or os.environ.get("LLM_URL", "") or _DEFAULT_BASE).strip() + raw = raw.rstrip("/") + for suffix in ("/api/generate", "/api/chat", "/api", "/v1"): + if raw.endswith(suffix): + raw = raw[: -len(suffix)].rstrip("/") + break + return raw or _DEFAULT_BASE + + +def _headers() -> dict[str, str]: + return { + "User-Agent": os.environ.get("OSINT_USER_AGENT") or _DEFAULT_UA, + "Accept": "application/json", + } + + +def chat(prompt: str, *, model: str, base_url: str | None = None, json_mode: bool = False) -> str: + """POST /api/generate. Returns response text, or \"\" on any failure.""" + if not (model or "").strip(): + return "" + origin = normalize_base(base_url) + url = f"{origin}/api/generate" + payload: dict = { + "model": model.strip(), + "prompt": prompt, + "stream": False, + "options": dict(_GENERATE_OPTIONS), + } + if json_mode: + payload["format"] = "json" + try: + with httpx.Client(timeout=_GENERATE_TIMEOUT) as client: + resp = client.post(url, headers=_headers(), json=payload) + if resp.status_code == 401 or resp.status_code >= 500: + return "" + data = resp.json() + text = data.get("response") if isinstance(data, dict) else None + return text if isinstance(text, str) else "" + except Exception: + return "" + + +def list_tags(base_url: str | None = None) -> list[str]: + """GET /api/tags model names. Empty list on failure.""" + origin = normalize_base(base_url) + url = f"{origin}/api/tags" + try: + with httpx.Client(timeout=_TAGS_TIMEOUT) as client: + resp = client.get(url, headers=_headers()) + if resp.status_code != 200: + return [] + models = (resp.json() or {}).get("models") or [] + names: list[str] = [] + for item in models: + if isinstance(item, dict): + name = item.get("name") or item.get("model") or "" + if name: + names.append(str(name)) + return names + except Exception: + return [] + + +def ollama_ready(model: str, *, base_url: str | None = None) -> bool: + """True when /api/tags is up and ``model`` is present (prefix match).""" + if not (model or "").strip(): + return False + names = list_tags(base_url) + if not names: + return False + want = model.strip() + return any(n == want or n.startswith(want) for n in names) diff --git a/news/summerizer/run_news_summarizer.py b/news/summerizer/run_news_summarizer.py index 91740c6..483ea9b 100644 --- a/news/summerizer/run_news_summarizer.py +++ b/news/summerizer/run_news_summarizer.py @@ -1,16 +1,11 @@ #!/usr/bin/env python3 """Scheduler loop for the news summarizer — hourly summarize at minute :05. -Replaces the k8s CronJob (`5 * * * *`) with an in-compose loop. Runs once on -boot (catches up on any articles scraped since the last summary), then fires -at each :NEWS_SUMMARIZE_MINUTE wall-clock boundary. - -The loop is serial, so a slow LLM pass never overlaps the next run. - Env (all optional, 12-factor): NEWS_SUMMARIZE_MINUTE minute of the hour to fire (default 5) NEWS_SUMMARIZE_RUN_ON_START "1" to summarize once immediately on boot (default 1) - NOUS_API_KEY optional in env; Keys UI / api_keys also works + LLM_URL Ollama origin; default http://127.0.0.1:11434 + SUMMARY_MODEL Ollama tag (else app_settings) """ from __future__ import annotations @@ -46,13 +41,10 @@ def run_summarize() -> None: def main() -> None: - if not os.getenv("NOUS_API_KEY", "").strip(): - logger.warning( - "NOUS_API_KEY unset in env — will read api_keys on each run; idle if both empty" - ) + llm_url = os.getenv("LLM_URL", "").strip() or "http://127.0.0.1:11434" logger.info( - "news summarizer loop starting (minute=%s, run_on_start=%s)", - MINUTE, RUN_ON_START, + "news summarizer loop starting (minute=%s, run_on_start=%s, llm_url=%s)", + MINUTE, RUN_ON_START, llm_url, ) if RUN_ON_START: run_summarize() diff --git a/news/summerizer/summarizer.py b/news/summerizer/summarizer.py index ea82a37..196f3b9 100644 --- a/news/summerizer/summarizer.py +++ b/news/summerizer/summarizer.py @@ -1,19 +1,17 @@ #!/usr/bin/env python3 -"""News summarizer — Nous map-reduce of scraped articles into brief/ticker/map. +"""News summarizer — local Ollama map-reduce into brief/ticker/map. Reads articles scraped within the last hour from the shared `articles` table, -maps them with Nous (per-article English fact blocks), reduces to one JSON -object (summary_en + ticker + map_items), and stores the brief in +maps them with a local Ollama model (per-article English fact blocks), reduces +to one JSON object (summary_en + ticker + map_items), and stores the brief in `article_summaries` plus flagged rows in `news_items`. Tables live in the EXISTING osint-db (alembic 003_news + 005_news_items, idempotent). -Everything is env-driven (12-factor). Secrets/config are resolved at the start -of each summarize_news() — env wins, else api_keys / app_settings: +LLM is the newsPipeline local_llm path: Ollama POST /api/generate. No API key. DB_HOST / DB_NAME / DB_USER / DB_PASSWORD / DB_PORT PostgreSQL (osint-db) - NOUS_API_KEY Nous Portal key (else api_keys.name='NOUS_API_KEY') - NOUS_BASE_URL default https://inference-api.nousresearch.com/v1 - SUMMARY_MODEL default Hermes-4.3-36B (else app_settings) + LLM_URL Ollama origin (default http://127.0.0.1:11434) + SUMMARY_MODEL Ollama tag (else app_settings; default qwen3:30b-a3b) BATCH_SIZE articles per map-phase batch (default 50) SUMMARY_WINDOW_HOURS look-back window in hours (default 1) OSINT_USER_AGENT default osint-dashboard-news-summarizer @@ -21,11 +19,6 @@ of each summarize_news() — env wins, else api_keys / app_settings: SUMMARY_PROMPT override reduce-phase prompt (uses {final_input}) NEWS_SUMMARIZE_FORCE "1" to ignore the current-UTC-hour idempotency skip INCLUDE_FUTURES "1" to prepend live futures prices (default 0) - -The futures/markets coupling from the original pipeline is gated behind -INCLUDE_FUTURES and OFF by default — it is irrelevant to the OSINT dashboard -and pulled yfinance into the image. Re-enable by installing yfinance and -setting INCLUDE_FUTURES=1. """ from __future__ import annotations @@ -37,7 +30,7 @@ from datetime import datetime import psycopg2 from intel import parse_reduce_json, select_map, select_ticker -from nous_client import chat +from ollama_client import chat, ollama_ready logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") logger = logging.getLogger("news.summarizer") @@ -51,8 +44,8 @@ DB_CONFIG = { "port": int(os.getenv("DB_PORT", "5432")), } -DEFAULT_NOUS_BASE_URL = "https://inference-api.nousresearch.com/v1" -DEFAULT_SUMMARY_MODEL = "Hermes-4.3-36B" +DEFAULT_LLM_URL = "http://127.0.0.1:11434" +DEFAULT_SUMMARY_MODEL = "qwen3:30b-a3b" BATCH_SIZE = int(os.getenv("BATCH_SIZE", "50")) SUMMARY_WINDOW_HOURS = int(os.getenv("SUMMARY_WINDOW_HOURS", "1")) INCLUDE_FUTURES = os.getenv("INCLUDE_FUTURES", "0").lower() in ("1", "true", "yes") @@ -134,20 +127,6 @@ def _kv(conn, table, name) -> str: return (row[0] or "").strip() if row else "" -def resolve_api_key() -> str: - env = os.getenv("NOUS_API_KEY", "").strip() - if env: - return env - try: - conn = psycopg2.connect(**DB_CONFIG) - try: - return _kv(conn, "api_keys", "NOUS_API_KEY") - finally: - conn.close() - except Exception: # noqa: BLE001 - return "" - - def resolve_model() -> str: env = os.getenv("SUMMARY_MODEL", "").strip() if env: @@ -164,15 +143,15 @@ def resolve_model() -> str: def resolve_base_url() -> str: - return os.getenv("NOUS_BASE_URL", DEFAULT_NOUS_BASE_URL).strip() or DEFAULT_NOUS_BASE_URL + return os.getenv("LLM_URL", DEFAULT_LLM_URL).strip() or DEFAULT_LLM_URL -def call_llm(prompt: str, *, api_key: str, model: str, base_url: str, json_mode: bool = False) -> str: - """Send a prompt to Nous chat completions and return the text (\"\" on failure).""" - if not api_key: - logger.warning("NOUS_API_KEY not set — skipping LLM call") +def call_llm(prompt: str, *, model: str, base_url: str, json_mode: bool = False) -> str: + """Send a prompt to Ollama generate and return the text (\"\" on failure).""" + if not model: + logger.warning("SUMMARY_MODEL empty — skipping LLM call") return "" - return chat(prompt, api_key=api_key, model=model, base_url=base_url, json_mode=json_mode) + return chat(prompt, model=model, base_url=base_url, json_mode=json_mode) # ── Futures (legacy, gated) ──────────────────────────────────────────────── @@ -426,11 +405,16 @@ def summarize_news() -> None: ) return - api_key = resolve_api_key() model = resolve_model() base_url = resolve_base_url() - if not api_key: - logger.warning("NOUS_API_KEY unset in env and api_keys — idle this run") + if not model: + logger.warning("SUMMARY_MODEL empty — idle this run") + return + if not ollama_ready(model, base_url=base_url): + logger.warning( + "Ollama not ready at %s (model %s missing or /api/tags down) — idle this run", + base_url, model, + ) return articles = get_recent_news() @@ -452,7 +436,6 @@ def summarize_news() -> None: ) summary = call_llm( build_map_prompt(batch), - api_key=api_key, model=model, base_url=base_url, json_mode=False, @@ -468,7 +451,6 @@ def summarize_news() -> None: logger.info("reduce phase over %d partial summaries", len(partial_summaries)) master_raw = call_llm( build_master_prompt(final_input), - api_key=api_key, model=model, base_url=base_url, json_mode=True, diff --git a/news/summerizer/tests/test_nous_client.py b/news/summerizer/tests/test_nous_client.py deleted file mode 100644 index 002faa4..0000000 --- a/news/summerizer/tests/test_nous_client.py +++ /dev/null @@ -1,97 +0,0 @@ -from unittest.mock import MagicMock - -import httpx -from nous_client import chat - -DEFAULT_UA = "osint-dashboard-news-summarizer" -BASE = "https://inference-api.nousresearch.com/v1" - - -def _ok_response(content="hello"): - resp = MagicMock() - resp.status_code = 200 - resp.json.return_value = {"choices": [{"message": {"content": content}}]} - return resp - - -def _install_fake(monkeypatch, post_impl): - captured = {} - - class FakeClient: - def __init__(self, timeout=None, **kwargs): - captured["timeout"] = timeout - - def __enter__(self): - return self - - def __exit__(self, *exc): - return False - - def post(self, url, *, headers=None, json=None, **kwargs): - captured["url"] = url - captured["headers"] = headers - captured["json"] = json - return post_impl(url, headers, json) - - monkeypatch.setattr(httpx, "Client", FakeClient) - return captured - - -def test_posts_chat_completions_with_auth_body_and_returns_content(monkeypatch): - captured = _install_fake(monkeypatch, lambda *a: _ok_response("the-content")) - out = chat( - "summarize this", - api_key="secret-key", - model="hermes-3", - base_url=BASE, - ) - assert out == "the-content" - assert captured["url"] == f"{BASE}/chat/completions" - assert captured["headers"]["Authorization"] == "Bearer secret-key" - assert captured["headers"]["User-Agent"] == DEFAULT_UA - assert captured["json"]["model"] == "hermes-3" - assert captured["json"]["messages"] == [{"role": "user", "content": "summarize this"}] - assert captured["json"]["temperature"] == 0.2 - assert captured["json"]["max_tokens"] == 4096 - assert "response_format" not in captured["json"] - - -def test_user_agent_equals_osint_user_agent_env(monkeypatch): - monkeypatch.setenv("OSINT_USER_AGENT", "custom-ua/2.0") - captured = _install_fake(monkeypatch, lambda *a: _ok_response("ok")) - chat("p", api_key="k", model="m", base_url=BASE) - assert captured["headers"]["User-Agent"] == "custom-ua/2.0" - - -def test_json_mode_sets_response_format(monkeypatch): - captured = _install_fake(monkeypatch, lambda *a: _ok_response("{}")) - chat("p", api_key="k", model="m", base_url=BASE, json_mode=True) - assert captured["json"]["response_format"] == {"type": "json_object"} - - -def test_401_returns_empty_string(monkeypatch): - def post_impl(*a): - resp = MagicMock() - resp.status_code = 401 - return resp - - _install_fake(monkeypatch, post_impl) - assert chat("p", api_key="bad", model="m", base_url=BASE) == "" - - -def test_5xx_returns_empty_string(monkeypatch): - def post_impl(*a): - resp = MagicMock() - resp.status_code = 503 - return resp - - _install_fake(monkeypatch, post_impl) - assert chat("p", api_key="k", model="m", base_url=BASE) == "" - - -def test_timeout_returns_empty_string(monkeypatch): - def post_impl(*a): - raise httpx.TimeoutException("timed out") - - _install_fake(monkeypatch, post_impl) - assert chat("p", api_key="k", model="m", base_url=BASE) == "" diff --git a/news/summerizer/tests/test_ollama_client.py b/news/summerizer/tests/test_ollama_client.py new file mode 100644 index 0000000..8339c86 --- /dev/null +++ b/news/summerizer/tests/test_ollama_client.py @@ -0,0 +1,111 @@ +from unittest.mock import MagicMock + +import httpx +from ollama_client import chat, list_tags, normalize_base, ollama_ready + +DEFAULT_UA = "osint-dashboard-news-summarizer" +BASE = "http://127.0.0.1:11434" + + +def _ok_response(content="hello"): + resp = MagicMock() + resp.status_code = 200 + resp.json.return_value = {"response": content} + return resp + + +def _install_fake(monkeypatch, post_impl=None, get_impl=None): + captured = {} + + class FakeClient: + def __init__(self, timeout=None, **kwargs): + captured["timeout"] = timeout + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def post(self, url, *, headers=None, json=None, **kwargs): + captured["url"] = url + captured["headers"] = headers + captured["json"] = json + if post_impl is None: + raise AssertionError("post not expected") + return post_impl(url, headers, json) + + def get(self, url, *, headers=None, **kwargs): + captured["url"] = url + captured["headers"] = headers + if get_impl is None: + raise AssertionError("get not expected") + return get_impl(url, headers) + + monkeypatch.setattr(httpx, "Client", FakeClient) + return captured + + +def test_normalize_base_strips_generate_path(): + assert normalize_base("http://ollama:11434/api/generate") == "http://ollama:11434" + assert normalize_base("http://ollama:11434/api/") == "http://ollama:11434" + assert normalize_base("http://ollama:11434") == "http://ollama:11434" + + +def test_posts_generate_and_returns_response(monkeypatch): + captured = _install_fake(monkeypatch, post_impl=lambda *a: _ok_response("the-content")) + out = chat("summarize this", model="qwen3:30b-a3b", base_url=BASE) + assert out == "the-content" + assert captured["url"] == f"{BASE}/api/generate" + assert captured["headers"]["User-Agent"] == DEFAULT_UA + assert captured["json"]["model"] == "qwen3:30b-a3b" + assert captured["json"]["prompt"] == "summarize this" + assert captured["json"]["stream"] is False + assert "format" not in captured["json"] + assert captured["json"]["options"]["num_ctx"] == 32768 + assert captured["timeout"] == 300.0 + + +def test_json_mode_sets_format_json(monkeypatch): + captured = _install_fake(monkeypatch, post_impl=lambda *a: _ok_response("{}")) + chat("p", model="m", base_url=BASE, json_mode=True) + assert captured["json"]["format"] == "json" + + +def test_empty_model_returns_empty(monkeypatch): + assert chat("p", model="", base_url=BASE) == "" + + +def test_5xx_returns_empty_string(monkeypatch): + def post_impl(*a): + resp = MagicMock() + resp.status_code = 503 + return resp + + _install_fake(monkeypatch, post_impl=post_impl) + assert chat("p", model="m", base_url=BASE) == "" + + +def test_timeout_returns_empty_string(monkeypatch): + def post_impl(*a): + raise httpx.TimeoutException("timed out") + + _install_fake(monkeypatch, post_impl=post_impl) + assert chat("p", model="m", base_url=BASE) == "" + + +def test_list_tags_and_ready(monkeypatch): + def get_impl(*a): + resp = MagicMock() + resp.status_code = 200 + resp.json.return_value = { + "models": [{"name": "qwen3:30b-a3b"}, {"name": "llama3.2:latest"}], + } + return resp + + captured = _install_fake(monkeypatch, get_impl=get_impl) + names = list_tags(BASE) + assert names == ["qwen3:30b-a3b", "llama3.2:latest"] + assert captured["url"] == f"{BASE}/api/tags" + assert ollama_ready("qwen3:30b-a3b", base_url=BASE) is True + assert ollama_ready("missing", base_url=BASE) is False diff --git a/tests/test_api_settings.py b/tests/test_api_settings.py index 6dad2f5..5d2ec40 100644 --- a/tests/test_api_settings.py +++ b/tests/test_api_settings.py @@ -64,31 +64,26 @@ def _put(path: str, json: dict) -> httpx.Response: return _request("PUT", path, json=json) -def test_get_news_models_without_key_returns_fallback(monkeypatch): - monkeypatch.delenv("NOUS_API_KEY", raising=False) - monkeypatch.setattr("keystore.get_api_key", _missing_key) +def test_get_news_models_ollama_down_returns_fallback(monkeypatch): + async def boom_http_get(url, *, headers, timeout): + raise httpx.ConnectError("ollama down") + + import settings_store + monkeypatch.setattr(settings_store, "_http_get", boom_http_get) + settings_store._models_cache = None resp = _get("/api/news/models") assert resp.status_code == 200 body = resp.json() assert body["source"] == "fallback" ids = [m["id"] for m in body["models"]] - assert "Hermes-4.3-36B" in ids from settings_store import FALLBACK_MODELS assert ids == FALLBACK_MODELS + assert "qwen3:30b-a3b" in ids -async def _missing_key(name: str): - return None - - -async def _present_key(name: str): - return "test-nous-api-key-1234" - - -def test_get_news_models_live_from_upstream(monkeypatch): - monkeypatch.setattr("keystore.get_api_key", _present_key) - monkeypatch.setenv("NOUS_BASE_URL", "https://inference-api.nousresearch.com/v1") +def test_get_news_models_live_from_ollama_tags(monkeypatch): + monkeypatch.setenv("LLM_URL", "http://ollama:11434") monkeypatch.delenv("OSINT_USER_AGENT", raising=False) captured: dict = {} @@ -100,7 +95,7 @@ def test_get_news_models_live_from_upstream(monkeypatch): return None def json(self): - return {"data": [{"id": "live-model-a"}, {"id": "Hermes-4.3-36B"}]} + return {"models": [{"name": "qwen3:30b-a3b"}, {"name": "llama3.2:latest"}]} async def fake_http_get(url, *, headers, timeout): captured["url"] = url @@ -117,30 +112,10 @@ def test_get_news_models_live_from_upstream(monkeypatch): body = resp.json() assert body["source"] == "live" ids = [m["id"] for m in body["models"]] - assert ids == ["live-model-a", "Hermes-4.3-36B"] - assert captured["url"] == "https://inference-api.nousresearch.com/v1/models" + assert ids == ["qwen3:30b-a3b", "llama3.2:latest"] + assert captured["url"] == "http://ollama:11434/api/tags" assert captured["headers"]["User-Agent"] == "osint-dashboard-news-summarizer" - assert captured["headers"]["Authorization"] == "Bearer test-nous-api-key-1234" - timeout = captured["timeout"] - assert timeout == 8 or getattr(timeout, "read", timeout) == 8 or float(timeout) == 8.0 - - -def test_get_news_models_upstream_failure_returns_fallback(monkeypatch): - monkeypatch.setattr("keystore.get_api_key", _present_key) - - async def boom_http_get(url, *, headers, timeout): - raise httpx.ConnectError("upstream down") - - import settings_store - monkeypatch.setattr(settings_store, "_http_get", boom_http_get) - settings_store._models_cache = None - - resp = _get("/api/news/models") - assert resp.status_code == 200 - body = resp.json() - assert body["source"] == "fallback" - ids = [m["id"] for m in body["models"]] - assert "Hermes-4.3-36B" in ids + assert "Authorization" not in captured["headers"] def test_put_settings_empty_returns_422(): @@ -156,17 +131,17 @@ def test_put_settings_whitespace_only_returns_422(): @requires_db def test_put_settings_round_trip(clean_settings, monkeypatch): monkeypatch.delenv("SUMMARY_MODEL", raising=False) - monkeypatch.delenv("NOUS_BASE_URL", raising=False) + monkeypatch.delenv("LLM_URL", raising=False) - put_resp = _put("/api/settings", {"summary_model": "google/gemini-2.5-flash"}) + put_resp = _put("/api/settings", {"summary_model": "qwen3:30b-a3b"}) assert put_resp.status_code == 200 put_body = put_resp.json() - assert put_body["summary_model"] == "google/gemini-2.5-flash" - assert put_body["nous_base_url"] == "https://inference-api.nousresearch.com/v1" + assert put_body["summary_model"] == "qwen3:30b-a3b" + assert put_body["llm_url"] == "http://127.0.0.1:11434" get_resp = _get("/api/settings") assert get_resp.status_code == 200 get_body = get_resp.json() - assert get_body["summary_model"] == "google/gemini-2.5-flash" - assert get_body["nous_base_url"] == "https://inference-api.nousresearch.com/v1" - assert set(get_body.keys()) == {"summary_model", "nous_base_url"} + assert get_body["summary_model"] == "qwen3:30b-a3b" + assert get_body["llm_url"] == "http://127.0.0.1:11434" + assert set(get_body.keys()) == {"summary_model", "llm_url"}