feat: 23:00 daily news recap; prompts drop futures

Summarizer still runs the 15-min analyst, plus a 24h breaking-news recap
at 23:00 America/New_York. HUD pins kind=daily_recap. Prompts ignore
futures/market tape.
This commit is contained in:
Sirius DevOps 2026-08-28 22:53:31 -04:00
parent d25ca8ed9c
commit 49ee5fe7a3
16 changed files with 431 additions and 75 deletions

View file

@ -0,0 +1,26 @@
"""article_summaries.kind — interval vs daily_recap
Revision ID: 008_summary_kind
Revises: 007_event_dedup
Create Date: 2026-08-28
"""
from alembic import op
revision = "008_summary_kind"
down_revision = "007_event_dedup"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.execute(
"""
ALTER TABLE article_summaries
ADD COLUMN IF NOT EXISTS kind TEXT
"""
)
def downgrade() -> None:
op.execute("ALTER TABLE article_summaries DROP COLUMN IF EXISTS kind")

View file

@ -1220,22 +1220,31 @@ async def list_news_summaries(
None, None,
description="Only summaries generated at/after this UTC instant.", description="Only summaries generated at/after this UTC instant.",
), ),
kind: str | None = Query(
None,
description="Filter: interval or daily_recap. Omit for all.",
),
limit: int = Query(20, ge=1, le=100), limit: int = Query(20, ge=1, le=100),
offset: int = Query(0, ge=0), offset: int = Query(0, ge=0),
): ):
"""Most recent master LLM summaries (newest first).""" """Most recent master LLM summaries (newest first)."""
if kind is not None and kind not in ("interval", "daily_recap"):
raise HTTPException(422, "kind must be interval or daily_recap")
async with async_session() as session: async with async_session() as session:
stmt = select(article_summaries).order_by( stmt = select(article_summaries).order_by(
article_summaries.c.batch_timestamp.desc().nullslast() article_summaries.c.batch_timestamp.desc().nullslast()
) )
if since: if since:
stmt = stmt.where(article_summaries.c.batch_timestamp >= since) stmt = stmt.where(article_summaries.c.batch_timestamp >= since)
if kind is not None:
stmt = stmt.where(article_summaries.c.kind == kind)
stmt = stmt.limit(limit).offset(offset) stmt = stmt.limit(limit).offset(offset)
rows = (await session.execute(stmt)).mappings().all() rows = (await session.execute(stmt)).mappings().all()
return [ return [
NewsSummaryOut( NewsSummaryOut(
id=r["id"], summary_text=r["summary_text"], id=r["id"], summary_text=r["summary_text"],
batch_timestamp=r["batch_timestamp"], model=r["model"], batch_timestamp=r["batch_timestamp"], model=r["model"],
kind=r["kind"],
) )
for r in rows for r in rows
] ]

View file

@ -219,6 +219,7 @@ article_summaries = Table(
Column("batch_timestamp", DateTime(timezone=True), Column("batch_timestamp", DateTime(timezone=True),
server_default=func.now(), nullable=False), server_default=func.now(), nullable=False),
Column("model", Text), # LLM id used for this batch; nullable for old rows Column("model", Text), # LLM id used for this batch; nullable for old rows
Column("kind", Text), # interval | daily_recap; nullable for old rows
) )
Index("ix_article_summaries_batch_timestamp", article_summaries.c.batch_timestamp) Index("ix_article_summaries_batch_timestamp", article_summaries.c.batch_timestamp)

View file

@ -274,6 +274,7 @@ class NewsSummaryOut(BaseModel):
summary_text: str summary_text: str
batch_timestamp: datetime batch_timestamp: datetime
model: Optional[str] = None model: Optional[str] = None
kind: Optional[str] = None
class NewsTickerItemOut(BaseModel): class NewsTickerItemOut(BaseModel):

View file

@ -1350,15 +1350,18 @@ async function loadNews(force) {
if (upd) upd.textContent = 'updating…'; if (upd) upd.textContent = 'updating…';
let articles = []; let articles = [];
let summaries = []; let summaries = [];
let recaps = [];
let tickerItems = []; let tickerItems = [];
try { try {
const [rArticles, rSumm, rTick] = await Promise.all([ const [rArticles, rSumm, rRecap, rTick] = await Promise.all([
fetch(`${API}/api/news?limit=100`), fetch(`${API}/api/news?limit=100`),
fetch(`${API}/api/news/summaries?limit=1`), fetch(`${API}/api/news/summaries?limit=1`),
fetch(`${API}/api/news/summaries?kind=daily_recap&limit=1`),
fetch(`${API}/api/news/ticker?limit=20`), fetch(`${API}/api/news/ticker?limit=20`),
]); ]);
if (rArticles.ok) articles = await rArticles.json(); if (rArticles.ok) articles = await rArticles.json();
if (rSumm.ok) summaries = await rSumm.json(); if (rSumm.ok) summaries = await rSumm.json();
if (rRecap.ok) recaps = await rRecap.json();
if (rTick.ok) tickerItems = await rTick.json(); if (rTick.ok) tickerItems = await rTick.json();
} catch (e) { } catch (e) {
const list = document.getElementById('news-list'); const list = document.getElementById('news-list');
@ -1369,7 +1372,7 @@ async function loadNews(force) {
console.error('News load failed', e); console.error('News load failed', e);
return; return;
} }
renderNewsSummary(summaries); renderNewsSummary(recaps.length ? recaps : summaries, recaps.length > 0);
renderNewsList(articles); renderNewsList(articles);
renderNewsTicker(articles, summaries, tickerItems); renderNewsTicker(articles, summaries, tickerItems);
if (upd) upd.textContent = 'updated ' + new Date().toLocaleTimeString(); if (upd) upd.textContent = 'updated ' + new Date().toLocaleTimeString();
@ -1377,7 +1380,7 @@ async function loadNews(force) {
newsInterval = setInterval(() => loadNews(false), NEWS_REFRESH_MS); newsInterval = setInterval(() => loadNews(false), NEWS_REFRESH_MS);
} }
} }
function renderNewsSummary(summaries) { function renderNewsSummary(summaries, isRecap) {
const bodyEl = document.getElementById('ns-body'); const bodyEl = document.getElementById('ns-body');
const timeEl = document.getElementById('ns-time'); const timeEl = document.getElementById('ns-time');
const badgeEl = document.getElementById('ns-badge'); const badgeEl = document.getElementById('ns-badge');
@ -1389,7 +1392,7 @@ function renderNewsSummary(summaries) {
return; return;
} }
const s = summaries[0]; const s = summaries[0];
badgeEl.textContent = 'LATEST'; badgeEl.textContent = isRecap || s.kind === 'daily_recap' ? 'DAILY RECAP' : 'LATEST';
timeEl.textContent = s.batch_timestamp ? 'Batch: ' + new Date(s.batch_timestamp).toLocaleString() : '—'; timeEl.textContent = s.batch_timestamp ? 'Batch: ' + new Date(s.batch_timestamp).toLocaleString() : '—';
bodyEl.innerHTML = newsSimpleMD(s.summary_text || ''); bodyEl.innerHTML = newsSimpleMD(s.summary_text || '');
} }

View file

@ -234,6 +234,9 @@ services:
NEWS_SUMMARIZE_INTERVAL_S: ${NEWS_SUMMARIZE_INTERVAL_S:-900} NEWS_SUMMARIZE_INTERVAL_S: ${NEWS_SUMMARIZE_INTERVAL_S:-900}
NEWS_SUMMARIZE_RUN_ON_START: ${NEWS_SUMMARIZE_RUN_ON_START:-1} NEWS_SUMMARIZE_RUN_ON_START: ${NEWS_SUMMARIZE_RUN_ON_START:-1}
NEWS_SUMMARIZE_FORCE: ${NEWS_SUMMARIZE_FORCE:-0} NEWS_SUMMARIZE_FORCE: ${NEWS_SUMMARIZE_FORCE:-0}
TZ: ${TZ:-America/New_York}
NEWS_RECAP_HOUR: ${NEWS_RECAP_HOUR:-23}
NEWS_RECAP_MINUTE: ${NEWS_RECAP_MINUTE:-0}
command: ["python", "run_news_summarizer.py"] command: ["python", "run_news_summarizer.py"]
volumes: volumes:

View file

@ -16,7 +16,7 @@ urls.txt (RSS + homepages)
news-scraper (Scrapy, continuous) ──► articles table (osint-db) news-scraper (Scrapy, continuous) ──► articles table (osint-db)
│ │ │ │
│ ▼ │ ▼
news-summarizer (Nous Portal, every 15m) ──► article_summaries + news_items news-summarizer (Nous Portal, every 15m + 23:00 recap) ──► article_summaries + news_items
GET /api/news · /api/news/summaries · /api/news/ticker · /api/news/map GET /api/news · /api/news/summaries · /api/news/ticker · /api/news/map
@ -45,11 +45,13 @@ feeds (`GET /api/news` exact key set is unchanged on purpose).
the `PostgresPipeline` writes to `articles` with URL-based dedup the `PostgresPipeline` writes to `articles` with URL-based dedup
(`ON CONFLICT (url) DO NOTHING`). (`ON CONFLICT (url) DO NOTHING`).
2. **Summarizer**`news/summerizer/run_news_summarizer.py` runs 2. **Summarizer**`news/summerizer/run_news_summarizer.py` runs
`summarizer.py` every `NEWS_SUMMARIZE_INTERVAL_S` (default 900). It reads `summarizer.py` every `NEWS_SUMMARIZE_INTERVAL_S` (default 900) over the
articles from the last `SUMMARY_WINDOW_MINUTES` (default 15), map-reduces last `SUMMARY_WINDOW_MINUTES` (default 15), and again at 23:00
them through Nous Portal (`SUMMARY_MODEL` / Settings, default `America/New_York` (`TZ`) over the last 24 hours as a daily recap
`Hermes-4.3-36B`), writes the English brief to `article_summaries` (column (`kind=daily_recap`). Both map-reduce through Nous Portal (`SUMMARY_MODEL`
`model` is the LLM id), and flagged ticker/map rows to `news_items`. / Settings, default `Hermes-4.3-36B`), write the English brief to
`article_summaries` (column `model` is the LLM id; `kind` is
`interval` or `daily_recap`), and flagged ticker/map rows to `news_items`.
Loops are serial (two crawls/summaries never overlap). Interval idempotency: Loops are serial (two crawls/summaries never overlap). Interval idempotency:
if `article_summaries` already has a row in the last interval, the summarizer if `article_summaries` already has a row in the last interval, the summarizer
@ -116,12 +118,15 @@ Key set **unchanged** (no `lat`/`lon` on articles; geo lives on `/api/news/map`)
"id": 1, "id": 1,
"summary_text": "English markdown brief…", "summary_text": "English markdown brief…",
"batch_timestamp": "2026-08-24T18:10:00Z", "batch_timestamp": "2026-08-24T18:10:00Z",
"model": "Hermes-4.3-36B" "model": "Hermes-4.3-36B",
"kind": "daily_recap"
} }
] ]
``` ```
`model` is additive. Empty DB → `[]` (no crash). `model` and `kind` are additive (`interval` | `daily_recap` | `null` for old rows).
`?kind=daily_recap` pins the nightly 24h recap. Empty DB → `[]` (no crash).
Malformed `kind``422`.
### GET /api/news/ticker — flagged HUD headlines ### GET /api/news/ticker — flagged HUD headlines
@ -237,10 +242,13 @@ lands in `article_summaries.summary_text`.
| `SUMMARY_WINDOW_MINUTES` | `15` | How far back the summarizer looks for new articles. | | `SUMMARY_WINDOW_MINUTES` | `15` | How far back the summarizer looks for new articles. |
| `NEWS_SCRAPE_INTERVAL_S` | `10` | Pause after each crawl before the next (scraper is otherwise continuous). | | `NEWS_SCRAPE_INTERVAL_S` | `10` | Pause after each crawl before the next (scraper is otherwise continuous). |
| `NEWS_SUMMARIZE_INTERVAL_S` | `900` | Seconds between analyst runs (default 15 min). | | `NEWS_SUMMARIZE_INTERVAL_S` | `900` | Seconds between analyst runs (default 15 min). |
| `TZ` | `America/New_York` | Timezone for the 23:00 daily recap. |
| `NEWS_RECAP_HOUR` | `23` | Local hour of the daily 24h recap. |
| `NEWS_RECAP_MINUTE` | `0` | Local minute of the daily recap. |
| `NEWS_SCRAPE_RUN_ON_START` | `1` | Run one scrape immediately on container start. | | `NEWS_SCRAPE_RUN_ON_START` | `1` | Run one scrape immediately on container start. |
| `NEWS_SUMMARIZE_RUN_ON_START` | `1` | Run one summarize immediately on container start. | | `NEWS_SUMMARIZE_RUN_ON_START` | `1` | Run one summarize immediately on container start. |
| `NEWS_SUMMARIZE_FORCE` | `0` | `1` ignores the interval idempotency skip (double-pins on recreate). | | `NEWS_SUMMARIZE_FORCE` | `0` | `1` ignores the interval/recap idempotency skip (double-pins on recreate). |
| `INCLUDE_FUTURES` | `0` | Legacy futures-prices coupling. OFF for OSINT; set `1` + install `yfinance` to enable. | | `INCLUDE_FUTURES` | `0` | Legacy. Ignored — prompts never inject futures/market tape. |
| `NEWS_LOG_LEVEL` | `INFO` | Scrapy log level. | | `NEWS_LOG_LEVEL` | `INFO` | Scrapy log level. |
| `OSINT_USER_AGENT` | `osint-dashboard-news-summarizer` | Sent on every outbound Nous call. | | `OSINT_USER_AGENT` | `osint-dashboard-news-summarizer` | Sent on every outbound Nous call. |
| `TELEGRAM_TOKEN` / `TELEGRAM_CHAT_ID` | *(blank)* | Reserved for the (out-of-scope) Telegram delivery bot. | | `TELEGRAM_TOKEN` / `TELEGRAM_CHAT_ID` | *(blank)* | Reserved for the (out-of-scope) Telegram delivery bot. |
@ -254,11 +262,10 @@ No Hermes-4 reasoning system prompt. Reduce uses `json_mode=True`.
## Prompts ## Prompts
Both prompts are env-overridable — the default `MAP_PROMPT` is OSINT-neutral Both prompts are env-overridable. Defaults focus on **breaking important news**
(facts, locations, entities, category, OSINT signal per article; English) and and explicitly ignore futures, commodity tape, and routine market moves.
the default `SUMMARY_PROMPT` demands the reduce JSON above (with a `RECAP_PROMPT` (23:00, 24h window) is the daily recap; `SUMMARY_PROMPT` is the
"no qualifying events" escape hatch). Upstream's futures/markets prompt 15-min analyst. `INCLUDE_FUTURES` is ignored.
language is gated behind `INCLUDE_FUTURES=1`.
## Tests ## Tests

View file

@ -7,7 +7,7 @@ WORKDIR /app
# libpq-dev + gcc for psycopg2 build/adapters; keep the image lean. # libpq-dev + gcc for psycopg2 build/adapters; keep the image lean.
RUN apt-get update && apt-get install -y --no-install-recommends \ RUN apt-get update && apt-get install -y --no-install-recommends \
libpq-dev gcc \ libpq-dev gcc tzdata \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
COPY requirements.txt . COPY requirements.txt .

View file

@ -1,37 +1,97 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Scheduler loop for the news summarizer — every NEWS_SUMMARIZE_INTERVAL_S. """Scheduler loop for the news summarizer.
Default 900s (15 minutes). Serial: a slow LLM pass never overlaps the next. 15-minute analyst (NEWS_SUMMARIZE_INTERVAL_S, default 900s) plus a daily
recap at 23:00 in TZ (default America/New_York) over the last 24 hours.
Serial: a slow LLM pass never overlaps the next.
Env (all optional, 12-factor): Env (all optional, 12-factor):
NEWS_SUMMARIZE_INTERVAL_S seconds between runs (default 900) NEWS_SUMMARIZE_INTERVAL_S seconds between analyst runs (default 900)
NEWS_SUMMARIZE_RUN_ON_START "1" to summarize once immediately on boot (default 1) NEWS_SUMMARIZE_RUN_ON_START "1" to summarize once immediately on boot (default 1)
NEWS_RECAP_HOUR / MINUTE wall-clock recap time (default 23:00)
TZ IANA tz (default America/New_York)
NOUS_API_KEY optional in env; Keys UI / api_keys also works NOUS_API_KEY optional in env; Keys UI / api_keys also works
""" """
from __future__ import annotations from __future__ import annotations
import datetime
import logging import logging
import os import os
import subprocess import subprocess
import sys import sys
import time import time
from datetime import datetime, timedelta
from zoneinfo import ZoneInfo
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
logger = logging.getLogger("news.summarizer.scheduler") logger = logging.getLogger("news.summarizer.scheduler")
INTERVAL_S = max(1, int(os.getenv("NEWS_SUMMARIZE_INTERVAL_S", "900"))) INTERVAL_S = max(1, int(os.getenv("NEWS_SUMMARIZE_INTERVAL_S", "900")))
RUN_ON_START = os.getenv("NEWS_SUMMARIZE_RUN_ON_START", "1").lower() in ("1", "true", "yes") RUN_ON_START = os.getenv("NEWS_SUMMARIZE_RUN_ON_START", "1").lower() in ("1", "true", "yes")
DEFAULT_TZ = "America/New_York"
DEFAULT_RECAP_HOUR = 23
DEFAULT_RECAP_MINUTE = 0
def run_summarize() -> None: def _tz() -> ZoneInfo:
logger.info("summarize starting at %s", datetime.datetime.now().isoformat(timespec="seconds")) name = (os.getenv("TZ") or DEFAULT_TZ).strip() or DEFAULT_TZ
return ZoneInfo(name)
def recap_hour_minute() -> tuple[int, int]:
hour = int(os.getenv("NEWS_RECAP_HOUR", str(DEFAULT_RECAP_HOUR)))
minute = int(os.getenv("NEWS_RECAP_MINUTE", str(DEFAULT_RECAP_MINUTE)))
return hour, minute
def next_recap_datetime(
now: datetime, hour: int | None = None, minute: int | None = None
) -> datetime:
"""Next 23:00 (or hour/minute) strictly after *now* in now's timezone."""
if now.tzinfo is None:
now = now.replace(tzinfo=_tz())
env_h, env_m = recap_hour_minute()
hour = env_h if hour is None else hour
minute = env_m if minute is None else minute
candidate = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
if now >= candidate:
candidate += timedelta(days=1)
return candidate
def next_event(
now: datetime,
last_periodic: datetime | None,
interval_s: int,
hour: int = DEFAULT_RECAP_HOUR,
minute: int = DEFAULT_RECAP_MINUTE,
) -> tuple[datetime, str]:
"""Return (when, 'recap'|'interval') for the sooner of recap vs interval."""
recap_at = next_recap_datetime(now, hour=hour, minute=minute)
periodic_at = now if last_periodic is None else last_periodic + timedelta(seconds=interval_s)
if recap_at <= periodic_at:
return recap_at, "recap"
return periodic_at, "interval"
def run_summarize(*, recap: bool = False) -> None:
kind = "recap" if recap else "interval"
logger.info(
"%s starting at %s", kind, datetime.now().isoformat(timespec="seconds")
)
env = os.environ.copy()
if recap:
env["NEWS_RECAP"] = "1"
else:
env.pop("NEWS_RECAP", None)
try: try:
proc = subprocess.run([sys.executable, "summarizer.py"], cwd="/app") proc = subprocess.run(
logger.info("summarize finished rc=%s", proc.returncode) [sys.executable, "summarizer.py"], cwd="/app", env=env
)
logger.info("%s finished rc=%s", kind, proc.returncode)
except Exception: # noqa: BLE001 — keep the loop alive across failures except Exception: # noqa: BLE001 — keep the loop alive across failures
logger.exception("summarize failed") logger.exception("%s failed", kind)
def main() -> None: def main() -> None:
@ -39,16 +99,32 @@ def main() -> None:
logger.warning( logger.warning(
"NOUS_API_KEY unset in env — will read api_keys on each run; idle if both empty" "NOUS_API_KEY unset in env — will read api_keys on each run; idle if both empty"
) )
tz = _tz()
hour, minute = recap_hour_minute()
logger.info( logger.info(
"news summarizer loop starting (interval_s=%s, run_on_start=%s)", "news summarizer loop starting (interval_s=%s, run_on_start=%s, recap=%02d:%02d %s)",
INTERVAL_S, RUN_ON_START, INTERVAL_S, RUN_ON_START, hour, minute, tz,
) )
last_periodic: datetime | None = None
if RUN_ON_START: if RUN_ON_START:
run_summarize() run_summarize(recap=False)
last_periodic = datetime.now(tz)
while True: while True:
logger.info("next summarize in %ss", INTERVAL_S) now = datetime.now(tz)
time.sleep(INTERVAL_S) when, kind = next_event(
run_summarize() now, last_periodic, INTERVAL_S, hour=hour, minute=minute
)
sleep_s = max(1, (when - now).total_seconds())
logger.info("next %s in %ss", kind, int(sleep_s))
time.sleep(sleep_s)
now = datetime.now(tz)
if kind == "recap":
run_summarize(recap=True)
# Recap covers the 15-min window; don't immediately fire interval.
last_periodic = now
else:
run_summarize(recap=False)
last_periodic = now
if __name__ == "__main__": if __name__ == "__main__":

View file

@ -16,13 +16,10 @@ of each summarize_news() — env wins, else api_keys / app_settings:
SUMMARY_MODEL default Hermes-4.3-36B (else app_settings) SUMMARY_MODEL default Hermes-4.3-36B (else app_settings)
BATCH_SIZE articles per map-phase batch (default 50) BATCH_SIZE articles per map-phase batch (default 50)
SUMMARY_WINDOW_MINUTES look-back window (default 15; SUMMARY_WINDOW_HOURS wins if set) SUMMARY_WINDOW_MINUTES look-back window (default 15; SUMMARY_WINDOW_HOURS wins if set)
NEWS_SUMMARIZE_FORCE "1" to ignore the interval idempotency skip NEWS_RECAP "1" for the 23:00 daily recap (24h window, recap prompt)
INCLUDE_FUTURES "1" to prepend live futures prices (default 0) NEWS_SUMMARIZE_FORCE "1" to ignore the interval/recap idempotency skip
TZ IANA tz for recap-day bounds (default America/New_York)
The futures/markets coupling from the original pipeline is gated behind INCLUDE_FUTURES legacy; ignored prompts never inject futures data
INCLUDE_FUTURES and OFF by default it is irrelevant to the OSINT dashboard
and pulled yfinance into the image. Re-enable by installing yfinance and
setting INCLUDE_FUTURES=1.
""" """
from __future__ import annotations from __future__ import annotations
@ -30,6 +27,7 @@ from __future__ import annotations
import logging import logging
import os import os
from datetime import datetime from datetime import datetime
from zoneinfo import ZoneInfo
import psycopg2 import psycopg2
@ -67,6 +65,18 @@ def _summarize_interval_seconds() -> int:
return max(1, int(os.getenv("NEWS_SUMMARIZE_INTERVAL_S", "900"))) return max(1, int(os.getenv("NEWS_SUMMARIZE_INTERVAL_S", "900")))
def is_recap_run() -> bool:
return os.getenv("NEWS_RECAP", "0").lower() in ("1", "true", "yes")
def effective_window_minutes(*, recap: bool | None = None) -> int:
if recap is None:
recap = is_recap_run()
if recap:
return 24 * 60
return _summary_window_minutes()
SUMMARY_WINDOW_MINUTES = _summary_window_minutes() SUMMARY_WINDOW_MINUTES = _summary_window_minutes()
INCLUDE_FUTURES = os.getenv("INCLUDE_FUTURES", "0").lower() in ("1", "true", "yes") INCLUDE_FUTURES = os.getenv("INCLUDE_FUTURES", "0").lower() in ("1", "true", "yes")
@ -84,6 +94,8 @@ FUTURES_TICKERS = {
MAP_PROMPT_DEFAULT = """\ MAP_PROMPT_DEFAULT = """\
You are a precise, factual OSINT news processor. Your ONLY source of information is the articles provided below. Do NOT add external knowledge, assumptions, training data, or invented facts. You are a precise, factual OSINT news processor. Your ONLY source of information is the articles provided below. Do NOT add external knowledge, assumptions, training data, or invented facts.
Focus on breaking important news (geopolitical, military/conflict, security, disasters, major political developments). Ignore futures prices, commodity tape, ticker chatter, and routine market moves unless they themselves are the breaking event.
Write every field in English. Translate if the article is not English. Write every field in English. Translate if the article is not English.
For EACH article in the batch: For EACH article in the batch:
@ -91,7 +103,7 @@ For EACH article in the batch:
2. Location: country/city/region or Unknown. If you can estimate coordinates, emit them as numbers; otherwise omit. 2. Location: country/city/region or Unknown. If you can estimate coordinates, emit them as numbers; otherwise omit.
3. Entities: list the key people, organizations, or governments mentioned (comma-separated, only names present in the text), else "None". 3. Entities: list the key people, organizations, or governments mentioned (comma-separated, only names present in the text), else "None".
4. Category: pick one politics, military/conflict, economy, technology, environment/disaster, health, crime, society, sport, other. 4. Category: pick one politics, military/conflict, economy, technology, environment/disaster, health, crime, society, sport, other.
5. OSINT signal: if the article describes an event with geopolitical, security, military, economic, or disaster significance, say so in one short sentence. Otherwise write: "No notable OSINT signal." 5. OSINT signal: if the article describes a breaking event with geopolitical, security, military, or disaster significance, say so in one short sentence. Otherwise write: "No notable OSINT signal."
6. Importance: critical (breaking geopolitical/military/disaster with immediate impact), high, medium, low, none. 6. Importance: critical (breaking geopolitical/military/disaster with immediate impact), high, medium, low, none.
If several articles cover the same story, add one short batch-level note at the end: "Batch theme: [one sentence]". If several articles cover the same story, add one short batch-level note at the end: "Batch theme: [one sentence]".
@ -117,10 +129,12 @@ Articles in this batch:
""" """
SUMMARY_PROMPT_DEFAULT = """\ SUMMARY_PROMPT_DEFAULT = """\
CRITICAL INSTRUCTION - REPEAT 3 TIMES: YOU MUST USE ONLY THE DATA PROVIDED BELOW. DO NOT INVENT, RECALL, OR ADD ANY EVENTS, NAMES, DATES, IMPLICATIONS, PROJECTS, OR DETAILS NOT EXPLICITLY PRESENT IN THE DATA. IF THE DATA HAS NO MAJOR GEOPOLITICAL/TECH/MILITARY/ECONOMIC/IMPACTFUL EVENTS OR UNUSUAL STORIES, set summary_en to exactly: "No qualifying impactful or unusual events in the recent hourly news data." and use empty ticker and map_items arrays. AND STOP. NO EXTERNAL KNOWLEDGE FROM TRAINING. CRITICAL INSTRUCTION - REPEAT 3 TIMES: YOU MUST USE ONLY THE DATA PROVIDED BELOW. DO NOT INVENT, RECALL, OR ADD ANY EVENTS, NAMES, DATES, IMPLICATIONS, PROJECTS, OR DETAILS NOT EXPLICITLY PRESENT IN THE DATA. IF THE DATA HAS NO BREAKING IMPORTANT NEWS, set summary_en to exactly: "No qualifying breaking news in the recent news data." and use empty ticker and map_items arrays. AND STOP. NO EXTERNAL KNOWLEDGE FROM TRAINING.
All text in English. All text in English.
Focus on breaking important news (geopolitical, military/conflict, security, disasters, major political developments). Ignore futures, commodity prices, and routine market data do not treat price ticks as news.
Demand a single JSON object (no markdown fences) with this exact shape: Demand a single JSON object (no markdown fences) with this exact shape:
{ {
@ -131,7 +145,32 @@ Demand a single JSON object (no markdown fences) with this exact shape:
ticker: only critical and high, max 12, 140 chars, no markdown. ticker: only critical and high, max 12, 140 chars, no markdown.
map_items: only critical and high where a real-world location is explicit in the data. Estimate lat/lon. If location is Unknown or not in the data, omit the item. Never invent a place. Max 20. map_items: only critical and high where a real-world location is explicit in the data. Estimate lat/lon. If location is Unknown or not in the data, omit the item. Never invent a place. Max 20.
summary_en: English markdown brief for an operator HUD. summary_en: English markdown brief of breaking important news for an operator HUD.
DATA:
{final_input}
"""
RECAP_PROMPT_DEFAULT = """\
You are writing a daily recap of the last 24 hours of news for an OSINT operator HUD.
CRITICAL INSTRUCTION: YOU MUST USE ONLY THE DATA PROVIDED BELOW. DO NOT INVENT, RECALL, OR ADD ANY EVENTS, NAMES, DATES, IMPLICATIONS, PROJECTS, OR DETAILS NOT EXPLICITLY PRESENT IN THE DATA. IF THE DATA HAS NO BREAKING IMPORTANT NEWS, set summary_en to exactly: "No qualifying breaking news in the last 24 hours." and use empty ticker and map_items arrays. AND STOP. NO EXTERNAL KNOWLEDGE FROM TRAINING.
All text in English.
Focus on breaking important news from the last 24 hours (geopolitical, military/conflict, security, disasters, major political developments). Ignore futures, commodity prices, and routine market data do not treat price ticks as news.
Demand a single JSON object (no markdown fences) with this exact shape:
{
"summary_en": "English markdown daily recap or the no-qualifying-events sentence",
"ticker": [{"headline": "", "importance": "critical", "url": "", "location_name": ""}],
"map_items": [{"headline": "", "importance": "critical", "location_name": "", "lat": 0, "lon": 0, "location_confidence": "city", "category": "military/conflict", "url": ""}]
}
ticker: only critical and high, max 12, 140 chars, no markdown.
map_items: only critical and high where a real-world location is explicit in the data. Estimate lat/lon. If location is Unknown or not in the data, omit the item. Never invent a place. Max 20.
summary_en: English markdown daily recap of the last 24 hours of breaking important news.
DATA: DATA:
{final_input} {final_input}
@ -263,6 +302,7 @@ def ensure_tables() -> None:
batch_timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW() batch_timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW()
); );
ALTER TABLE article_summaries ADD COLUMN IF NOT EXISTS model TEXT; ALTER TABLE article_summaries ADD COLUMN IF NOT EXISTS model TEXT;
ALTER TABLE article_summaries ADD COLUMN IF NOT EXISTS kind TEXT;
CREATE TABLE IF NOT EXISTS news_items ( CREATE TABLE IF NOT EXISTS news_items (
id SERIAL PRIMARY KEY, id SERIAL PRIMARY KEY,
summary_id INTEGER REFERENCES article_summaries(id) ON DELETE CASCADE, summary_id INTEGER REFERENCES article_summaries(id) ON DELETE CASCADE,
@ -294,8 +334,9 @@ def ensure_tables() -> None:
logger.error("Error ensuring news tables: %s", exc) logger.error("Error ensuring news tables: %s", exc)
def get_recent_news() -> list[dict]: def get_recent_news(window_minutes: int | None = None) -> list[dict]:
"""Fetch articles from the last SUMMARY_WINDOW_MINUTES (content > 100 chars).""" """Fetch articles from the look-back window (content > 100 chars)."""
mins = window_minutes if window_minutes is not None else effective_window_minutes()
query = """ query = """
SELECT title, content, url, domain SELECT title, content, url, domain
FROM articles FROM articles
@ -306,7 +347,7 @@ def get_recent_news() -> list[dict]:
try: try:
conn = psycopg2.connect(**DB_CONFIG) conn = psycopg2.connect(**DB_CONFIG)
cur = conn.cursor() cur = conn.cursor()
cur.execute(query, (SUMMARY_WINDOW_MINUTES,)) cur.execute(query, (mins,))
rows = cur.fetchall() rows = cur.fetchall()
cur.close() cur.close()
conn.close() conn.close()
@ -340,7 +381,34 @@ def _already_summarized_this_interval() -> bool:
return False return False
def save_batch(summary_en: str, model: str, ticker: list, map_items: list) -> None: def _already_recapped_today() -> bool:
"""True when a daily_recap row already exists for the local calendar day."""
if os.getenv("NEWS_SUMMARIZE_FORCE", "") == "1":
return False
tz_name = (os.getenv("TZ") or "America/New_York").strip() or "America/New_York"
start = datetime.now(ZoneInfo(tz_name)).replace(
hour=0, minute=0, second=0, microsecond=0
)
query = (
"SELECT 1 FROM article_summaries "
"WHERE kind = 'daily_recap' AND batch_timestamp >= %s"
)
try:
conn = psycopg2.connect(**DB_CONFIG)
cur = conn.cursor()
cur.execute(query, (start,))
row = cur.fetchone()
cur.close()
conn.close()
return row is not None
except Exception as exc: # noqa: BLE001
logger.error("Error checking recap idempotency: %s", exc)
return False
def save_batch(
summary_en: str, model: str, ticker: list, map_items: list, *, kind: str = "interval"
) -> None:
"""Insert the master brief plus flagged ticker/map rows.""" """Insert the master brief plus flagged ticker/map rows."""
ticker_rows = select_ticker(ticker or []) ticker_rows = select_ticker(ticker or [])
map_rows = select_map(map_items or []) map_rows = select_map(map_items or [])
@ -358,8 +426,8 @@ def save_batch(summary_en: str, model: str, ticker: list, map_items: list) -> No
conn = psycopg2.connect(**DB_CONFIG) conn = psycopg2.connect(**DB_CONFIG)
cur = conn.cursor() cur = conn.cursor()
cur.execute( cur.execute(
"INSERT INTO article_summaries (summary_text, model) VALUES (%s, %s) RETURNING id", "INSERT INTO article_summaries (summary_text, model, kind) VALUES (%s, %s, %s) RETURNING id",
(text, model), (text, model, kind),
) )
summary_id = cur.fetchone()[0] summary_id = cur.fetchone()[0]
for row in ticker_rows: for row in ticker_rows:
@ -396,8 +464,8 @@ def save_batch(summary_en: str, model: str, ticker: list, map_items: list) -> No
) )
conn.commit() conn.commit()
logger.info( logger.info(
"Master summary saved id=%s model=%s ticker=%d map=%d", "Master summary saved id=%s model=%s kind=%s ticker=%d map=%d",
summary_id, model, len(ticker_rows), len(map_rows), summary_id, model, kind, len(ticker_rows), len(map_rows),
) )
cur.close() cur.close()
conn.close() conn.close()
@ -413,26 +481,35 @@ def build_map_prompt(batch: list[dict]) -> str:
for a in batch for a in batch
) )
template = os.getenv("MAP_PROMPT", MAP_PROMPT_DEFAULT) template = os.getenv("MAP_PROMPT", MAP_PROMPT_DEFAULT)
prefix = build_futures_context() + "\n" if INCLUDE_FUTURES else ""
try: try:
return prefix + template.format(batch_text=batch_text) return template.format(batch_text=batch_text)
except KeyError: except KeyError:
return prefix + template return template
def build_master_prompt(final_input: str) -> str: def build_master_prompt(final_input: str, recap: bool = False) -> str:
template = os.getenv("SUMMARY_PROMPT", SUMMARY_PROMPT_DEFAULT) if recap:
prefix = build_futures_context() + "\n" if INCLUDE_FUTURES else "" template = os.getenv("RECAP_PROMPT", RECAP_PROMPT_DEFAULT)
try: else:
return prefix + template.format(final_input=final_input) template = os.getenv("SUMMARY_PROMPT", SUMMARY_PROMPT_DEFAULT)
except KeyError: if "{final_input}" in template:
return prefix + template return template.replace("{final_input}", final_input)
return template
def summarize_news() -> None: def summarize_news() -> None:
"""Map-reduce summarize recent articles and store brief + ticker + map.""" """Map-reduce summarize recent articles and store brief + ticker + map."""
recap = is_recap_run()
window = effective_window_minutes(recap=recap)
ensure_tables() ensure_tables()
if _already_summarized_this_interval(): if recap:
if _already_recapped_today():
logger.info(
"Skipping recap: article_summaries already has daily_recap today "
"(set NEWS_SUMMARIZE_FORCE=1 to override)"
)
return
elif _already_summarized_this_interval():
logger.info( logger.info(
"Skipping summarize: article_summaries already has a row in the last %ss " "Skipping summarize: article_summaries already has a row in the last %ss "
"(set NEWS_SUMMARIZE_FORCE=1 to override)", "(set NEWS_SUMMARIZE_FORCE=1 to override)",
@ -447,14 +524,14 @@ def summarize_news() -> None:
logger.warning("NOUS_API_KEY unset in env and api_keys — idle this run") logger.warning("NOUS_API_KEY unset in env and api_keys — idle this run")
return return
articles = get_recent_news() articles = get_recent_news(window)
if not articles: if not articles:
logger.info("No new articles found in the last %s min.", SUMMARY_WINDOW_MINUTES) logger.info("No new articles found in the last %s min.", window)
return return
logger.info( logger.info(
"Processing %d articles with %s (batch_size=%d, futures=%s)...", "Processing %d articles with %s (batch_size=%d, recap=%s, window_min=%s)...",
len(articles), model, BATCH_SIZE, INCLUDE_FUTURES, len(articles), model, BATCH_SIZE, recap, window,
) )
partial_summaries: list[str] = [] partial_summaries: list[str] = []
@ -481,7 +558,7 @@ def summarize_news() -> None:
logger.info("reduce phase over %d partial summaries", len(partial_summaries)) logger.info("reduce phase over %d partial summaries", len(partial_summaries))
master_raw = call_llm( master_raw = call_llm(
build_master_prompt(final_input), build_master_prompt(final_input, recap=recap),
api_key=api_key, api_key=api_key,
model=model, model=model,
base_url=base_url, base_url=base_url,
@ -491,7 +568,13 @@ def summarize_news() -> None:
logger.warning("Reduce phase returned empty — nothing to persist.") logger.warning("Reduce phase returned empty — nothing to persist.")
return return
parsed = parse_reduce_json(master_raw) parsed = parse_reduce_json(master_raw)
save_batch(parsed["summary_en"], model, parsed["ticker"], parsed["map_items"]) save_batch(
parsed["summary_en"],
model,
parsed["ticker"],
parsed["map_items"],
kind="daily_recap" if recap else "interval",
)
if __name__ == "__main__": if __name__ == "__main__":

View file

@ -0,0 +1,11 @@
"""Keep summarizer unit tests importable without Postgres drivers."""
from __future__ import annotations
import sys
from types import ModuleType
if "psycopg2" not in sys.modules:
fake = ModuleType("psycopg2")
fake.connect = lambda **kwargs: None # type: ignore[attr-defined]
sys.modules["psycopg2"] = fake

View file

@ -0,0 +1,32 @@
"""Default prompts: breaking news, ignore futures/market tape."""
from summarizer import MAP_PROMPT_DEFAULT, RECAP_PROMPT_DEFAULT, SUMMARY_PROMPT_DEFAULT
def _assert_breaking_not_futures(prompt: str) -> None:
p = prompt.lower()
assert "breaking" in p
assert "futures" in p
assert "ignore" in p or "do not" in p or "not" in p
assert "es=f" not in p
assert "yfinance" not in p
def test_map_prompt_focuses_on_breaking_news_not_futures():
_assert_breaking_not_futures(MAP_PROMPT_DEFAULT)
def test_summary_prompt_focuses_on_breaking_news_not_futures():
_assert_breaking_not_futures(SUMMARY_PROMPT_DEFAULT)
p = SUMMARY_PROMPT_DEFAULT.lower()
assert "commodity" in p or "market" in p
def test_recap_prompt_is_daily_24h_breaking_news():
p = RECAP_PROMPT_DEFAULT.lower()
_assert_breaking_not_futures(RECAP_PROMPT_DEFAULT)
assert "daily" in p
assert "24" in p
assert "summary_en" in p
assert "ticker" in p
assert "map_items" in p

View file

@ -0,0 +1,42 @@
"""Wall-clock scheduling: 15-min analyst + 23:00 America/New_York recap."""
from datetime import datetime, timedelta
from zoneinfo import ZoneInfo
from run_news_summarizer import next_event, next_recap_datetime
TZ = ZoneInfo("America/New_York")
def test_next_recap_is_11pm_same_day_before_2300():
now = datetime(2026, 8, 28, 15, 4, tzinfo=TZ)
got = next_recap_datetime(now)
assert got == datetime(2026, 8, 28, 23, 0, tzinfo=TZ)
def test_next_recap_is_11pm_next_day_at_or_after_2300():
now = datetime(2026, 8, 28, 23, 0, tzinfo=TZ)
got = next_recap_datetime(now)
assert got == datetime(2026, 8, 29, 23, 0, tzinfo=TZ)
def test_next_recap_honors_custom_hour():
now = datetime(2026, 8, 28, 10, 0, tzinfo=TZ)
got = next_recap_datetime(now, hour=22, minute=30)
assert got == datetime(2026, 8, 28, 22, 30, tzinfo=TZ)
def test_next_event_picks_recap_when_sooner_than_interval():
now = datetime(2026, 8, 28, 22, 50, tzinfo=TZ)
last_periodic = now - timedelta(seconds=100)
when, kind = next_event(now, last_periodic=last_periodic, interval_s=900)
assert kind == "recap"
assert when == datetime(2026, 8, 28, 23, 0, tzinfo=TZ)
def test_next_event_picks_interval_when_recap_is_hours_away():
now = datetime(2026, 8, 28, 10, 0, tzinfo=TZ)
last_periodic = now
when, kind = next_event(now, last_periodic=last_periodic, interval_s=900)
assert kind == "interval"
assert when == now + timedelta(seconds=900)

View file

@ -0,0 +1,45 @@
"""Summarizer window, recap flag, and no futures injection into prompts."""
from summarizer import (
build_map_prompt,
build_master_prompt,
effective_window_minutes,
is_recap_run,
)
def test_interval_window_defaults_to_15_minutes(monkeypatch):
monkeypatch.delenv("NEWS_RECAP", raising=False)
monkeypatch.delenv("SUMMARY_WINDOW_HOURS", raising=False)
monkeypatch.setenv("SUMMARY_WINDOW_MINUTES", "15")
assert is_recap_run() is False
assert effective_window_minutes() == 15
def test_recap_window_is_24_hours(monkeypatch):
monkeypatch.setenv("NEWS_RECAP", "1")
monkeypatch.setenv("SUMMARY_WINDOW_MINUTES", "15")
assert is_recap_run() is True
assert effective_window_minutes() == 24 * 60
def test_build_map_prompt_does_not_inject_futures():
prompt = build_map_prompt(
[{"title": "Blast", "domain": "ex.com", "url": "https://ex.com/1", "content": "x" * 120}]
)
assert "FUTURES PRICES" not in prompt
assert "ES=F" not in prompt
assert "Blast" in prompt
def test_build_master_prompt_interval_uses_summary_not_recap():
prompt = build_master_prompt("partial facts", recap=False)
assert "partial facts" in prompt
assert "daily recap" not in prompt.lower()
def test_build_master_prompt_recap_uses_daily_template():
prompt = build_master_prompt("partial facts", recap=True)
assert "partial facts" in prompt
assert "daily recap" in prompt.lower()
assert "24" in prompt

View file

@ -68,14 +68,14 @@ def _seed_article(title: str, url: str, domain: str, ts: str, content: str = "bo
asyncio.run(run()) asyncio.run(run())
def _seed_summary(text: str, ts: str, model: str | None = None) -> int: def _seed_summary(text: str, ts: str, model: str | None = None, kind: str | None = None) -> int:
async def run() -> int: async def run() -> int:
conn = await asyncpg.connect(**_conn_kwargs()) conn = await asyncpg.connect(**_conn_kwargs())
try: try:
row = await conn.fetchrow( row = await conn.fetchrow(
"INSERT INTO article_summaries (summary_text, batch_timestamp, model) " "INSERT INTO article_summaries (summary_text, batch_timestamp, model, kind) "
"VALUES ($1, $2, $3) RETURNING id", "VALUES ($1, $2, $3, $4) RETURNING id",
text, datetime.fromisoformat(ts), model, text, datetime.fromisoformat(ts), model, kind,
) )
return int(row["id"]) return int(row["id"])
finally: finally:
@ -164,9 +164,21 @@ def test_api_news_summaries_contract(clean_news):
assert isinstance(body, list) assert isinstance(body, list)
assert len(body) == 1 assert len(body) == 1
s = body[0] s = body[0]
assert set(s.keys()) == {"id", "summary_text", "batch_timestamp", "model"} assert set(s.keys()) == {"id", "summary_text", "batch_timestamp", "model", "kind"}
assert s["summary_text"] == "master summary markdown…" assert s["summary_text"] == "master summary markdown…"
assert s["batch_timestamp"].startswith("2026-08-24T18:05") assert s["batch_timestamp"].startswith("2026-08-24T18:05")
assert s["kind"] is None
@requires_db
def test_api_news_summaries_kind_filter(clean_news):
_seed_summary("interval brief", "2026-08-28T22:05:00+00:00", kind="interval")
_seed_summary("daily recap", "2026-08-28T03:00:00+00:00", kind="daily_recap")
recap = _get("/api/news/summaries?kind=daily_recap").json()
assert len(recap) == 1
assert recap[0]["summary_text"] == "daily recap"
assert recap[0]["kind"] == "daily_recap"
assert _get("/api/news/summaries?kind=nope").status_code == 422
@requires_db @requires_db

View file

@ -14,6 +14,11 @@ def test_summarizer_dockerfile_copies_intel_modules():
assert "nous_client.py" in df assert "nous_client.py" in df
def test_news_panel_pins_daily_recap():
assert "kind=daily_recap" in HTML
assert "DAILY RECAP" in HTML
def test_nginx_ws_snippet_has_upgrade_headers(): def test_nginx_ws_snippet_has_upgrade_headers():
conf = (ROOT / "deploy/osint-ws.nginx.conf").read_text() conf = (ROOT / "deploy/osint-ws.nginx.conf").read_text()
assert "proxy_http_version 1.1" in conf assert "proxy_http_version 1.1" in conf