From 49ee5fe7a3c040242bdbbb24ca7bb5dceb8b1bcd Mon Sep 17 00:00:00 2001 From: Sirius DevOps Date: Fri, 28 Aug 2026 22:53:31 -0400 Subject: [PATCH] feat: 23:00 daily news recap; prompts drop futures Summarizer still runs the 15-min analyst, plus a 24h breaking-news recap at 23:00 America/New_York. HUD pins kind=daily_recap. Prompts ignore futures/market tape. --- alembic/versions/008_summary_kind.py | 26 ++++ app/main.py | 9 ++ app/models.py | 1 + app/schemas.py | 1 + app/static/index.html | 11 +- docker-compose.yml | 3 + docs/news.md | 37 +++--- news/summerizer/Dockerfile | 2 +- news/summerizer/run_news_summarizer.py | 106 +++++++++++++--- news/summerizer/summarizer.py | 153 +++++++++++++++++------ news/summerizer/tests/conftest.py | 11 ++ news/summerizer/tests/test_prompts.py | 32 +++++ news/summerizer/tests/test_scheduler.py | 42 +++++++ news/summerizer/tests/test_summarizer.py | 45 +++++++ tests/test_api_news.py | 22 +++- tests/test_frontend_reliability.py | 5 + 16 files changed, 431 insertions(+), 75 deletions(-) create mode 100644 alembic/versions/008_summary_kind.py create mode 100644 news/summerizer/tests/conftest.py create mode 100644 news/summerizer/tests/test_prompts.py create mode 100644 news/summerizer/tests/test_scheduler.py create mode 100644 news/summerizer/tests/test_summarizer.py diff --git a/alembic/versions/008_summary_kind.py b/alembic/versions/008_summary_kind.py new file mode 100644 index 0000000..e5bced4 --- /dev/null +++ b/alembic/versions/008_summary_kind.py @@ -0,0 +1,26 @@ +"""article_summaries.kind — interval vs daily_recap + +Revision ID: 008_summary_kind +Revises: 007_event_dedup +Create Date: 2026-08-28 +""" + +from alembic import op + +revision = "008_summary_kind" +down_revision = "007_event_dedup" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute( + """ + ALTER TABLE article_summaries + ADD COLUMN IF NOT EXISTS kind TEXT + """ + ) + + +def downgrade() -> None: + op.execute("ALTER TABLE article_summaries DROP COLUMN IF EXISTS kind") diff --git a/app/main.py b/app/main.py index 355d379..5b31141 100644 --- a/app/main.py +++ b/app/main.py @@ -1220,22 +1220,31 @@ async def list_news_summaries( None, description="Only summaries generated at/after this UTC instant.", ), + kind: str | None = Query( + None, + description="Filter: interval or daily_recap. Omit for all.", + ), limit: int = Query(20, ge=1, le=100), offset: int = Query(0, ge=0), ): """Most recent master LLM summaries (newest first).""" + if kind is not None and kind not in ("interval", "daily_recap"): + raise HTTPException(422, "kind must be interval or daily_recap") async with async_session() as session: stmt = select(article_summaries).order_by( article_summaries.c.batch_timestamp.desc().nullslast() ) if since: stmt = stmt.where(article_summaries.c.batch_timestamp >= since) + if kind is not None: + stmt = stmt.where(article_summaries.c.kind == kind) stmt = stmt.limit(limit).offset(offset) rows = (await session.execute(stmt)).mappings().all() return [ NewsSummaryOut( id=r["id"], summary_text=r["summary_text"], batch_timestamp=r["batch_timestamp"], model=r["model"], + kind=r["kind"], ) for r in rows ] diff --git a/app/models.py b/app/models.py index dac5d0a..f3b528d 100644 --- a/app/models.py +++ b/app/models.py @@ -219,6 +219,7 @@ article_summaries = Table( Column("batch_timestamp", DateTime(timezone=True), server_default=func.now(), nullable=False), Column("model", Text), # LLM id used for this batch; nullable for old rows + Column("kind", Text), # interval | daily_recap; nullable for old rows ) Index("ix_article_summaries_batch_timestamp", article_summaries.c.batch_timestamp) diff --git a/app/schemas.py b/app/schemas.py index b61e296..799aeb3 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -274,6 +274,7 @@ class NewsSummaryOut(BaseModel): summary_text: str batch_timestamp: datetime model: Optional[str] = None + kind: Optional[str] = None class NewsTickerItemOut(BaseModel): diff --git a/app/static/index.html b/app/static/index.html index 9974edb..efd9e61 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -1350,15 +1350,18 @@ async function loadNews(force) { if (upd) upd.textContent = 'updating…'; let articles = []; let summaries = []; + let recaps = []; let tickerItems = []; try { - const [rArticles, rSumm, rTick] = await Promise.all([ + const [rArticles, rSumm, rRecap, rTick] = await Promise.all([ fetch(`${API}/api/news?limit=100`), fetch(`${API}/api/news/summaries?limit=1`), + fetch(`${API}/api/news/summaries?kind=daily_recap&limit=1`), fetch(`${API}/api/news/ticker?limit=20`), ]); if (rArticles.ok) articles = await rArticles.json(); if (rSumm.ok) summaries = await rSumm.json(); + if (rRecap.ok) recaps = await rRecap.json(); if (rTick.ok) tickerItems = await rTick.json(); } catch (e) { const list = document.getElementById('news-list'); @@ -1369,7 +1372,7 @@ async function loadNews(force) { console.error('News load failed', e); return; } - renderNewsSummary(summaries); + renderNewsSummary(recaps.length ? recaps : summaries, recaps.length > 0); renderNewsList(articles); renderNewsTicker(articles, summaries, tickerItems); if (upd) upd.textContent = 'updated ' + new Date().toLocaleTimeString(); @@ -1377,7 +1380,7 @@ async function loadNews(force) { newsInterval = setInterval(() => loadNews(false), NEWS_REFRESH_MS); } } -function renderNewsSummary(summaries) { +function renderNewsSummary(summaries, isRecap) { const bodyEl = document.getElementById('ns-body'); const timeEl = document.getElementById('ns-time'); const badgeEl = document.getElementById('ns-badge'); @@ -1389,7 +1392,7 @@ function renderNewsSummary(summaries) { return; } const s = summaries[0]; - badgeEl.textContent = 'LATEST'; + badgeEl.textContent = isRecap || s.kind === 'daily_recap' ? 'DAILY RECAP' : 'LATEST'; timeEl.textContent = s.batch_timestamp ? 'Batch: ' + new Date(s.batch_timestamp).toLocaleString() : '—'; bodyEl.innerHTML = newsSimpleMD(s.summary_text || ''); } diff --git a/docker-compose.yml b/docker-compose.yml index f810304..e854010 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -234,6 +234,9 @@ services: NEWS_SUMMARIZE_INTERVAL_S: ${NEWS_SUMMARIZE_INTERVAL_S:-900} NEWS_SUMMARIZE_RUN_ON_START: ${NEWS_SUMMARIZE_RUN_ON_START:-1} NEWS_SUMMARIZE_FORCE: ${NEWS_SUMMARIZE_FORCE:-0} + TZ: ${TZ:-America/New_York} + NEWS_RECAP_HOUR: ${NEWS_RECAP_HOUR:-23} + NEWS_RECAP_MINUTE: ${NEWS_RECAP_MINUTE:-0} command: ["python", "run_news_summarizer.py"] volumes: diff --git a/docs/news.md b/docs/news.md index fe06f6c..5ac373b 100644 --- a/docs/news.md +++ b/docs/news.md @@ -16,7 +16,7 @@ urls.txt (RSS + homepages) news-scraper (Scrapy, continuous) ──► articles table (osint-db) │ │ │ ▼ -news-summarizer (Nous Portal, every 15m) ──► article_summaries + news_items +news-summarizer (Nous Portal, every 15m + 23:00 recap) ──► article_summaries + news_items │ ▼ GET /api/news · /api/news/summaries · /api/news/ticker · /api/news/map @@ -45,11 +45,13 @@ feeds (`GET /api/news` exact key set is unchanged on purpose). the `PostgresPipeline` writes to `articles` with URL-based dedup (`ON CONFLICT (url) DO NOTHING`). 2. **Summarizer** — `news/summerizer/run_news_summarizer.py` runs - `summarizer.py` every `NEWS_SUMMARIZE_INTERVAL_S` (default 900). It reads - articles from the last `SUMMARY_WINDOW_MINUTES` (default 15), map-reduces - them through Nous Portal (`SUMMARY_MODEL` / Settings, default - `Hermes-4.3-36B`), writes the English brief to `article_summaries` (column - `model` is the LLM id), and flagged ticker/map rows to `news_items`. + `summarizer.py` every `NEWS_SUMMARIZE_INTERVAL_S` (default 900) over the + last `SUMMARY_WINDOW_MINUTES` (default 15), and again at 23:00 + `America/New_York` (`TZ`) over the last 24 hours as a daily recap + (`kind=daily_recap`). Both map-reduce through Nous Portal (`SUMMARY_MODEL` + / Settings, default `Hermes-4.3-36B`), write the English brief to + `article_summaries` (column `model` is the LLM id; `kind` is + `interval` or `daily_recap`), and flagged ticker/map rows to `news_items`. Loops are serial (two crawls/summaries never overlap). Interval idempotency: if `article_summaries` already has a row in the last interval, the summarizer @@ -116,12 +118,15 @@ Key set **unchanged** (no `lat`/`lon` on articles; geo lives on `/api/news/map`) "id": 1, "summary_text": "English markdown brief…", "batch_timestamp": "2026-08-24T18:10:00Z", - "model": "Hermes-4.3-36B" + "model": "Hermes-4.3-36B", + "kind": "daily_recap" } ] ``` -`model` is additive. Empty DB → `[]` (no crash). +`model` and `kind` are additive (`interval` | `daily_recap` | `null` for old rows). +`?kind=daily_recap` pins the nightly 24h recap. Empty DB → `[]` (no crash). +Malformed `kind` → `422`. ### GET /api/news/ticker — flagged HUD headlines @@ -237,10 +242,13 @@ lands in `article_summaries.summary_text`. | `SUMMARY_WINDOW_MINUTES` | `15` | How far back the summarizer looks for new articles. | | `NEWS_SCRAPE_INTERVAL_S` | `10` | Pause after each crawl before the next (scraper is otherwise continuous). | | `NEWS_SUMMARIZE_INTERVAL_S` | `900` | Seconds between analyst runs (default 15 min). | +| `TZ` | `America/New_York` | Timezone for the 23:00 daily recap. | +| `NEWS_RECAP_HOUR` | `23` | Local hour of the daily 24h recap. | +| `NEWS_RECAP_MINUTE` | `0` | Local minute of the daily recap. | | `NEWS_SCRAPE_RUN_ON_START` | `1` | Run one scrape immediately on container start. | | `NEWS_SUMMARIZE_RUN_ON_START` | `1` | Run one summarize immediately on container start. | -| `NEWS_SUMMARIZE_FORCE` | `0` | `1` ignores the interval idempotency skip (double-pins on recreate). | -| `INCLUDE_FUTURES` | `0` | Legacy futures-prices coupling. OFF for OSINT; set `1` + install `yfinance` to enable. | +| `NEWS_SUMMARIZE_FORCE` | `0` | `1` ignores the interval/recap idempotency skip (double-pins on recreate). | +| `INCLUDE_FUTURES` | `0` | Legacy. Ignored — prompts never inject futures/market tape. | | `NEWS_LOG_LEVEL` | `INFO` | Scrapy log level. | | `OSINT_USER_AGENT` | `osint-dashboard-news-summarizer` | Sent on every outbound Nous call. | | `TELEGRAM_TOKEN` / `TELEGRAM_CHAT_ID` | *(blank)* | Reserved for the (out-of-scope) Telegram delivery bot. | @@ -254,11 +262,10 @@ No Hermes-4 reasoning system prompt. Reduce uses `json_mode=True`. ## Prompts -Both prompts are env-overridable — the default `MAP_PROMPT` is OSINT-neutral -(facts, locations, entities, category, OSINT signal per article; English) and -the default `SUMMARY_PROMPT` demands the reduce JSON above (with a -"no qualifying events" escape hatch). Upstream's futures/markets prompt -language is gated behind `INCLUDE_FUTURES=1`. +Both prompts are env-overridable. Defaults focus on **breaking important news** +and explicitly ignore futures, commodity tape, and routine market moves. +`RECAP_PROMPT` (23:00, 24h window) is the daily recap; `SUMMARY_PROMPT` is the +15-min analyst. `INCLUDE_FUTURES` is ignored. ## Tests diff --git a/news/summerizer/Dockerfile b/news/summerizer/Dockerfile index 3b795c2..19a145b 100644 --- a/news/summerizer/Dockerfile +++ b/news/summerizer/Dockerfile @@ -7,7 +7,7 @@ WORKDIR /app # libpq-dev + gcc for psycopg2 build/adapters; keep the image lean. RUN apt-get update && apt-get install -y --no-install-recommends \ - libpq-dev gcc \ + libpq-dev gcc tzdata \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt . diff --git a/news/summerizer/run_news_summarizer.py b/news/summerizer/run_news_summarizer.py index 1032af7..dca6d68 100644 --- a/news/summerizer/run_news_summarizer.py +++ b/news/summerizer/run_news_summarizer.py @@ -1,37 +1,97 @@ #!/usr/bin/env python3 -"""Scheduler loop for the news summarizer — every NEWS_SUMMARIZE_INTERVAL_S. +"""Scheduler loop for the news summarizer. -Default 900s (15 minutes). Serial: a slow LLM pass never overlaps the next. +15-minute analyst (NEWS_SUMMARIZE_INTERVAL_S, default 900s) plus a daily +recap at 23:00 in TZ (default America/New_York) over the last 24 hours. + +Serial: a slow LLM pass never overlaps the next. Env (all optional, 12-factor): - NEWS_SUMMARIZE_INTERVAL_S seconds between runs (default 900) + NEWS_SUMMARIZE_INTERVAL_S seconds between analyst runs (default 900) NEWS_SUMMARIZE_RUN_ON_START "1" to summarize once immediately on boot (default 1) + NEWS_RECAP_HOUR / MINUTE wall-clock recap time (default 23:00) + TZ IANA tz (default America/New_York) NOUS_API_KEY optional in env; Keys UI / api_keys also works """ from __future__ import annotations -import datetime import logging import os import subprocess import sys import time +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") logger = logging.getLogger("news.summarizer.scheduler") INTERVAL_S = max(1, int(os.getenv("NEWS_SUMMARIZE_INTERVAL_S", "900"))) RUN_ON_START = os.getenv("NEWS_SUMMARIZE_RUN_ON_START", "1").lower() in ("1", "true", "yes") +DEFAULT_TZ = "America/New_York" +DEFAULT_RECAP_HOUR = 23 +DEFAULT_RECAP_MINUTE = 0 -def run_summarize() -> None: - logger.info("summarize starting at %s", datetime.datetime.now().isoformat(timespec="seconds")) +def _tz() -> ZoneInfo: + name = (os.getenv("TZ") or DEFAULT_TZ).strip() or DEFAULT_TZ + return ZoneInfo(name) + + +def recap_hour_minute() -> tuple[int, int]: + hour = int(os.getenv("NEWS_RECAP_HOUR", str(DEFAULT_RECAP_HOUR))) + minute = int(os.getenv("NEWS_RECAP_MINUTE", str(DEFAULT_RECAP_MINUTE))) + return hour, minute + + +def next_recap_datetime( + now: datetime, hour: int | None = None, minute: int | None = None +) -> datetime: + """Next 23:00 (or hour/minute) strictly after *now* in now's timezone.""" + if now.tzinfo is None: + now = now.replace(tzinfo=_tz()) + env_h, env_m = recap_hour_minute() + hour = env_h if hour is None else hour + minute = env_m if minute is None else minute + candidate = now.replace(hour=hour, minute=minute, second=0, microsecond=0) + if now >= candidate: + candidate += timedelta(days=1) + return candidate + + +def next_event( + now: datetime, + last_periodic: datetime | None, + interval_s: int, + hour: int = DEFAULT_RECAP_HOUR, + minute: int = DEFAULT_RECAP_MINUTE, +) -> tuple[datetime, str]: + """Return (when, 'recap'|'interval') for the sooner of recap vs interval.""" + recap_at = next_recap_datetime(now, hour=hour, minute=minute) + periodic_at = now if last_periodic is None else last_periodic + timedelta(seconds=interval_s) + if recap_at <= periodic_at: + return recap_at, "recap" + return periodic_at, "interval" + + +def run_summarize(*, recap: bool = False) -> None: + kind = "recap" if recap else "interval" + logger.info( + "%s starting at %s", kind, datetime.now().isoformat(timespec="seconds") + ) + env = os.environ.copy() + if recap: + env["NEWS_RECAP"] = "1" + else: + env.pop("NEWS_RECAP", None) try: - proc = subprocess.run([sys.executable, "summarizer.py"], cwd="/app") - logger.info("summarize finished rc=%s", proc.returncode) + proc = subprocess.run( + [sys.executable, "summarizer.py"], cwd="/app", env=env + ) + logger.info("%s finished rc=%s", kind, proc.returncode) except Exception: # noqa: BLE001 — keep the loop alive across failures - logger.exception("summarize failed") + logger.exception("%s failed", kind) def main() -> None: @@ -39,16 +99,32 @@ def main() -> None: logger.warning( "NOUS_API_KEY unset in env — will read api_keys on each run; idle if both empty" ) + tz = _tz() + hour, minute = recap_hour_minute() logger.info( - "news summarizer loop starting (interval_s=%s, run_on_start=%s)", - INTERVAL_S, RUN_ON_START, + "news summarizer loop starting (interval_s=%s, run_on_start=%s, recap=%02d:%02d %s)", + INTERVAL_S, RUN_ON_START, hour, minute, tz, ) + last_periodic: datetime | None = None if RUN_ON_START: - run_summarize() + run_summarize(recap=False) + last_periodic = datetime.now(tz) while True: - logger.info("next summarize in %ss", INTERVAL_S) - time.sleep(INTERVAL_S) - run_summarize() + now = datetime.now(tz) + when, kind = next_event( + now, last_periodic, INTERVAL_S, hour=hour, minute=minute + ) + sleep_s = max(1, (when - now).total_seconds()) + logger.info("next %s in %ss", kind, int(sleep_s)) + time.sleep(sleep_s) + now = datetime.now(tz) + if kind == "recap": + run_summarize(recap=True) + # Recap covers the 15-min window; don't immediately fire interval. + last_periodic = now + else: + run_summarize(recap=False) + last_periodic = now if __name__ == "__main__": diff --git a/news/summerizer/summarizer.py b/news/summerizer/summarizer.py index c57476f..82c624a 100644 --- a/news/summerizer/summarizer.py +++ b/news/summerizer/summarizer.py @@ -16,13 +16,10 @@ of each summarize_news() — env wins, else api_keys / app_settings: SUMMARY_MODEL default Hermes-4.3-36B (else app_settings) BATCH_SIZE articles per map-phase batch (default 50) SUMMARY_WINDOW_MINUTES look-back window (default 15; SUMMARY_WINDOW_HOURS wins if set) - NEWS_SUMMARIZE_FORCE "1" to ignore the interval idempotency skip - INCLUDE_FUTURES "1" to prepend live futures prices (default 0) - -The futures/markets coupling from the original pipeline is gated behind -INCLUDE_FUTURES and OFF by default — it is irrelevant to the OSINT dashboard -and pulled yfinance into the image. Re-enable by installing yfinance and -setting INCLUDE_FUTURES=1. + NEWS_RECAP "1" for the 23:00 daily recap (24h window, recap prompt) + NEWS_SUMMARIZE_FORCE "1" to ignore the interval/recap idempotency skip + TZ IANA tz for recap-day bounds (default America/New_York) + INCLUDE_FUTURES legacy; ignored — prompts never inject futures data """ from __future__ import annotations @@ -30,6 +27,7 @@ from __future__ import annotations import logging import os from datetime import datetime +from zoneinfo import ZoneInfo import psycopg2 @@ -67,6 +65,18 @@ def _summarize_interval_seconds() -> int: return max(1, int(os.getenv("NEWS_SUMMARIZE_INTERVAL_S", "900"))) +def is_recap_run() -> bool: + return os.getenv("NEWS_RECAP", "0").lower() in ("1", "true", "yes") + + +def effective_window_minutes(*, recap: bool | None = None) -> int: + if recap is None: + recap = is_recap_run() + if recap: + return 24 * 60 + return _summary_window_minutes() + + SUMMARY_WINDOW_MINUTES = _summary_window_minutes() INCLUDE_FUTURES = os.getenv("INCLUDE_FUTURES", "0").lower() in ("1", "true", "yes") @@ -84,6 +94,8 @@ FUTURES_TICKERS = { MAP_PROMPT_DEFAULT = """\ You are a precise, factual OSINT news processor. Your ONLY source of information is the articles provided below. Do NOT add external knowledge, assumptions, training data, or invented facts. +Focus on breaking important news (geopolitical, military/conflict, security, disasters, major political developments). Ignore futures prices, commodity tape, ticker chatter, and routine market moves unless they themselves are the breaking event. + Write every field in English. Translate if the article is not English. For EACH article in the batch: @@ -91,7 +103,7 @@ For EACH article in the batch: 2. Location: country/city/region or Unknown. If you can estimate coordinates, emit them as numbers; otherwise omit. 3. Entities: list the key people, organizations, or governments mentioned (comma-separated, only names present in the text), else "None". 4. Category: pick one — politics, military/conflict, economy, technology, environment/disaster, health, crime, society, sport, other. -5. OSINT signal: if the article describes an event with geopolitical, security, military, economic, or disaster significance, say so in one short sentence. Otherwise write: "No notable OSINT signal." +5. OSINT signal: if the article describes a breaking event with geopolitical, security, military, or disaster significance, say so in one short sentence. Otherwise write: "No notable OSINT signal." 6. Importance: critical (breaking geopolitical/military/disaster with immediate impact), high, medium, low, none. If several articles cover the same story, add one short batch-level note at the end: "Batch theme: [one sentence]". @@ -117,10 +129,12 @@ Articles in this batch: """ SUMMARY_PROMPT_DEFAULT = """\ -CRITICAL INSTRUCTION - REPEAT 3 TIMES: YOU MUST USE ONLY THE DATA PROVIDED BELOW. DO NOT INVENT, RECALL, OR ADD ANY EVENTS, NAMES, DATES, IMPLICATIONS, PROJECTS, OR DETAILS NOT EXPLICITLY PRESENT IN THE DATA. IF THE DATA HAS NO MAJOR GEOPOLITICAL/TECH/MILITARY/ECONOMIC/IMPACTFUL EVENTS OR UNUSUAL STORIES, set summary_en to exactly: "No qualifying impactful or unusual events in the recent hourly news data." and use empty ticker and map_items arrays. AND STOP. NO EXTERNAL KNOWLEDGE FROM TRAINING. +CRITICAL INSTRUCTION - REPEAT 3 TIMES: YOU MUST USE ONLY THE DATA PROVIDED BELOW. DO NOT INVENT, RECALL, OR ADD ANY EVENTS, NAMES, DATES, IMPLICATIONS, PROJECTS, OR DETAILS NOT EXPLICITLY PRESENT IN THE DATA. IF THE DATA HAS NO BREAKING IMPORTANT NEWS, set summary_en to exactly: "No qualifying breaking news in the recent news data." and use empty ticker and map_items arrays. AND STOP. NO EXTERNAL KNOWLEDGE FROM TRAINING. All text in English. +Focus on breaking important news (geopolitical, military/conflict, security, disasters, major political developments). Ignore futures, commodity prices, and routine market data — do not treat price ticks as news. + Demand a single JSON object (no markdown fences) with this exact shape: { @@ -131,7 +145,32 @@ Demand a single JSON object (no markdown fences) with this exact shape: ticker: only critical and high, max 12, ≤140 chars, no markdown. map_items: only critical and high where a real-world location is explicit in the data. Estimate lat/lon. If location is Unknown or not in the data, omit the item. Never invent a place. Max 20. -summary_en: English markdown brief for an operator HUD. +summary_en: English markdown brief of breaking important news for an operator HUD. + +DATA: +{final_input} +""" + +RECAP_PROMPT_DEFAULT = """\ +You are writing a daily recap of the last 24 hours of news for an OSINT operator HUD. + +CRITICAL INSTRUCTION: YOU MUST USE ONLY THE DATA PROVIDED BELOW. DO NOT INVENT, RECALL, OR ADD ANY EVENTS, NAMES, DATES, IMPLICATIONS, PROJECTS, OR DETAILS NOT EXPLICITLY PRESENT IN THE DATA. IF THE DATA HAS NO BREAKING IMPORTANT NEWS, set summary_en to exactly: "No qualifying breaking news in the last 24 hours." and use empty ticker and map_items arrays. AND STOP. NO EXTERNAL KNOWLEDGE FROM TRAINING. + +All text in English. + +Focus on breaking important news from the last 24 hours (geopolitical, military/conflict, security, disasters, major political developments). Ignore futures, commodity prices, and routine market data — do not treat price ticks as news. + +Demand a single JSON object (no markdown fences) with this exact shape: + +{ + "summary_en": "English markdown daily recap or the no-qualifying-events sentence", + "ticker": [{"headline": "", "importance": "critical", "url": "", "location_name": ""}], + "map_items": [{"headline": "", "importance": "critical", "location_name": "", "lat": 0, "lon": 0, "location_confidence": "city", "category": "military/conflict", "url": ""}] +} + +ticker: only critical and high, max 12, ≤140 chars, no markdown. +map_items: only critical and high where a real-world location is explicit in the data. Estimate lat/lon. If location is Unknown or not in the data, omit the item. Never invent a place. Max 20. +summary_en: English markdown daily recap of the last 24 hours of breaking important news. DATA: {final_input} @@ -263,6 +302,7 @@ def ensure_tables() -> None: batch_timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW() ); ALTER TABLE article_summaries ADD COLUMN IF NOT EXISTS model TEXT; + ALTER TABLE article_summaries ADD COLUMN IF NOT EXISTS kind TEXT; CREATE TABLE IF NOT EXISTS news_items ( id SERIAL PRIMARY KEY, summary_id INTEGER REFERENCES article_summaries(id) ON DELETE CASCADE, @@ -294,8 +334,9 @@ def ensure_tables() -> None: logger.error("Error ensuring news tables: %s", exc) -def get_recent_news() -> list[dict]: - """Fetch articles from the last SUMMARY_WINDOW_MINUTES (content > 100 chars).""" +def get_recent_news(window_minutes: int | None = None) -> list[dict]: + """Fetch articles from the look-back window (content > 100 chars).""" + mins = window_minutes if window_minutes is not None else effective_window_minutes() query = """ SELECT title, content, url, domain FROM articles @@ -306,7 +347,7 @@ def get_recent_news() -> list[dict]: try: conn = psycopg2.connect(**DB_CONFIG) cur = conn.cursor() - cur.execute(query, (SUMMARY_WINDOW_MINUTES,)) + cur.execute(query, (mins,)) rows = cur.fetchall() cur.close() conn.close() @@ -340,7 +381,34 @@ def _already_summarized_this_interval() -> bool: return False -def save_batch(summary_en: str, model: str, ticker: list, map_items: list) -> None: +def _already_recapped_today() -> bool: + """True when a daily_recap row already exists for the local calendar day.""" + if os.getenv("NEWS_SUMMARIZE_FORCE", "") == "1": + return False + tz_name = (os.getenv("TZ") or "America/New_York").strip() or "America/New_York" + start = datetime.now(ZoneInfo(tz_name)).replace( + hour=0, minute=0, second=0, microsecond=0 + ) + query = ( + "SELECT 1 FROM article_summaries " + "WHERE kind = 'daily_recap' AND batch_timestamp >= %s" + ) + try: + conn = psycopg2.connect(**DB_CONFIG) + cur = conn.cursor() + cur.execute(query, (start,)) + row = cur.fetchone() + cur.close() + conn.close() + return row is not None + except Exception as exc: # noqa: BLE001 + logger.error("Error checking recap idempotency: %s", exc) + return False + + +def save_batch( + summary_en: str, model: str, ticker: list, map_items: list, *, kind: str = "interval" +) -> None: """Insert the master brief plus flagged ticker/map rows.""" ticker_rows = select_ticker(ticker or []) map_rows = select_map(map_items or []) @@ -358,8 +426,8 @@ def save_batch(summary_en: str, model: str, ticker: list, map_items: list) -> No conn = psycopg2.connect(**DB_CONFIG) cur = conn.cursor() cur.execute( - "INSERT INTO article_summaries (summary_text, model) VALUES (%s, %s) RETURNING id", - (text, model), + "INSERT INTO article_summaries (summary_text, model, kind) VALUES (%s, %s, %s) RETURNING id", + (text, model, kind), ) summary_id = cur.fetchone()[0] for row in ticker_rows: @@ -396,8 +464,8 @@ def save_batch(summary_en: str, model: str, ticker: list, map_items: list) -> No ) conn.commit() logger.info( - "Master summary saved id=%s model=%s ticker=%d map=%d", - summary_id, model, len(ticker_rows), len(map_rows), + "Master summary saved id=%s model=%s kind=%s ticker=%d map=%d", + summary_id, model, kind, len(ticker_rows), len(map_rows), ) cur.close() conn.close() @@ -413,26 +481,35 @@ def build_map_prompt(batch: list[dict]) -> str: for a in batch ) template = os.getenv("MAP_PROMPT", MAP_PROMPT_DEFAULT) - prefix = build_futures_context() + "\n" if INCLUDE_FUTURES else "" try: - return prefix + template.format(batch_text=batch_text) + return template.format(batch_text=batch_text) except KeyError: - return prefix + template + return template -def build_master_prompt(final_input: str) -> str: - template = os.getenv("SUMMARY_PROMPT", SUMMARY_PROMPT_DEFAULT) - prefix = build_futures_context() + "\n" if INCLUDE_FUTURES else "" - try: - return prefix + template.format(final_input=final_input) - except KeyError: - return prefix + template +def build_master_prompt(final_input: str, recap: bool = False) -> str: + if recap: + template = os.getenv("RECAP_PROMPT", RECAP_PROMPT_DEFAULT) + else: + template = os.getenv("SUMMARY_PROMPT", SUMMARY_PROMPT_DEFAULT) + if "{final_input}" in template: + return template.replace("{final_input}", final_input) + return template def summarize_news() -> None: """Map-reduce summarize recent articles and store brief + ticker + map.""" + recap = is_recap_run() + window = effective_window_minutes(recap=recap) ensure_tables() - if _already_summarized_this_interval(): + if recap: + if _already_recapped_today(): + logger.info( + "Skipping recap: article_summaries already has daily_recap today " + "(set NEWS_SUMMARIZE_FORCE=1 to override)" + ) + return + elif _already_summarized_this_interval(): logger.info( "Skipping summarize: article_summaries already has a row in the last %ss " "(set NEWS_SUMMARIZE_FORCE=1 to override)", @@ -447,14 +524,14 @@ def summarize_news() -> None: logger.warning("NOUS_API_KEY unset in env and api_keys — idle this run") return - articles = get_recent_news() + articles = get_recent_news(window) if not articles: - logger.info("No new articles found in the last %s min.", SUMMARY_WINDOW_MINUTES) + logger.info("No new articles found in the last %s min.", window) return logger.info( - "Processing %d articles with %s (batch_size=%d, futures=%s)...", - len(articles), model, BATCH_SIZE, INCLUDE_FUTURES, + "Processing %d articles with %s (batch_size=%d, recap=%s, window_min=%s)...", + len(articles), model, BATCH_SIZE, recap, window, ) partial_summaries: list[str] = [] @@ -481,7 +558,7 @@ def summarize_news() -> None: logger.info("reduce phase over %d partial summaries", len(partial_summaries)) master_raw = call_llm( - build_master_prompt(final_input), + build_master_prompt(final_input, recap=recap), api_key=api_key, model=model, base_url=base_url, @@ -491,7 +568,13 @@ def summarize_news() -> None: logger.warning("Reduce phase returned empty — nothing to persist.") return parsed = parse_reduce_json(master_raw) - save_batch(parsed["summary_en"], model, parsed["ticker"], parsed["map_items"]) + save_batch( + parsed["summary_en"], + model, + parsed["ticker"], + parsed["map_items"], + kind="daily_recap" if recap else "interval", + ) if __name__ == "__main__": diff --git a/news/summerizer/tests/conftest.py b/news/summerizer/tests/conftest.py new file mode 100644 index 0000000..8cb64cd --- /dev/null +++ b/news/summerizer/tests/conftest.py @@ -0,0 +1,11 @@ +"""Keep summarizer unit tests importable without Postgres drivers.""" + +from __future__ import annotations + +import sys +from types import ModuleType + +if "psycopg2" not in sys.modules: + fake = ModuleType("psycopg2") + fake.connect = lambda **kwargs: None # type: ignore[attr-defined] + sys.modules["psycopg2"] = fake diff --git a/news/summerizer/tests/test_prompts.py b/news/summerizer/tests/test_prompts.py new file mode 100644 index 0000000..e6ca187 --- /dev/null +++ b/news/summerizer/tests/test_prompts.py @@ -0,0 +1,32 @@ +"""Default prompts: breaking news, ignore futures/market tape.""" + +from summarizer import MAP_PROMPT_DEFAULT, RECAP_PROMPT_DEFAULT, SUMMARY_PROMPT_DEFAULT + + +def _assert_breaking_not_futures(prompt: str) -> None: + p = prompt.lower() + assert "breaking" in p + assert "futures" in p + assert "ignore" in p or "do not" in p or "not" in p + assert "es=f" not in p + assert "yfinance" not in p + + +def test_map_prompt_focuses_on_breaking_news_not_futures(): + _assert_breaking_not_futures(MAP_PROMPT_DEFAULT) + + +def test_summary_prompt_focuses_on_breaking_news_not_futures(): + _assert_breaking_not_futures(SUMMARY_PROMPT_DEFAULT) + p = SUMMARY_PROMPT_DEFAULT.lower() + assert "commodity" in p or "market" in p + + +def test_recap_prompt_is_daily_24h_breaking_news(): + p = RECAP_PROMPT_DEFAULT.lower() + _assert_breaking_not_futures(RECAP_PROMPT_DEFAULT) + assert "daily" in p + assert "24" in p + assert "summary_en" in p + assert "ticker" in p + assert "map_items" in p diff --git a/news/summerizer/tests/test_scheduler.py b/news/summerizer/tests/test_scheduler.py new file mode 100644 index 0000000..b2ef43c --- /dev/null +++ b/news/summerizer/tests/test_scheduler.py @@ -0,0 +1,42 @@ +"""Wall-clock scheduling: 15-min analyst + 23:00 America/New_York recap.""" + +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo + +from run_news_summarizer import next_event, next_recap_datetime + +TZ = ZoneInfo("America/New_York") + + +def test_next_recap_is_11pm_same_day_before_2300(): + now = datetime(2026, 8, 28, 15, 4, tzinfo=TZ) + got = next_recap_datetime(now) + assert got == datetime(2026, 8, 28, 23, 0, tzinfo=TZ) + + +def test_next_recap_is_11pm_next_day_at_or_after_2300(): + now = datetime(2026, 8, 28, 23, 0, tzinfo=TZ) + got = next_recap_datetime(now) + assert got == datetime(2026, 8, 29, 23, 0, tzinfo=TZ) + + +def test_next_recap_honors_custom_hour(): + now = datetime(2026, 8, 28, 10, 0, tzinfo=TZ) + got = next_recap_datetime(now, hour=22, minute=30) + assert got == datetime(2026, 8, 28, 22, 30, tzinfo=TZ) + + +def test_next_event_picks_recap_when_sooner_than_interval(): + now = datetime(2026, 8, 28, 22, 50, tzinfo=TZ) + last_periodic = now - timedelta(seconds=100) + when, kind = next_event(now, last_periodic=last_periodic, interval_s=900) + assert kind == "recap" + assert when == datetime(2026, 8, 28, 23, 0, tzinfo=TZ) + + +def test_next_event_picks_interval_when_recap_is_hours_away(): + now = datetime(2026, 8, 28, 10, 0, tzinfo=TZ) + last_periodic = now + when, kind = next_event(now, last_periodic=last_periodic, interval_s=900) + assert kind == "interval" + assert when == now + timedelta(seconds=900) diff --git a/news/summerizer/tests/test_summarizer.py b/news/summerizer/tests/test_summarizer.py new file mode 100644 index 0000000..117044f --- /dev/null +++ b/news/summerizer/tests/test_summarizer.py @@ -0,0 +1,45 @@ +"""Summarizer window, recap flag, and no futures injection into prompts.""" + +from summarizer import ( + build_map_prompt, + build_master_prompt, + effective_window_minutes, + is_recap_run, +) + + +def test_interval_window_defaults_to_15_minutes(monkeypatch): + monkeypatch.delenv("NEWS_RECAP", raising=False) + monkeypatch.delenv("SUMMARY_WINDOW_HOURS", raising=False) + monkeypatch.setenv("SUMMARY_WINDOW_MINUTES", "15") + assert is_recap_run() is False + assert effective_window_minutes() == 15 + + +def test_recap_window_is_24_hours(monkeypatch): + monkeypatch.setenv("NEWS_RECAP", "1") + monkeypatch.setenv("SUMMARY_WINDOW_MINUTES", "15") + assert is_recap_run() is True + assert effective_window_minutes() == 24 * 60 + + +def test_build_map_prompt_does_not_inject_futures(): + prompt = build_map_prompt( + [{"title": "Blast", "domain": "ex.com", "url": "https://ex.com/1", "content": "x" * 120}] + ) + assert "FUTURES PRICES" not in prompt + assert "ES=F" not in prompt + assert "Blast" in prompt + + +def test_build_master_prompt_interval_uses_summary_not_recap(): + prompt = build_master_prompt("partial facts", recap=False) + assert "partial facts" in prompt + assert "daily recap" not in prompt.lower() + + +def test_build_master_prompt_recap_uses_daily_template(): + prompt = build_master_prompt("partial facts", recap=True) + assert "partial facts" in prompt + assert "daily recap" in prompt.lower() + assert "24" in prompt diff --git a/tests/test_api_news.py b/tests/test_api_news.py index 340cbd0..0ba569d 100644 --- a/tests/test_api_news.py +++ b/tests/test_api_news.py @@ -68,14 +68,14 @@ def _seed_article(title: str, url: str, domain: str, ts: str, content: str = "bo asyncio.run(run()) -def _seed_summary(text: str, ts: str, model: str | None = None) -> int: +def _seed_summary(text: str, ts: str, model: str | None = None, kind: str | None = None) -> int: async def run() -> int: conn = await asyncpg.connect(**_conn_kwargs()) try: row = await conn.fetchrow( - "INSERT INTO article_summaries (summary_text, batch_timestamp, model) " - "VALUES ($1, $2, $3) RETURNING id", - text, datetime.fromisoformat(ts), model, + "INSERT INTO article_summaries (summary_text, batch_timestamp, model, kind) " + "VALUES ($1, $2, $3, $4) RETURNING id", + text, datetime.fromisoformat(ts), model, kind, ) return int(row["id"]) finally: @@ -164,9 +164,21 @@ def test_api_news_summaries_contract(clean_news): assert isinstance(body, list) assert len(body) == 1 s = body[0] - assert set(s.keys()) == {"id", "summary_text", "batch_timestamp", "model"} + assert set(s.keys()) == {"id", "summary_text", "batch_timestamp", "model", "kind"} assert s["summary_text"] == "master summary markdown…" assert s["batch_timestamp"].startswith("2026-08-24T18:05") + assert s["kind"] is None + + +@requires_db +def test_api_news_summaries_kind_filter(clean_news): + _seed_summary("interval brief", "2026-08-28T22:05:00+00:00", kind="interval") + _seed_summary("daily recap", "2026-08-28T03:00:00+00:00", kind="daily_recap") + recap = _get("/api/news/summaries?kind=daily_recap").json() + assert len(recap) == 1 + assert recap[0]["summary_text"] == "daily recap" + assert recap[0]["kind"] == "daily_recap" + assert _get("/api/news/summaries?kind=nope").status_code == 422 @requires_db diff --git a/tests/test_frontend_reliability.py b/tests/test_frontend_reliability.py index 7735d5b..908f779 100644 --- a/tests/test_frontend_reliability.py +++ b/tests/test_frontend_reliability.py @@ -14,6 +14,11 @@ def test_summarizer_dockerfile_copies_intel_modules(): assert "nous_client.py" in df +def test_news_panel_pins_daily_recap(): + assert "kind=daily_recap" in HTML + assert "DAILY RECAP" in HTML + + def test_nginx_ws_snippet_has_upgrade_headers(): conf = (ROOT / "deploy/osint-ws.nginx.conf").read_text() assert "proxy_http_version 1.1" in conf -- 2.45.3