sirius/osint-dashboard
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: continuous news scrape + 15-min analyst, expand urls.txt #11

Merged
sirius merged 2 commits from feat/news-cadence-feeds into master 2026-08-28 20:56:02 -04:00
Conversation 0 Commits 2 Files changed 11 +434 -376
11 changed files with 434 additions and 376 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
.env.example
View file

@ -67,10 +67,8 @@ INGEST_FIRES=1
# its lookup/fallback is a planned follow-up. # its lookup/fallback is a planned follow-up.
# ── News pipeline (scraper + summarizer, profile `ingest`) ──────────────── # ── News pipeline (scraper + summarizer, profile `ingest`) ────────────────
# Hourly: the scraper crawls 257 RSS sources at minute :00 and the summarizer # Scraper crawls urls.txt continuously (default 10s between crawls).
# runs the Nous map-reduce at minute :05, both writing to the shared osint-db # Summarizer runs Nous map-reduce every 15 min (NEWS_SUMMARIZE_INTERVAL_S=900).
# (tables `articles` + `article_summaries`, created by alembic 003_news).
# Consume via GET /api/news and GET /api/news/summaries.
# NOUS_API_KEY is also (preferably) set in the Keys UI; env is an override. # NOUS_API_KEY is also (preferably) set in the Keys UI; env is an override.
# Unset in both env and api_keys = summarizer logs and idles (never crashes). # Unset in both env and api_keys = summarizer logs and idles (never crashes).
NOUS_API_KEY= NOUS_API_KEY=
@ -81,18 +79,15 @@ NOUS_BASE_URL=https://inference-api.nousresearch.com/v1
# Hermes-4.3-36B remains after a Postgres miss. Env wins when set. # Hermes-4.3-36B remains after a Postgres miss. Env wins when set.
# SUMMARY_MODEL= # SUMMARY_MODEL=
NEWS_BATCH_SIZE=50 NEWS_BATCH_SIZE=50
SUMMARY_WINDOW_HOURS=1 SUMMARY_WINDOW_MINUTES=15
# Futures/markets coupling from the upstream pipeline is OFF by default # Futures/markets coupling from the upstream pipeline is OFF by default
# (irrelevant to OSINT). Set INCLUDE_FUTURES=1 + install yfinance to enable. # (irrelevant to OSINT). Set INCLUDE_FUTURES=1 + install yfinance to enable.
INCLUDE_FUTURES=0 INCLUDE_FUTURES=0
# Wall-clock scheduling (k8s CronJob replacement): scrape minute, summarize minute NEWS_SCRAPE_INTERVAL_S=10
NEWS_SCRAPE_MINUTE=0 NEWS_SUMMARIZE_INTERVAL_S=900
NEWS_SUMMARIZE_MINUTE=5
# Run once immediately on container start (seeds data fast), then align to the
# scheduled minute.
NEWS_SCRAPE_RUN_ON_START=1 NEWS_SCRAPE_RUN_ON_START=1
NEWS_SUMMARIZE_RUN_ON_START=1 NEWS_SUMMARIZE_RUN_ON_START=1
# "1" ignores the current-UTC-hour idempotency skip (double-pins on recreate). # "1" ignores the interval idempotency skip (double-pins on recreate).
NEWS_SUMMARIZE_FORCE=0 NEWS_SUMMARIZE_FORCE=0
NEWS_LOG_LEVEL=INFO NEWS_LOG_LEVEL=INFO
# Reserved for the (out-of-scope) Telegram delivery bot. # Reserved for the (out-of-scope) Telegram delivery bot.

2
app/keystore.py
View file

@ -58,7 +58,7 @@ KEY_REGISTRY: dict[str, dict] = {
"example": "32-char hex string (e.g. 5f3c…9a02)", "example": "32-char hex string (e.g. 5f3c…9a02)",
}, },
"NOUS_API_KEY": { "NOUS_API_KEY": {
"description": "Nous Portal API key — hourly news summarizer (inference-api.nousresearch.com).", "description": "Nous Portal API key — 15-min news summarizer (inference-api.nousresearch.com).",
"pattern": r"^.{16,}$", "pattern": r"^.{16,}$",
"example": "key from https://portal.nousresearch.com (API keys page)", "example": "key from https://portal.nousresearch.com (API keys page)",
}, },

2
app/main.py
View file

@ -1084,7 +1084,7 @@ async def camera_hls_segment(camera_id: UUID, u: str = Query(..., min_length=8))
# ── News pipeline (scraper + summarizer) ────────────────────────────────── # ── News pipeline (scraper + summarizer) ──────────────────────────────────
# Backing data for the frontend news panel. Written by the vendored # Backing data for the frontend news panel. Written by the vendored
# news-scraper (hourly Scrapy crawl) and news-summarizer (hourly Gemini # news-scraper (continuous Scrapy crawl) and news-summarizer (15-min Nous
# map-reduce) services into the shared osint-db. # map-reduce) services into the shared osint-db.
@app.get("/api/news", response_model=list[NewsArticleOut]) @app.get("/api/news", response_model=list[NewsArticleOut])

12
app/static/index.html
View file

@ -731,7 +731,7 @@
<label class="lp-name"><input type="checkbox" id="lp-news-on" checked onchange="toggleNewsPins()"> <span class="lp-dot news"></span> Critical News</label> <label class="lp-name"><input type="checkbox" id="lp-news-on" checked onchange="toggleNewsPins()"> <span class="lp-dot news"></span> Critical News</label>
<span class="lp-count" id="lp-news-count">0</span> <span class="lp-count" id="lp-news-count">0</span>
</div> </div>
<div class="lp-note">LLM-estimated locations from the hourly brief. Pins only for critical/high.</div> <div class="lp-note">LLM-estimated locations from the 15-min brief. Pins only for critical/high.</div>
</div> </div>
<div class="lp-layer"> <div class="lp-layer">
@ -828,7 +828,7 @@
<section class="view" id="view-news" aria-label="News brief"> <section class="view" id="view-news" aria-label="News brief">
<div class="subview"> <div class="subview">
<h2><span class="tick">◉</span> Global News Brief</h2> <h2><span class="tick">◉</span> Global News Brief</h2>
<p class="sub">Scraped on the 15-minute cycle · latest LLM executive summary pinned above</p> <p class="sub">Scraped continuously · latest LLM executive summary pinned above</p>
<div class="news-head"> <div class="news-head">
<div class="news-refresh"> <div class="news-refresh">
<span class="news-updated" id="news-updated">—</span> <span class="news-updated" id="news-updated">—</span>
@ -994,7 +994,7 @@
</div> </div>
<div class="panel"> <div class="panel">
<h3>News Summarizer</h3> <h3>News Summarizer</h3>
<p class="set-note" style="margin-bottom:0.9rem">Takes effect on the next hourly run (:05). No container restart.</p> <p class="set-note" style="margin-bottom:0.9rem">Takes effect on the next 15-min run. No container restart.</p>
<div class="set-row"> <div class="set-row">
<label>Provider</label> <label>Provider</label>
<span>Nous Portal (inference-api.nousresearch.com)</span> <span>Nous Portal (inference-api.nousresearch.com)</span>
@ -1062,7 +1062,7 @@
<div class="sys-row"><span class="k">Active sources</span><span class="v" id="sys-sources">—</span></div> <div class="sys-row"><span class="k">Active sources</span><span class="v" id="sys-sources">—</span></div>
<div class="sys-row"><span class="k">Open alerts</span><span class="v" id="sys-alerts">—</span></div> <div class="sys-row"><span class="k">Open alerts</span><span class="v" id="sys-alerts">—</span></div>
<div class="sys-row"><span class="k">Tracked entities</span><span class="v" id="sys-entities">—</span></div> <div class="sys-row"><span class="k">Tracked entities</span><span class="v" id="sys-entities">—</span></div>
<div class="sys-row"><span class="k">News cycle</span><span class="v">hourly :05</span></div> <div class="sys-row"><span class="k">News cycle</span><span class="v">analyst 15m · scraper continuous</span></div>
<div class="sys-row"><span class="k">Market feed</span><span class="v" id="sys-market">STANDBY</span></div> <div class="sys-row"><span class="k">Market feed</span><span class="v" id="sys-market">STANDBY</span></div>
</div> </div>
<div class="set-actions"> <div class="set-actions">
@ -1088,7 +1088,7 @@
<div class="tick-wrap"> <div class="tick-wrap">
<div class="tick-track" id="nt-track"></div> <div class="tick-track" id="nt-track"></div>
</div> </div>
<div class="tick-tag" id="nt-tag">HOURLY</div> <div class="tick-tag" id="nt-tag">15 MIN</div>
</div> </div>
</footer> </footer>
@ -1385,7 +1385,7 @@ function renderNewsList(articles) {
if (!listEl) return; if (!listEl) return;
document.getElementById('news-count').textContent = articles.length + ' articles'; document.getElementById('news-count').textContent = articles.length + ' articles';
if (!articles || !articles.length) { if (!articles || !articles.length) {
listEl.innerHTML = '<div class="news-empty">No articles scraped yet — the scraper runs hourly at :00. Check news-scraper logs.</div>'; listEl.innerHTML = '<div class="news-empty">No articles scraped yet — the scraper runs continuously. Check news-scraper logs.</div>';
return; return;
} }
listEl.innerHTML = articles.map(a => { listEl.innerHTML = articles.map(a => {

8
docker-compose.yml
View file

@ -176,7 +176,7 @@ services:
volumes: volumes:
- camera-snapshots:/data/snapshots - camera-snapshots:/data/snapshots
# ── News pipeline: hourly scraper (:00) + summarizer (:05) ─────────────── # ── News pipeline: continuous scraper + 15-min summarizer ───────────────
# Both services point at the EXISTING osint-db (tables articles + # Both services point at the EXISTING osint-db (tables articles +
# article_summaries, created by idempotent alembic migration 003_news). # article_summaries, created by idempotent alembic migration 003_news).
# Scheduling replaces the upstream k8s CronJobs with in-compose wall-clock # Scheduling replaces the upstream k8s CronJobs with in-compose wall-clock
@ -200,7 +200,7 @@ services:
DB_PORT: ${DB_PORT:-5432} DB_PORT: ${DB_PORT:-5432}
DB_NAME: ${DB_NAME:-osint_data} DB_NAME: ${DB_NAME:-osint_data}
LOG_LEVEL: ${NEWS_LOG_LEVEL:-INFO} LOG_LEVEL: ${NEWS_LOG_LEVEL:-INFO}
NEWS_SCRAPE_MINUTE: ${NEWS_SCRAPE_MINUTE:-0} NEWS_SCRAPE_INTERVAL_S: ${NEWS_SCRAPE_INTERVAL_S:-10}
NEWS_SCRAPE_RUN_ON_START: ${NEWS_SCRAPE_RUN_ON_START:-1} NEWS_SCRAPE_RUN_ON_START: ${NEWS_SCRAPE_RUN_ON_START:-1}
# Override the image ENTRYPOINT ["scrapy"] with the scheduler loop. # Override the image ENTRYPOINT ["scrapy"] with the scheduler loop.
entrypoint: [] entrypoint: []
@ -229,9 +229,9 @@ services:
SUMMARY_MODEL: ${SUMMARY_MODEL:-} SUMMARY_MODEL: ${SUMMARY_MODEL:-}
OSINT_USER_AGENT: ${OSINT_USER_AGENT:-osint-dashboard-news-summarizer} OSINT_USER_AGENT: ${OSINT_USER_AGENT:-osint-dashboard-news-summarizer}
BATCH_SIZE: ${NEWS_BATCH_SIZE:-50} BATCH_SIZE: ${NEWS_BATCH_SIZE:-50}
SUMMARY_WINDOW_HOURS: ${SUMMARY_WINDOW_HOURS:-1} SUMMARY_WINDOW_MINUTES: ${SUMMARY_WINDOW_MINUTES:-15}
INCLUDE_FUTURES: ${INCLUDE_FUTURES:-0} INCLUDE_FUTURES: ${INCLUDE_FUTURES:-0}
NEWS_SUMMARIZE_MINUTE: ${NEWS_SUMMARIZE_MINUTE:-5} NEWS_SUMMARIZE_INTERVAL_S: ${NEWS_SUMMARIZE_INTERVAL_S:-900}
NEWS_SUMMARIZE_RUN_ON_START: ${NEWS_SUMMARIZE_RUN_ON_START:-1} NEWS_SUMMARIZE_RUN_ON_START: ${NEWS_SUMMARIZE_RUN_ON_START:-1}
NEWS_SUMMARIZE_FORCE: ${NEWS_SUMMARIZE_FORCE:-0} NEWS_SUMMARIZE_FORCE: ${NEWS_SUMMARIZE_FORCE:-0}
command: ["python", "run_news_summarizer.py"] command: ["python", "run_news_summarizer.py"]

60
docs/news.md
View file

@ -1,22 +1,22 @@
# News pipeline — scraper + Nous Portal summarizer # News pipeline — scraper + Nous Portal summarizer
The OSINT dashboard ingests ~257 global news RSS sources hourly and produces The OSINT dashboard ingests a large curated feed list (`news/scraper/urls.txt`)
an English LLM brief plus flagged ticker/map rows. Both services were vendored continuously and produces an English LLM brief plus flagged ticker/map rows
from the upstream `~/Projects/newsPipeline` project and re-integrated here to every 15 minutes. Both services were vendored from the upstream
replace the old k8s CronJob choreography with in-compose scheduling against `~/Projects/newsPipeline` project and re-integrated here against the EXISTING
the EXISTING osint-db — **no second Postgres**. The LLM is **Nous Portal** osint-db — **no second Postgres**. The LLM is **Nous Portal**
(`inference-api.nousresearch.com`) — not Gemini. (`inference-api.nousresearch.com`) — not Gemini.
## Architecture ## Architecture
``` ```
257 RSS feeds (news/scraper/urls.txt) urls.txt (RSS + homepages)
│ │
▼ ▼
news-scraper (Scrapy, hourly :00) ──► articles table (osint-db) news-scraper (Scrapy, continuous) ──► articles table (osint-db)
│ │ │ │
│ ▼ │ ▼
news-summarizer (Nous Portal map-reduce, :05) ──► article_summaries + news_items news-summarizer (Nous Portal, every 15m) ──► article_summaries + news_items
│ │
▼ ▼
GET /api/news · /api/news/summaries · /api/news/ticker · /api/news/map GET /api/news · /api/news/summaries · /api/news/ticker · /api/news/map
@ -25,8 +25,8 @@ news-summarizer (Nous Portal map-reduce, :05) ──► article_summaries + news
| Component | Image | Container | Scheduling | | Component | Image | Container | Scheduling |
|---|---|---|---| |---|---|---|---|
| Scraper | `localhost/osint-news-scraper` | `osint-news-scraper` | wall-clock loop, minute `NEWS_SCRAPE_MINUTE` (default :00) | | Scraper | `localhost/osint-news-scraper` | `osint-news-scraper` | loop, `NEWS_SCRAPE_INTERVAL_S` (default 10s after each crawl) |
| Summarizer | `localhost/osint-news-summarizer` | `osint-news-summarizer` | wall-clock loop, minute `NEWS_SUMMARIZE_MINUTE` (default :05) | | Summarizer | `localhost/osint-news-summarizer` | `osint-news-summarizer` | loop, `NEWS_SUMMARIZE_INTERVAL_S` (default 900s) |
Both services live under the `ingest` compose profile (same as the ingester Both services live under the `ingest` compose profile (same as the ingester
and camera-scraper): `docker compose --profile ingest up -d`. and camera-scraper): `docker compose --profile ingest up -d`.
@ -39,26 +39,22 @@ feeds (`GET /api/news` exact key set is unchanged on purpose).
## Data flow ## Data flow
1. **Scraper** — `news/scraper/run_news_scraper.py` runs 1. **Scraper** — `news/scraper/run_news_scraper.py` runs
`scrapy crawl articles` (spider `news/scraper/newsScraper/spiders/news_spider.py`) `scrapy crawl articles` back-to-back (default 10s pause). The spider reads
at the top of each hour. The spider reads the RSS feed URLs from `urls.txt`, URLs from `urls.txt` (homepages autodiscover RSS; feed URLs are parsed
follows each `<item>` link, extracts the main article body, and the directly), follows each `<item>` link, extracts the main article body, and
`PostgresPipeline` writes to `articles` with URL-based dedup the `PostgresPipeline` writes to `articles` with URL-based dedup
(`ON CONFLICT (url) DO NOTHING`). (`ON CONFLICT (url) DO NOTHING`).
2. **Summarizer** — `news/summerizer/run_news_summarizer.py` runs 2. **Summarizer** — `news/summerizer/run_news_summarizer.py` runs
`summarizer.py` at :05 past each hour. It reads articles from the last `summarizer.py` every `NEWS_SUMMARIZE_INTERVAL_S` (default 900). It reads
`SUMMARY_WINDOW_HOURS`, map-reduces them through Nous Portal articles from the last `SUMMARY_WINDOW_MINUTES` (default 15), map-reduces
(`SUMMARY_MODEL` / Settings, default `Hermes-4.3-36B`), writes the English them through Nous Portal (`SUMMARY_MODEL` / Settings, default
brief to `article_summaries` (column `model` is the LLM id), and flagged `Hermes-4.3-36B`), writes the English brief to `article_summaries` (column
ticker/map rows to `news_items`. `model` is the LLM id), and flagged ticker/map rows to `news_items`.
Scheduling is done with small in-compose wall-clock loops (not host cron): each Loops are serial (two crawls/summaries never overlap). Interval idempotency:
loop runs once on boot (`*_RUN_ON_START=1`, seeds data fast) then sleeps until if `article_summaries` already has a row in the last interval, the summarizer
the next scheduled minute. The loop is serial, so a run that overruns its slot **skips** (prevents double-pins on `RUN_ON_START` recreate). Set
simply shifts to the next boundary — two crawls/summaries never overlap. `NEWS_SUMMARIZE_FORCE=1` to ignore that skip.
Hour-truncation idempotency: if `article_summaries` already has a row for the
current UTC hour, the summarizer **skips** (prevents double-pins on
`RUN_ON_START` recreate). Set `NEWS_SUMMARIZE_FORCE=1` to ignore that skip.
The `articles` and `article_summaries` tables are created by the idempotent The `articles` and `article_summaries` tables are created by the idempotent
alembic migration `003_news` (also created by the scraper's own alembic migration `003_news` (also created by the scraper's own
@ -238,13 +234,13 @@ lands in `article_summaries.summary_text`.
| `NOUS_BASE_URL` | `https://inference-api.nousresearch.com/v1` | Read-only in Settings. | | `NOUS_BASE_URL` | `https://inference-api.nousresearch.com/v1` | Read-only in Settings. |
| `SUMMARY_MODEL` | `Hermes-4.3-36B` | Compose default. Operator-facing choice is Settings → `app_settings.SUMMARY_MODEL`. | | `SUMMARY_MODEL` | `Hermes-4.3-36B` | Compose default. Operator-facing choice is Settings → `app_settings.SUMMARY_MODEL`. |
| `NEWS_BATCH_SIZE` | `50` | Articles per map-phase batch (compose maps to container `BATCH_SIZE`). | | `NEWS_BATCH_SIZE` | `50` | Articles per map-phase batch (compose maps to container `BATCH_SIZE`). |
| `SUMMARY_WINDOW_HOURS` | `1` | How far back the summarizer looks for new articles. | | `SUMMARY_WINDOW_MINUTES` | `15` | How far back the summarizer looks for new articles. |
| `INCLUDE_FUTURES` | `0` | Legacy futures-prices coupling (upstream pipeline). OFF for OSINT; set `1` + install `yfinance` to enable. | | `NEWS_SCRAPE_INTERVAL_S` | `10` | Pause after each crawl before the next (scraper is otherwise continuous). |
| `NEWS_SCRAPE_MINUTE` | `0` | Wall-clock minute the scraper fires. | | `NEWS_SUMMARIZE_INTERVAL_S` | `900` | Seconds between analyst runs (default 15 min). |
| `NEWS_SUMMARIZE_MINUTE` | `5` | Wall-clock minute the summarizer fires. |
| `NEWS_SCRAPE_RUN_ON_START` | `1` | Run one scrape immediately on container start. | | `NEWS_SCRAPE_RUN_ON_START` | `1` | Run one scrape immediately on container start. |
| `NEWS_SUMMARIZE_RUN_ON_START` | `1` | Run one summarize immediately on container start. | | `NEWS_SUMMARIZE_RUN_ON_START` | `1` | Run one summarize immediately on container start. |
| `NEWS_SUMMARIZE_FORCE` | `0` | `1` ignores the current-UTC-hour idempotency skip (double-pins on recreate). | | `NEWS_SUMMARIZE_FORCE` | `0` | `1` ignores the interval idempotency skip (double-pins on recreate). |
| `INCLUDE_FUTURES` | `0` | Legacy futures-prices coupling. OFF for OSINT; set `1` + install `yfinance` to enable. |
| `NEWS_LOG_LEVEL` | `INFO` | Scrapy log level. | | `NEWS_LOG_LEVEL` | `INFO` | Scrapy log level. |
| `OSINT_USER_AGENT` | `osint-dashboard-news-summarizer` | Sent on every outbound Nous call. | | `OSINT_USER_AGENT` | `osint-dashboard-news-summarizer` | Sent on every outbound Nous call. |
| `TELEGRAM_TOKEN` / `TELEGRAM_CHAT_ID` | *(blank)* | Reserved for the (out-of-scope) Telegram delivery bot. | | `TELEGRAM_TOKEN` / `TELEGRAM_CHAT_ID` | *(blank)* | Reserved for the (out-of-scope) Telegram delivery bot. |

2
news/scraper/newsScraper/spiders/news_spider.py
View file

@ -8,7 +8,7 @@ import re
class NewsRSSSpider(Spider): class NewsRSSSpider(Spider):
"""Crawl the curated news sources in urls.txt and extract articles. """Crawl the curated news sources in urls.txt and extract articles.
urls.txt contains 257 news HOMEPAGES (not feed URLs), so this spider urls.txt contains curated news HOMEPAGES and RSS/Atom feeds, so this spider
implements feed autodiscovery: it fetches each start URL, finds the implements feed autodiscovery: it fetches each start URL, finds the
RSS/Atom feed link (`<link rel="alternate" type="application/rss+xml">` RSS/Atom feed link (`<link rel="alternate" type="application/rss+xml">`
or a visible /rss|/feed link), follows it, and then follows each feed or a visible /rss|/feed link), follows it, and then follows each feed

34
news/scraper/run_news_scraper.py
View file

@ -1,18 +1,12 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Scheduler loop for the news scraper — hourly scrape at minute :00. """Scheduler loop for the news scraper — crawl continuously.
Replaces the k8s CronJob (`0 * * * *`) with an in-compose loop so the whole As soon as one Scrapy pass finishes, wait NEWS_SCRAPE_INTERVAL_S seconds
news pipeline lives inside docker-compose. Each iteration: and start the next. Two crawls never overlap (the loop is serial).
1. (optionally, on first boot) runs the Scrapy crawl once to seed data fast
2. sleeps until the next :NEWS_SCRAPE_MINUTE wall-clock boundary
Because the loop is serial, a crawl that overruns its hour simply delays the
next run to the following boundary — two crawls never overlap.
Env (all optional, 12-factor): Env (all optional, 12-factor):
NEWS_SCRAPE_MINUTE minute of the hour to fire (default 0) NEWS_SCRAPE_INTERVAL_S seconds between crawls (default 10)
NEWS_SCRAPE_RUN_ON_START "1" to crawl once immediately on boot (default 1) NEWS_SCRAPE_RUN_ON_START "1" to crawl immediately on boot (default 1)
""" """
from __future__ import annotations from __future__ import annotations
@ -27,19 +21,12 @@ import time
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
logger = logging.getLogger("news.scraper") logger = logging.getLogger("news.scraper")
MINUTE = int(os.getenv("NEWS_SCRAPE_MINUTE", "0")) INTERVAL_S = max(0, int(os.getenv("NEWS_SCRAPE_INTERVAL_S", "10")))
RUN_ON_START = os.getenv("NEWS_SCRAPE_RUN_ON_START", "1").lower() in ("1", "true", "yes") RUN_ON_START = os.getenv("NEWS_SCRAPE_RUN_ON_START", "1").lower() in ("1", "true", "yes")
CRAWL_CMD = ["scrapy", "crawl", "articles"] CRAWL_CMD = ["scrapy", "crawl", "articles"]
def seconds_until_next(minute: int) -> float:
"""Seconds until the next occurrence of ``minute`` past the hour (local time)."""
now = datetime.datetime.now()
nxt = now.replace(minute=minute, second=0, microsecond=0) + datetime.timedelta(hours=1)
return (nxt - now).total_seconds()
def run_crawl() -> None: def run_crawl() -> None:
logger.info("scrape starting at %s", datetime.datetime.now().isoformat(timespec="seconds")) logger.info("scrape starting at %s", datetime.datetime.now().isoformat(timespec="seconds"))
try: try:
@ -51,15 +38,14 @@ def run_crawl() -> None:
def main() -> None: def main() -> None:
logger.info( logger.info(
"news scraper loop starting (minute=%s, run_on_start=%s)", "news scraper loop starting (interval_s=%s, run_on_start=%s)",
MINUTE, RUN_ON_START, INTERVAL_S, RUN_ON_START,
) )
if RUN_ON_START: if RUN_ON_START:
run_crawl() run_crawl()
while True: while True:
delay = seconds_until_next(MINUTE) logger.info("next scrape in %ss", INTERVAL_S)
logger.info("next scrape at :%02d (in %.0fs)", MINUTE, delay) time.sleep(INTERVAL_S)
time.sleep(delay)
run_crawl() run_crawl()

593
news/scraper/urls.txt
View file

@ -1,257 +1,336 @@
# --- NORTH AMERICA --- https://www.investing.com/rss/news.rss
# USA https://www.ftchinese.com/rss
https://www.npr.org https://www.alwatan.com
https://www.pbs.org/newshour https://albiladpress.com
https://www.usatoday.com https://www.aletihad.ae/
https://www.cbsnews.com https://www.albayan.ae
https://www.nbcnews.com https://www.aljazeera.com/xml/rss/all.xml
https://english.alarabiya.net/.mrss/en.xml
# Canada https://english.aawsat.com/home/rss
https://www.cbc.ca/news https://www.newarab.com/rss
https://www.ctvnews.ca https://www.skynewsarabia.com/rss/feeds/rss-1.xml
https://globalnews.ca https://www.thenationalnews.com/arc/outboundfeeds/rss/
https://nationalpost.com https://www.arabnews.com/rss.xml
https://www.thestar.com https://gulfnews.com/rss
https://www.kuwaittimes.com/feed/
# Mexico https://www.omanobserver.om/feed/
https://www.eluniversal.com.mx https://www.khaleejtimes.com/rss/news
https://www.milenio.com http://www.akhbar-alkhaleej.com/rss/all
https://www.jornada.com.mx https://today.lorientleyour.com/rss
https://www.excelsior.com.mx https://www.annahar.com/english/rss
https://aristeguinoticias.com https://english.almayadeen.net/rss
https://english.ahram.org.eg/rss/0/Home.aspx
# --- SOUTH AMERICA --- https://www.dailynewsegypt.com/feed/
# Brazil http://www.jordantimes.com/rss
https://g1.globo.com https://www.alraimedia.com/rss
https://www.uol.com.br https://alghad.com/feed/
https://agenciabrasil.ebc.com.br https://nypost.com/feed/
https://www.metropoles.com https://gothamist.com/feed/
https://www.terra.com.br/noticias https://www.cityandstateny.com/rss
https://feeds.nytimes.com/nyt/rss/HomePage
# Argentina https://www.thecity.nyc/rss/index.xml
https://www.infobae.com https://brooklyneagle.com/feed/
https://www.clarin.com https://www.reutersagency.com/feed/
https://www.lanacion.com.ar https://newsatme.com/api/v1/rss/ap/world
https://www.pagina12.com.ar https://feeds.bbci.co.uk/news/world/rss.xml
https://www.cronista.com https://rss.dw.com/rdf/rss-en-all
https://www.france24.com/en/rss
# Colombia https://www3.nhk.or.jp/rss/news/shakaitokushu.xml
https://www.eltiempo.com https://www.cbc.ca/cctoc/rss/topstories.north
https://www.elespectador.com https://www.defensenews.com/arc/outboundfeeds/rss/
https://www.semana.com https://therecord.media/feed
https://www.bluradio.com https://www.cfr.org/rss/newsletters/daily-news-brief
https://www.rcnradio.com https://warontherocks.com/feed/
https://www.thecipherbrief.com/feed
# --- EUROPE --- https://www.foreignaffairs.com/rss.xml
# United Kingdom https://geopoliticalfutures.com/feed
https://www.bbc.com/news # --- TACTICAL CYBER & VULNERABILITIES ---
https://www.theguardian.com/uk https://www.bleepingcomputer.com/feed/
https://news.sky.com https://www.cisa.gov/cybersecurity-advisory-feeds
https://www.independent.co.uk https://krebsonsecurity.com/feed/
https://metro.co.uk https://thehackernews.com/feeds/posts/default
https://www.darkreading.com/rss.xml
# France https://www.mandiant.com/resources/blog/rss.xml
https://www.france24.com/en https://schneier.com/feed/atom/
https://www.lefigaro.fr https://www.securityweek.com/feed/
https://www.20minutes.fr # --- REGIONAL THREAT LANDSCAPE ---
https://www.francetvinfo.fr https://www.thenationalnews.com/rss/
https://www.lemonde.fr https://www.scmp.com/rss/91/feed
https://www.batimes.com.ar/rss
# Germany https://brazilian.report/feed/
https://www.dw.com/en https://www.khon2.com/feed/
https://www.tagesschau.de https://www.staradvertiser.com/feed/
https://www.spiegel.de https://www.westhawaiitoday.com/feed/
https://www.zeit.de https://mauinow.com/feed/
https://www.bild.de https://www.idahofallsidaho.gov/RSSFeed.aspx?ModID=1&CID=All-newsflash.xml
https://www.eastidahonews.com/feed/
# Spain https://localnews8.com/feed/
https://elpais.com https://www.boisestatepublicradio.org/news.rss
https://www.elmundo.es https://www.illinoistimes.com/springfield/Rss.xml
https://www.rtve.es/noticias https://www.thecentersquare.com/search/?f=rss&t=article&l=20&s=start_time&fulltext=showtext&sd=desc&c%5B%5D=Illinois
https://www.20minutos.es https://chicago.suntimes.com/rss/index.xml
https://www.elconfidencial.com https://wgntv.com/feed/
http://feeds.indiana.statenews.net/rss/7b3aa09cdd5d5eac
# Italy https://fox59.com/feed/
https://www.ansa.it https://www.nwitimes.com/search/?f=rss&t=article&c=news/local&l=50&s=start_time&sd=desc
https://www.corriere.it https://www.wishtv.com/feed/
https://www.repubblica.it https://www.kcci.com/topstories-rss
https://www.lastampa.it https://www.myiowainfo.com/feed/
https://tg24.sky.it https://feeds.feedburner.com/radioiowanews
https://www.mississippivalleypublishing.com/search/?f=rss&t=article&c=the_hawk_eye&l=50&s=start_time&sd=desc
# Russia (State & Independent mix) https://www.ksn.com/feed/
https://tass.com https://www.ksnt.com/feed/
https://www.interfax.ru https://www.hdnews.net/feed/
https://www.rt.com https://themercury.com/search/?f=rss&t=article&c=news&l=50&s=start_time&sd=desc
https://www.themoscowtimes.com https://www.wdrb.com/search/?f=rss&t=article&c=news&l=50&s=start_time&sd=desc
https://meduza.io/en https://www.wtvq.com/feed/
https://www.wnky.com/feed/
# --- ASIA --- https://www.wlky.com/topstories-rss
# China https://thehayride.com/feed/
https://www.xinhuanet.com/english https://wgno.com/feed/
https://www.chinadaily.com.cn https://feeds.feedburner.com/wbrz/news
https://www.globaltimes.cn https://thelensnola.org/feed/
https://www.cgtn.com https://www.pressherald.com/news/feed/
https://www.scmp.com https://www.centralmaine.com/feed/
https://www.bangordailynews.com/feed/
# India https://www.sunjournal.com/news/feed/
https://www.ndtv.com https://www.wbaltv.com/topstories-rss
https://timesofindia.indiatimes.com https://www.manisteenews.com/news/feed/Latest-News-Feed-2564.php
https://indianexpress.com https://www.theoaklandpress.com/feed/
https://www.thehindu.com https://www.macombdaily.com/feed/
https://www.hindustantimes.com https://www.startribune.com/local/index.rss2
https://www.wctrib.com/index.rss
# Japan https://www.austindailyherald.com/feed/
https://www3.nhk.or.jp/nhkworld https://helenair.com/search/?f=rss&t=article&l=50&s=start_time&sd=desc
https://www.japantimes.co.jp https://www.ktvq.com/news.rss
https://www.asahi.com/ajw https://mtstandard.com/search/?f=rss&t=article&l=50&s=start_time&sd=desc
https://mainichi.jp/english https://www.ketv.com/topstories-rss
https://english.kyodonews.net https://nebraskaexaminer.com/feed/
https://kearneyhub.com/rss
# South Korea https://www.wowt.com/rss
https://en.yna.co.kr https://thenevadaindependent.com/feed/
https://www.koreaherald.com https://www.8newsnow.com/feed/
https://koreajoongangdaily.joins.com https://www.reviewjournal.com/feed/
https://www.donga.com/en https://thisisreno.com/feed/
https://english.chosun.com https://www.conwaydailysun.com/search/?f=rss&t=article&c=berlin_sun/community/news&l=50&s=start_time&sd=desc
https://newhampshirebulletin.com/feed/
# --- AFRICA --- https://www.nhgazette.com/feed/
# South Africa https://www.nhbr.com/feed/
https://www.news24.com https://www.nj.com/arc/outboundfeeds/rss/?outputType=xml
https://www.iol.co.za https://www.njspotlightnews.org/feed/
https://www.dailymaverick.co.za https://njmonthly.com/feed/
https://www.sabcnews.com https://www.trentonian.com/feed/
https://www.timeslive.co.za https://www.krqe.com/feed/
https://www.santafenewmexican.com/search/?f=rss&t=article&l=50&s=start_time&sd=desc
# Nigeria https://www.easternnewmexiconews.com/rss
https://www.vanguardngr.com https://www.koat.com/topstories-rss
https://punchng.com https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml
https://dailypost.ng https://www.thecity.nyc/feed/
https://saharareporters.com https://www.nbcnewyork.com/?rss=y
https://thenationonlineng.net https://www.wral.com/news/rss/48/
https://www.cbs17.com/news/north-carolina-news/feed/
# --- MIDDLE EAST --- https://abc11.com/feed/
# General Region https://myfox8.com/news/feed/
https://www.aljazeera.com https://www.kxnet.com/feed/
https://english.alarabiya.net https://www.wday.com/feed/
https://www.timesofisrael.com https://www.jamestownsun.com/index.rss
https://www.tehrantimes.com https://www.inforum.com/index.rss
https://www.middleeasteye.net http://rssfeeds.wkyc.com/wkyc/news
https://theohiostar.com/feed/
# --- OCEANIA --- https://feeds.feedblitz.com/wtol/news
# Australia https://www.wcpo.com/news.rss
https://www.abc.net.au/news https://kfor.com/feed/
https://www.news.com.au https://oklahomawatch.org/feed/
https://www.9news.com.au https://freepressokc.com/feed/
https://www.smh.com.au https://osagenews.org/feed/
https://www.theage.com.au http://rssfeeds.kgw.com/kgw/local
# --- USA: MAJOR CITIES & LOCAL --- https://www.koin.com/feed/
https://www.latimes.com https://www.bendsource.com/bend/Rss.xml/feed
https://www.chicagotribune.com https://eugeneweekly.com/feed/
https://www.sfchronicle.com https://www.wtae.com/topstories-rss
https://www.bostonglobe.com https://www.montgomerycountypa.gov/RSSFeed.aspx?ModID=76&CID=All-0
https://www.seattletimes.com https://www.mainlinemedianews.com/feed/
https://www.houstonchronicle.com https://www.dailylocal.com/feed/
https://www.inquirer.com https://www.wpri.com/feed/
https://www.denverpost.com https://www.abc6.com/feed/
https://www.miamiherald.com https://whdh.com/regional/rhode-island/feed/
https://www.dallasnews.com https://warwickpost.com/feed/
https://www.startribune.com https://www.wyff4.com/topstories-rss
https://www.detroitnews.com https://www.wispolitics.com/feed/
https://www.ajc.com https://wiseye.org/feed/
https://www.nydailynews.com https://wisconsinexaminer.com/feed/
https://nypost.com https://trib.com/search/?f=rss&t=article&c=news/state-and-regional&l=50&s=start_time&sd=desc
https://www.mercurynews.com https://wyofile.com/feed/
https://www.baltimoresun.com https://www.wyomingnews.com/search/?f=rss&t=article&c=news&l=50&s=start_time&sd=desc
https://www.oregonlive.com https://www.wyodaily.com/rss
https://www.cleveland.com https://www.wnct.com/news/north-carolina/feed/
https://www.tampabay.com https://www.usnews.com/rss/news/north-carolina
https://indyweek.com/feed/
# --- EUROPE: LOCAL & INDEPENDENT --- https://portcitydaily.com/feed/
https://www.manchestereveningnews.co.uk https://www.theguardian.com/uk/rss
https://www.scotsman.com https://feeds.bbci.co.uk/news/england/rss.xml
https://www.belfasttelegraph.co.uk https://www.lemonde.fr/rss/une.xml
https://www.irishtimes.com https://www.ansa.it/sito/notizie/rss.xml
https://www.berliner-zeitung.de https://www.ilgiornale.it/feed
https://www.leparisien.fr https://www.larepublica.it/rss/homepage/rss2.xml
https://www.corriere.it https://www.sueddeutsche.de/rss
https://www.elperiodico.com https://www.welt.de/feeds/top-news.rss
https://kyivindependent.com https://www.rfi.fr/en/rss
https://www.pravda.com.ua/en https://www.bangkokpost.com/rss
https://balkaninsight.com https://thephnompenhpost.com/rss
https://www.ekathimerini.com https://www.thejakartapost.com/rss
https://www.swissinfo.ch https://www.straitstimes.com/news/singapore/rss.xml
https://www.thelocal.se https://www.channelnewsasia.com/rss
https://www.thelocal.fr https://www.antaranews.com/rss/
https://www.thelocal.de https://www.irrawaddy.com/feed
https://www.novinite.com https://news.abs-cbn.com/rss
https://www.romania-insider.com https://www.hindustantimes.com/feeds/rss
https://hungarytoday.hu https://www.africanews.com/feed/rss
https://polandin.com https://www.clarin.com/rss
https://www.lanacion.com.ar/rss
# --- MIDDLE EAST & CONFLICT ZONES --- https://www.eluniversal.com.mx/rss
https://www.haaretz.com https://www.excelsior.com.mx/rss
https://www.jpost.com https://www.eltiempo.com/rss
https://www.timesofisrael.com https://www.elespectador.com/rss
https://www.rudaw.net/english https://www.larepublica.pe/rss
https://www.kurdistan24.net/en https://www.elcomercio.com/rss
https://www.middleeasteye.net https://www.abc.net.au/news/feed/
https://www.al-monitor.com https://www.smh.com.au/rss/world.xml
https://www.dailysabah.com https://www.theage.com.au/rss
https://www.duvarenglish.com https://www.brisbanetimes.com.au/rss
https://english.aawsat.com https://www.stuff.co.nz/rss
https://www.arabnews.com https://www.nzherald.co.nz/arcio/rss/
https://www.thenationalnews.com https://www.rnz.co.nz/rss
https://www.jordantimes.com https://globalvoices.org/regions/africa/feed/
https://www.naharnet.com https://globalvoices.org/regions/asia/feed/
https://www.tehrantimes.com https://globalvoices.org/regions/latin-america/feed/
https://globalvoices.org/regions/eastern-europe/feed/
# --- ASIA: HOTSPOTS & LOCAL --- https://globalvoices.org/regions/middle-east-north-africa/feed/
https://www.taipeitimes.com https://globalvoices.org/regions/south-asia/feed/
https://focustaiwan.tw https://globalvoices.org/regions/sub-saharan-africa/feed/
https://hongkongfp.com https://globalvoices.org/regions/west-africa/feed/
https://www.bangkokpost.com https://globalvoices.org/regions/east-asia/feed/
https://www.thejakartapost.com https://globalvoices.org/regions/southeast-asia/feed/
https://www.straitstimes.com https://globalvoices.org/regions/central-asia/feed/
https://www.khmertimeskh.com https://globalvoices.org/regions/pacific/feed/
https://www.irrawaddy.com https://globalvoices.org/regions/caribbean/feed/
https://www.myanmarnow.org/en https://www.townandcountry-mo.gov/rss.aspx
https://www.rappler.com https://feeds.smh.com.au/rssheadlines/national.xml
https://www.philstar.com https://www.abc.net.au/local/rss/sydney/
https://english.hani.co.kr https://www.voanews.com/rssfeeds
https://www.japantoday.com https://rss.feedspot.com/southeast_asian_rss_feeds
https://www.caixinglobal.com https://www.crisisgroup.org/rss
https://thediplomat.com https://news.panasonic.com/global/rss/area01/index.xml
https://news.panasonic.com/global/rss/area04/index.xml
# --- LATIN AMERICA & AFRICA: LOCAL --- https://allafrica.com/tools/headlines/rdf/latest/headlines.rdf
https://buenosairesherald.com https://www.afro.who.int/rss-feeds
https://riotimesonline.com https://pressat.co.uk/rss-list
https://mercopress.com https://www.monitor.co.ug/rss
https://www.elmostrador.cl https://www.standardmedia.co.ke/rss
https://www.jornada.com.mx https://www.ft.com/rss/home
https://www.theeastafrican.co.ke https://www.economist.com/rss/the-world-this-week
https://allafrica.com https://feeds.bloomberg.com/economics/news.rss
https://www.premiumtimesng.com https://feeds.bloomberg.com/markets/news.rss
https://www.dailytrust.com https://www.reuters.com/arc/outboundfeeds/newsroom/business/
https://www.newtimes.co.rw https://www.cnbc.com/id/10000113/device/rss/rss.html
https://www.herald.co.zw https://feeds.a.dj.com/rss/RSSWorldBusiness.xml
https://www.namibian.com.na https://www.marketwatch.com/rss/topstories
https://www.graphic.com.gh https://www.investing.com/rss/news_14.rss
https://www.thecitizen.co.tz https://feeds.bbci.co.uk/news/business/rss.xml
https://www.monitor.co.ug https://feeds.feedburner.com/TheHackersNews
https://www.darkreading.com/rss/all.xml
# --- ALTERNATIVE, INVESTIGATIVE & "FRINGE" --- https://isc.sans.edu/rssfeed_full.xml
https://theintercept.com https://securelist.com/feed/
https://www.propublica.org https://feeds.feedburner.com/eset/blog
https://www.democracynow.org https://news.sophos.com/en-us/feed/
https://reason.com https://www.schneier.com/feed/atom/
https://www.motherjones.com https://www.securitymagazine.com/rss/topic/2236-cybersecurity-news
https://www.vox.com https://www.reuters.com/arc/outboundfeeds/rss/?outputType=xml
https://slate.com https://www.investing.com/rss/news_462.rss
https://www.axios.com https://www.investing.com/rss/news_1.rss
https://www.politico.com https://www.investing.com/rss/stock_Futures.rss
https://www.vice.com https://www.ecb.europa.eu/rss/fxref-ecbpress.en.xml
https://www.bellingcat.com https://www.federalreserve.gov/feeds/news-events.xml
https://www.project-syndicate.org https://www.boj.or.jp/en/rss/whatsnew.xml
https://cryptonews.com https://www.bankofengland.co.uk/rss/news
https://www.coindesk.com https://www.centralbanking.com/feeds/rss
https://techcrunch.com https://oilprice.com/rss/
https://www.spglobal.com/commodityinsights/en/rss
https://www.eia.gov/tools/rssfeeds/
https://www.cmegroup.com/rss
https://globalvoices.org/-/topics/economics-business/feed/
http://globalization.einnews.com/rss
https://financefeeds.com/feed/
https://newsquawk.com/blog/feed.rss
https://www.coindesk.com/arc/outboundfeeds/rss/
https://ishookfinance.com/feed/
https://www.scmp.com/rss/92/feed
https://www.scmp.com/rss/93/feed
https://www.scmp.com/rss/94/feed
https://www.scmp.com/rss/317/feed
https://asia.nikkei.com/rss
https://www.caixin.com/rss/index_EN.xml
https://www.straitstimes.com/news/asia/rss.xml
https://www.straitstimes.com/business/rss.xml
https://www.reuters.com/arc/outboundfeeds/rss/?outputType=xml&section=asia
https://www.reuters.com/arc/outboundfeeds/rss/?outputType=xml&section=china
https://www.bloomberg.com/feeds/asia.rss
https://www.bloomberg.com/feeds/markets.rss
https://www.ft.com/asia-pacific?format=rss
https://www.ft.com/china?format=rss
https://english.kyodonews.net/rss/news.xml
https://en.yna.co.kr/RSS/news.xml
https://www.thejakartapost.com/rss/business
https://www.nationthailand.com/rss/business
https://www.aramco.com/api/v1/com/rss/news?sc_lang=en
https://www.worldoil.com/rss?feed=topic:saudi+arabia
https://www.worldoil.com/rss?feed=topic:iraq
https://www.worldoil.com/rss?feed=topic:uae
https://www.worldoil.com/rss?feed=topic:russia
https://www.worldoil.com/rss?feed=topic:canada
https://www.worldoil.com/rss?feed=topic:oil+sands
https://www.rigzone.com/news/europe_russia/production/rss/
https://www.argusmedia.com/en/news-and-insights/latest-market-news/rss
https://www.eia.gov/rss/
https://www.opec.org/opec_web/en/pressreleases.rss
https://www.opec.org
https://www.rosneft.com/press/news/rss/
https://feeds.content.dowjones.io/public/rss/RSSMarketsMain
https://feeds.content.dowjones.io/public/rss/socialeconomyfeed
https://feeds.content.dowjones.io/public/rss/WSJcomUSBusiness
https://feeds.content.dowjones.io/public/rss/RSSWorldNews
http://feeds.feedburner.com/EconomicEventsAgriculture
http://feeds.feedburner.com/EconomicEventsEnergy
http://feeds.feedburner.com/EconomicEventsInterestRates
http://feeds.feedburner.com/mediaroom/CMsF
http://feeds.feedburner.com/CMEClearPortNoticesRss
http://feeds.feedburner.com/GlobexAdvisories
https://feeds.content.dowjones.io/public/rss/mw_topstories
https://feeds.content.dowjones.io/public/rss/mw_realtimeheadlines
http://feeds.marketwatch.com/marketwatch/bulletins
https://feeds.content.dowjones.io/public/rss/mw_marketpulse
https://www.nasdaqtrader.com/rss.aspx?feed=currentheadlines&categorylist=51
https://www.nasdaqtrader.com/rss.aspx?feed=currentheadlines&categorylist=11
https://www.investing.com/rss/stock_Options.rss
https://www.investing.com/rss/news_11.rss
https://www.investing.com/rss/news_25.rss
https://www.nasdaq.com/feed/rssoutbound?category=Markets
https://www.nasdaq.com/feed/rssoutbound?category=Commodities
https://www.barchart.com/news/rss/financials/options-news
https://www.barchart.com/news/rss/commodities/futures-news
https://www.spglobal.com/spdji/en/rss
https://www.litefinance.org/rss/analytics/
https://www.mrt.com/arc/outboundfeeds/rss/category/business/oil/?outputType=xml
https://www.oaoa.com/category/local-news/inthepipeline/rss
https://pboilandgasmagazine.com/feed/
https://www.rigzone.com/news/rss.asp
https://rbnenergy.com/blogcast.rss
https://www.eia.gov/rss/todayinenergy.xml
https://www.firstalert7.com/news/energy
https://www.energyvoice.com/feed/?category=oilandgas/north-sea
https://www.rigzone.com/news/rss/north_sea
https://www.oedigital.com/feeds/rss
https://www.sodir.no/en/whats-new/news/rss
https://www.worldoil.com/rss?feed=topic:offshore
https://oilandgas.einnews.com/rss/north-sea-offshore
https://www.energyvoice.com/feed/

28
news/summerizer/run_news_summarizer.py
View file

@ -1,14 +1,10 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Scheduler loop for the news summarizer — hourly summarize at minute :05. """Scheduler loop for the news summarizer — every NEWS_SUMMARIZE_INTERVAL_S.
Replaces the k8s CronJob (`5 * * * *`) with an in-compose loop. Runs once on Default 900s (15 minutes). Serial: a slow LLM pass never overlaps the next.
boot (catches up on any articles scraped since the last summary), then fires
at each :NEWS_SUMMARIZE_MINUTE wall-clock boundary.
The loop is serial, so a slow LLM pass never overlaps the next run.
Env (all optional, 12-factor): Env (all optional, 12-factor):
NEWS_SUMMARIZE_MINUTE minute of the hour to fire (default 5) NEWS_SUMMARIZE_INTERVAL_S seconds between runs (default 900)
NEWS_SUMMARIZE_RUN_ON_START "1" to summarize once immediately on boot (default 1) NEWS_SUMMARIZE_RUN_ON_START "1" to summarize once immediately on boot (default 1)
NOUS_API_KEY optional in env; Keys UI / api_keys also works NOUS_API_KEY optional in env; Keys UI / api_keys also works
""" """
@ -25,17 +21,10 @@ import time
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
logger = logging.getLogger("news.summarizer.scheduler") logger = logging.getLogger("news.summarizer.scheduler")
MINUTE = int(os.getenv("NEWS_SUMMARIZE_MINUTE", "5")) INTERVAL_S = max(1, int(os.getenv("NEWS_SUMMARIZE_INTERVAL_S", "900")))
RUN_ON_START = os.getenv("NEWS_SUMMARIZE_RUN_ON_START", "1").lower() in ("1", "true", "yes") RUN_ON_START = os.getenv("NEWS_SUMMARIZE_RUN_ON_START", "1").lower() in ("1", "true", "yes")
def seconds_until_next(minute: int) -> float:
"""Seconds until the next occurrence of ``minute`` past the hour (local time)."""
now = datetime.datetime.now()
nxt = now.replace(minute=minute, second=0, microsecond=0) + datetime.timedelta(hours=1)
return (nxt - now).total_seconds()
def run_summarize() -> None: def run_summarize() -> None:
logger.info("summarize starting at %s", datetime.datetime.now().isoformat(timespec="seconds")) logger.info("summarize starting at %s", datetime.datetime.now().isoformat(timespec="seconds"))
try: try:
@ -51,15 +40,14 @@ def main() -> None:
"NOUS_API_KEY unset in env — will read api_keys on each run; idle if both empty" "NOUS_API_KEY unset in env — will read api_keys on each run; idle if both empty"
) )
logger.info( logger.info(
"news summarizer loop starting (minute=%s, run_on_start=%s)", "news summarizer loop starting (interval_s=%s, run_on_start=%s)",
MINUTE, RUN_ON_START, INTERVAL_S, RUN_ON_START,
) )
if RUN_ON_START: if RUN_ON_START:
run_summarize() run_summarize()
while True: while True:
delay = seconds_until_next(MINUTE) logger.info("next summarize in %ss", INTERVAL_S)
logger.info("next summarize at :%02d (in %.0fs)", MINUTE, delay) time.sleep(INTERVAL_S)
time.sleep(delay)
run_summarize() run_summarize()

52
news/summerizer/summarizer.py
View file

@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""News summarizer — Nous map-reduce of scraped articles into brief/ticker/map. """News summarizer — Nous map-reduce of scraped articles into brief/ticker/map.
Reads articles scraped within the last hour from the shared `articles` table, Reads articles scraped within the last SUMMARY_WINDOW_MINUTES from the shared `articles` table,
maps them with Nous (per-article English fact blocks), reduces to one JSON maps them with Nous (per-article English fact blocks), reduces to one JSON
object (summary_en + ticker + map_items), and stores the brief in object (summary_en + ticker + map_items), and stores the brief in
`article_summaries` plus flagged rows in `news_items`. Tables live in the `article_summaries` plus flagged rows in `news_items`. Tables live in the
@ -15,11 +15,8 @@ of each summarize_news() — env wins, else api_keys / app_settings:
NOUS_BASE_URL default https://inference-api.nousresearch.com/v1 NOUS_BASE_URL default https://inference-api.nousresearch.com/v1
SUMMARY_MODEL default Hermes-4.3-36B (else app_settings) SUMMARY_MODEL default Hermes-4.3-36B (else app_settings)
BATCH_SIZE articles per map-phase batch (default 50) BATCH_SIZE articles per map-phase batch (default 50)
SUMMARY_WINDOW_HOURS look-back window in hours (default 1) SUMMARY_WINDOW_MINUTES look-back window (default 15; SUMMARY_WINDOW_HOURS wins if set)
OSINT_USER_AGENT default osint-dashboard-news-summarizer NEWS_SUMMARIZE_FORCE "1" to ignore the interval idempotency skip
MAP_PROMPT override map-phase prompt (uses {batch_text})
SUMMARY_PROMPT override reduce-phase prompt (uses {final_input})
NEWS_SUMMARIZE_FORCE "1" to ignore the current-UTC-hour idempotency skip
INCLUDE_FUTURES "1" to prepend live futures prices (default 0) INCLUDE_FUTURES "1" to prepend live futures prices (default 0)
The futures/markets coupling from the original pipeline is gated behind The futures/markets coupling from the original pipeline is gated behind
@ -54,7 +51,23 @@ DB_CONFIG = {
DEFAULT_NOUS_BASE_URL = "https://inference-api.nousresearch.com/v1" DEFAULT_NOUS_BASE_URL = "https://inference-api.nousresearch.com/v1"
DEFAULT_SUMMARY_MODEL = "Hermes-4.3-36B" DEFAULT_SUMMARY_MODEL = "Hermes-4.3-36B"
BATCH_SIZE = int(os.getenv("BATCH_SIZE", "50")) BATCH_SIZE = int(os.getenv("BATCH_SIZE", "50"))
SUMMARY_WINDOW_HOURS = int(os.getenv("SUMMARY_WINDOW_HOURS", "1"))
def _summary_window_minutes() -> int:
hours = (os.getenv("SUMMARY_WINDOW_HOURS") or "").strip()
if hours:
return max(1, int(hours) * 60)
mins = (os.getenv("SUMMARY_WINDOW_MINUTES") or "").strip()
if mins:
return max(1, int(mins))
return 15
def _summarize_interval_seconds() -> int:
return max(1, int(os.getenv("NEWS_SUMMARIZE_INTERVAL_S", "900")))
SUMMARY_WINDOW_MINUTES = _summary_window_minutes()
INCLUDE_FUTURES = os.getenv("INCLUDE_FUTURES", "0").lower() in ("1", "true", "yes") INCLUDE_FUTURES = os.getenv("INCLUDE_FUTURES", "0").lower() in ("1", "true", "yes")
# Only touched when INCLUDE_FUTURES=1 (legacy markets coupling, OSINT-off). # Only touched when INCLUDE_FUTURES=1 (legacy markets coupling, OSINT-off).
@ -282,18 +295,18 @@ def ensure_tables() -> None:
def get_recent_news() -> list[dict]: def get_recent_news() -> list[dict]:
"""Fetch articles from the last SUMMARY_WINDOW_HOURS (content > 100 chars).""" """Fetch articles from the last SUMMARY_WINDOW_MINUTES (content > 100 chars)."""
query = """ query = """
SELECT title, content, url, domain SELECT title, content, url, domain
FROM articles FROM articles
WHERE timestamp > NOW() - make_interval(hours => %s) WHERE timestamp > NOW() - make_interval(mins => %s)
AND content IS NOT NULL AND length(content) > 100 AND content IS NOT NULL AND length(content) > 100
ORDER BY timestamp DESC; ORDER BY timestamp DESC;
""" """
try: try:
conn = psycopg2.connect(**DB_CONFIG) conn = psycopg2.connect(**DB_CONFIG)
cur = conn.cursor() cur = conn.cursor()
cur.execute(query, (SUMMARY_WINDOW_HOURS,)) cur.execute(query, (SUMMARY_WINDOW_MINUTES,))
rows = cur.fetchall() rows = cur.fetchall()
cur.close() cur.close()
conn.close() conn.close()
@ -306,24 +319,24 @@ def get_recent_news() -> list[dict]:
return [] return []
def _already_summarized_this_hour() -> bool: def _already_summarized_this_interval() -> bool:
"""True when article_summaries already has a row for the current UTC hour.""" """True when article_summaries already has a row in the last interval."""
if os.getenv("NEWS_SUMMARIZE_FORCE", "") == "1": if os.getenv("NEWS_SUMMARIZE_FORCE", "") == "1":
return False return False
query = ( query = (
"SELECT 1 FROM article_summaries " "SELECT 1 FROM article_summaries "
"WHERE batch_timestamp >= date_trunc('hour', NOW() AT TIME ZONE 'utc')" "WHERE batch_timestamp >= NOW() - make_interval(secs => %s)"
) )
try: try:
conn = psycopg2.connect(**DB_CONFIG) conn = psycopg2.connect(**DB_CONFIG)
cur = conn.cursor() cur = conn.cursor()
cur.execute(query) cur.execute(query, (_summarize_interval_seconds(),))
row = cur.fetchone() row = cur.fetchone()
cur.close() cur.close()
conn.close() conn.close()
return row is not None return row is not None
except Exception as exc: # noqa: BLE001 except Exception as exc: # noqa: BLE001
logger.error("Error checking hourly idempotency: %s", exc) logger.error("Error checking interval idempotency: %s", exc)
return False return False
@ -419,10 +432,11 @@ def build_master_prompt(final_input: str) -> str:
def summarize_news() -> None: def summarize_news() -> None:
"""Map-reduce summarize recent articles and store brief + ticker + map.""" """Map-reduce summarize recent articles and store brief + ticker + map."""
ensure_tables() ensure_tables()
if _already_summarized_this_hour(): if _already_summarized_this_interval():
logger.info( logger.info(
"Skipping summarize: article_summaries already has a row this UTC hour " "Skipping summarize: article_summaries already has a row in the last %ss "
"(set NEWS_SUMMARIZE_FORCE=1 to override)" "(set NEWS_SUMMARIZE_FORCE=1 to override)",
_summarize_interval_seconds(),
) )
return return
@ -435,7 +449,7 @@ def summarize_news() -> None:
articles = get_recent_news() articles = get_recent_news()
if not articles: if not articles:
logger.info("No new articles found in the last %sh.", SUMMARY_WINDOW_HOURS) logger.info("No new articles found in the last %s min.", SUMMARY_WINDOW_MINUTES)
return return
logger.info( logger.info(