feat: continuous news scrape + 15-min analyst, expand urls.txt #11
11 changed files with 434 additions and 376 deletions
17
.env.example
17
.env.example
|
|
@ -67,10 +67,8 @@ INGEST_FIRES=1
|
|||
# its lookup/fallback is a planned follow-up.
|
||||
|
||||
# ── News pipeline (scraper + summarizer, profile `ingest`) ────────────────
|
||||
# Hourly: the scraper crawls 257 RSS sources at minute :00 and the summarizer
|
||||
# runs the Nous map-reduce at minute :05, both writing to the shared osint-db
|
||||
# (tables `articles` + `article_summaries`, created by alembic 003_news).
|
||||
# Consume via GET /api/news and GET /api/news/summaries.
|
||||
# Scraper crawls urls.txt continuously (default 10s between crawls).
|
||||
# Summarizer runs Nous map-reduce every 15 min (NEWS_SUMMARIZE_INTERVAL_S=900).
|
||||
# NOUS_API_KEY is also (preferably) set in the Keys UI; env is an override.
|
||||
# Unset in both env and api_keys = summarizer logs and idles (never crashes).
|
||||
NOUS_API_KEY=
|
||||
|
|
@ -81,18 +79,15 @@ NOUS_BASE_URL=https://inference-api.nousresearch.com/v1
|
|||
# Hermes-4.3-36B remains after a Postgres miss. Env wins when set.
|
||||
# SUMMARY_MODEL=
|
||||
NEWS_BATCH_SIZE=50
|
||||
SUMMARY_WINDOW_HOURS=1
|
||||
SUMMARY_WINDOW_MINUTES=15
|
||||
# Futures/markets coupling from the upstream pipeline is OFF by default
|
||||
# (irrelevant to OSINT). Set INCLUDE_FUTURES=1 + install yfinance to enable.
|
||||
INCLUDE_FUTURES=0
|
||||
# Wall-clock scheduling (k8s CronJob replacement): scrape minute, summarize minute
|
||||
NEWS_SCRAPE_MINUTE=0
|
||||
NEWS_SUMMARIZE_MINUTE=5
|
||||
# Run once immediately on container start (seeds data fast), then align to the
|
||||
# scheduled minute.
|
||||
NEWS_SCRAPE_INTERVAL_S=10
|
||||
NEWS_SUMMARIZE_INTERVAL_S=900
|
||||
NEWS_SCRAPE_RUN_ON_START=1
|
||||
NEWS_SUMMARIZE_RUN_ON_START=1
|
||||
# "1" ignores the current-UTC-hour idempotency skip (double-pins on recreate).
|
||||
# "1" ignores the interval idempotency skip (double-pins on recreate).
|
||||
NEWS_SUMMARIZE_FORCE=0
|
||||
NEWS_LOG_LEVEL=INFO
|
||||
# Reserved for the (out-of-scope) Telegram delivery bot.
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ KEY_REGISTRY: dict[str, dict] = {
|
|||
"example": "32-char hex string (e.g. 5f3c…9a02)",
|
||||
},
|
||||
"NOUS_API_KEY": {
|
||||
"description": "Nous Portal API key — hourly news summarizer (inference-api.nousresearch.com).",
|
||||
"description": "Nous Portal API key — 15-min news summarizer (inference-api.nousresearch.com).",
|
||||
"pattern": r"^.{16,}$",
|
||||
"example": "key from https://portal.nousresearch.com (API keys page)",
|
||||
},
|
||||
|
|
|
|||
|
|
@ -1084,7 +1084,7 @@ async def camera_hls_segment(camera_id: UUID, u: str = Query(..., min_length=8))
|
|||
|
||||
# ── News pipeline (scraper + summarizer) ──────────────────────────────────
|
||||
# Backing data for the frontend news panel. Written by the vendored
|
||||
# news-scraper (hourly Scrapy crawl) and news-summarizer (hourly Gemini
|
||||
# news-scraper (continuous Scrapy crawl) and news-summarizer (15-min Nous
|
||||
# map-reduce) services into the shared osint-db.
|
||||
|
||||
@app.get("/api/news", response_model=list[NewsArticleOut])
|
||||
|
|
|
|||
|
|
@ -731,7 +731,7 @@
|
|||
<label class="lp-name"><input type="checkbox" id="lp-news-on" checked onchange="toggleNewsPins()"> <span class="lp-dot news"></span> Critical News</label>
|
||||
<span class="lp-count" id="lp-news-count">0</span>
|
||||
</div>
|
||||
<div class="lp-note">LLM-estimated locations from the hourly brief. Pins only for critical/high.</div>
|
||||
<div class="lp-note">LLM-estimated locations from the 15-min brief. Pins only for critical/high.</div>
|
||||
</div>
|
||||
|
||||
<div class="lp-layer">
|
||||
|
|
@ -828,7 +828,7 @@
|
|||
<section class="view" id="view-news" aria-label="News brief">
|
||||
<div class="subview">
|
||||
<h2><span class="tick">◉</span> Global News Brief</h2>
|
||||
<p class="sub">Scraped on the 15-minute cycle · latest LLM executive summary pinned above</p>
|
||||
<p class="sub">Scraped continuously · latest LLM executive summary pinned above</p>
|
||||
<div class="news-head">
|
||||
<div class="news-refresh">
|
||||
<span class="news-updated" id="news-updated">—</span>
|
||||
|
|
@ -994,7 +994,7 @@
|
|||
</div>
|
||||
<div class="panel">
|
||||
<h3>News Summarizer</h3>
|
||||
<p class="set-note" style="margin-bottom:0.9rem">Takes effect on the next hourly run (:05). No container restart.</p>
|
||||
<p class="set-note" style="margin-bottom:0.9rem">Takes effect on the next 15-min run. No container restart.</p>
|
||||
<div class="set-row">
|
||||
<label>Provider</label>
|
||||
<span>Nous Portal (inference-api.nousresearch.com)</span>
|
||||
|
|
@ -1062,7 +1062,7 @@
|
|||
<div class="sys-row"><span class="k">Active sources</span><span class="v" id="sys-sources">—</span></div>
|
||||
<div class="sys-row"><span class="k">Open alerts</span><span class="v" id="sys-alerts">—</span></div>
|
||||
<div class="sys-row"><span class="k">Tracked entities</span><span class="v" id="sys-entities">—</span></div>
|
||||
<div class="sys-row"><span class="k">News cycle</span><span class="v">hourly :05</span></div>
|
||||
<div class="sys-row"><span class="k">News cycle</span><span class="v">analyst 15m · scraper continuous</span></div>
|
||||
<div class="sys-row"><span class="k">Market feed</span><span class="v" id="sys-market">STANDBY</span></div>
|
||||
</div>
|
||||
<div class="set-actions">
|
||||
|
|
@ -1088,7 +1088,7 @@
|
|||
<div class="tick-wrap">
|
||||
<div class="tick-track" id="nt-track"></div>
|
||||
</div>
|
||||
<div class="tick-tag" id="nt-tag">HOURLY</div>
|
||||
<div class="tick-tag" id="nt-tag">15 MIN</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
|
|
@ -1385,7 +1385,7 @@ function renderNewsList(articles) {
|
|||
if (!listEl) return;
|
||||
document.getElementById('news-count').textContent = articles.length + ' articles';
|
||||
if (!articles || !articles.length) {
|
||||
listEl.innerHTML = '<div class="news-empty">No articles scraped yet — the scraper runs hourly at :00. Check news-scraper logs.</div>';
|
||||
listEl.innerHTML = '<div class="news-empty">No articles scraped yet — the scraper runs continuously. Check news-scraper logs.</div>';
|
||||
return;
|
||||
}
|
||||
listEl.innerHTML = articles.map(a => {
|
||||
|
|
|
|||
|
|
@ -176,7 +176,7 @@ services:
|
|||
volumes:
|
||||
- camera-snapshots:/data/snapshots
|
||||
|
||||
# ── News pipeline: hourly scraper (:00) + summarizer (:05) ───────────────
|
||||
# ── News pipeline: continuous scraper + 15-min summarizer ───────────────
|
||||
# Both services point at the EXISTING osint-db (tables articles +
|
||||
# article_summaries, created by idempotent alembic migration 003_news).
|
||||
# Scheduling replaces the upstream k8s CronJobs with in-compose wall-clock
|
||||
|
|
@ -200,7 +200,7 @@ services:
|
|||
DB_PORT: ${DB_PORT:-5432}
|
||||
DB_NAME: ${DB_NAME:-osint_data}
|
||||
LOG_LEVEL: ${NEWS_LOG_LEVEL:-INFO}
|
||||
NEWS_SCRAPE_MINUTE: ${NEWS_SCRAPE_MINUTE:-0}
|
||||
NEWS_SCRAPE_INTERVAL_S: ${NEWS_SCRAPE_INTERVAL_S:-10}
|
||||
NEWS_SCRAPE_RUN_ON_START: ${NEWS_SCRAPE_RUN_ON_START:-1}
|
||||
# Override the image ENTRYPOINT ["scrapy"] with the scheduler loop.
|
||||
entrypoint: []
|
||||
|
|
@ -229,9 +229,9 @@ services:
|
|||
SUMMARY_MODEL: ${SUMMARY_MODEL:-}
|
||||
OSINT_USER_AGENT: ${OSINT_USER_AGENT:-osint-dashboard-news-summarizer}
|
||||
BATCH_SIZE: ${NEWS_BATCH_SIZE:-50}
|
||||
SUMMARY_WINDOW_HOURS: ${SUMMARY_WINDOW_HOURS:-1}
|
||||
SUMMARY_WINDOW_MINUTES: ${SUMMARY_WINDOW_MINUTES:-15}
|
||||
INCLUDE_FUTURES: ${INCLUDE_FUTURES:-0}
|
||||
NEWS_SUMMARIZE_MINUTE: ${NEWS_SUMMARIZE_MINUTE:-5}
|
||||
NEWS_SUMMARIZE_INTERVAL_S: ${NEWS_SUMMARIZE_INTERVAL_S:-900}
|
||||
NEWS_SUMMARIZE_RUN_ON_START: ${NEWS_SUMMARIZE_RUN_ON_START:-1}
|
||||
NEWS_SUMMARIZE_FORCE: ${NEWS_SUMMARIZE_FORCE:-0}
|
||||
command: ["python", "run_news_summarizer.py"]
|
||||
|
|
|
|||
60
docs/news.md
60
docs/news.md
|
|
@ -1,22 +1,22 @@
|
|||
# News pipeline — scraper + Nous Portal summarizer
|
||||
|
||||
The OSINT dashboard ingests ~257 global news RSS sources hourly and produces
|
||||
an English LLM brief plus flagged ticker/map rows. Both services were vendored
|
||||
from the upstream `~/Projects/newsPipeline` project and re-integrated here to
|
||||
replace the old k8s CronJob choreography with in-compose scheduling against
|
||||
the EXISTING osint-db — **no second Postgres**. The LLM is **Nous Portal**
|
||||
The OSINT dashboard ingests a large curated feed list (`news/scraper/urls.txt`)
|
||||
continuously and produces an English LLM brief plus flagged ticker/map rows
|
||||
every 15 minutes. Both services were vendored from the upstream
|
||||
`~/Projects/newsPipeline` project and re-integrated here against the EXISTING
|
||||
osint-db — **no second Postgres**. The LLM is **Nous Portal**
|
||||
(`inference-api.nousresearch.com`) — not Gemini.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
257 RSS feeds (news/scraper/urls.txt)
|
||||
urls.txt (RSS + homepages)
|
||||
│
|
||||
▼
|
||||
news-scraper (Scrapy, hourly :00) ──► articles table (osint-db)
|
||||
news-scraper (Scrapy, continuous) ──► articles table (osint-db)
|
||||
│ │
|
||||
│ ▼
|
||||
news-summarizer (Nous Portal map-reduce, :05) ──► article_summaries + news_items
|
||||
news-summarizer (Nous Portal, every 15m) ──► article_summaries + news_items
|
||||
│
|
||||
▼
|
||||
GET /api/news · /api/news/summaries · /api/news/ticker · /api/news/map
|
||||
|
|
@ -25,8 +25,8 @@ news-summarizer (Nous Portal map-reduce, :05) ──► article_summaries + news
|
|||
|
||||
| Component | Image | Container | Scheduling |
|
||||
|---|---|---|---|
|
||||
| Scraper | `localhost/osint-news-scraper` | `osint-news-scraper` | wall-clock loop, minute `NEWS_SCRAPE_MINUTE` (default :00) |
|
||||
| Summarizer | `localhost/osint-news-summarizer` | `osint-news-summarizer` | wall-clock loop, minute `NEWS_SUMMARIZE_MINUTE` (default :05) |
|
||||
| Scraper | `localhost/osint-news-scraper` | `osint-news-scraper` | loop, `NEWS_SCRAPE_INTERVAL_S` (default 10s after each crawl) |
|
||||
| Summarizer | `localhost/osint-news-summarizer` | `osint-news-summarizer` | loop, `NEWS_SUMMARIZE_INTERVAL_S` (default 900s) |
|
||||
|
||||
Both services live under the `ingest` compose profile (same as the ingester
|
||||
and camera-scraper): `docker compose --profile ingest up -d`.
|
||||
|
|
@ -39,26 +39,22 @@ feeds (`GET /api/news` exact key set is unchanged on purpose).
|
|||
## Data flow
|
||||
|
||||
1. **Scraper** — `news/scraper/run_news_scraper.py` runs
|
||||
`scrapy crawl articles` (spider `news/scraper/newsScraper/spiders/news_spider.py`)
|
||||
at the top of each hour. The spider reads the RSS feed URLs from `urls.txt`,
|
||||
follows each `<item>` link, extracts the main article body, and the
|
||||
`PostgresPipeline` writes to `articles` with URL-based dedup
|
||||
`scrapy crawl articles` back-to-back (default 10s pause). The spider reads
|
||||
URLs from `urls.txt` (homepages autodiscover RSS; feed URLs are parsed
|
||||
directly), follows each `<item>` link, extracts the main article body, and
|
||||
the `PostgresPipeline` writes to `articles` with URL-based dedup
|
||||
(`ON CONFLICT (url) DO NOTHING`).
|
||||
2. **Summarizer** — `news/summerizer/run_news_summarizer.py` runs
|
||||
`summarizer.py` at :05 past each hour. It reads articles from the last
|
||||
`SUMMARY_WINDOW_HOURS`, map-reduces them through Nous Portal
|
||||
(`SUMMARY_MODEL` / Settings, default `Hermes-4.3-36B`), writes the English
|
||||
brief to `article_summaries` (column `model` is the LLM id), and flagged
|
||||
ticker/map rows to `news_items`.
|
||||
`summarizer.py` every `NEWS_SUMMARIZE_INTERVAL_S` (default 900). It reads
|
||||
articles from the last `SUMMARY_WINDOW_MINUTES` (default 15), map-reduces
|
||||
them through Nous Portal (`SUMMARY_MODEL` / Settings, default
|
||||
`Hermes-4.3-36B`), writes the English brief to `article_summaries` (column
|
||||
`model` is the LLM id), and flagged ticker/map rows to `news_items`.
|
||||
|
||||
Scheduling is done with small in-compose wall-clock loops (not host cron): each
|
||||
loop runs once on boot (`*_RUN_ON_START=1`, seeds data fast) then sleeps until
|
||||
the next scheduled minute. The loop is serial, so a run that overruns its slot
|
||||
simply shifts to the next boundary — two crawls/summaries never overlap.
|
||||
|
||||
Hour-truncation idempotency: if `article_summaries` already has a row for the
|
||||
current UTC hour, the summarizer **skips** (prevents double-pins on
|
||||
`RUN_ON_START` recreate). Set `NEWS_SUMMARIZE_FORCE=1` to ignore that skip.
|
||||
Loops are serial (two crawls/summaries never overlap). Interval idempotency:
|
||||
if `article_summaries` already has a row in the last interval, the summarizer
|
||||
**skips** (prevents double-pins on `RUN_ON_START` recreate). Set
|
||||
`NEWS_SUMMARIZE_FORCE=1` to ignore that skip.
|
||||
|
||||
The `articles` and `article_summaries` tables are created by the idempotent
|
||||
alembic migration `003_news` (also created by the scraper's own
|
||||
|
|
@ -238,13 +234,13 @@ lands in `article_summaries.summary_text`.
|
|||
| `NOUS_BASE_URL` | `https://inference-api.nousresearch.com/v1` | Read-only in Settings. |
|
||||
| `SUMMARY_MODEL` | `Hermes-4.3-36B` | Compose default. Operator-facing choice is Settings → `app_settings.SUMMARY_MODEL`. |
|
||||
| `NEWS_BATCH_SIZE` | `50` | Articles per map-phase batch (compose maps to container `BATCH_SIZE`). |
|
||||
| `SUMMARY_WINDOW_HOURS` | `1` | How far back the summarizer looks for new articles. |
|
||||
| `INCLUDE_FUTURES` | `0` | Legacy futures-prices coupling (upstream pipeline). OFF for OSINT; set `1` + install `yfinance` to enable. |
|
||||
| `NEWS_SCRAPE_MINUTE` | `0` | Wall-clock minute the scraper fires. |
|
||||
| `NEWS_SUMMARIZE_MINUTE` | `5` | Wall-clock minute the summarizer fires. |
|
||||
| `SUMMARY_WINDOW_MINUTES` | `15` | How far back the summarizer looks for new articles. |
|
||||
| `NEWS_SCRAPE_INTERVAL_S` | `10` | Pause after each crawl before the next (scraper is otherwise continuous). |
|
||||
| `NEWS_SUMMARIZE_INTERVAL_S` | `900` | Seconds between analyst runs (default 15 min). |
|
||||
| `NEWS_SCRAPE_RUN_ON_START` | `1` | Run one scrape immediately on container start. |
|
||||
| `NEWS_SUMMARIZE_RUN_ON_START` | `1` | Run one summarize immediately on container start. |
|
||||
| `NEWS_SUMMARIZE_FORCE` | `0` | `1` ignores the current-UTC-hour idempotency skip (double-pins on recreate). |
|
||||
| `NEWS_SUMMARIZE_FORCE` | `0` | `1` ignores the interval idempotency skip (double-pins on recreate). |
|
||||
| `INCLUDE_FUTURES` | `0` | Legacy futures-prices coupling. OFF for OSINT; set `1` + install `yfinance` to enable. |
|
||||
| `NEWS_LOG_LEVEL` | `INFO` | Scrapy log level. |
|
||||
| `OSINT_USER_AGENT` | `osint-dashboard-news-summarizer` | Sent on every outbound Nous call. |
|
||||
| `TELEGRAM_TOKEN` / `TELEGRAM_CHAT_ID` | *(blank)* | Reserved for the (out-of-scope) Telegram delivery bot. |
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import re
|
|||
class NewsRSSSpider(Spider):
|
||||
"""Crawl the curated news sources in urls.txt and extract articles.
|
||||
|
||||
urls.txt contains 257 news HOMEPAGES (not feed URLs), so this spider
|
||||
urls.txt contains curated news HOMEPAGES and RSS/Atom feeds, so this spider
|
||||
implements feed autodiscovery: it fetches each start URL, finds the
|
||||
RSS/Atom feed link (`<link rel="alternate" type="application/rss+xml">`
|
||||
or a visible /rss|/feed link), follows it, and then follows each feed
|
||||
|
|
|
|||
|
|
@ -1,18 +1,12 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Scheduler loop for the news scraper — hourly scrape at minute :00.
|
||||
"""Scheduler loop for the news scraper — crawl continuously.
|
||||
|
||||
Replaces the k8s CronJob (`0 * * * *`) with an in-compose loop so the whole
|
||||
news pipeline lives inside docker-compose. Each iteration:
|
||||
|
||||
1. (optionally, on first boot) runs the Scrapy crawl once to seed data fast
|
||||
2. sleeps until the next :NEWS_SCRAPE_MINUTE wall-clock boundary
|
||||
|
||||
Because the loop is serial, a crawl that overruns its hour simply delays the
|
||||
next run to the following boundary — two crawls never overlap.
|
||||
As soon as one Scrapy pass finishes, wait NEWS_SCRAPE_INTERVAL_S seconds
|
||||
and start the next. Two crawls never overlap (the loop is serial).
|
||||
|
||||
Env (all optional, 12-factor):
|
||||
NEWS_SCRAPE_MINUTE minute of the hour to fire (default 0)
|
||||
NEWS_SCRAPE_RUN_ON_START "1" to crawl once immediately on boot (default 1)
|
||||
NEWS_SCRAPE_INTERVAL_S seconds between crawls (default 10)
|
||||
NEWS_SCRAPE_RUN_ON_START "1" to crawl immediately on boot (default 1)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -27,19 +21,12 @@ import time
|
|||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
logger = logging.getLogger("news.scraper")
|
||||
|
||||
MINUTE = int(os.getenv("NEWS_SCRAPE_MINUTE", "0"))
|
||||
INTERVAL_S = max(0, int(os.getenv("NEWS_SCRAPE_INTERVAL_S", "10")))
|
||||
RUN_ON_START = os.getenv("NEWS_SCRAPE_RUN_ON_START", "1").lower() in ("1", "true", "yes")
|
||||
|
||||
CRAWL_CMD = ["scrapy", "crawl", "articles"]
|
||||
|
||||
|
||||
def seconds_until_next(minute: int) -> float:
|
||||
"""Seconds until the next occurrence of ``minute`` past the hour (local time)."""
|
||||
now = datetime.datetime.now()
|
||||
nxt = now.replace(minute=minute, second=0, microsecond=0) + datetime.timedelta(hours=1)
|
||||
return (nxt - now).total_seconds()
|
||||
|
||||
|
||||
def run_crawl() -> None:
|
||||
logger.info("scrape starting at %s", datetime.datetime.now().isoformat(timespec="seconds"))
|
||||
try:
|
||||
|
|
@ -51,15 +38,14 @@ def run_crawl() -> None:
|
|||
|
||||
def main() -> None:
|
||||
logger.info(
|
||||
"news scraper loop starting (minute=%s, run_on_start=%s)",
|
||||
MINUTE, RUN_ON_START,
|
||||
"news scraper loop starting (interval_s=%s, run_on_start=%s)",
|
||||
INTERVAL_S, RUN_ON_START,
|
||||
)
|
||||
if RUN_ON_START:
|
||||
run_crawl()
|
||||
while True:
|
||||
delay = seconds_until_next(MINUTE)
|
||||
logger.info("next scrape at :%02d (in %.0fs)", MINUTE, delay)
|
||||
time.sleep(delay)
|
||||
logger.info("next scrape in %ss", INTERVAL_S)
|
||||
time.sleep(INTERVAL_S)
|
||||
run_crawl()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,257 +1,336 @@
|
|||
# --- NORTH AMERICA ---
|
||||
# USA
|
||||
https://www.npr.org
|
||||
https://www.pbs.org/newshour
|
||||
https://www.usatoday.com
|
||||
https://www.cbsnews.com
|
||||
https://www.nbcnews.com
|
||||
|
||||
# Canada
|
||||
https://www.cbc.ca/news
|
||||
https://www.ctvnews.ca
|
||||
https://globalnews.ca
|
||||
https://nationalpost.com
|
||||
https://www.thestar.com
|
||||
|
||||
# Mexico
|
||||
https://www.eluniversal.com.mx
|
||||
https://www.milenio.com
|
||||
https://www.jornada.com.mx
|
||||
https://www.excelsior.com.mx
|
||||
https://aristeguinoticias.com
|
||||
|
||||
# --- SOUTH AMERICA ---
|
||||
# Brazil
|
||||
https://g1.globo.com
|
||||
https://www.uol.com.br
|
||||
https://agenciabrasil.ebc.com.br
|
||||
https://www.metropoles.com
|
||||
https://www.terra.com.br/noticias
|
||||
|
||||
# Argentina
|
||||
https://www.infobae.com
|
||||
https://www.clarin.com
|
||||
https://www.lanacion.com.ar
|
||||
https://www.pagina12.com.ar
|
||||
https://www.cronista.com
|
||||
|
||||
# Colombia
|
||||
https://www.eltiempo.com
|
||||
https://www.elespectador.com
|
||||
https://www.semana.com
|
||||
https://www.bluradio.com
|
||||
https://www.rcnradio.com
|
||||
|
||||
# --- EUROPE ---
|
||||
# United Kingdom
|
||||
https://www.bbc.com/news
|
||||
https://www.theguardian.com/uk
|
||||
https://news.sky.com
|
||||
https://www.independent.co.uk
|
||||
https://metro.co.uk
|
||||
|
||||
# France
|
||||
https://www.france24.com/en
|
||||
https://www.lefigaro.fr
|
||||
https://www.20minutes.fr
|
||||
https://www.francetvinfo.fr
|
||||
https://www.lemonde.fr
|
||||
|
||||
# Germany
|
||||
https://www.dw.com/en
|
||||
https://www.tagesschau.de
|
||||
https://www.spiegel.de
|
||||
https://www.zeit.de
|
||||
https://www.bild.de
|
||||
|
||||
# Spain
|
||||
https://elpais.com
|
||||
https://www.elmundo.es
|
||||
https://www.rtve.es/noticias
|
||||
https://www.20minutos.es
|
||||
https://www.elconfidencial.com
|
||||
|
||||
# Italy
|
||||
https://www.ansa.it
|
||||
https://www.corriere.it
|
||||
https://www.repubblica.it
|
||||
https://www.lastampa.it
|
||||
https://tg24.sky.it
|
||||
|
||||
# Russia (State & Independent mix)
|
||||
https://tass.com
|
||||
https://www.interfax.ru
|
||||
https://www.rt.com
|
||||
https://www.themoscowtimes.com
|
||||
https://meduza.io/en
|
||||
|
||||
# --- ASIA ---
|
||||
# China
|
||||
https://www.xinhuanet.com/english
|
||||
https://www.chinadaily.com.cn
|
||||
https://www.globaltimes.cn
|
||||
https://www.cgtn.com
|
||||
https://www.scmp.com
|
||||
|
||||
# India
|
||||
https://www.ndtv.com
|
||||
https://timesofindia.indiatimes.com
|
||||
https://indianexpress.com
|
||||
https://www.thehindu.com
|
||||
https://www.hindustantimes.com
|
||||
|
||||
# Japan
|
||||
https://www3.nhk.or.jp/nhkworld
|
||||
https://www.japantimes.co.jp
|
||||
https://www.asahi.com/ajw
|
||||
https://mainichi.jp/english
|
||||
https://english.kyodonews.net
|
||||
|
||||
# South Korea
|
||||
https://en.yna.co.kr
|
||||
https://www.koreaherald.com
|
||||
https://koreajoongangdaily.joins.com
|
||||
https://www.donga.com/en
|
||||
https://english.chosun.com
|
||||
|
||||
# --- AFRICA ---
|
||||
# South Africa
|
||||
https://www.news24.com
|
||||
https://www.iol.co.za
|
||||
https://www.dailymaverick.co.za
|
||||
https://www.sabcnews.com
|
||||
https://www.timeslive.co.za
|
||||
|
||||
# Nigeria
|
||||
https://www.vanguardngr.com
|
||||
https://punchng.com
|
||||
https://dailypost.ng
|
||||
https://saharareporters.com
|
||||
https://thenationonlineng.net
|
||||
|
||||
# --- MIDDLE EAST ---
|
||||
# General Region
|
||||
https://www.aljazeera.com
|
||||
https://english.alarabiya.net
|
||||
https://www.timesofisrael.com
|
||||
https://www.tehrantimes.com
|
||||
https://www.middleeasteye.net
|
||||
|
||||
# --- OCEANIA ---
|
||||
# Australia
|
||||
https://www.abc.net.au/news
|
||||
https://www.news.com.au
|
||||
https://www.9news.com.au
|
||||
https://www.smh.com.au
|
||||
https://www.theage.com.au
|
||||
# --- USA: MAJOR CITIES & LOCAL ---
|
||||
https://www.latimes.com
|
||||
https://www.chicagotribune.com
|
||||
https://www.sfchronicle.com
|
||||
https://www.bostonglobe.com
|
||||
https://www.seattletimes.com
|
||||
https://www.houstonchronicle.com
|
||||
https://www.inquirer.com
|
||||
https://www.denverpost.com
|
||||
https://www.miamiherald.com
|
||||
https://www.dallasnews.com
|
||||
https://www.startribune.com
|
||||
https://www.detroitnews.com
|
||||
https://www.ajc.com
|
||||
https://www.nydailynews.com
|
||||
https://nypost.com
|
||||
https://www.mercurynews.com
|
||||
https://www.baltimoresun.com
|
||||
https://www.oregonlive.com
|
||||
https://www.cleveland.com
|
||||
https://www.tampabay.com
|
||||
|
||||
# --- EUROPE: LOCAL & INDEPENDENT ---
|
||||
https://www.manchestereveningnews.co.uk
|
||||
https://www.scotsman.com
|
||||
https://www.belfasttelegraph.co.uk
|
||||
https://www.irishtimes.com
|
||||
https://www.berliner-zeitung.de
|
||||
https://www.leparisien.fr
|
||||
https://www.corriere.it
|
||||
https://www.elperiodico.com
|
||||
https://kyivindependent.com
|
||||
https://www.pravda.com.ua/en
|
||||
https://balkaninsight.com
|
||||
https://www.ekathimerini.com
|
||||
https://www.swissinfo.ch
|
||||
https://www.thelocal.se
|
||||
https://www.thelocal.fr
|
||||
https://www.thelocal.de
|
||||
https://www.novinite.com
|
||||
https://www.romania-insider.com
|
||||
https://hungarytoday.hu
|
||||
https://polandin.com
|
||||
|
||||
# --- MIDDLE EAST & CONFLICT ZONES ---
|
||||
https://www.haaretz.com
|
||||
https://www.jpost.com
|
||||
https://www.timesofisrael.com
|
||||
https://www.rudaw.net/english
|
||||
https://www.kurdistan24.net/en
|
||||
https://www.middleeasteye.net
|
||||
https://www.al-monitor.com
|
||||
https://www.dailysabah.com
|
||||
https://www.duvarenglish.com
|
||||
https://english.aawsat.com
|
||||
https://www.arabnews.com
|
||||
https://www.thenationalnews.com
|
||||
https://www.jordantimes.com
|
||||
https://www.naharnet.com
|
||||
https://www.tehrantimes.com
|
||||
|
||||
# --- ASIA: HOTSPOTS & LOCAL ---
|
||||
https://www.taipeitimes.com
|
||||
https://focustaiwan.tw
|
||||
https://hongkongfp.com
|
||||
https://www.bangkokpost.com
|
||||
https://www.thejakartapost.com
|
||||
https://www.straitstimes.com
|
||||
https://www.khmertimeskh.com
|
||||
https://www.irrawaddy.com
|
||||
https://www.myanmarnow.org/en
|
||||
https://www.rappler.com
|
||||
https://www.philstar.com
|
||||
https://english.hani.co.kr
|
||||
https://www.japantoday.com
|
||||
https://www.caixinglobal.com
|
||||
https://thediplomat.com
|
||||
|
||||
# --- LATIN AMERICA & AFRICA: LOCAL ---
|
||||
https://buenosairesherald.com
|
||||
https://riotimesonline.com
|
||||
https://mercopress.com
|
||||
https://www.elmostrador.cl
|
||||
https://www.jornada.com.mx
|
||||
https://www.theeastafrican.co.ke
|
||||
https://allafrica.com
|
||||
https://www.premiumtimesng.com
|
||||
https://www.dailytrust.com
|
||||
https://www.newtimes.co.rw
|
||||
https://www.herald.co.zw
|
||||
https://www.namibian.com.na
|
||||
https://www.graphic.com.gh
|
||||
https://www.thecitizen.co.tz
|
||||
https://www.monitor.co.ug
|
||||
|
||||
# --- ALTERNATIVE, INVESTIGATIVE & "FRINGE" ---
|
||||
https://theintercept.com
|
||||
https://www.propublica.org
|
||||
https://www.democracynow.org
|
||||
https://reason.com
|
||||
https://www.motherjones.com
|
||||
https://www.vox.com
|
||||
https://slate.com
|
||||
https://www.axios.com
|
||||
https://www.politico.com
|
||||
https://www.vice.com
|
||||
https://www.bellingcat.com
|
||||
https://www.project-syndicate.org
|
||||
https://cryptonews.com
|
||||
https://www.coindesk.com
|
||||
https://techcrunch.com
|
||||
https://www.investing.com/rss/news.rss
|
||||
https://www.ftchinese.com/rss
|
||||
https://www.alwatan.com
|
||||
https://albiladpress.com
|
||||
https://www.aletihad.ae/
|
||||
https://www.albayan.ae
|
||||
https://www.aljazeera.com/xml/rss/all.xml
|
||||
https://english.alarabiya.net/.mrss/en.xml
|
||||
https://english.aawsat.com/home/rss
|
||||
https://www.newarab.com/rss
|
||||
https://www.skynewsarabia.com/rss/feeds/rss-1.xml
|
||||
https://www.thenationalnews.com/arc/outboundfeeds/rss/
|
||||
https://www.arabnews.com/rss.xml
|
||||
https://gulfnews.com/rss
|
||||
https://www.kuwaittimes.com/feed/
|
||||
https://www.omanobserver.om/feed/
|
||||
https://www.khaleejtimes.com/rss/news
|
||||
http://www.akhbar-alkhaleej.com/rss/all
|
||||
https://today.lorientleyour.com/rss
|
||||
https://www.annahar.com/english/rss
|
||||
https://english.almayadeen.net/rss
|
||||
https://english.ahram.org.eg/rss/0/Home.aspx
|
||||
https://www.dailynewsegypt.com/feed/
|
||||
http://www.jordantimes.com/rss
|
||||
https://www.alraimedia.com/rss
|
||||
https://alghad.com/feed/
|
||||
https://nypost.com/feed/
|
||||
https://gothamist.com/feed/
|
||||
https://www.cityandstateny.com/rss
|
||||
https://feeds.nytimes.com/nyt/rss/HomePage
|
||||
https://www.thecity.nyc/rss/index.xml
|
||||
https://brooklyneagle.com/feed/
|
||||
https://www.reutersagency.com/feed/
|
||||
https://newsatme.com/api/v1/rss/ap/world
|
||||
https://feeds.bbci.co.uk/news/world/rss.xml
|
||||
https://rss.dw.com/rdf/rss-en-all
|
||||
https://www.france24.com/en/rss
|
||||
https://www3.nhk.or.jp/rss/news/shakaitokushu.xml
|
||||
https://www.cbc.ca/cctoc/rss/topstories.north
|
||||
https://www.defensenews.com/arc/outboundfeeds/rss/
|
||||
https://therecord.media/feed
|
||||
https://www.cfr.org/rss/newsletters/daily-news-brief
|
||||
https://warontherocks.com/feed/
|
||||
https://www.thecipherbrief.com/feed
|
||||
https://www.foreignaffairs.com/rss.xml
|
||||
https://geopoliticalfutures.com/feed
|
||||
# --- TACTICAL CYBER & VULNERABILITIES ---
|
||||
https://www.bleepingcomputer.com/feed/
|
||||
https://www.cisa.gov/cybersecurity-advisory-feeds
|
||||
https://krebsonsecurity.com/feed/
|
||||
https://thehackernews.com/feeds/posts/default
|
||||
https://www.darkreading.com/rss.xml
|
||||
https://www.mandiant.com/resources/blog/rss.xml
|
||||
https://schneier.com/feed/atom/
|
||||
https://www.securityweek.com/feed/
|
||||
# --- REGIONAL THREAT LANDSCAPE ---
|
||||
https://www.thenationalnews.com/rss/
|
||||
https://www.scmp.com/rss/91/feed
|
||||
https://www.batimes.com.ar/rss
|
||||
https://brazilian.report/feed/
|
||||
https://www.khon2.com/feed/
|
||||
https://www.staradvertiser.com/feed/
|
||||
https://www.westhawaiitoday.com/feed/
|
||||
https://mauinow.com/feed/
|
||||
https://www.idahofallsidaho.gov/RSSFeed.aspx?ModID=1&CID=All-newsflash.xml
|
||||
https://www.eastidahonews.com/feed/
|
||||
https://localnews8.com/feed/
|
||||
https://www.boisestatepublicradio.org/news.rss
|
||||
https://www.illinoistimes.com/springfield/Rss.xml
|
||||
https://www.thecentersquare.com/search/?f=rss&t=article&l=20&s=start_time&fulltext=showtext&sd=desc&c%5B%5D=Illinois
|
||||
https://chicago.suntimes.com/rss/index.xml
|
||||
https://wgntv.com/feed/
|
||||
http://feeds.indiana.statenews.net/rss/7b3aa09cdd5d5eac
|
||||
https://fox59.com/feed/
|
||||
https://www.nwitimes.com/search/?f=rss&t=article&c=news/local&l=50&s=start_time&sd=desc
|
||||
https://www.wishtv.com/feed/
|
||||
https://www.kcci.com/topstories-rss
|
||||
https://www.myiowainfo.com/feed/
|
||||
https://feeds.feedburner.com/radioiowanews
|
||||
https://www.mississippivalleypublishing.com/search/?f=rss&t=article&c=the_hawk_eye&l=50&s=start_time&sd=desc
|
||||
https://www.ksn.com/feed/
|
||||
https://www.ksnt.com/feed/
|
||||
https://www.hdnews.net/feed/
|
||||
https://themercury.com/search/?f=rss&t=article&c=news&l=50&s=start_time&sd=desc
|
||||
https://www.wdrb.com/search/?f=rss&t=article&c=news&l=50&s=start_time&sd=desc
|
||||
https://www.wtvq.com/feed/
|
||||
https://www.wnky.com/feed/
|
||||
https://www.wlky.com/topstories-rss
|
||||
https://thehayride.com/feed/
|
||||
https://wgno.com/feed/
|
||||
https://feeds.feedburner.com/wbrz/news
|
||||
https://thelensnola.org/feed/
|
||||
https://www.pressherald.com/news/feed/
|
||||
https://www.centralmaine.com/feed/
|
||||
https://www.bangordailynews.com/feed/
|
||||
https://www.sunjournal.com/news/feed/
|
||||
https://www.wbaltv.com/topstories-rss
|
||||
https://www.manisteenews.com/news/feed/Latest-News-Feed-2564.php
|
||||
https://www.theoaklandpress.com/feed/
|
||||
https://www.macombdaily.com/feed/
|
||||
https://www.startribune.com/local/index.rss2
|
||||
https://www.wctrib.com/index.rss
|
||||
https://www.austindailyherald.com/feed/
|
||||
https://helenair.com/search/?f=rss&t=article&l=50&s=start_time&sd=desc
|
||||
https://www.ktvq.com/news.rss
|
||||
https://mtstandard.com/search/?f=rss&t=article&l=50&s=start_time&sd=desc
|
||||
https://www.ketv.com/topstories-rss
|
||||
https://nebraskaexaminer.com/feed/
|
||||
https://kearneyhub.com/rss
|
||||
https://www.wowt.com/rss
|
||||
https://thenevadaindependent.com/feed/
|
||||
https://www.8newsnow.com/feed/
|
||||
https://www.reviewjournal.com/feed/
|
||||
https://thisisreno.com/feed/
|
||||
https://www.conwaydailysun.com/search/?f=rss&t=article&c=berlin_sun/community/news&l=50&s=start_time&sd=desc
|
||||
https://newhampshirebulletin.com/feed/
|
||||
https://www.nhgazette.com/feed/
|
||||
https://www.nhbr.com/feed/
|
||||
https://www.nj.com/arc/outboundfeeds/rss/?outputType=xml
|
||||
https://www.njspotlightnews.org/feed/
|
||||
https://njmonthly.com/feed/
|
||||
https://www.trentonian.com/feed/
|
||||
https://www.krqe.com/feed/
|
||||
https://www.santafenewmexican.com/search/?f=rss&t=article&l=50&s=start_time&sd=desc
|
||||
https://www.easternnewmexiconews.com/rss
|
||||
https://www.koat.com/topstories-rss
|
||||
https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml
|
||||
https://www.thecity.nyc/feed/
|
||||
https://www.nbcnewyork.com/?rss=y
|
||||
https://www.wral.com/news/rss/48/
|
||||
https://www.cbs17.com/news/north-carolina-news/feed/
|
||||
https://abc11.com/feed/
|
||||
https://myfox8.com/news/feed/
|
||||
https://www.kxnet.com/feed/
|
||||
https://www.wday.com/feed/
|
||||
https://www.jamestownsun.com/index.rss
|
||||
https://www.inforum.com/index.rss
|
||||
http://rssfeeds.wkyc.com/wkyc/news
|
||||
https://theohiostar.com/feed/
|
||||
https://feeds.feedblitz.com/wtol/news
|
||||
https://www.wcpo.com/news.rss
|
||||
https://kfor.com/feed/
|
||||
https://oklahomawatch.org/feed/
|
||||
https://freepressokc.com/feed/
|
||||
https://osagenews.org/feed/
|
||||
http://rssfeeds.kgw.com/kgw/local
|
||||
https://www.koin.com/feed/
|
||||
https://www.bendsource.com/bend/Rss.xml/feed
|
||||
https://eugeneweekly.com/feed/
|
||||
https://www.wtae.com/topstories-rss
|
||||
https://www.montgomerycountypa.gov/RSSFeed.aspx?ModID=76&CID=All-0
|
||||
https://www.mainlinemedianews.com/feed/
|
||||
https://www.dailylocal.com/feed/
|
||||
https://www.wpri.com/feed/
|
||||
https://www.abc6.com/feed/
|
||||
https://whdh.com/regional/rhode-island/feed/
|
||||
https://warwickpost.com/feed/
|
||||
https://www.wyff4.com/topstories-rss
|
||||
https://www.wispolitics.com/feed/
|
||||
https://wiseye.org/feed/
|
||||
https://wisconsinexaminer.com/feed/
|
||||
https://trib.com/search/?f=rss&t=article&c=news/state-and-regional&l=50&s=start_time&sd=desc
|
||||
https://wyofile.com/feed/
|
||||
https://www.wyomingnews.com/search/?f=rss&t=article&c=news&l=50&s=start_time&sd=desc
|
||||
https://www.wyodaily.com/rss
|
||||
https://www.wnct.com/news/north-carolina/feed/
|
||||
https://www.usnews.com/rss/news/north-carolina
|
||||
https://indyweek.com/feed/
|
||||
https://portcitydaily.com/feed/
|
||||
https://www.theguardian.com/uk/rss
|
||||
https://feeds.bbci.co.uk/news/england/rss.xml
|
||||
https://www.lemonde.fr/rss/une.xml
|
||||
https://www.ansa.it/sito/notizie/rss.xml
|
||||
https://www.ilgiornale.it/feed
|
||||
https://www.larepublica.it/rss/homepage/rss2.xml
|
||||
https://www.sueddeutsche.de/rss
|
||||
https://www.welt.de/feeds/top-news.rss
|
||||
https://www.rfi.fr/en/rss
|
||||
https://www.bangkokpost.com/rss
|
||||
https://thephnompenhpost.com/rss
|
||||
https://www.thejakartapost.com/rss
|
||||
https://www.straitstimes.com/news/singapore/rss.xml
|
||||
https://www.channelnewsasia.com/rss
|
||||
https://www.antaranews.com/rss/
|
||||
https://www.irrawaddy.com/feed
|
||||
https://news.abs-cbn.com/rss
|
||||
https://www.hindustantimes.com/feeds/rss
|
||||
https://www.africanews.com/feed/rss
|
||||
https://www.clarin.com/rss
|
||||
https://www.lanacion.com.ar/rss
|
||||
https://www.eluniversal.com.mx/rss
|
||||
https://www.excelsior.com.mx/rss
|
||||
https://www.eltiempo.com/rss
|
||||
https://www.elespectador.com/rss
|
||||
https://www.larepublica.pe/rss
|
||||
https://www.elcomercio.com/rss
|
||||
https://www.abc.net.au/news/feed/
|
||||
https://www.smh.com.au/rss/world.xml
|
||||
https://www.theage.com.au/rss
|
||||
https://www.brisbanetimes.com.au/rss
|
||||
https://www.stuff.co.nz/rss
|
||||
https://www.nzherald.co.nz/arcio/rss/
|
||||
https://www.rnz.co.nz/rss
|
||||
https://globalvoices.org/regions/africa/feed/
|
||||
https://globalvoices.org/regions/asia/feed/
|
||||
https://globalvoices.org/regions/latin-america/feed/
|
||||
https://globalvoices.org/regions/eastern-europe/feed/
|
||||
https://globalvoices.org/regions/middle-east-north-africa/feed/
|
||||
https://globalvoices.org/regions/south-asia/feed/
|
||||
https://globalvoices.org/regions/sub-saharan-africa/feed/
|
||||
https://globalvoices.org/regions/west-africa/feed/
|
||||
https://globalvoices.org/regions/east-asia/feed/
|
||||
https://globalvoices.org/regions/southeast-asia/feed/
|
||||
https://globalvoices.org/regions/central-asia/feed/
|
||||
https://globalvoices.org/regions/pacific/feed/
|
||||
https://globalvoices.org/regions/caribbean/feed/
|
||||
https://www.townandcountry-mo.gov/rss.aspx
|
||||
https://feeds.smh.com.au/rssheadlines/national.xml
|
||||
https://www.abc.net.au/local/rss/sydney/
|
||||
https://www.voanews.com/rssfeeds
|
||||
https://rss.feedspot.com/southeast_asian_rss_feeds
|
||||
https://www.crisisgroup.org/rss
|
||||
https://news.panasonic.com/global/rss/area01/index.xml
|
||||
https://news.panasonic.com/global/rss/area04/index.xml
|
||||
https://allafrica.com/tools/headlines/rdf/latest/headlines.rdf
|
||||
https://www.afro.who.int/rss-feeds
|
||||
https://pressat.co.uk/rss-list
|
||||
https://www.monitor.co.ug/rss
|
||||
https://www.standardmedia.co.ke/rss
|
||||
https://www.ft.com/rss/home
|
||||
https://www.economist.com/rss/the-world-this-week
|
||||
https://feeds.bloomberg.com/economics/news.rss
|
||||
https://feeds.bloomberg.com/markets/news.rss
|
||||
https://www.reuters.com/arc/outboundfeeds/newsroom/business/
|
||||
https://www.cnbc.com/id/10000113/device/rss/rss.html
|
||||
https://feeds.a.dj.com/rss/RSSWorldBusiness.xml
|
||||
https://www.marketwatch.com/rss/topstories
|
||||
https://www.investing.com/rss/news_14.rss
|
||||
https://feeds.bbci.co.uk/news/business/rss.xml
|
||||
https://feeds.feedburner.com/TheHackersNews
|
||||
https://www.darkreading.com/rss/all.xml
|
||||
https://isc.sans.edu/rssfeed_full.xml
|
||||
https://securelist.com/feed/
|
||||
https://feeds.feedburner.com/eset/blog
|
||||
https://news.sophos.com/en-us/feed/
|
||||
https://www.schneier.com/feed/atom/
|
||||
https://www.securitymagazine.com/rss/topic/2236-cybersecurity-news
|
||||
https://www.reuters.com/arc/outboundfeeds/rss/?outputType=xml
|
||||
https://www.investing.com/rss/news_462.rss
|
||||
https://www.investing.com/rss/news_1.rss
|
||||
https://www.investing.com/rss/stock_Futures.rss
|
||||
https://www.ecb.europa.eu/rss/fxref-ecbpress.en.xml
|
||||
https://www.federalreserve.gov/feeds/news-events.xml
|
||||
https://www.boj.or.jp/en/rss/whatsnew.xml
|
||||
https://www.bankofengland.co.uk/rss/news
|
||||
https://www.centralbanking.com/feeds/rss
|
||||
https://oilprice.com/rss/
|
||||
https://www.spglobal.com/commodityinsights/en/rss
|
||||
https://www.eia.gov/tools/rssfeeds/
|
||||
https://www.cmegroup.com/rss
|
||||
https://globalvoices.org/-/topics/economics-business/feed/
|
||||
http://globalization.einnews.com/rss
|
||||
https://financefeeds.com/feed/
|
||||
https://newsquawk.com/blog/feed.rss
|
||||
https://www.coindesk.com/arc/outboundfeeds/rss/
|
||||
https://ishookfinance.com/feed/
|
||||
https://www.scmp.com/rss/92/feed
|
||||
https://www.scmp.com/rss/93/feed
|
||||
https://www.scmp.com/rss/94/feed
|
||||
https://www.scmp.com/rss/317/feed
|
||||
https://asia.nikkei.com/rss
|
||||
https://www.caixin.com/rss/index_EN.xml
|
||||
https://www.straitstimes.com/news/asia/rss.xml
|
||||
https://www.straitstimes.com/business/rss.xml
|
||||
https://www.reuters.com/arc/outboundfeeds/rss/?outputType=xml§ion=asia
|
||||
https://www.reuters.com/arc/outboundfeeds/rss/?outputType=xml§ion=china
|
||||
https://www.bloomberg.com/feeds/asia.rss
|
||||
https://www.bloomberg.com/feeds/markets.rss
|
||||
https://www.ft.com/asia-pacific?format=rss
|
||||
https://www.ft.com/china?format=rss
|
||||
https://english.kyodonews.net/rss/news.xml
|
||||
https://en.yna.co.kr/RSS/news.xml
|
||||
https://www.thejakartapost.com/rss/business
|
||||
https://www.nationthailand.com/rss/business
|
||||
https://www.aramco.com/api/v1/com/rss/news?sc_lang=en
|
||||
https://www.worldoil.com/rss?feed=topic:saudi+arabia
|
||||
https://www.worldoil.com/rss?feed=topic:iraq
|
||||
https://www.worldoil.com/rss?feed=topic:uae
|
||||
https://www.worldoil.com/rss?feed=topic:russia
|
||||
https://www.worldoil.com/rss?feed=topic:canada
|
||||
https://www.worldoil.com/rss?feed=topic:oil+sands
|
||||
https://www.rigzone.com/news/europe_russia/production/rss/
|
||||
https://www.argusmedia.com/en/news-and-insights/latest-market-news/rss
|
||||
https://www.eia.gov/rss/
|
||||
https://www.opec.org/opec_web/en/pressreleases.rss
|
||||
https://www.opec.org
|
||||
https://www.rosneft.com/press/news/rss/
|
||||
https://feeds.content.dowjones.io/public/rss/RSSMarketsMain
|
||||
https://feeds.content.dowjones.io/public/rss/socialeconomyfeed
|
||||
https://feeds.content.dowjones.io/public/rss/WSJcomUSBusiness
|
||||
https://feeds.content.dowjones.io/public/rss/RSSWorldNews
|
||||
http://feeds.feedburner.com/EconomicEventsAgriculture
|
||||
http://feeds.feedburner.com/EconomicEventsEnergy
|
||||
http://feeds.feedburner.com/EconomicEventsInterestRates
|
||||
http://feeds.feedburner.com/mediaroom/CMsF
|
||||
http://feeds.feedburner.com/CMEClearPortNoticesRss
|
||||
http://feeds.feedburner.com/GlobexAdvisories
|
||||
https://feeds.content.dowjones.io/public/rss/mw_topstories
|
||||
https://feeds.content.dowjones.io/public/rss/mw_realtimeheadlines
|
||||
http://feeds.marketwatch.com/marketwatch/bulletins
|
||||
https://feeds.content.dowjones.io/public/rss/mw_marketpulse
|
||||
https://www.nasdaqtrader.com/rss.aspx?feed=currentheadlines&categorylist=51
|
||||
https://www.nasdaqtrader.com/rss.aspx?feed=currentheadlines&categorylist=11
|
||||
https://www.investing.com/rss/stock_Options.rss
|
||||
https://www.investing.com/rss/news_11.rss
|
||||
https://www.investing.com/rss/news_25.rss
|
||||
https://www.nasdaq.com/feed/rssoutbound?category=Markets
|
||||
https://www.nasdaq.com/feed/rssoutbound?category=Commodities
|
||||
https://www.barchart.com/news/rss/financials/options-news
|
||||
https://www.barchart.com/news/rss/commodities/futures-news
|
||||
https://www.spglobal.com/spdji/en/rss
|
||||
https://www.litefinance.org/rss/analytics/
|
||||
https://www.mrt.com/arc/outboundfeeds/rss/category/business/oil/?outputType=xml
|
||||
https://www.oaoa.com/category/local-news/inthepipeline/rss
|
||||
https://pboilandgasmagazine.com/feed/
|
||||
https://www.rigzone.com/news/rss.asp
|
||||
https://rbnenergy.com/blogcast.rss
|
||||
https://www.eia.gov/rss/todayinenergy.xml
|
||||
https://www.firstalert7.com/news/energy
|
||||
https://www.energyvoice.com/feed/?category=oilandgas/north-sea
|
||||
https://www.rigzone.com/news/rss/north_sea
|
||||
https://www.oedigital.com/feeds/rss
|
||||
https://www.sodir.no/en/whats-new/news/rss
|
||||
https://www.worldoil.com/rss?feed=topic:offshore
|
||||
https://oilandgas.einnews.com/rss/north-sea-offshore
|
||||
https://www.energyvoice.com/feed/
|
||||
|
|
|
|||
|
|
@ -1,14 +1,10 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Scheduler loop for the news summarizer — hourly summarize at minute :05.
|
||||
"""Scheduler loop for the news summarizer — every NEWS_SUMMARIZE_INTERVAL_S.
|
||||
|
||||
Replaces the k8s CronJob (`5 * * * *`) with an in-compose loop. Runs once on
|
||||
boot (catches up on any articles scraped since the last summary), then fires
|
||||
at each :NEWS_SUMMARIZE_MINUTE wall-clock boundary.
|
||||
|
||||
The loop is serial, so a slow LLM pass never overlaps the next run.
|
||||
Default 900s (15 minutes). Serial: a slow LLM pass never overlaps the next.
|
||||
|
||||
Env (all optional, 12-factor):
|
||||
NEWS_SUMMARIZE_MINUTE minute of the hour to fire (default 5)
|
||||
NEWS_SUMMARIZE_INTERVAL_S seconds between runs (default 900)
|
||||
NEWS_SUMMARIZE_RUN_ON_START "1" to summarize once immediately on boot (default 1)
|
||||
NOUS_API_KEY optional in env; Keys UI / api_keys also works
|
||||
"""
|
||||
|
|
@ -25,17 +21,10 @@ import time
|
|||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
logger = logging.getLogger("news.summarizer.scheduler")
|
||||
|
||||
MINUTE = int(os.getenv("NEWS_SUMMARIZE_MINUTE", "5"))
|
||||
INTERVAL_S = max(1, int(os.getenv("NEWS_SUMMARIZE_INTERVAL_S", "900")))
|
||||
RUN_ON_START = os.getenv("NEWS_SUMMARIZE_RUN_ON_START", "1").lower() in ("1", "true", "yes")
|
||||
|
||||
|
||||
def seconds_until_next(minute: int) -> float:
|
||||
"""Seconds until the next occurrence of ``minute`` past the hour (local time)."""
|
||||
now = datetime.datetime.now()
|
||||
nxt = now.replace(minute=minute, second=0, microsecond=0) + datetime.timedelta(hours=1)
|
||||
return (nxt - now).total_seconds()
|
||||
|
||||
|
||||
def run_summarize() -> None:
|
||||
logger.info("summarize starting at %s", datetime.datetime.now().isoformat(timespec="seconds"))
|
||||
try:
|
||||
|
|
@ -51,15 +40,14 @@ def main() -> None:
|
|||
"NOUS_API_KEY unset in env — will read api_keys on each run; idle if both empty"
|
||||
)
|
||||
logger.info(
|
||||
"news summarizer loop starting (minute=%s, run_on_start=%s)",
|
||||
MINUTE, RUN_ON_START,
|
||||
"news summarizer loop starting (interval_s=%s, run_on_start=%s)",
|
||||
INTERVAL_S, RUN_ON_START,
|
||||
)
|
||||
if RUN_ON_START:
|
||||
run_summarize()
|
||||
while True:
|
||||
delay = seconds_until_next(MINUTE)
|
||||
logger.info("next summarize at :%02d (in %.0fs)", MINUTE, delay)
|
||||
time.sleep(delay)
|
||||
logger.info("next summarize in %ss", INTERVAL_S)
|
||||
time.sleep(INTERVAL_S)
|
||||
run_summarize()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
"""News summarizer — Nous map-reduce of scraped articles into brief/ticker/map.
|
||||
|
||||
Reads articles scraped within the last hour from the shared `articles` table,
|
||||
Reads articles scraped within the last SUMMARY_WINDOW_MINUTES from the shared `articles` table,
|
||||
maps them with Nous (per-article English fact blocks), reduces to one JSON
|
||||
object (summary_en + ticker + map_items), and stores the brief in
|
||||
`article_summaries` plus flagged rows in `news_items`. Tables live in the
|
||||
|
|
@ -15,11 +15,8 @@ of each summarize_news() — env wins, else api_keys / app_settings:
|
|||
NOUS_BASE_URL default https://inference-api.nousresearch.com/v1
|
||||
SUMMARY_MODEL default Hermes-4.3-36B (else app_settings)
|
||||
BATCH_SIZE articles per map-phase batch (default 50)
|
||||
SUMMARY_WINDOW_HOURS look-back window in hours (default 1)
|
||||
OSINT_USER_AGENT default osint-dashboard-news-summarizer
|
||||
MAP_PROMPT override map-phase prompt (uses {batch_text})
|
||||
SUMMARY_PROMPT override reduce-phase prompt (uses {final_input})
|
||||
NEWS_SUMMARIZE_FORCE "1" to ignore the current-UTC-hour idempotency skip
|
||||
SUMMARY_WINDOW_MINUTES look-back window (default 15; SUMMARY_WINDOW_HOURS wins if set)
|
||||
NEWS_SUMMARIZE_FORCE "1" to ignore the interval idempotency skip
|
||||
INCLUDE_FUTURES "1" to prepend live futures prices (default 0)
|
||||
|
||||
The futures/markets coupling from the original pipeline is gated behind
|
||||
|
|
@ -54,7 +51,23 @@ DB_CONFIG = {
|
|||
DEFAULT_NOUS_BASE_URL = "https://inference-api.nousresearch.com/v1"
|
||||
DEFAULT_SUMMARY_MODEL = "Hermes-4.3-36B"
|
||||
BATCH_SIZE = int(os.getenv("BATCH_SIZE", "50"))
|
||||
SUMMARY_WINDOW_HOURS = int(os.getenv("SUMMARY_WINDOW_HOURS", "1"))
|
||||
|
||||
|
||||
def _summary_window_minutes() -> int:
|
||||
hours = (os.getenv("SUMMARY_WINDOW_HOURS") or "").strip()
|
||||
if hours:
|
||||
return max(1, int(hours) * 60)
|
||||
mins = (os.getenv("SUMMARY_WINDOW_MINUTES") or "").strip()
|
||||
if mins:
|
||||
return max(1, int(mins))
|
||||
return 15
|
||||
|
||||
|
||||
def _summarize_interval_seconds() -> int:
|
||||
return max(1, int(os.getenv("NEWS_SUMMARIZE_INTERVAL_S", "900")))
|
||||
|
||||
|
||||
SUMMARY_WINDOW_MINUTES = _summary_window_minutes()
|
||||
INCLUDE_FUTURES = os.getenv("INCLUDE_FUTURES", "0").lower() in ("1", "true", "yes")
|
||||
|
||||
# Only touched when INCLUDE_FUTURES=1 (legacy markets coupling, OSINT-off).
|
||||
|
|
@ -282,18 +295,18 @@ def ensure_tables() -> None:
|
|||
|
||||
|
||||
def get_recent_news() -> list[dict]:
|
||||
"""Fetch articles from the last SUMMARY_WINDOW_HOURS (content > 100 chars)."""
|
||||
"""Fetch articles from the last SUMMARY_WINDOW_MINUTES (content > 100 chars)."""
|
||||
query = """
|
||||
SELECT title, content, url, domain
|
||||
FROM articles
|
||||
WHERE timestamp > NOW() - make_interval(hours => %s)
|
||||
WHERE timestamp > NOW() - make_interval(mins => %s)
|
||||
AND content IS NOT NULL AND length(content) > 100
|
||||
ORDER BY timestamp DESC;
|
||||
"""
|
||||
try:
|
||||
conn = psycopg2.connect(**DB_CONFIG)
|
||||
cur = conn.cursor()
|
||||
cur.execute(query, (SUMMARY_WINDOW_HOURS,))
|
||||
cur.execute(query, (SUMMARY_WINDOW_MINUTES,))
|
||||
rows = cur.fetchall()
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
|
@ -306,24 +319,24 @@ def get_recent_news() -> list[dict]:
|
|||
return []
|
||||
|
||||
|
||||
def _already_summarized_this_hour() -> bool:
|
||||
"""True when article_summaries already has a row for the current UTC hour."""
|
||||
def _already_summarized_this_interval() -> bool:
|
||||
"""True when article_summaries already has a row in the last interval."""
|
||||
if os.getenv("NEWS_SUMMARIZE_FORCE", "") == "1":
|
||||
return False
|
||||
query = (
|
||||
"SELECT 1 FROM article_summaries "
|
||||
"WHERE batch_timestamp >= date_trunc('hour', NOW() AT TIME ZONE 'utc')"
|
||||
"WHERE batch_timestamp >= NOW() - make_interval(secs => %s)"
|
||||
)
|
||||
try:
|
||||
conn = psycopg2.connect(**DB_CONFIG)
|
||||
cur = conn.cursor()
|
||||
cur.execute(query)
|
||||
cur.execute(query, (_summarize_interval_seconds(),))
|
||||
row = cur.fetchone()
|
||||
cur.close()
|
||||
conn.close()
|
||||
return row is not None
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.error("Error checking hourly idempotency: %s", exc)
|
||||
logger.error("Error checking interval idempotency: %s", exc)
|
||||
return False
|
||||
|
||||
|
||||
|
|
@ -419,10 +432,11 @@ def build_master_prompt(final_input: str) -> str:
|
|||
def summarize_news() -> None:
|
||||
"""Map-reduce summarize recent articles and store brief + ticker + map."""
|
||||
ensure_tables()
|
||||
if _already_summarized_this_hour():
|
||||
if _already_summarized_this_interval():
|
||||
logger.info(
|
||||
"Skipping summarize: article_summaries already has a row this UTC hour "
|
||||
"(set NEWS_SUMMARIZE_FORCE=1 to override)"
|
||||
"Skipping summarize: article_summaries already has a row in the last %ss "
|
||||
"(set NEWS_SUMMARIZE_FORCE=1 to override)",
|
||||
_summarize_interval_seconds(),
|
||||
)
|
||||
return
|
||||
|
||||
|
|
@ -435,7 +449,7 @@ def summarize_news() -> None:
|
|||
|
||||
articles = get_recent_news()
|
||||
if not articles:
|
||||
logger.info("No new articles found in the last %sh.", SUMMARY_WINDOW_HOURS)
|
||||
logger.info("No new articles found in the last %s min.", SUMMARY_WINDOW_MINUTES)
|
||||
return
|
||||
|
||||
logger.info(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue