Add news pipeline: hourly scraper + Gemini summarizer in compose

Vendor the newsPipeline scraper + summarizer into the repo and wire them into docker-compose against the EXISTING osint-db (no second Postgres), replacing the upstream k8s CronJobs with in-compose wall-clock loops (:00 scrape, :05 summarize). - news/scraper: vendored Scrapy project (257 RSS feeds) + hourly loop scheduler (run_news_scraper.py) - news/summerizer: vendored Gemini map-reduce summarizer, cleaned: * fix broken google-genai response handling (_extract_text, defensive) * fix malformed INSERT/GRANT query in save_summary_to_db * OSINT-neutral default MAP_PROMPT; futures/markets language gated behind INCLUDE_FUTURES=0 (yfinance lazy-imported) * env-configurable model, batch size, lookback window + hourly loop scheduler (run_news_summarizer.py, :05) - alembic 003_news: idempotent articles + article_summaries tables - API: GET /api/news and GET /api/news/summaries (+ models, schemas) - tests/test_api_news.py: 5 DB-backed contract tests (all pass vs real PG) - docs/news.md + .env.example updates Both services run under the `ingest` compose profile (matching the ingester/camera-scraper pattern) and build arm64 on the Pi via the existing Forgejo CI workflow. telebot left out of scope (reserved env only).
2026-08-24 17:28:46 -04:00 · 2026-08-24 17:28:46 -04:00 · 91f436390b
commit 91f436390b
parent 172273bed6
27 changed files with 1772 additions and 2 deletions
--- a/.env.example
+++ b/.env.example
@ -48,3 +48,29 @@ INGEST_FIRES=1
 # (GET/POST/DELETE /api/keys/{name}) — see app/keystore.py. The FIRMS ingestor
 # currently reads FIRMS_MAP_KEY from .env (above); wiring the Keys-UI store as
 # its lookup/fallback is a planned follow-up.
+
+# ── News pipeline (scraper + summarizer, profile `ingest`) ────────────────
+# Hourly: the scraper crawls 257 RSS sources at minute :00 and the summarizer
+# runs the Gemini map-reduce at minute :05, both writing to the shared osint-db
+# (tables `articles` + `article_summaries`, created by alembic 003_news).
+# Consume via GET /api/news and GET /api/news/summaries.
+# GEMINI_API_KEY is REQUIRED for summarization; unset = summarizer idles.
+GEMINI_API_KEY=
+# Optional LLM knobs
+SUMMARY_MODEL=gemini-2.0-flash
+NEWS_BATCH_SIZE=50
+SUMMARY_WINDOW_HOURS=1
+# Futures/markets coupling from the upstream pipeline is OFF by default
+# (irrelevant to OSINT). Set INCLUDE_FUTURES=1 + install yfinance to enable.
+INCLUDE_FUTURES=0
+# Wall-clock scheduling (k8s CronJob replacement): scrape minute, summarize minute
+NEWS_SCRAPE_MINUTE=0
+NEWS_SUMMARIZE_MINUTE=5
+# Run once immediately on container start (seeds data fast), then align to the
+# scheduled minute.
+NEWS_SCRAPE_RUN_ON_START=1
+NEWS_SUMMARIZE_RUN_ON_START=1
+NEWS_LOG_LEVEL=INFO
+# Reserved for the (out-of-scope) Telegram delivery bot.
+TELEGRAM_TOKEN=
+TELEGRAM_CHAT_ID=
--- a/alembic/versions/003_news.py
+++ b/alembic/versions/003_news.py
@ -0,0 +1,57 @@
+"""news tables: scraped articles + LLM article_summaries
+
+Revision ID: 003_news
+Revises: 002_cameras
+Create Date: 2026-08-24
+"""
+
+from alembic import op
+import sqlalchemy as sa  # noqa: F401
+
+# revision identifiers, used by Alembic.
+revision = '003_news'
+down_revision = '002_cameras'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Idempotent DDL: the news scraper's Scrapy pipeline also issues
+    # `CREATE TABLE IF NOT EXISTS articles`, so either the scraper or the app
+    # may create these first depending on container startup order. IF NOT
+    # EXISTS makes both orders safe — whichever runs first wins, the other
+    # no-ops. Same table shapes as the upstream newsPipeline services.
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS articles (
+            id SERIAL PRIMARY KEY,
+            title TEXT,
+            url TEXT UNIQUE,
+            content TEXT,
+            domain TEXT,
+            timestamp TIMESTAMPTZ
+        )
+        """
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_articles_timestamp ON articles (timestamp)"
+    )
+
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS article_summaries (
+            id SERIAL PRIMARY KEY,
+            summary_text TEXT NOT NULL,
+            batch_timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW()
+        )
+        """
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_article_summaries_batch_timestamp "
+        "ON article_summaries (batch_timestamp)"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS article_summaries")
+    op.execute("DROP TABLE IF EXISTS articles")
--- a/app/main.py
+++ b/app/main.py
@ -26,12 +26,13 @@ from sqlalchemy.ext.asyncio import AsyncSession

 from database import async_session, init_extensions
 from models import (
-    alerts, documents, entities, entity_events, events, feed_sources, fires
+    alerts, documents, entities, entity_events, events, feed_sources, fires,
+    articles, article_summaries,
 )
 from schemas import (
    AlertCreate, AlertOut, AlertSeverity, AlertType, AlertUpdate,
    DashboardSummary, EntityCreate, EntityKind, EntityOut,
-    EventCreate, EventOut, FireOut,
+    EventCreate, EventOut, FireOut, NewsArticleOut, NewsSummaryOut,
    FeedSourceCreate, FeedSourceOut,
    KeyOut, KeyValueIn,
    SearchResult, SentimentSummary, SourceType,
@ -767,6 +768,72 @@ async def camera_snapshot(camera_id: UUID):
    return Response(content=data, media_type="image/jpeg")


+# ── News pipeline (scraper + summarizer) ──────────────────────────────────
+# Backing data for the frontend news panel. Written by the vendored
+# news-scraper (hourly Scrapy crawl) and news-summarizer (hourly Gemini
+# map-reduce) services into the shared osint-db.
+
+@app.get("/api/news", response_model=list[NewsArticleOut])
+async def list_news(
+    domain: str | None = Query(
+        None, description="Filter by source domain (e.g. 'www.reuters.com')"
+    ),
+    since: datetime | None = Query(
+        None,
+        description="Only articles captured at/after this UTC instant "
+                    "(ISO 8601, e.g. '2026-08-24T12:00:00Z').",
+    ),
+    limit: int = Query(50, ge=1, le=500),
+    offset: int = Query(0, ge=0),
+):
+    """Most recent scraped news articles (newest first)."""
+    async with async_session() as session:
+        stmt = select(articles).order_by(
+            articles.c.timestamp.desc().nullslast()
+        )
+        if domain:
+            stmt = stmt.where(articles.c.domain == domain)
+        if since:
+            stmt = stmt.where(articles.c.timestamp >= since)
+        stmt = stmt.limit(limit).offset(offset)
+        rows = (await session.execute(stmt)).mappings().all()
+    return [
+        NewsArticleOut(
+            id=r["id"], title=r["title"], url=r["url"],
+            content=r["content"], domain=r["domain"],
+            timestamp=r["timestamp"],
+        )
+        for r in rows
+    ]
+
+
+@app.get("/api/news/summaries", response_model=list[NewsSummaryOut])
+async def list_news_summaries(
+    since: datetime | None = Query(
+        None,
+        description="Only summaries generated at/after this UTC instant.",
+    ),
+    limit: int = Query(20, ge=1, le=100),
+    offset: int = Query(0, ge=0),
+):
+    """Most recent master LLM summaries (newest first)."""
+    async with async_session() as session:
+        stmt = select(article_summaries).order_by(
+            article_summaries.c.batch_timestamp.desc().nullslast()
+        )
+        if since:
+            stmt = stmt.where(article_summaries.c.batch_timestamp >= since)
+        stmt = stmt.limit(limit).offset(offset)
+        rows = (await session.execute(stmt)).mappings().all()
+    return [
+        NewsSummaryOut(
+            id=r["id"], summary_text=r["summary_text"],
+            batch_timestamp=r["batch_timestamp"],
+        )
+        for r in rows
+    ]
+
+
 # ── Frontend ──────────────────────────────────────────────────────────────

@app.get("/", response_class=HTMLResponse)
--- a/app/models.py
+++ b/app/models.py
@ -182,3 +182,33 @@ fires = Table(

 # Bounding-box index for `bbox=` filtering (lon first for max/min-lon scans).
 Index("ix_fires_bbox", fires.c.longitude, fires.c.latitude)
+
+
+# ── News pipeline (scraper + summarizer) ──────────────────────────────────
+# Written by the vendored news-scraper (Scrapy) / news-summarizer (Gemini)
+# services; schema must match the idempotent alembic migration 003_news.
+
+articles = Table(
+    "articles",
+    metadata,
+    Column("id", Integer, primary_key=True, autoincrement=True),
+    Column("title", Text),
+    Column("url", Text, unique=True),       # dedup key
+    Column("content", Text),                # full extracted article text
+    Column("domain", Text),                 # source domain
+    Column("timestamp", DateTime(timezone=True)),  # capture time
+)
+
+Index("ix_articles_timestamp", articles.c.timestamp)
+
+
+article_summaries = Table(
+    "article_summaries",
+    metadata,
+    Column("id", Integer, primary_key=True, autoincrement=True),
+    Column("summary_text", Text, nullable=False),
+    Column("batch_timestamp", DateTime(timezone=True),
+           server_default=func.now(), nullable=False),
+)
+
+Index("ix_article_summaries_batch_timestamp", article_summaries.c.batch_timestamp)
--- a/app/schemas.py
+++ b/app/schemas.py
@ -253,6 +253,27 @@ class FireOut(BaseModel):
    daynight: Optional[str] = None      # D / N


+# ─── News pipeline (scraper + summarizer) ────────────────────────────────
+
+class NewsArticleOut(BaseModel):
+    """One scraped article as exposed by GET /api/news."""
+
+    id: int
+    title: Optional[str] = None
+    url: Optional[str] = None
+    content: Optional[str] = None
+    domain: Optional[str] = None
+    timestamp: Optional[datetime] = None
+
+
+class NewsSummaryOut(BaseModel):
+    """One master LLM summary as exposed by GET /api/news/summaries."""
+
+    id: int
+    summary_text: str
+    batch_timestamp: datetime
+
+
 # ─── Aggregations ────────────────────────────────────────────────────────

 class SentimentSummary(BaseModel):
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -145,6 +145,64 @@ services:
    volumes:
      - camera-snapshots:/data/snapshots

+  # ── News pipeline: hourly scraper (:00) + summarizer (:05) ───────────────
+  # Both services point at the EXISTING osint-db (tables articles +
+  # article_summaries, created by idempotent alembic migration 003_news).
+  # Scheduling replaces the upstream k8s CronJobs with in-compose wall-clock
+  # loops (run_news_scraper.py / run_news_summarizer.py).
+  news-scraper:
+    build:
+      context: ./news/scraper
+      dockerfile: Dockerfile
+      platforms: ["linux/arm64"]
+    image: localhost/osint-news-scraper:latest
+    container_name: osint-news-scraper
+    restart: unless-stopped
+    profiles: ["ingest"]
+    depends_on:
+      db:
+        condition: service_healthy
+    environment:
+      DB_USER: ${DB_USER:-osint}
+      DB_PASSWORD: ${DB_PASSWORD:-osint}
+      DB_HOST: db
+      DB_PORT: ${DB_PORT:-5432}
+      DB_NAME: ${DB_NAME:-osint_data}
+      LOG_LEVEL: ${NEWS_LOG_LEVEL:-INFO}
+      NEWS_SCRAPE_MINUTE: ${NEWS_SCRAPE_MINUTE:-0}
+      NEWS_SCRAPE_RUN_ON_START: ${NEWS_SCRAPE_RUN_ON_START:-1}
+    # Override the image ENTRYPOINT ["scrapy"] with the scheduler loop.
+    entrypoint: []
+    command: ["python", "run_news_scraper.py"]
+
+  news-summarizer:
+    build:
+      context: ./news/summerizer
+      dockerfile: Dockerfile
+      platforms: ["linux/arm64"]
+    image: localhost/osint-news-summarizer:latest
+    container_name: osint-news-summarizer
+    restart: unless-stopped
+    profiles: ["ingest"]
+    depends_on:
+      db:
+        condition: service_healthy
+    environment:
+      DB_USER: ${DB_USER:-osint}
+      DB_PASSWORD: ${DB_PASSWORD:-osint}
+      DB_HOST: db
+      DB_PORT: ${DB_PORT:-5432}
+      DB_NAME: ${DB_NAME:-osint_data}
+      # Required to do real work; unset → the loop logs and idles.
+      GEMINI_API_KEY: ${GEMINI_API_KEY:-}
+      SUMMARY_MODEL: ${SUMMARY_MODEL:-gemini-2.0-flash}
+      BATCH_SIZE: ${NEWS_BATCH_SIZE:-50}
+      SUMMARY_WINDOW_HOURS: ${SUMMARY_WINDOW_HOURS:-1}
+      INCLUDE_FUTURES: ${INCLUDE_FUTURES:-0}
+      NEWS_SUMMARIZE_MINUTE: ${NEWS_SUMMARIZE_MINUTE:-5}
+      NEWS_SUMMARIZE_RUN_ON_START: ${NEWS_SUMMARIZE_RUN_ON_START:-1}
+    command: ["python", "run_news_summarizer.py"]
+
 volumes:
  osint-pgdata:
  camera-snapshots:
--- a/docs/news.md
+++ b/docs/news.md
@ -0,0 +1,139 @@
+# News pipeline — scraper + summarizer
+
+The OSINT dashboard ingests ~257 global news RSS sources hourly and produces
+LLM master summaries. Both services were vendored from the upstream
+`~/Projects/newsPipeline` project and re-integrated here to replace the old
+k8s CronJob choreography with in-compose scheduling against the EXISTING
+osint-db — **no second Postgres**.
+
+## Architecture
+
+```
+257 RSS feeds (news/scraper/urls.txt)
+        │
+        ▼
+news-scraper   (Scrapy, hourly :00) ──► articles table (osint-db)
+        │                                      │
+        │                                      ▼
+news-summarizer (Gemini map-reduce, hourly :05) ──► article_summaries table
+                                                      │
+                                                      ▼
+                              GET /api/news  ·  GET /api/news/summaries
+```
+
+| Component | Image | Container | Scheduling |
+|---|---|---|---|
+| Scraper | `localhost/osint-news-scraper` | `osint-news-scraper` | wall-clock loop, minute `NEWS_SCRAPE_MINUTE` (default :00) |
+| Summarizer | `localhost/osint-news-summarizer` | `osint-news-summarizer` | wall-clock loop, minute `NEWS_SUMMARIZE_MINUTE` (default :05) |
+
+Both services live under the `ingest` compose profile (same as the ingester
+and camera-scraper): `docker compose --profile ingest up -d`.
+
+## Data flow
+
+1. **Scraper** — `news/scraper/run_news_scraper.py` runs
+   `scrapy crawl articles` (spider `news/scraper/newsScraper/spiders/news_spider.py`)
+   at the top of each hour. The spider reads the RSS feed URLs from `urls.txt`,
+   follows each `<item>` link, extracts the main article body, and the
+   `PostgresPipeline` writes to `articles` with URL-based dedup
+   (`ON CONFLICT (url) DO NOTHING`).
+2. **Summarizer** — `news/summerizer/run_news_summarizer.py` runs
+   `summarizer.py` at :05 past each hour. It reads articles from the last
+   `SUMMARY_WINDOW_HOURS`, map-reduces them through Gemini
+   (`SUMMARY_MODEL`, default `gemini-2.0-flash`), and inserts one master
+   summary into `article_summaries`.
+
+Scheduling is done with small in-compose wall-clock loops (not host cron): each
+loop runs once on boot (`*_RUN_ON_START=1`, seeds data fast) then sleeps until
+the next scheduled minute. The loop is serial, so a run that overruns its slot
+simply shifts to the next boundary — two crawls/summaries never overlap.
+
+The `articles` and `article_summaries` tables are created by the idempotent
+alembic migration `003_news` (also created by the scraper's own
+`CREATE TABLE IF NOT EXISTS`, so container startup order doesn't matter).
+
+## Endpoints
+
+### GET /api/news — recent articles
+
+| Query param | Meaning | Default |
+|---|---|---|
+| `domain` | filter by source domain (e.g. `www.reuters.com`) | none |
+| `since` | only articles captured at/after this UTC instant (ISO-8601) | none |
+| `limit` | max rows | `50` (max `500`) |
+| `offset` | pagination offset | `0` |
+
+```json
+[
+  {
+    "id": 1,
+    "title": "…",
+    "url": "https://…",
+    "content": "full extracted article text…",
+    "domain": "www.reuters.com",
+    "timestamp": "2026-08-24T18:10:00Z"
+  }
+]
+```
+
+### GET /api/news/summaries — master LLM summaries
+
+| Query param | Meaning | Default |
+|---|---|---|
+| `since` | only summaries generated at/after this UTC instant | none |
+| `limit` | max rows | `20` (max `100`) |
+| `offset` | pagination offset | `0` |
+
+```json
+[
+  {
+    "id": 1,
+    "summary_text": "master LLM summary (markdown)…",
+    "batch_timestamp": "2026-08-24T18:10:00Z"
+  }
+]
+```
+
+## Configuration (all via env / `.env`)
+
+| Var | Default | Notes |
+|---|---|---|
+| `GEMINI_API_KEY` | *(blank)* | **Required for summaries.** Unset = summarizer logs and idles (never crashes). |
+| `SUMMARY_MODEL` | `gemini-2.0-flash` | Gemini model id. |
+| `NEWS_BATCH_SIZE` | `50` | Articles per map-phase batch. |
+| `SUMMARY_WINDOW_HOURS` | `1` | How far back the summarizer looks for new articles. |
+| `INCLUDE_FUTURES` | `0` | Legacy futures-prices coupling (upstream pipeline). OFF for OSINT; set `1` + install `yfinance` to enable. |
+| `NEWS_SCRAPE_MINUTE` | `0` | Wall-clock minute the scraper fires. |
+| `NEWS_SUMMARIZE_MINUTE` | `5` | Wall-clock minute the summarizer fires. |
+| `NEWS_SCRAPE_RUN_ON_START` | `1` | Run one scrape immediately on container start. |
+| `NEWS_SUMMARIZE_RUN_ON_START` | `1` | Run one summarize immediately on container start. |
+| `NEWS_LOG_LEVEL` | `INFO` | Scrapy log level. |
+| `TELEGRAM_TOKEN` / `TELEGRAM_CHAT_ID` | *(blank)* | Reserved for the (out-of-scope) Telegram delivery bot. |
+
+DB_* for both services is mapped to the shared osint-db credentials
+(`DB_HOST=db`, same `DB_USER/DB_PASSWORD/DB_NAME` as the rest of the stack).
+
+## Prompts
+
+Both prompts are env-overridable — the default `MAP_PROMPT` is OSINT-neutral
+(facts, locations, entities, category, OSINT signal per article) and the default
+`SUMMARY_PROMPT` produces a concise executive summary of the most impactful
+items (with a "no qualifying events" escape hatch). Upstream's futures/markets
+prompt language is gated behind `INCLUDE_FUTURES=1`.
+
+## Tests
+
+`tests/test_api_news.py` — DB-backed API contract tests (auto-skip without a
+reachable test database, same as the FIRMS tests):
+
+```bash
+DB_HOST=... DB_PORT=... DB_USER=osint DB_PASSWORD=... DB_NAME=osint_data \
+  pytest tests/test_api_news.py -v
+```
+
+## Live verification
+
+End-to-end (real crawl → DB → API) is verified after deploy on the Pi: check
+`docker compose --profile ingest logs -f news-scraper news-summarizer`, then
+`curl -s localhost:8000/api/news | head`. Summaries additionally require
+`GEMINI_API_KEY` to be set in `.env` on the Pi.
--- a/news/scraper/.dockerignore
+++ b/news/scraper/.dockerignore
@ -0,0 +1,19 @@
+# Secrets and Config
+.env
+*.key
+*.pem
+
+# Python artifacts
+**/__pycache__/
+*.py[cod]
+*$py.class
+.pytest_cache/
+
+# Scrapy runtime output (created inside the container at /app/data)
+data/
+*.jsonl
+.scrapy/
+
+# Git
+.git
+.gitignore
--- a/news/scraper/.gitignore
+++ b/news/scraper/.gitignore
@ -0,0 +1,4 @@
+# Scrapy runtime output (only created when run locally, not in the container)
+data/
+*.jsonl
+.scrapy/
--- a/news/scraper/Dockerfile
+++ b/news/scraper/Dockerfile
@ -0,0 +1,44 @@
+FROM python:3.12-slim AS builder
+
+# Prevent Python from writing .pyc files and enable unbuffered logging
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /install
+
+# Install system dependencies required for building lxml and other Scrapy deps
+RUN apt-get update && apt-get install -y \
+    gcc \
+    libxml2-dev \
+    libxslt-dev \
+    libffi-dev \
+    libssl-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install dependencies to a temporary location
+COPY requirements.txt .
+RUN pip install --no-cache-dir --prefix=/install -r requirements.txt
+
+
+# --- Stage 2: Runtime ---
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Copy the installed python packages from the builder stage
+COPY --from=builder /install /usr/local
+
+# Copy the project files
+COPY . .
+
+
+# Security: Run as a non-privileged user
+RUN useradd -m scraper
+RUN mkdir -p /app/data && chown -R scraper:scraper /app/data
+USER scraper
+
+# Set the entrypoint to the scrapy command
+ENTRYPOINT ["scrapy"]
+
+# Default command if none is provided
+CMD ["crawl", "articles"]
--- a/news/scraper/newsScraper/init.py
+++ b/news/scraper/newsScraper/init.py
--- a/news/scraper/newsScraper/items.py
+++ b/news/scraper/newsScraper/items.py
@ -0,0 +1,12 @@
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class NewsscraperItem(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    pass
--- a/news/scraper/newsScraper/middlewares.py
+++ b/news/scraper/newsScraper/middlewares.py
@ -0,0 +1,113 @@
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+# useful for handling different item types with a single interface
+from itemadapter import ItemAdapter
+
+
+class NewsscraperSpiderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, or item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request or item objects.
+        pass
+
+    async def process_start(self, start):
+        # Called with an async iterator over the spider start() method or the
+        # matching method of an earlier spider middleware.
+        async for item_or_request in start:
+            yield item_or_request
+
+    def spider_opened(self, spider):
+        spider.logger.info("Spider opened: %s" % spider.name)
+
+
+class NewsscraperDownloaderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info("Spider opened: %s" % spider.name)
+
+class ProxyMiddleware:
+    def __init__(self, proxy_url):
+        self.proxy_url = proxy_url
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(proxy_url=crawler.settings.get('PROXY_URL'))
+
+    def process_request(self, request, spider):
+        # Only attach the proxy if PROXY_URL was successfully built
+        if self.proxy_url:
+            request.meta['proxy'] = self.proxy_url
--- a/news/scraper/newsScraper/pipelines.py
+++ b/news/scraper/newsScraper/pipelines.py
@ -0,0 +1,92 @@
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+
+import logging 
+import psycopg2
+import os
+from scrapy.exceptions import DropItem
+class PostgresPipeline:
+    
+    def __init__(self, db_config):
+        # 1. Store the config
+        self.db_config = db_config
+        # 2. Initialize the set here so it exists when process_item is called
+        self.seen_urls = set()
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        db_config = {
+        'host': crawler.settings.get('DB_HOST'),
+        'database': crawler.settings.get('DB_NAME'),
+        'user': crawler.settings.get('DB_USER'),
+        'password': crawler.settings.get('DB_PASSWORD'),
+        }
+        return cls(db_config=db_config)
+
+
+
+
+    def open_spider(self, spider):
+        # Connect using environment variables
+        self.connection = psycopg2.connect(
+            host=os.getenv('DB_HOST'),
+            database=os.getenv('DB_NAME'),
+            user=os.getenv('DB_USER'),
+            password=os.getenv('DB_PASSWORD'),
+            port=os.getenv('DB_PORT', '5432')
+        )
+        self.cur = self.connection.cursor()
+        
+        # Create table if it doesn't exist
+        self.cur.execute("""
+            CREATE TABLE IF NOT EXISTS articles (
+                id SERIAL PRIMARY KEY,
+                title TEXT,
+                url TEXT UNIQUE,
+                content TEXT,
+                domain TEXT,
+                timestamp TIMESTAMPTZ
+            )
+        """)
+        self.connection.commit()
+
+    def process_item(self, item, spider):
+        if item ['url'] in self.seen_urls:
+            raise DropItem()
+        try:
+            self.cur.execute("""
+                INSERT INTO articles (title, url, content, domain, timestamp)
+                VALUES (%s, %s, %s, %s, %s)
+                ON CONFLICT (url) DO NOTHING
+            """, (
+                item['title'],
+                item['url'],
+                item['text'],
+                item['domain'],
+                item['timestamp']
+            ))
+            if self.cur.rowcount == 0:
+                e = DropItem("Duplicate URL (database conflict)")
+                e.log_level = logging.DEBUG
+                raise e 
+            self.connection.commit()
+            return item 
+        except Exception as e:
+            spider.logger.error(f"Error saving to Postgres: {e}")
+            self.connection.rollback()
+            raise 
+
+    def close_spider(self, spider):
+        self.cur.close()
+        self.connection.close()
+
+from itemadapter import ItemAdapter
+
+
+class NewsscraperPipeline:
+    def process_item(self, item, spider):
+        return item
+
+
--- a/news/scraper/newsScraper/settings.py
+++ b/news/scraper/newsScraper/settings.py
@ -0,0 +1,94 @@
+import os 
+from dotenv import load_dotenv 
+load_dotenv() 
+
+PROXY_USER = os.getenv('PROXY_USER', '').strip()
+PROXY_PASS = os.getenv('PROXY_PASS', '').strip()
+PROXY_ENDPOINT = os.getenv('PROXY_ENDPOINT', '').strip()
+
+def get_proxy_url():
+    if not PROXY_ENDPOINT: 
+        return None 
+    endpoint = PROXY_ENDPOINT.replace('http://', '').replace('https://','')
+
+    if PROXY_USER and PROXY_PASS:
+        return f"http://{PROXY_USER}:{PROXY_PASS}@{endpoint}"
+    else:
+        return f"http://{endpoint}"
+
+PROXY_URL = get_proxy_url() 
+
+DOWNLOADER_MIDDLEWARES = {
+    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 110,
+}
+
+BOT_NAME = "newsScraper"
+
+SPIDER_MODULES = ["newsScraper.spiders"]
+NEWSPIDER_MODULE = "newsScraper.spiders"
+
+ADDONS = {}
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+USER_AGENT = os.getenv('USER_AGENT', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+DEFAULT_REQUEST_HEADERS = {
+   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+   'Accept-Language': 'en-US,en;q=0.5',
+   'Accept-Encoding': 'gzip, deflate, br',
+   'DNT': '1', 
+   'Connection': 'keep-alive',
+   'Upgrade-Insecure-Requests': '1',
+}
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = True 
+
+# Concurrency and throttling settings
+CONCURRENT_REQUESTS = os.getenv('CONCURRENT_REQUESTS', '100').strip() 
+CONCURRENT_REQUESTS_PER_DOMAIN = 2 
+DOWNLOAD_DELAY = 3 
+REACTOR_THREADPOOL_MAXSIZE = 100
+LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
+RETRY_ENABLED = True 
+DOWNLOAD_TIMEOUT = 60
+AJAXCRAWL_ENABLED = False
+AUTO_THROTTLE_ENABLED = True   
+AUTOTHROTTLE_ENABLED = True
+AUTOTHROTTLE_TARGET_CONCURRENCY = 2.0
+DNSCACHE_ENABLED = True 
+DNSCACHE_SIZE = 20000
+DNS_TIMEOUT = 20 
+DNS_RESOLVER = 'scrapy.resolver.CachingThreadedResolver'
+DEPTH_LIMIT = os.getenv('DEPTH_LIMIT', 1)
+
+TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+
+
+FEEDS = {
+    'data/hourly_news.jsonl':{
+        'format': 'jsonlines',
+        'encoding': 'utf8',
+            'overwrite': True, 
+    }
+}
+FEED_EXPORT_ENCODING = "utf-8"
+
+ITEM_PIPELINES = {
+    'newsScraper.pipelines.PostgresPipeline': 300,
+}
+
+# Database Config (These should be in your .env / K8s Secrets)
+
+DB_HOST = os.getenv('DB_HOST', 'postgres-service')
+DB_NAME = os.getenv('DB_NAME', 'news_db')
+DB_USER = os.getenv('DB_USER', 'admin')
+DB_PASSWORD = os.getenv('DB_PASSWORD')
+
+db_config = {
+    'host': DB_HOST,
+    'database': DB_NAME,
+    'user': DB_USER,
+    'password': DB_PASSWORD,
+    'port': 5432 # Default postgres port
+}
--- a/news/scraper/newsScraper/spiders/init.py
+++ b/news/scraper/newsScraper/spiders/init.py
@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.
--- a/news/scraper/newsScraper/spiders/news_spider.py
+++ b/news/scraper/newsScraper/spiders/news_spider.py
@ -0,0 +1,50 @@
+import scrapy
+from scrapy.spiders import XMLFeedSpider
+from urllib.parse import urlparse
+import datetime
+import re
+
+class NewsRSSSpider(XMLFeedSpider):
+    name = "articles"
+    iterator = 'xml' 
+    itertag = 'item'  # Standard RSS tag for an article
+    namespaces = [
+        ('dc', 'http://purl.org/dc/elements/1.1/'),
+        ('content', 'http://purl.org/rss/1.0/modules/content/'),
+        ('media', 'http://search.yahoo.com/mrss/')
+    ]
+    
+    def __init__(self, filename='urls.txt', *args, **kwargs):
+        super(NewsRSSSpider, self).__init__(*args, **kwargs)
+        with open(filename, 'r') as f:
+            # We filter for RSS feeds only here
+            self.start_urls = [line.strip() for line in f if '/rss' in line or '/feed' in line]
+        
+    def parse_node(self, response, node):
+        """This runs for every <item> found in the RSS XML"""
+        title = node.xpath('title/text()').get()
+        link = node.xpath('link/text()').get()
+        pub_date = node.xpath('pubDate/text()').get()
+        
+        # We now yield a Request to the actual article to get the full text
+        # Since these are RSS links, they are usually 'clean' HTML
+        if link:
+            yield scrapy.Request(link, callback=self.parse_article, meta={'title': title, 'date': pub_date})
+
+    def parse_article(self, response):
+        
+        title = response.meta.get('title')
+        
+        # Greedy search for the main text body
+        article_text = response.xpath('//article//p/text() | //main//p/text() | //div[contains(@class, "body")]//p/text()').getall()
+        pure_text = " ".join(article_text)
+        pure_text = re.sub(r'\s+', ' ', pure_text).strip()
+
+        if len(pure_text) > 300:
+            yield {
+                'title': title,
+                'url': response.url,
+                'text': pure_text,
+                'domain': urlparse(response.url).netloc,
+                'timestamp': datetime.datetime.now().isoformat()
+            }
--- a/news/scraper/requirements.txt
+++ b/news/scraper/requirements.txt
@ -0,0 +1,44 @@
+attrs==25.4.0
+Automat==25.4.16
+brotli==1.2.0
+certifi==2026.1.4
+cffi==2.0.0
+charset-normalizer==3.4.4
+constantly==23.10.4
+cryptography==46.0.3
+cssselect==1.3.0
+defusedxml==0.7.1
+filelock==3.20.2
+hyperlink==21.0.0
+idna==3.11
+Incremental==24.11.0
+itemadapter==0.13.1
+itemloaders==1.3.2
+jmespath==1.0.1
+lxml==6.0.2
+packaging==25.0
+parsel==1.10.0
+Protego==0.5.0
+pyasn1==0.6.1
+pyasn1_modules==0.4.2
+pycparser==2.23
+PyDispatcher==2.0.7
+pyOpenSSL==25.3.0
+queuelib==1.8.0
+requests==2.32.5
+requests-file==3.0.1
+Scrapy==2.14.0
+scrapy-user-agents==0.1.1
+service-identity==24.2.0
+tldextract==5.3.1
+Twisted==25.5.0
+typing_extensions==4.15.0
+ua-parser==1.0.1
+ua-parser-builtins==202601
+urllib3==2.6.3
+user-agents==2.2.0
+w3lib==2.3.1
+zope.interface==8.1.1
+psycopg2-binary==2.9.11
+python-dotenv==0.20.0
+scrapy-playwright==0.0.45
--- a/news/scraper/run_news_scraper.py
+++ b/news/scraper/run_news_scraper.py
@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Scheduler loop for the news scraper — hourly scrape at minute :00.
+
+Replaces the k8s CronJob (`0 * * * *`) with an in-compose loop so the whole
+news pipeline lives inside docker-compose. Each iteration:
+
+  1. (optionally, on first boot) runs the Scrapy crawl once to seed data fast
+  2. sleeps until the next :NEWS_SCRAPE_MINUTE wall-clock boundary
+
+Because the loop is serial, a crawl that overruns its hour simply delays the
+next run to the following boundary — two crawls never overlap.
+
+Env (all optional, 12-factor):
+  NEWS_SCRAPE_MINUTE        minute of the hour to fire (default 0)
+  NEWS_SCRAPE_RUN_ON_START  "1" to crawl once immediately on boot (default 1)
+"""
+
+from __future__ import annotations
+
+import datetime
+import logging
+import os
+import subprocess
+import sys
+import time
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+logger = logging.getLogger("news.scraper")
+
+MINUTE = int(os.getenv("NEWS_SCRAPE_MINUTE", "0"))
+RUN_ON_START = os.getenv("NEWS_SCRAPE_RUN_ON_START", "1").lower() in ("1", "true", "yes")
+
+CRAWL_CMD = ["scrapy", "crawl", "articles"]
+
+
+def seconds_until_next(minute: int) -> float:
+    """Seconds until the next occurrence of ``minute`` past the hour (local time)."""
+    now = datetime.datetime.now()
+    nxt = now.replace(minute=minute, second=0, microsecond=0) + datetime.timedelta(hours=1)
+    return (nxt - now).total_seconds()
+
+
+def run_crawl() -> None:
+    logger.info("scrape starting at %s", datetime.datetime.now().isoformat(timespec="seconds"))
+    try:
+        proc = subprocess.run(CRAWL_CMD, cwd="/app")
+        logger.info("scrape finished rc=%s", proc.returncode)
+    except Exception:  # noqa: BLE001 — keep the loop alive across failures
+        logger.exception("scrape failed")
+
+
+def main() -> None:
+    logger.info(
+        "news scraper loop starting (minute=%s, run_on_start=%s)",
+        MINUTE, RUN_ON_START,
+    )
+    if RUN_ON_START:
+        run_crawl()
+    while True:
+        delay = seconds_until_next(MINUTE)
+        logger.info("next scrape at :%02d (in %.0fs)", MINUTE, delay)
+        time.sleep(delay)
+        run_crawl()
+
+
+if __name__ == "__main__":
+    sys.exit(main())
--- a/news/scraper/scrapy.cfg
+++ b/news/scraper/scrapy.cfg
@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = newsScraper.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = newsScraper
--- a/news/scraper/urls.txt
+++ b/news/scraper/urls.txt
@ -0,0 +1,257 @@
+# --- NORTH AMERICA ---
+# USA
+https://www.npr.org
+https://www.pbs.org/newshour
+https://www.usatoday.com
+https://www.cbsnews.com
+https://www.nbcnews.com
+
+# Canada
+https://www.cbc.ca/news
+https://www.ctvnews.ca
+https://globalnews.ca
+https://nationalpost.com
+https://www.thestar.com
+
+# Mexico
+https://www.eluniversal.com.mx
+https://www.milenio.com
+https://www.jornada.com.mx
+https://www.excelsior.com.mx
+https://aristeguinoticias.com
+
+# --- SOUTH AMERICA ---
+# Brazil
+https://g1.globo.com
+https://www.uol.com.br
+https://agenciabrasil.ebc.com.br
+https://www.metropoles.com
+https://www.terra.com.br/noticias
+
+# Argentina
+https://www.infobae.com
+https://www.clarin.com
+https://www.lanacion.com.ar
+https://www.pagina12.com.ar
+https://www.cronista.com
+
+# Colombia
+https://www.eltiempo.com
+https://www.elespectador.com
+https://www.semana.com
+https://www.bluradio.com
+https://www.rcnradio.com
+
+# --- EUROPE ---
+# United Kingdom
+https://www.bbc.com/news
+https://www.theguardian.com/uk
+https://news.sky.com
+https://www.independent.co.uk
+https://metro.co.uk
+
+# France
+https://www.france24.com/en
+https://www.lefigaro.fr
+https://www.20minutes.fr
+https://www.francetvinfo.fr
+https://www.lemonde.fr
+
+# Germany
+https://www.dw.com/en
+https://www.tagesschau.de
+https://www.spiegel.de
+https://www.zeit.de
+https://www.bild.de
+
+# Spain
+https://elpais.com
+https://www.elmundo.es
+https://www.rtve.es/noticias
+https://www.20minutos.es
+https://www.elconfidencial.com
+
+# Italy
+https://www.ansa.it
+https://www.corriere.it
+https://www.repubblica.it
+https://www.lastampa.it
+https://tg24.sky.it
+
+# Russia (State & Independent mix)
+https://tass.com
+https://www.interfax.ru
+https://www.rt.com
+https://www.themoscowtimes.com
+https://meduza.io/en
+
+# --- ASIA ---
+# China
+https://www.xinhuanet.com/english
+https://www.chinadaily.com.cn
+https://www.globaltimes.cn
+https://www.cgtn.com
+https://www.scmp.com
+
+# India
+https://www.ndtv.com
+https://timesofindia.indiatimes.com
+https://indianexpress.com
+https://www.thehindu.com
+https://www.hindustantimes.com
+
+# Japan
+https://www3.nhk.or.jp/nhkworld
+https://www.japantimes.co.jp
+https://www.asahi.com/ajw
+https://mainichi.jp/english
+https://english.kyodonews.net
+
+# South Korea
+https://en.yna.co.kr
+https://www.koreaherald.com
+https://koreajoongangdaily.joins.com
+https://www.donga.com/en
+https://english.chosun.com
+
+# --- AFRICA ---
+# South Africa
+https://www.news24.com
+https://www.iol.co.za
+https://www.dailymaverick.co.za
+https://www.sabcnews.com
+https://www.timeslive.co.za
+
+# Nigeria
+https://www.vanguardngr.com
+https://punchng.com
+https://dailypost.ng
+https://saharareporters.com
+https://thenationonlineng.net
+
+# --- MIDDLE EAST ---
+# General Region
+https://www.aljazeera.com
+https://english.alarabiya.net
+https://www.timesofisrael.com
+https://www.tehrantimes.com
+https://www.middleeasteye.net
+
+# --- OCEANIA ---
+# Australia
+https://www.abc.net.au/news
+https://www.news.com.au
+https://www.9news.com.au
+https://www.smh.com.au
+https://www.theage.com.au
+# --- USA: MAJOR CITIES & LOCAL ---
+https://www.latimes.com
+https://www.chicagotribune.com
+https://www.sfchronicle.com
+https://www.bostonglobe.com
+https://www.seattletimes.com
+https://www.houstonchronicle.com
+https://www.inquirer.com
+https://www.denverpost.com
+https://www.miamiherald.com
+https://www.dallasnews.com
+https://www.startribune.com
+https://www.detroitnews.com
+https://www.ajc.com
+https://www.nydailynews.com
+https://nypost.com
+https://www.mercurynews.com
+https://www.baltimoresun.com
+https://www.oregonlive.com
+https://www.cleveland.com
+https://www.tampabay.com
+
+# --- EUROPE: LOCAL & INDEPENDENT ---
+https://www.manchestereveningnews.co.uk
+https://www.scotsman.com
+https://www.belfasttelegraph.co.uk
+https://www.irishtimes.com
+https://www.berliner-zeitung.de
+https://www.leparisien.fr
+https://www.corriere.it
+https://www.elperiodico.com
+https://kyivindependent.com
+https://www.pravda.com.ua/en
+https://balkaninsight.com
+https://www.ekathimerini.com
+https://www.swissinfo.ch
+https://www.thelocal.se
+https://www.thelocal.fr
+https://www.thelocal.de
+https://www.novinite.com
+https://www.romania-insider.com
+https://hungarytoday.hu
+https://polandin.com
+
+# --- MIDDLE EAST & CONFLICT ZONES ---
+https://www.haaretz.com
+https://www.jpost.com
+https://www.timesofisrael.com
+https://www.rudaw.net/english
+https://www.kurdistan24.net/en
+https://www.middleeasteye.net
+https://www.al-monitor.com
+https://www.dailysabah.com
+https://www.duvarenglish.com
+https://english.aawsat.com
+https://www.arabnews.com
+https://www.thenationalnews.com
+https://www.jordantimes.com
+https://www.naharnet.com
+https://www.tehrantimes.com
+
+# --- ASIA: HOTSPOTS & LOCAL ---
+https://www.taipeitimes.com
+https://focustaiwan.tw
+https://hongkongfp.com
+https://www.bangkokpost.com
+https://www.thejakartapost.com
+https://www.straitstimes.com
+https://www.khmertimeskh.com
+https://www.irrawaddy.com
+https://www.myanmarnow.org/en
+https://www.rappler.com
+https://www.philstar.com
+https://english.hani.co.kr
+https://www.japantoday.com
+https://www.caixinglobal.com
+https://thediplomat.com
+
+# --- LATIN AMERICA & AFRICA: LOCAL ---
+https://buenosairesherald.com
+https://riotimesonline.com
+https://mercopress.com
+https://www.elmostrador.cl
+https://www.jornada.com.mx
+https://www.theeastafrican.co.ke
+https://allafrica.com
+https://www.premiumtimesng.com
+https://www.dailytrust.com
+https://www.newtimes.co.rw
+https://www.herald.co.zw
+https://www.namibian.com.na
+https://www.graphic.com.gh
+https://www.thecitizen.co.tz
+https://www.monitor.co.ug
+
+# --- ALTERNATIVE, INVESTIGATIVE & "FRINGE" ---
+https://theintercept.com
+https://www.propublica.org
+https://www.democracynow.org
+https://reason.com
+https://www.motherjones.com
+https://www.vox.com
+https://slate.com
+https://www.axios.com
+https://www.politico.com
+https://www.vice.com
+https://www.bellingcat.com
+https://www.project-syndicate.org
+https://cryptonews.com
+https://www.coindesk.com
+https://techcrunch.com
--- a/news/summerizer/.dockerignore
+++ b/news/summerizer/.dockerignore
@ -0,0 +1,14 @@
+# Secrets and Config
+.env
+*.key
+*.pem
+
+# Python artifacts
+**/__pycache__/
+*.py[cod]
+*$py.class
+.pytest_cache/
+
+# Git
+.git
+.gitignore
--- a/news/summerizer/Dockerfile
+++ b/news/summerizer/Dockerfile
@ -0,0 +1,24 @@
+FROM python:3.11-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+# libpq-dev + gcc for psycopg2 build/adapters; keep the image lean.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libpq-dev gcc \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY summarizer.py run_news_summarizer.py ./
+
+# Security: run as a non-privileged user.
+RUN useradd -m summarizer_user
+USER summarizer_user
+
+# Default: single pass (as the old k8s CronJob ran). The compose service
+# overrides `command` to the scheduler loop (run_news_summarizer.py).
+CMD ["python", "summarizer.py"]
--- a/news/summerizer/requirements.txt
+++ b/news/summerizer/requirements.txt
@ -0,0 +1,6 @@
+# Database adapter for PostgreSQL (shared osint-db).
+psycopg2-binary==2.9.11
+
+# Gemini LLM SDK (google-genai). yfinance is NOT a dependency: futures prices
+# are gated behind INCLUDE_FUTURES=1 and lazy-imported (install it to enable).
+google-genai
--- a/news/summerizer/run_news_summarizer.py
+++ b/news/summerizer/run_news_summarizer.py
@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Scheduler loop for the news summarizer — hourly summarize at minute :05.
+
+Replaces the k8s CronJob (`5 * * * *`) with an in-compose loop. Runs once on
+boot (catches up on any articles scraped since the last summary), then fires
+at each :NEWS_SUMMARIZE_MINUTE wall-clock boundary.
+
+The loop is serial, so a slow LLM pass never overlaps the next run.
+
+Env (all optional, 12-factor):
+  NEWS_SUMMARIZE_MINUTE        minute of the hour to fire (default 5)
+  NEWS_SUMMARIZE_RUN_ON_START  "1" to summarize once immediately on boot (default 1)
+  GEMINI_API_KEY               required to do real work; unset = idle
+"""
+
+from __future__ import annotations
+
+import datetime
+import logging
+import os
+import subprocess
+import sys
+import time
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
+logger = logging.getLogger("news.summarizer.scheduler")
+
+MINUTE = int(os.getenv("NEWS_SUMMARIZE_MINUTE", "5"))
+RUN_ON_START = os.getenv("NEWS_SUMMARIZE_RUN_ON_START", "1").lower() in ("1", "true", "yes")
+
+
+def seconds_until_next(minute: int) -> float:
+    """Seconds until the next occurrence of ``minute`` past the hour (local time)."""
+    now = datetime.datetime.now()
+    nxt = now.replace(minute=minute, second=0, microsecond=0) + datetime.timedelta(hours=1)
+    return (nxt - now).total_seconds()
+
+
+def run_summarize() -> None:
+    logger.info("summarize starting at %s", datetime.datetime.now().isoformat(timespec="seconds"))
+    try:
+        proc = subprocess.run([sys.executable, "summarizer.py"], cwd="/app")
+        logger.info("summarize finished rc=%s", proc.returncode)
+    except Exception:  # noqa: BLE001 — keep the loop alive across failures
+        logger.exception("summarize failed")
+
+
+def main() -> None:
+    if not os.getenv("GEMINI_API_KEY", "").strip():
+        logger.warning(
+            "GEMINI_API_KEY not set — summarizer will idle (set it in .env and "
+            "recreate the service to enable)"
+        )
+    logger.info(
+        "news summarizer loop starting (minute=%s, run_on_start=%s)",
+        MINUTE, RUN_ON_START,
+    )
+    if RUN_ON_START:
+        run_summarize()
+    while True:
+        delay = seconds_until_next(MINUTE)
+        logger.info("next summarize at :%02d (in %.0fs)", MINUTE, delay)
+        time.sleep(delay)
+        run_summarize()
+
+
+if __name__ == "__main__":
+    sys.exit(main())
--- a/news/summerizer/summarizer.py
+++ b/news/summerizer/summarizer.py
@ -0,0 +1,307 @@
+#!/usr/bin/env python3
+"""News summarizer — LLM (Gemini) map-reduce summarization of scraped articles.
+
+Reads articles scraped within the last hour from the shared `articles` table,
+summarizes them with Gemini (map phase per batch, reduce phase into one master
+summary), and stores the result in `article_summaries` — both tables live in
+the EXISTING osint-db (created by alembic migration 003_news, idempotent).
+
+Everything is env-driven (12-factor):
+  DB_HOST / DB_NAME / DB_USER / DB_PASSWORD / DB_PORT   PostgreSQL (osint-db)
+  GEMINI_API_KEY           Google AI Studio key (required to actually run)
+  SUMMARY_MODEL            Gemini model id (default gemini-2.0-flash)
+  BATCH_SIZE               articles per map-phase batch (default 50)
+  SUMMARY_WINDOW_HOURS     look-back window in hours (default 1)
+  MAP_PROMPT               override map-phase prompt (uses {batch_text})
+  SUMMARY_PROMPT           override reduce-phase prompt (uses {final_input})
+  INCLUDE_FUTURES          "1" to prepend live futures prices (default 0)
+
+The futures/markets coupling from the original pipeline is gated behind
+INCLUDE_FUTURES and OFF by default — it is irrelevant to the OSINT dashboard
+and pulled yfinance into the image. Re-enable by installing yfinance and
+setting INCLUDE_FUTURES=1.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from datetime import datetime
+
+import psycopg2
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+logger = logging.getLogger("news.summarizer")
+
+# ── Configuration (12-factor, container-friendly defaults) ─────────────────
+DB_CONFIG = {
+    "host": os.getenv("DB_HOST", "db").strip(),
+    "database": os.getenv("DB_NAME", "osint_data").strip(),
+    "user": os.getenv("DB_USER", "osint").strip(),
+    "password": os.getenv("DB_PASSWORD", "").strip(),
+    "port": int(os.getenv("DB_PORT", "5432")),
+}
+
+GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "").strip()
+MODEL_NAME = os.getenv("SUMMARY_MODEL", "gemini-2.0-flash").strip()
+BATCH_SIZE = int(os.getenv("BATCH_SIZE", "50"))
+SUMMARY_WINDOW_HOURS = int(os.getenv("SUMMARY_WINDOW_HOURS", "1"))
+INCLUDE_FUTURES = os.getenv("INCLUDE_FUTURES", "0").lower() in ("1", "true", "yes")
+
+# Only touched when INCLUDE_FUTURES=1 (legacy markets coupling, OSINT-off).
+FUTURES_TICKERS = {
+    "Equity Indices": ["ES=F", "NQ=F", "YM=F", "RTY=F"],
+    "Energy": ["CL=F", "NG=F", "HO=F", "RB=F"],
+    "Metals": ["GC=F", "SI=F", "HG=F"],
+    "Agriculture": ["ZC=F", "ZS=F", "ZW=F", "ZL=F", "KE=F"],
+    "Currencies": ["6E=F", "6J=F", "6B=F"],
+}
+
+# ── OSINT-neutral default prompts (env-overridable via MAP_PROMPT/SUMMARY_PROMPT) ──
+
+MAP_PROMPT_DEFAULT = """\
+You are a precise, factual OSINT news processor. Your ONLY source of information is the articles provided below. Do NOT add external knowledge, assumptions, training data, or invented facts.
+
+For EACH article in the batch:
+1. Extract 2-4 key factual bullet points (who, what, when, where, numbers, quotes — stay very close to the text).
+2. Location: name the country / city / region mentioned if determinable from the text, else "Unknown".
+3. Entities: list the key people, organizations, or governments mentioned (comma-separated, only names present in the text), else "None".
+4. Category: pick one — politics, military/conflict, economy, technology, environment/disaster, health, crime, society, sport, other.
+5. OSINT signal: if the article describes an event with geopolitical, security, military, economic, or disaster significance, say so in one short sentence. Otherwise write: "No notable OSINT signal."
+
+If several articles cover the same story, add one short batch-level note at the end: "Batch theme: [one sentence]".
+
+Output format — strictly one block per article:
+
+Article 1:
+- Fact bullet 1
+- Fact bullet 2
+- Location: ...
+- Entities: ...
+- Category: ...
+- OSINT signal: ...
+
+Article 2:
+...
+
+Articles in this batch:
+{batch_text}
+"""
+
+SUMMARY_PROMPT_DEFAULT = """\
+CRITICAL INSTRUCTION - REPEAT 3 TIMES: YOU MUST USE ONLY THE DATA PROVIDED BELOW. DO NOT INVENT, RECALL, OR ADD ANY EVENTS, NAMES, DATES, IMPLICATIONS, PROJECTS, OR DETAILS NOT EXPLICITLY PRESENT IN THE DATA. IF THE DATA HAS NO MAJOR GEOPOLITICAL/TECH/MILITARY/ECONOMIC/IMPACTFUL EVENTS OR UNUSUAL STORIES, OUTPUT ONLY: "No qualifying impactful or unusual events in the recent hourly news data." AND STOP. NO EXTERNAL KNOWLEDGE FROM TRAINING.
+
+Write a concise executive summary of the most impactful items as a short markdown list, one line per story, using only the data.
+
+DATA:
+{final_input}
+"""
+
+
+# ── LLM helpers ────────────────────────────────────────────────────────────
+
+_client = None
+
+
+def _get_client():
+    """Lazily build the Gemini client (avoids import/init when key unset)."""
+    global _client
+    if _client is None:
+        from google import genai
+
+        _client = genai.Client(api_key=GEMINI_API_KEY)
+    return _client
+
+
+def _extract_text(resp) -> str:
+    """Defensively pull text out of the google-genai GenerateContentResponse.
+
+    The modern SDK returns the response directly (``resp.text``); some older
+    wrappers exposed it as ``resp.response``. Handle both plus a candidates
+    fallback so a provider/SDK change degrades to "" instead of crashing.
+    """
+    if not resp:
+        return ""
+    if hasattr(resp, "text") and resp.text:
+        return resp.text
+    inner = getattr(resp, "response", None)
+    if inner is not None and hasattr(inner, "text") and inner.text:
+        return inner.text
+    try:
+        parts = []
+        for cand in getattr(resp, "candidates", None) or []:
+            content = getattr(cand, "content", None)
+            for part in getattr(content, "parts", None) or []:
+                if getattr(part, "text", None):
+                    parts.append(part.text)
+        return "\n".join(parts)
+    except Exception:  # noqa: BLE001
+        return str(resp)
+
+
+def call_llm(prompt: str) -> str:
+    """Send a prompt to Gemini and return the text ("" on any failure)."""
+    if not GEMINI_API_KEY:
+        logger.warning("GEMINI_API_KEY not set — skipping LLM call")
+        return ""
+    try:
+        resp = _get_client().models.generate_content(model=MODEL_NAME, contents=prompt)
+        return _extract_text(resp)
+    except Exception as exc:  # noqa: BLE001
+        logger.error("Gemini API error: %s", exc)
+        return ""
+
+
+# ── Futures (legacy, gated) ────────────────────────────────────────────────
+
+def fetch_current_futures_prices() -> dict:
+    """Live futures prices. Only meaningful when INCLUDE_FUTURES=1."""
+    if not INCLUDE_FUTURES:
+        return {}
+    try:
+        import yfinance as yf  # noqa: PLC0415
+    except ImportError:
+        logger.warning(
+            "INCLUDE_FUTURES=1 but yfinance is not installed — install it to enable futures prices"
+        )
+        return {}
+
+    prices: dict = {}
+    for category, tickers in FUTURES_TICKERS.items():
+        for ticker in tickers:
+            try:
+                data = yf.Ticker(ticker).history(period="1d", interval="1m")
+                if not data.empty:
+                    last_price = data["Close"].iloc[-1]
+                    prices[ticker] = {
+                        "price": round(last_price, 2),
+                        "change_pct": round(
+                            (last_price - data["Open"].iloc[0]) / data["Open"].iloc[0] * 100, 2
+                        ) if len(data) > 1 else 0,
+                        "timestamp": datetime.utcnow().strftime("%Y-%m-%d %H:%M UTC"),
+                        "category": category,
+                    }
+                else:
+                    prices[ticker] = {"price": None, "error": "No data"}
+            except Exception as exc:  # noqa: BLE001
+                prices[ticker] = {"price": None, "error": str(exc)}
+    return prices
+
+
+def build_futures_context() -> str:
+    ctx = f"CURRENT FUTURES PRICES (as of {datetime.now().strftime('%Y-%m-%d %H:%M UTC')}):\n"
+    for ticker, info in fetch_current_futures_prices().items():
+        if info.get("price") is not None:
+            ctx += (
+                f"- {ticker} ({info['category']}): ${info['price']:.2f}  "
+                f"({info['change_pct']:+.2f}% today)\n"
+            )
+        else:
+            ctx += f"- {ticker}: unavailable ({info.get('error', 'unknown error')})\n"
+    return ctx
+
+
+# ── DB helpers ─────────────────────────────────────────────────────────────
+
+def get_recent_news() -> list[dict]:
+    """Fetch articles from the last SUMMARY_WINDOW_HOURS (content > 100 chars)."""
+    query = """
+        SELECT title, content, url, domain
+        FROM articles
+        WHERE timestamp > NOW() - make_interval(hours => %s)
+          AND content IS NOT NULL AND length(content) > 100
+        ORDER BY timestamp DESC;
+    """
+    try:
+        conn = psycopg2.connect(**DB_CONFIG)
+        cur = conn.cursor()
+        cur.execute(query, (SUMMARY_WINDOW_HOURS,))
+        rows = cur.fetchall()
+        cur.close()
+        conn.close()
+        return [
+            {"title": r[0], "content": r[1], "url": r[2], "domain": r[3]}
+            for r in rows
+        ]
+    except Exception as exc:  # noqa: BLE001
+        logger.error("Database error reading articles: %s", exc)
+        return []
+
+
+def save_summary_to_db(summary_text: str) -> None:
+    """Insert one master summary row (table created by alembic 003_news)."""
+    if not summary_text or len(summary_text.strip()) < 10:
+        logger.info("Summary too short or empty. Skipping save.")
+        return
+    try:
+        conn = psycopg2.connect(**DB_CONFIG)
+        cur = conn.cursor()
+        cur.execute(
+            "INSERT INTO article_summaries (summary_text) VALUES (%s)",
+            (summary_text.strip(),),
+        )
+        conn.commit()
+        logger.info("Master summary saved to database successfully.")
+        cur.close()
+        conn.close()
+    except Exception as exc:  # noqa: BLE001
+        logger.error("Error saving summary to DB: %s", exc)
+
+
+# ── Orchestration ──────────────────────────────────────────────────────────
+
+def build_map_prompt(batch: list[dict]) -> str:
+    batch_text = "\n\n".join(
+        f"Title: {a['title']}\nSource: {a['domain']}\nURL: {a['url']}\nContent: {a['content'][:1500]}"
+        for a in batch
+    )
+    template = os.getenv("MAP_PROMPT", MAP_PROMPT_DEFAULT)
+    prefix = build_futures_context() + "\n" if INCLUDE_FUTURES else ""
+    try:
+        return prefix + template.format(batch_text=batch_text)
+    except KeyError:
+        return prefix + template
+
+
+def build_master_prompt(final_input: str) -> str:
+    template = os.getenv("SUMMARY_PROMPT", SUMMARY_PROMPT_DEFAULT)
+    prefix = build_futures_context() + "\n" if INCLUDE_FUTURES else ""
+    try:
+        return prefix + template.format(final_input=final_input)
+    except KeyError:
+        return prefix + template
+
+
+def summarize_news() -> None:
+    """Map-reduce summarize recent articles and store the master summary."""
+    articles = get_recent_news()
+    if not articles:
+        logger.info("No new articles found in the last %sh.", SUMMARY_WINDOW_HOURS)
+        return
+
+    logger.info(
+        "Processing %d articles with %s (batch_size=%d, futures=%s)...",
+        len(articles), MODEL_NAME, BATCH_SIZE, INCLUDE_FUTURES,
+    )
+
+    partial_summaries: list[str] = []
+    for i in range(0, len(articles), BATCH_SIZE):
+        batch = articles[i : i + BATCH_SIZE]
+        logger.info("map batch %d/%d (%d articles)", i // BATCH_SIZE + 1, -(-len(articles) // BATCH_SIZE), len(batch))
+        summary = call_llm(build_map_prompt(batch))
+        if summary:
+            partial_summaries.append(summary)
+
+    final_input = "\n\n".join(partial_summaries)
+    if not final_input.strip():
+        logger.warning("No partial summaries produced — nothing to reduce.")
+        return
+
+    logger.info("reduce phase over %d partial summaries", len(partial_summaries))
+    master_summary = call_llm(build_master_prompt(final_input))
+    if master_summary:
+        save_summary_to_db(master_summary)
+
+
+if __name__ == "__main__":
+    summarize_news()
--- a/tests/test_api_news.py
+++ b/tests/test_api_news.py
@ -0,0 +1,142 @@
+"""Integration tests for the news pipeline API (GET /api/news + summaries).
+
+DB-backed: marked `requires_db` and auto-skip when the test database is
+unreachable (see tests/conftest.py). Seeding writes directly to the shared
+`articles` / `article_summaries` tables, exactly as the scraper + summarizer
+services would.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from datetime import datetime, timezone
+
+import asyncpg
+import httpx
+import pytest
+
+from conftest import requires_db
+
+from main import app
+
+BASE = "http://test"
+
+
+def _conn_kwargs() -> dict:
+    return {
+        "host": os.environ["DB_HOST"],
+        "port": int(os.environ["DB_PORT"]),
+        "user": os.environ["DB_USER"],
+        "password": os.environ["DB_PASSWORD"],
+        "database": os.environ["DB_NAME"],
+    }
+
+
+def _truncate() -> None:
+    async def run():
+        conn = await asyncpg.connect(**_conn_kwargs())
+        try:
+            await conn.execute("TRUNCATE articles, article_summaries")
+        finally:
+            await conn.close()
+
+    asyncio.run(run())
+
+
+@pytest.fixture()
+def clean_news():
+    _truncate()
+    yield
+    _truncate()
+
+
+def _seed_article(title: str, url: str, domain: str, ts: str, content: str = "body text") -> None:
+    async def run():
+        conn = await asyncpg.connect(**_conn_kwargs())
+        try:
+            await conn.execute(
+                "INSERT INTO articles (title, url, content, domain, timestamp) "
+                "VALUES ($1, $2, $3, $4, $5)",
+                title, url, content, domain, datetime.fromisoformat(ts),
+            )
+        finally:
+            await conn.close()
+
+    asyncio.run(run())
+
+
+def _seed_summary(text: str, ts: str) -> None:
+    async def run():
+        conn = await asyncpg.connect(**_conn_kwargs())
+        try:
+            await conn.execute(
+                "INSERT INTO article_summaries (summary_text, batch_timestamp) "
+                "VALUES ($1, $2)",
+                text, datetime.fromisoformat(ts),
+            )
+        finally:
+            await conn.close()
+
+    asyncio.run(run())
+
+
+def _get(path: str) -> httpx.Response:
+    async def _get_async() -> httpx.Response:
+        transport = httpx.ASGITransport(app=app)
+        async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
+            return await client.get(path)
+
+    return asyncio.run(_get_async())
+
+
+@requires_db
+def test_api_news_returns_newest_first(clean_news):
+    _seed_article("older", "https://a.example/1", "a.example", "2026-08-24T17:00:00+00:00")
+    _seed_article("newer", "https://a.example/2", "a.example", "2026-08-24T18:00:00+00:00")
+    body = _get("/api/news").json()
+    assert isinstance(body, list)
+    assert len(body) == 2
+    # newest first
+    assert [a["title"] for a in body] == ["newer", "older"]
+    # exact JSON contract the frontend news panel needs
+    assert set(body[0].keys()) == {
+        "id", "title", "url", "content", "domain", "timestamp",
+    }
+    assert body[0]["domain"] == "a.example"
+
+
+@requires_db
+def test_api_news_domain_filter(clean_news):
+    _seed_article("x", "https://x.example/1", "x.example", "2026-08-24T17:00:00+00:00")
+    _seed_article("y", "https://y.example/1", "y.example", "2026-08-24T17:00:00+00:00")
+    body = _get("/api/news?domain=y.example").json()
+    assert len(body) == 1
+    assert body[0]["domain"] == "y.example"
+
+
+@requires_db
+def test_api_news_since_filter(clean_news):
+    _seed_article("before", "https://a.example/1", "a.example", "2026-08-24T17:00:00+00:00")
+    _seed_article("after", "https://a.example/2", "a.example", "2026-08-24T18:30:00+00:00")
+    body = _get("/api/news?since=2026-08-24T18:00:00Z").json()
+    assert len(body) == 1
+    assert body[0]["title"] == "after"
+
+
+@requires_db
+def test_api_news_summaries_contract(clean_news):
+    _seed_summary("master summary markdown…", "2026-08-24T18:05:00+00:00")
+    body = _get("/api/news/summaries").json()
+    assert isinstance(body, list)
+    assert len(body) == 1
+    s = body[0]
+    assert set(s.keys()) == {"id", "summary_text", "batch_timestamp"}
+    assert s["summary_text"] == "master summary markdown…"
+    assert s["batch_timestamp"].startswith("2026-08-24T18:05")
+
+
+@requires_db
+def test_api_news_empty(clean_news):
+    assert _get("/api/news").json() == []
+    assert _get("/api/news/summaries").json() == []