Merge pull request 'feat(map): quiet HUD chrome — VIIRS default, collapsed layer rail' (#49) from feat/hud-chrome-quiet into master

Reviewed-on: #49

.env.example

@@ -31,20 +31,6 @@ NOMINATIM_URL=https://nominatim.openstreetmap.org
 NOMINATIM_MIN_INTERVAL=1.1
 SNAPSHOT_TTL_SECONDS=300
 
-# ── masscan active camera discovery (host-level systemd service, NOT compose) ─
-# Continuous rolling sweep for open RTSP port 554 across a range. Runs on the
-# Pi host via deploy/osint-masscan.service (needs root + raw sockets). Results
-# land in the same `cameras` table as the scraper (discovery_source=masscan).
-# NOTE: 200 pps is the residential-safe default. 1k/10k pps saturated a home
-# uplink. A full 0.0.0.0/0 sweep at 200 pps takes ~8 months (rolling).
-MASSCAN_RANGE=0.0.0.0/0
-MASSCAN_PORTS=554
-MASSCAN_RATE=200
-MASSCAN_RETRIES=1
-MASSCAN_WAIT=0
-MASSCAN_EXCLUDEFILE=/etc/osint-dashboard/masscan-excludes.txt
-MASSCAN_FLUSH_EVERY=250
-
 # ── NASA FIRMS (active fire / hotspot ingest) ──────────────────────────────
 # MAP_KEY is FREE — get one at https://firms.modaps.eosdis.nasa.gov/api/map_key_info/
 # (1-minute signup, no payment). Leave blank to keep fire ingest idle.
@@ -59,6 +45,24 @@ FIRMS_INTERVAL=900
 # Set to 0 to disable the fire loop entirely.
 INGEST_FIRES=1
 
+# ── VesselAPI (commercial REST AIS — Hormuz, 5×/day, 150 calls/mo cap) ─────
+# Independent of AISStream (open/shared live US-coast WebSocket). Both stay
+# on when their keys are set; missing one never disables the other.
+# Prefer pasting VESSELAPI_API_KEY on the dashboard Keys page.
+# The poller idles when the key is unset. Never called from map pans
+# (GET /api/vessels serves the shared last-known cache only).
+VESSELAPI_API_KEY=
+# Bounding box(es) as minlat,minlon,maxlat,maxlon (lat/lon order). Semicolon-
+# separated for multiple boxes. Default = Strait of Hormuz (span 3.6 ≤ 4° cap).
+VESSELAPI_BBOX=25.5,55.4,27.3,57.2
+# Poll cadence in seconds (17280 = 4.8h → 5 polls/day = 150/mo).
+VESSELAPI_INTERVAL=17280
+# Local hard cap on successful 2xx calls per UTC day (persisted in Postgres).
+VESSELAPI_MAX_CALLS_PER_DAY=5
+# 1 = run the poller inside the dashboard process (default); ingester off.
+VESSELAPI_IN_APP=1
+VESSELAPI_IN_INGEST=0
+
 # ── API keys (managed from the dashboard UI) ──────────────────────────────
 # Keys such as NOUS_API_KEY and TELEGRAM_TOKEN are stored in the Postgres
 # `api_keys` table and managed from the dashboard's "Keys" tab
@@ -67,10 +71,8 @@ INGEST_FIRES=1
 # its lookup/fallback is a planned follow-up.
 
 # ── News pipeline (scraper + summarizer, profile `ingest`) ────────────────
-# Hourly: the scraper crawls 257 RSS sources at minute :00 and the summarizer
-# runs the Nous map-reduce at minute :05, both writing to the shared osint-db
-# (tables `articles` + `article_summaries`, created by alembic 003_news).
-# Consume via GET /api/news and GET /api/news/summaries.
+# Scraper crawls urls.txt continuously (default 10s between crawls).
+# Summarizer runs Nous map-reduce every 15 min (NEWS_SUMMARIZE_INTERVAL_S=900).
 # NOUS_API_KEY is also (preferably) set in the Keys UI; env is an override.
 # Unset in both env and api_keys = summarizer logs and idles (never crashes).
 NOUS_API_KEY=
@@ -81,18 +83,15 @@ NOUS_BASE_URL=https://inference-api.nousresearch.com/v1
 # Hermes-4.3-36B remains after a Postgres miss. Env wins when set.
 # SUMMARY_MODEL=
 NEWS_BATCH_SIZE=50
-SUMMARY_WINDOW_HOURS=1
+SUMMARY_WINDOW_MINUTES=15
 # Futures/markets coupling from the upstream pipeline is OFF by default
 # (irrelevant to OSINT). Set INCLUDE_FUTURES=1 + install yfinance to enable.
 INCLUDE_FUTURES=0
-# Wall-clock scheduling (k8s CronJob replacement): scrape minute, summarize minute
-NEWS_SCRAPE_MINUTE=0
-NEWS_SUMMARIZE_MINUTE=5
-# Run once immediately on container start (seeds data fast), then align to the
-# scheduled minute.
+NEWS_SCRAPE_INTERVAL_S=10
+NEWS_SUMMARIZE_INTERVAL_S=900
 NEWS_SCRAPE_RUN_ON_START=1
 NEWS_SUMMARIZE_RUN_ON_START=1
-# "1" ignores the current-UTC-hour idempotency skip (double-pins on recreate).
+# "1" ignores the interval idempotency skip (double-pins on recreate).
 NEWS_SUMMARIZE_FORCE=0
 NEWS_LOG_LEVEL=INFO
 # Reserved for the (out-of-scope) Telegram delivery bot.

.forgejo/workflows/build.yml

@@ -1,6 +1,10 @@
-# Build all OSINT images, publish to the Forgejo container registry, then
+# Build changed OSINT images, publish to the Forgejo container registry, then
 # redeploy on the Pi runner (docker.sock mounted).
 #
+# Unchanged images are skipped. Dockerfile.pg / osint-dashboard-pg is NOT
+# rebuilt or pulled on a normal merge — Postgres stays up. Rebuild it only
+# when Dockerfile.pg changes, or via workflow_dispatch rebuild_pg.
+#
 # Public pull host: forgejo.siriusdevops.com (NOT ghcr.io)
 # CI push host: 127.0.0.1:3000 — Cloudflare 413s layers ≳100MB on the public
 # hostname, even from the Pi (hairpins out through the tunnel).
@@ -19,6 +23,15 @@ on:
   push:
     branches: [main, master]
   workflow_dispatch:
+    inputs:
+      rebuild_pg:
+        description: Rebuild Timescale+PostGIS (Dockerfile.pg)
+        type: boolean
+        default: false
+      rebuild_all:
+        description: Rebuild every app image (ignore path filter)
+        type: boolean
+        default: false
 
 env:
   PUBLIC_REGISTRY: ${{ vars.FORGEJO_REGISTRY || 'forgejo.siriusdevops.com' }}
@@ -36,6 +49,68 @@ jobs:
     steps:
       - name: Checkout
         uses: https://code.forgejo.org/actions/checkout@v4
+        with:
+          fetch-depth: 50
+
+      - name: Plan image builds
+        id: plan
+        run: |
+          set -euo pipefail
+          APP=0
+          SCRAPER=0
+          SUM=0
+          PG=0
+          COMPOSE=0
+
+          mark() {
+            case "$1" in
+              Dockerfile.pg)
+                PG=1 ;;
+              Dockerfile|app/*|alembic/*|alembic.ini)
+                APP=1 ;;
+              news/scraper/*)
+                SCRAPER=1 ;;
+              news/summerizer/*)
+                SUM=1 ;;
+              docker-compose.yml|scripts/compose-reup.sh)
+                COMPOSE=1 ;;
+            esac
+          }
+
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            APP=1; SCRAPER=1; SUM=1
+            if [ "${{ github.event.inputs.rebuild_all }}" = "true" ]; then
+              APP=1; SCRAPER=1; SUM=1; PG=1
+            fi
+            if [ "${{ github.event.inputs.rebuild_pg }}" = "true" ]; then
+              PG=1
+            fi
+          else
+            BEFORE="${{ github.event.before }}"
+            SHA="${GITHUB_SHA}"
+            ZEROS="0000000000000000000000000000000000000000"
+            if [ -z "$BEFORE" ] || [ "$BEFORE" = "$ZEROS" ]; then
+              echo "No previous SHA — build app images, skip pg"
+              APP=1; SCRAPER=1; SUM=1
+            elif ! git cat-file -e "${BEFORE}^{commit}" 2>/dev/null; then
+              echo "Previous SHA $BEFORE not in history — build app images, skip pg"
+              APP=1; SCRAPER=1; SUM=1
+            else
+              while IFS= read -r f; do
+                [ -z "$f" ] && continue
+                mark "$f"
+              done < <(git diff --name-only "$BEFORE" "$SHA")
+            fi
+          fi
+
+          {
+            echo "app=$APP"
+            echo "scraper=$SCRAPER"
+            echo "summarizer=$SUM"
+            echo "pg=$PG"
+            echo "compose=$COMPOSE"
+          } >> "$GITHUB_OUTPUT"
+          echo "plan app=$APP scraper=$SCRAPER summarizer=$SUM pg=$PG compose=$COMPOSE"
 
       - name: Image refs
         id: img
@@ -59,6 +134,7 @@ jobs:
           echo "SHA tag: $SHA"
 
       - name: Login to Forgejo registry
+        if: steps.plan.outputs.app == '1' || steps.plan.outputs.scraper == '1' || steps.plan.outputs.summarizer == '1' || steps.plan.outputs.pg == '1'
         run: |
           set -euo pipefail
           # GITHUB_TOKEN login "succeeds" but blob uploads 401 (Forgejo packages
@@ -71,6 +147,7 @@ jobs:
             -u sirius --password-stdin
 
       - name: Build application image (api / ingester / cameras)
+        if: steps.plan.outputs.app == '1'
         run: |
           set -ex
           APP="${{ steps.img.outputs.app }}"
@@ -81,6 +158,7 @@ jobs:
           docker push "${APP}:${SHA}"
 
       - name: Build news-scraper image
+        if: steps.plan.outputs.scraper == '1'
         run: |
           set -ex
           IMG="${{ steps.img.outputs.scraper }}"
@@ -91,6 +169,7 @@ jobs:
           docker push "${IMG}:${SHA}"
 
       - name: Build news-summarizer image
+        if: steps.plan.outputs.summarizer == '1'
         run: |
           set -ex
           IMG="${{ steps.img.outputs.summarizer }}"
@@ -101,22 +180,17 @@ jobs:
           docker push "${IMG}:${SHA}"
 
       - name: Build / refresh Timescale+PostGIS image
+        if: steps.plan.outputs.pg == '1'
         run: |
           set -ex
           PG="${{ steps.img.outputs.pg }}"
           SHA="${{ steps.img.outputs.sha }}"
-          # Prefer rebuild so registry always has a current pg image. If packagecloud
-          # is unreachable, fall back to whatever local image already exists.
           if docker build -f Dockerfile.pg -t "${PG}:latest" -t "${PG}:${SHA}" \
                -t "localhost/osint-dashboard-pg:latest" .; then
             docker push "${PG}:latest"
             docker push "${PG}:${SHA}"
           elif docker image inspect "localhost/osint-dashboard-pg:latest" >/dev/null 2>&1; then
-            echo "WARN: Dockerfile.pg build failed; retagging existing local pg image into registry"
-            docker tag "localhost/osint-dashboard-pg:latest" "${PG}:latest"
-            docker tag "localhost/osint-dashboard-pg:latest" "${PG}:${SHA}"
-            docker push "${PG}:latest"
-            docker push "${PG}:${SHA}"
+            echo "WARN: Dockerfile.pg build failed; keeping existing local pg image"
           else
             echo "ERROR: cannot build or find osint-dashboard-pg image"
             exit 1
@@ -126,33 +200,50 @@ jobs:
         run: |
           set -ex
           cd "${GITHUB_WORKSPACE}"
-          # Pull from Forgejo registry into local tags compose expects, then up.
-          # Compose file still uses localhost/* for stable local names; we mirror
-          # registry tags so a cold host can recover via docker pull.
-          REG="${{ steps.img.outputs.reg }}"
-          PUB="${{ steps.img.outputs.pub }}"
-          OWN="${{ env.OWNER }}"
-          for name in osint-dashboard osint-dashboard-pg osint-news-scraper osint-news-summarizer; do
-            docker pull "${REG}/${OWN}/${name}:latest" || true
-            docker tag "${REG}/${OWN}/${name}:latest" "localhost/${name}:latest" || true
-            docker tag "${REG}/${OWN}/${name}:latest" "${PUB}/${OWN}/${name}:latest" || true
-          done
-          # Do NOT set COMPOSE_PROJECT_NAME differently — volumes must stay
-          # osint-dashboard_osint-pgdata (pinned by `name:` in compose).
-          docker compose build --no-cache app ingester camera-service news-scraper news-summarizer || \
-            docker compose build app ingester camera-service news-scraper news-summarizer
-          docker compose up -d --force-recreate
+          APP="${{ steps.plan.outputs.app }}"
+          SCRAPER="${{ steps.plan.outputs.scraper }}"
+          SUM="${{ steps.plan.outputs.summarizer }}"
+          PG="${{ steps.plan.outputs.pg }}"
+          COMPOSE="${{ steps.plan.outputs.compose }}"
+
+          SVCS=()
+          [ "$APP" = "1" ] && SVCS+=(app ingester camera-service)
+          [ "$SCRAPER" = "1" ] && SVCS+=(news-scraper)
+          [ "$SUM" = "1" ] && SVCS+=(news-summarizer)
+          if [ "$COMPOSE" = "1" ]; then
+            # compose/script change: bounce workers so env/command updates apply.
+            # Still do not bounce Postgres.
+            for s in app ingester camera-service news-scraper news-summarizer; do
+              case " ${SVCS[*]} " in
+                *" $s "*) ;;
+                *) SVCS+=("$s") ;;
+              esac
+            done
+          fi
+
+          chmod +x scripts/compose-reup.sh
+          if [ "$PG" = "1" ]; then
+            FORCE_RECREATE_DB=1 COMPOSE_PROJECT_NAME=osint-dashboard COMPOSE_PROFILES=ingest \
+              scripts/compose-reup.sh "${SVCS[@]}" db
+          elif [ "${#SVCS[@]}" -gt 0 ]; then
+            COMPOSE_PROJECT_NAME=osint-dashboard COMPOSE_PROFILES=ingest \
+              scripts/compose-reup.sh "${SVCS[@]}"
+          else
+            echo "No image or compose changes — leave running containers alone"
+            docker compose --profile ingest ps
+          fi
           docker image prune -f
-          echo "osint-dashboard deployed; images also on ${PUB}/${OWN}/"
+          echo "osint-dashboard deploy done; db image left in place unless pg=1"
 
       - name: Summary
         if: always()
         run: |
           {
-            echo "## Forgejo registry images"
-            echo "Pushed via ${{ steps.img.outputs.reg }} (loopback). Pull publicly:"
-            echo "- \`${{ steps.img.outputs.pub }}/sirius/osint-dashboard:latest\`"
-            echo "- \`${{ steps.img.outputs.pub }}/sirius/osint-dashboard-pg:latest\`"
-            echo "- \`${{ steps.img.outputs.pub }}/sirius/osint-news-scraper:latest\`"
-            echo "- \`${{ steps.img.outputs.pub }}/sirius/osint-news-summarizer:latest\`"
+            echo "## Image plan"
+            echo "- app: \`${{ steps.plan.outputs.app }}\`"
+            echo "- news-scraper: \`${{ steps.plan.outputs.scraper }}\`"
+            echo "- news-summarizer: \`${{ steps.plan.outputs.summarizer }}\`"
+            echo "- pg (Timescale): \`${{ steps.plan.outputs.pg }}\`"
+            echo
+            echo "Postgres is rebuilt/pulled only when \`Dockerfile.pg\` changes (or workflow_dispatch rebuild_pg)."
           } >> "$GITHUB_STEP_SUMMARY"

alembic/versions/007_event_dedup.py

@@ -0,0 +1,115 @@
+"""event_dedup + Timescale compression/retention
+
+Revision ID: 007_event_dedup
+Revises: 006_merge_heads
+Create Date: 2026-08-28
+"""
+
+from alembic import op
+
+revision = "007_event_dedup"
+down_revision = "006_merge_heads"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute("""
+        CREATE TABLE IF NOT EXISTS event_dedup (
+            url TEXT PRIMARY KEY,
+            created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+        )
+    """)
+
+    # Keep the earliest row per URL, drop the 10× USGS/camera dupes.
+    op.execute("""
+        DELETE FROM events a
+        USING events b
+        WHERE a.url IS NOT NULL AND a.url <> ''
+          AND a.url = b.url
+          AND (a.ingested_at, a.id) > (b.ingested_at, b.id)
+    """)
+    op.execute("""
+        INSERT INTO event_dedup (url)
+        SELECT DISTINCT url FROM events
+        WHERE url IS NOT NULL AND url <> ''
+        ON CONFLICT (url) DO NOTHING
+    """)
+
+    # Compression + retention. Policies no-op if Timescale rejects (fresh PG).
+    op.execute("""
+        DO $$
+        BEGIN
+            PERFORM add_compression_policy('events', INTERVAL '7 days', if_not_exists => TRUE);
+        EXCEPTION WHEN OTHERS THEN
+            BEGIN
+                ALTER TABLE events SET (
+                    timescaledb.compress,
+                    timescaledb.compress_segmentby = 'source_type',
+                    timescaledb.compress_orderby = 'ingested_at DESC'
+                );
+                PERFORM add_compression_policy('events', INTERVAL '7 days', if_not_exists => TRUE);
+            EXCEPTION WHEN OTHERS THEN
+                NULL;
+            END;
+        END
+        $$;
+    """)
+    op.execute("""
+        DO $$
+        BEGIN
+            PERFORM add_retention_policy('events', INTERVAL '180 days', if_not_exists => TRUE);
+        EXCEPTION WHEN OTHERS THEN
+            NULL;
+        END
+        $$;
+    """)
+    op.execute("""
+        DO $$
+        BEGIN
+            ALTER TABLE fires SET (
+                timescaledb.compress,
+                timescaledb.compress_segmentby = 'satellite',
+                timescaledb.compress_orderby = 'acq_time DESC'
+            );
+            PERFORM add_compression_policy('fires', INTERVAL '7 days', if_not_exists => TRUE);
+            PERFORM add_retention_policy('fires', INTERVAL '90 days', if_not_exists => TRUE);
+        EXCEPTION WHEN OTHERS THEN
+            NULL;
+        END
+        $$;
+    """)
+    op.execute("""
+        DO $$
+        BEGIN
+            ALTER TABLE aircraft_positions SET (
+                timescaledb.compress,
+                timescaledb.compress_segmentby = 'hex',
+                timescaledb.compress_orderby = 'ts DESC'
+            );
+            PERFORM add_compression_policy('aircraft_positions', INTERVAL '1 day', if_not_exists => TRUE);
+            PERFORM add_retention_policy('aircraft_positions', INTERVAL '14 days', if_not_exists => TRUE);
+        EXCEPTION WHEN OTHERS THEN
+            NULL;
+        END
+        $$;
+    """)
+    op.execute("""
+        DO $$
+        BEGIN
+            ALTER TABLE vessel_positions SET (
+                timescaledb.compress,
+                timescaledb.compress_segmentby = 'mmsi',
+                timescaledb.compress_orderby = 'ts DESC'
+            );
+            PERFORM add_compression_policy('vessel_positions', INTERVAL '1 day', if_not_exists => TRUE);
+            PERFORM add_retention_policy('vessel_positions', INTERVAL '14 days', if_not_exists => TRUE);
+        EXCEPTION WHEN OTHERS THEN
+            NULL;
+        END
+        $$;
+    """)
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS event_dedup")

alembic/versions/008_summary_kind.py

@@ -0,0 +1,26 @@
+"""article_summaries.kind — interval vs daily_recap
+
+Revision ID: 008_summary_kind
+Revises: 007_event_dedup
+Create Date: 2026-08-28
+"""
+
+from alembic import op
+
+revision = "008_summary_kind"
+down_revision = "007_event_dedup"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        ALTER TABLE article_summaries
+            ADD COLUMN IF NOT EXISTS kind TEXT
+        """
+    )
+
+
+def downgrade() -> None:
+    op.execute("ALTER TABLE article_summaries DROP COLUMN IF EXISTS kind")

alembic/versions/009_vessels.py

@@ -0,0 +1,41 @@
+"""vessels — daily VesselAPI snapshots for DVR as-of
+
+Revision ID: 009_vessels
+Revises: 008_summary_kind
+Create Date: 2026-08-29
+"""
+
+from alembic import op
+
+revision = "009_vessels"
+down_revision = "008_summary_kind"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS vessels (
+            mmsi     TEXT NOT NULL,
+            poll_at  TIMESTAMPTZ NOT NULL,
+            lat      DOUBLE PRECISION NOT NULL,
+            lon      DOUBLE PRECISION NOT NULL,
+            heading  DOUBLE PRECISION,
+            speed    DOUBLE PRECISION,
+            label    TEXT,
+            extra    JSONB,
+            PRIMARY KEY (mmsi, poll_at)
+        )
+        """
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_vessels_poll_at ON vessels (poll_at DESC)"
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_vessels_bbox ON vessels (lon, lat)"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP TABLE IF EXISTS vessels")

alembic/versions/010_bbox_gist.py

@@ -0,0 +1,29 @@
+"""GIST bbox indexes for events/fires map-pan queries.
+
+Revision ID: 010_bbox_gist
+Revises: 009_vessels
+Create Date: 2026-09-01
+"""
+
+from alembic import op
+
+revision = "010_bbox_gist"
+down_revision = "009_vessels"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_events_geom_gist ON events "
+        "USING gist (ST_SetSRID(ST_MakePoint(location_lon, location_lat), 4326))"
+    )
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_fires_geom_gist ON fires "
+        "USING gist (ST_SetSRID(ST_MakePoint(longitude, latitude), 4326))"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP INDEX IF EXISTS ix_fires_geom_gist")
+    op.execute("DROP INDEX IF EXISTS ix_events_geom_gist")

alembic/versions/011_geofence_alerts_fence.py

@@ -0,0 +1,24 @@
+"""geofence_alerts (geofence_id, created_at DESC) for fence-scoped hit log
+
+Revision ID: 011_geofence_alerts_fence
+Revises: 010_bbox_gist
+Create Date: 2026-09-01
+"""
+
+from alembic import op
+
+revision = "011_geofence_alerts_fence"
+down_revision = "010_bbox_gist"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_geofence_alerts_fence_created "
+        "ON geofence_alerts (geofence_id, created_at DESC)"
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP INDEX IF EXISTS ix_geofence_alerts_fence_created")

app/bg_jobs.py

@@ -1,50 +1,19 @@
-"""Background masscan / ffmpeg — never block a FastAPI request on a scan.
+"""Background ffmpeg — never block a FastAPI request on a frame grab.
 
-masscan is capped at 200 pps (home uplink saturates at 1k+). ffmpeg frame
-grabs are scheduled with asyncio.create_task and shared per URL.
+ffmpeg frame grabs are scheduled with asyncio.create_task and shared per URL.
 """
 
 from __future__ import annotations
 
 import asyncio
-import logging
 import shutil
 from cachetools import TTLCache
 
-logger = logging.getLogger("osint.bg_jobs")
-
-MASSCAN_PPS_CAP = 200
-
-_masscan_task: asyncio.Task | None = None
 _ffmpeg_cache: TTLCache = TTLCache(maxsize=100, ttl=300)
 _ffmpeg_tasks: dict[str, asyncio.Task] = {}
 _FFMPEG = shutil.which("ffmpeg")
 
 
-def schedule_masscan_pass() -> bool:
-    """Kick one capped masscan pass. Returns False if a pass is already running."""
-    global _masscan_task
-    if _masscan_task is not None and not _masscan_task.done():
-        return False
-    _masscan_task = asyncio.create_task(_run_masscan_capped())
-    return True
-
-
-async def _run_masscan_capped() -> None:
-    import masscan_config as cfg
-    from run_masscan_service import _verify_excludefile, run_pass
-
-    orig = cfg.MASSCAN_RATE
-    if orig > MASSCAN_PPS_CAP:
-        logger.warning("capping masscan rate %s pps -> %s", orig, MASSCAN_PPS_CAP)
-        cfg.MASSCAN_RATE = MASSCAN_PPS_CAP
-    try:
-        _verify_excludefile()
-        await run_pass()
-    finally:
-        cfg.MASSCAN_RATE = orig
-
-
 def cached_ffmpeg_jpeg(url: str) -> bytes | None:
     return _ffmpeg_cache.get(url)
 

app/camera_config.py

@@ -16,6 +16,8 @@ CALTRANS_CCTV_URLS = tuple(
     f"https://cwwp2.dot.ca.gov/data/d{n}/cctv/cctvStatusD{n:02d}.json"
     for n in range(1, 13)
 )
+# MDOT MiDrive official DOT CCTV list (fields carry rendered HTML).
+MDOT_CAMERA_URL = "https://mdotjboss.state.mi.us/MiDrive/camera/list"
 _DEFAULT_SOURCE_URL = ",".join((
     # Publicly published open-camera list (markdown bullets of stream URLs).
     "https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md",
@@ -25,6 +27,10 @@ _DEFAULT_SOURCE_URL = ",".join((
     "https://raw.githubusercontent.com/willytop8/Live-Environment-Streams/main/streams.geojson",
     # Official Caltrans CWWP2 JPEG + HLS CCTV (districts 1–12).
     *CALTRANS_CCTV_URLS,
+    # Oregon DOT TripCheck public CCTV JPEG inventory (Esri JSON).
+    "https://www.tripcheck.com/Scripts/map/data/cctvinventory.js",
+    # Official MDOT MiDrive CCTV (JPEG stills, Michigan).
+    MDOT_CAMERA_URL,
 ))
 CAMERA_SOURCE_URLS = [
     u.strip()
@@ -55,3 +61,15 @@ SNAPSHOT_TIMEOUT = float(os.getenv("SNAPSHOT_TIMEOUT", "8.0"))
 
 # NATS subject cameras are published on (consumed by the shared ingester).
 CAMERA_NATS_SUBJECT = os.getenv("CAMERA_NATS_SUBJECT", "events.camera")
+
+
+# ── UDOT IBI 511 traffic cameras ──────────────────────────────────────────
+# DataTables endpoint (POST form-encoded; server caps at 100 rows/page no
+# matter what `length` is sent). No API key. Snapshot stills live at a stable
+# /map/Cctv/{id} URL — same URL always serves the latest frame, so we store
+# the URL and never scrape every frame ourselves.
+UDOT_IBI_URL = "https://prod-ut.ibi511.com/List/GetData/Cameras"
+UDOT_IBI_BASE = "https://prod-ut.ibi511.com"
+UDOT_IBI_PAGE_SIZE = 100
+# Safety cap on pages per cycle so a runaway recordsTotal cannot fan out.
+UDOT_IBI_MAX_PAGES = int(os.getenv("UDOT_IBI_MAX_PAGES", "40"))

app/camera_preview.py

@@ -1,7 +1,7 @@
 """Resolve a browser-renderable preview for a camera.
 
 HTTP/MJPEG cameras already expose a snapshot_url the existing proxy can
-stream. masscan finds are stored as `rtsp://IP/` with no snapshot_url, so
+stream. Some scraper sources store `rtsp://` URLs with no snapshot_url, so
 the map popup used to skip the <img> entirely and the leftover source link
 handed the browser an rtsp:// URL (which opens VLC).
 
@@ -42,16 +42,6 @@ _HTTP_PATHS = (
     "/tmpfs/auto.jpg",
 )
 
-# Browser-playable MJPEG paths the /stream proxy can pass through.
-_MJPEG_PATHS = (
-    "/mjpg/video.mjpg",
-    "/video.mjpg",
-    "/cgi-bin/mjpg/video.cgi",
-    "/axis-cgi/mjpg/video.cgi",
-    "/nphMotionJpeg",
-    "/mjpeg.cgi",
-)
-
 _FFMPEG = shutil.which("ffmpeg")
 
 
@@ -87,55 +77,6 @@ async def _http_get_image(url: str, timeout: float = 2.5) -> bytes | None:
         return None
 
 
-async def _http_feed_url(url: str, timeout: float = 2.5) -> str | None:
-    """Return url if it looks like an unauthenticated image/MJPEG feed."""
-    try:
-        async with httpx.AsyncClient(
-            timeout=timeout, follow_redirects=True,
-            headers={"User-Agent": USER_AGENT},
-        ) as c:
-            async with c.stream("GET", url) as r:
-                if r.status_code != 200:
-                    return None
-                ctype = (r.headers.get("content-type") or "").lower()
-                if "html" in ctype or ctype.startswith("text/"):
-                    return None
-                if any(x in ctype for x in ("image/", "multipart", "mjpeg", "octet-stream")):
-                    # Read a little to reject empty/error bodies.
-                    chunk = b""
-                    async for b in r.aiter_bytes():
-                        chunk += b
-                        if len(chunk) >= 64:
-                            break
-                    if len(chunk) < 64:
-                        return None
-                    if b"html" in chunk[:64].lower():
-                        return None
-                    return url
-    except Exception:  # noqa: BLE001
-        return None
-    return None
-
-
-async def probe_public_feed(host: str) -> str | None:
-    """Unauthenticated HTTP still or MJPEG URL for this host, or None.
-
-    Used at masscan ingest time so dead RTSP-only hosts never hit the map.
-    No credentials, no RTSP path-walking (too slow / rarely public).
-    """
-    urls = [f"http://{host}{p}" for p in _HTTP_PATHS]
-    urls.append(f"http://{host}:8080/shot.jpg")
-    urls.extend(f"http://{host}{p}" for p in _MJPEG_PATHS)
-    results = await asyncio.gather(
-        *(_http_feed_url(u) for u in urls),
-        return_exceptions=True,
-    )
-    for url, hit in zip(urls, results):
-        if isinstance(hit, str) and hit:
-            return hit
-    return None
-
-
 async def ffmpeg_snapshot(url: str, timeout: float = 8.0) -> bytes | None:
     """Grab a single JPEG frame from an RTSP URL. None if ffmpeg missing/fails.
 

app/camera_scraper.py

@@ -25,6 +25,7 @@ import hashlib
 import ipaddress
 import json
 import logging
+import math
 import re
 import time
 from datetime import datetime, timezone
@@ -37,6 +38,7 @@ from camera_config import (
     CAMERA_SOURCE_URLS, CAMERA_REQUEST_DELAY, CAMERA_MAX_PER_SOURCE,
     NOMINATIM_URL, NOMINATIM_MIN_INTERVAL, USER_AGENT,
     SNAPSHOT_CACHE_DIR, SNAPSHOT_TTL_SECONDS, SNAPSHOT_TIMEOUT,
+    UDOT_IBI_URL, UDOT_IBI_BASE, UDOT_IBI_PAGE_SIZE, UDOT_IBI_MAX_PAGES,
 )
 from camera_models import cameras
 from database import async_session
@@ -114,6 +116,15 @@ class RateLimitedClient:
         self._last[host] = time.monotonic()
         return await self.client.get(url, **kw)
 
+    async def post(self, url: str, **kw) -> httpx.Response:
+        host = urlparse(url).netloc
+        now = time.monotonic()
+        wait = self._last.get(host, 0.0) + self._delay - now
+        if wait > 0:
+            await asyncio.sleep(wait)
+        self._last[host] = time.monotonic()
+        return await self.client.post(url, **kw)
+
     async def aclose(self):
         await self.client.aclose()
 
@@ -379,6 +390,201 @@ def parse_caltrans_json(text: str, source_name: str) -> list[dict]:
     return out
 
 
+# ── UDOT IBI 511 ──────────────────────────────────────────────────────────
+# Utah bbox (lat 36.9–42.1, lon -114.2–-108.9). WKT is `POINT (lng lat)`.
+_UDOT_IBI_MIN_LAT, _UDOT_IBI_MAX_LAT = 36.9, 42.1
+_UDOT_IBI_MIN_LON, _UDOT_IBI_MAX_LON = -114.2, -108.9
+_UDOT_WKT_POINT_RE = re.compile(
+    r"POINT\s*\(\s*(-?\d+(?:\.\d+)?)\s+(-?\d+(?:\.\d+)?)\s*\)", re.I,
+)
+
+
+def parse_udot_ibi_page(text: str, source_name: str = "udot") -> list[dict]:
+    """Parse one UDOT IBI 511 DataTables camera page (`{"data": [...]}`).
+
+    Skips rows whose first image is `blocked` or `disabled`, and drops any
+    point outside the Utah bbox. The `/map/Cctv/{id}` URL is a stable identity
+    (always serves the latest frame), so it is stored as both source_url and
+    snapshot_url — we never scrape frames ourselves.
+    """
+    try:
+        payload = json.loads(text)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    rows = payload.get("data") if isinstance(payload, dict) else None
+    if not isinstance(rows, list):
+        return []
+    out: list[dict] = []
+    for row in rows:
+        if not isinstance(row, dict):
+            continue
+        cam_id = row.get("id")
+        images = row.get("images") or []
+        if cam_id is None or not images:
+            continue
+        img = images[0] or {}
+        if img.get("blocked") or img.get("disabled"):
+            continue
+        lon = lat = None
+        try:
+            wkt = (row.get("latLng") or {}).get("geography") or {}
+            wkt = wkt.get("wellKnownText") or ""
+            m = _UDOT_WKT_POINT_RE.match(str(wkt).strip())
+            if m:
+                lon, lat = float(m.group(1)), float(m.group(2))
+        except (AttributeError, TypeError, ValueError):
+            lon = lat = None
+        if lat is None or lon is None:
+            continue
+        if not (_UDOT_IBI_MIN_LAT <= lat <= _UDOT_IBI_MAX_LAT
+                and _UDOT_IBI_MIN_LON <= lon <= _UDOT_IBI_MAX_LON):
+            continue
+        snap = f"{UDOT_IBI_BASE}/map/Cctv/{cam_id}"
+        roadway, direction, location = (
+            row.get("roadway"), row.get("direction"), row.get("location"),
+        )
+        name = ", ".join(
+            str(b) for b in (roadway, direction, location)
+            if b and str(b).strip() and str(b).strip().lower() != "unknown"
+        ) or None
+        out.append({
+            "source_url": snap,
+            "snapshot_url": snap,
+            "discovery_source": source_name,
+            "location_lat": lat,
+            "location_lon": lon,
+            "location_name": name,
+            "vendor": "UDOT",
+            "device_type": "http",
+            "raw": {
+                "udot_id": cam_id,
+                "agency": row.get("source"),
+                "source_id": row.get("sourceId"),
+                "roadway": roadway,
+                "direction": direction,
+            },
+        })
+    return out
+
+
+# Oregon DOT TripCheck inventory bounding box (approx state extent).
+ODOT_BBOX = (41.9, 46.3, -124.6, -116.4)  # lat_min, lat_max, lon_min, lon_max
+
+
+def parse_odot_json(text: str, source_name: str) -> list[dict]:
+    """Parse Oregon DOT TripCheck cctvinventory Esri-style JSON.
+
+    Store the JPEG still as snapshot_url (map thumbs); never RTSP. Keep only
+    rows with finite coordinates inside Oregon and a usable filename.
+    """
+    try:
+        payload = json.loads(text)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    lat_min, lat_max, lon_min, lon_max = ODOT_BBOX
+    out: list[dict] = []
+    for feat in payload.get("features") or []:
+        attrs = (feat or {}).get("attributes") or {}
+        filename = (attrs.get("filename") or "").strip()
+        if not filename:
+            continue
+        try:
+            lat = float(attrs.get("latitude"))
+            lon = float(attrs.get("longitude"))
+        except (TypeError, ValueError):
+            continue
+        if not (math.isfinite(lat) and math.isfinite(lon)):
+            continue
+        if not (lat_min <= lat <= lat_max and lon_min <= lon <= lon_max):
+            continue
+        jpeg = f"https://tripcheck.com/RoadCams/cams/{filename}"
+        title = (attrs.get("title") or "").strip()
+        out.append({
+            "source_url": jpeg,
+            "snapshot_url": jpeg,
+            "discovery_source": "odot",
+            "location_lat": lat,
+            "location_lon": lon,
+            "location_name": title or None,
+            "vendor": "ODOT",
+            "device_type": "http",
+        })
+    return out
+
+
+# MDOT MiDrive field extractors (fields carry rendered HTML).
+_MDOT_LAT_RE = re.compile(r"lat=(-?\d+(?:\.\d+)?)", re.I)
+_MDOT_LON_RE = re.compile(r"lon=(-?\d+(?:\.\d+)?)", re.I)
+_MDOT_ID_RE = re.compile(r"[?&]id=(\d+)", re.I)
+_MDOT_IMG_RE = re.compile(r'<img[^>]+src=["\']([^"\']+)["\']', re.I)
+
+# Michigan bbox (docs/osiris-ideas.md §3.2): lat 41.6–48.3, lon -90.5–-82.1.
+MDOT_LAT_RANGE = (41.6, 48.3)
+MDOT_LON_RANGE = (-90.5, -82.1)
+
+
+def parse_mdot_json(text: str, source_name: str) -> list[dict]:
+    """Parse MDOT MiDrive `camera/list` JSON (fields carry rendered HTML).
+
+    Coordinates and the stable id live in the `county` field's map link
+    (`/MiDrive/map?...lat=&lon=&id=`); the `image` field carries an `<img>`
+    whose src is the JPEG still. Out-of-bbox and coord-less rows are dropped.
+    """
+    try:
+        payload = json.loads(text)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    if not isinstance(payload, list):
+        return []
+    out: list[dict] = []
+    for row in payload:
+        if not isinstance(row, dict):
+            continue
+        county_html = row.get("county") or ""
+        m_lat = _MDOT_LAT_RE.search(county_html)
+        m_lon = _MDOT_LON_RE.search(county_html)
+        m_id = _MDOT_ID_RE.search(county_html)
+        if not (m_lat and m_lon and m_id):
+            continue  # missing coordinates / stable id → drop
+        try:
+            lat = float(m_lat.group(1))
+            lon = float(m_lon.group(1))
+        except ValueError:
+            continue
+        if not (MDOT_LAT_RANGE[0] <= lat <= MDOT_LAT_RANGE[1]
+                and MDOT_LON_RANGE[0] <= lon <= MDOT_LON_RANGE[1]):
+            continue  # out of Michigan bbox → drop
+        img_m = _MDOT_IMG_RE.search(row.get("image") or "")
+        if not img_m:
+            continue
+        snap = img_m.group(1).strip()
+        low = snap.lower()
+        if not (low.startswith("http://") or low.startswith("https://")):
+            continue
+        if low.startswith("rtsp"):
+            continue
+        cam_id = m_id.group(1)
+        route = (row.get("route") or "").strip()
+        loc = (row.get("location") or "").strip().lstrip("@").strip()
+        county_name = county_html.split("<a", 1)[0].strip()
+        bits = [
+            f"{route} @ {loc}" if (route and loc) else (route or loc or None),
+            county_name or None,
+        ]
+        name = ", ".join(b for b in bits if b) or None
+        out.append({
+            "source_url": f"https://mdotjboss.state.mi.us/MiDrive/camera/{cam_id}",
+            "snapshot_url": snap,
+            "discovery_source": "mdot",
+            "location_lat": lat,
+            "location_lon": lon,
+            "location_name": name,
+            "vendor": "MDOT",
+            "device_type": "http",
+        })
+    return out
+
+
 def parse_live_streams_geojson(text: str, source_name: str) -> list[dict]:
     """Parse willytop8/Live-Environment-Streams GeoJSON.
 
@@ -490,6 +696,10 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder,
     body = resp.text
     if "cwwp2.dot.ca.gov" in src_url or "cctvStatus" in src_url:
         cams = parse_caltrans_json(body, name)
+    elif "cctvinventory" in src_url or "tripcheck.com" in src_url:
+        cams = parse_odot_json(body, name)
+    elif "mdotjboss.state.mi.us" in src_url or "/MiDrive/camera/list" in src_url:
+        cams = parse_mdot_json(body, name)
     elif ("getCameraDataByLoc" in src_url
             or ("json" in ctype and '"locs"' in body[:4000] and '"cams"' in body[:8000])):
         cams = parse_alertwest_json(body, name)
@@ -549,6 +759,54 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder,
     return out
 
 
+# ── UDOT IBI 511 paginated fetcher ────────────────────────────────────────
+
+async def scrape_udot_ibi(client: RateLimitedClient) -> list[dict]:
+    """Page through the UDOT IBI 511 DataTables endpoint and normalize.
+
+    POSTs `start`/`length` form fields (server caps at 100 rows/page), walking
+    pages until `recordsTotal` is exhausted or UDOT_IBI_MAX_PAGES is hit.
+    """
+    out: list[dict] = []
+    seen: set[str] = set()
+    start = 0
+    for _ in range(UDOT_IBI_MAX_PAGES):
+        try:
+            resp = await client.post(
+                UDOT_IBI_URL,
+                data={
+                    "start": str(start),
+                    "length": str(UDOT_IBI_PAGE_SIZE),
+                    "lang": "en-US",
+                },
+                headers={"X-Requested-With": "XMLHttpRequest"},
+            )
+            resp.raise_for_status()
+            body = resp.text
+        except Exception:  # noqa: BLE001
+            logger.exception("failed to fetch UDOT IBI page start=%d", start)
+            break
+        try:
+            payload = json.loads(body)
+        except ValueError:
+            logger.warning("UDOT IBI non-JSON response at start=%d", start)
+            break
+        total = int(payload.get("recordsTotal") or 0)
+        rows = payload.get("data") or []
+        if not isinstance(rows, list) or not rows:
+            break
+        for cam in parse_udot_ibi_page(body, "udot"):
+            if cam["source_url"] in seen:
+                continue
+            seen.add(cam["source_url"])
+            out.append(cam)
+        if start + len(rows) >= total:
+            break
+        start += len(rows)
+    logger.info("UDOT IBI yielded %d cameras", len(out))
+    return out
+
+
 # ── Persistence ────────────────────────────────────────────────────────────
 
 async def upsert_cameras(cams: list[dict]) -> int:
@@ -600,6 +858,7 @@ async def run_cycle() -> int:
     try:
         results = await asyncio.gather(
             *(scrape_source(client, geo, s) for s in CAMERA_SOURCE_URLS),
+            scrape_udot_ibi(client),
             return_exceptions=True,
         )
         all_cams: list[dict] = []

app/chokepoints.py

@@ -0,0 +1,62 @@
+"""Static chokepoint preset catalog — one-tap fly-to targets for the map.
+
+Pure data, no upstream calls and no VesselAPI quota spend. ``vesselapi`` is
+``True`` only for Hormuz (the single box the VesselAPI poller already covers);
+every other strait is AISStream-only until a human later spends quota. Never
+call VesselAPI from here.
+
+Bounding boxes are ``minlat,minlon,maxlat,maxlon`` (VesselAPI order) and each
+stays within the ``|dLat|+|dLon| <= 4`` span rule enforced by
+``vesselapi.validate_bbox_span``.
+"""
+
+from __future__ import annotations
+
+# id → preset. ``center`` is ``[lat, lon]`` for Leaflet ``setView``.
+_CHOKEPOINTS: tuple[dict, ...] = (
+    {
+        "id": "hormuz",
+        "title": "Strait of Hormuz",
+        "bbox": "25.5,55.4,27.3,57.2",
+        "center": [26.4, 56.5],
+        "zoom": 9,
+        "vesselapi": True,
+    },
+    {
+        "id": "bab_el_mandeb",
+        "title": "Bab el-Mandeb",
+        "bbox": "12.0,42.8,13.5,44.3",
+        "center": [12.7, 43.4],
+        "zoom": 9,
+        "vesselapi": False,
+    },
+    {
+        "id": "suez",
+        "title": "Suez / N. Red Sea",
+        "bbox": "29.5,32.0,31.0,33.5",
+        "center": [30.0,32.5],
+        "zoom": 9,
+        "vesselapi": False,
+    },
+    {
+        "id": "malacca",
+        "title": "Malacca / Singapore",
+        "bbox": "1.0,103.0,2.5,104.5",
+        "center": [1.3, 103.8],
+        "zoom": 9,
+        "vesselapi": False,
+    },
+    {
+        "id": "taiwan",
+        "title": "Taiwan Strait",
+        "bbox": "23.5,119.0,25.0,120.5",
+        "center": [24.2, 119.8],
+        "zoom": 9,
+        "vesselapi": False,
+    },
+)
+
+
+def chokepoints() -> list[dict]:
+    """Return a fresh copy of the catalog (callers must not mutate the source)."""
+    return [dict(p) for p in _CHOKEPOINTS]

app/config.py

@@ -71,6 +71,16 @@ FIRMS_DATASETS = [d.strip() for d in _FIRMS_DATASETS_RAW.split(",") if d.strip()
 OSINT_USER_AGENT = os.getenv(
     "OSINT_USER_AGENT", "osint-dashboard/1.0 (self-hosted; lancewalters94@gmail.com)"
 )
+# Nominatim reverse (GET /api/place). Camera scraper has its own copy in camera_config.
+NOMINATIM_URL = os.getenv("NOMINATIM_URL", "https://nominatim.openstreetmap.org")
+NOMINATIM_MIN_INTERVAL = float(os.getenv("NOMINATIM_MIN_INTERVAL", "1.0"))
+
+# Self-hosted TiTiler (warps Sentinel-1 signed COGs into XYZ tiles on the Pi).
+# TITILER_PUBLIC_BASE is the same-origin path prefix the browser hits through
+# the osint.rpi.local nginx vhost (`location /titiler/` → 127.0.0.1:8001).
+# TITILER_INTERNAL_URL is the compose-DNS address, used only for health checks.
+TITILER_PUBLIC_BASE = os.getenv("TITILER_PUBLIC_BASE", "/titiler").rstrip("/")
+TITILER_INTERNAL_URL = os.getenv("TITILER_INTERNAL_URL", "http://titiler:8000")
 
 # AISStream (server-side WebSocket only). Idle when unset.
 AISSTREAM_API_KEY = os.getenv("AISSTREAM_API_KEY", "")
@@ -81,3 +91,19 @@ AISSTREAM_BBOX = os.getenv("AISSTREAM_BBOX", "24,-125,50,-66")
 # without the ingest profile). Set 0 if the ingester owns the only connection.
 AISSTREAM_IN_APP = os.getenv("AISSTREAM_IN_APP", "1").lower() in ("1", "true", "yes")
 AISSTREAM_IN_INGEST = os.getenv("AISSTREAM_IN_INGEST", "0").lower() in ("1", "true", "yes")
+
+# VesselAPI (quota-capped REST AIS poller — free tier 150 calls/mo).
+# AISStream keeps US coasts; VesselAPI fills the Middle East blind spot. The
+# poller idles when VESSELAPI_API_KEY is unset (never from GET /api/vessels).
+VESSELAPI_API_KEY = os.getenv("VESSELAPI_API_KEY", "")
+# Bounding box(es) as minlat,minlon,maxlat,maxlon — note lat/lon order (same as
+# AISSTREAM_BBOX). Semicolon-separated for multiple boxes. Default: Strait of
+# Hormuz (|dLat|+|dLon| = 3.6 ≤ 4° span cap). VesselAPI 400s any box over 4°.
+VESSELAPI_BBOX = os.getenv("VESSELAPI_BBOX", "25.5,55.4,27.3,57.2")
+# Poll cadence in seconds. 17280 = 4.8h → 5 polls/day (150/mo free tier).
+VESSELAPI_INTERVAL = int(os.getenv("VESSELAPI_INTERVAL", "17280"))
+# Local hard cap on successful 2xx calls per UTC day (persisted in Postgres).
+VESSELAPI_MAX_CALLS_PER_DAY = int(os.getenv("VESSELAPI_MAX_CALLS_PER_DAY", "5"))
+# Run the VesselAPI poller inside the dashboard process (default on, like AIS).
+VESSELAPI_IN_APP = os.getenv("VESSELAPI_IN_APP", "1").lower() in ("1", "true", "yes")
+VESSELAPI_IN_INGEST = os.getenv("VESSELAPI_IN_INGEST", "0").lower() in ("1", "true", "yes")

app/conflicts.py

@@ -0,0 +1,168 @@
+"""Curated OSINT conflict-zone catalog + point-in-bbox event counting.
+
+A static, human-curated list of active conflict theatres (war / high /
+elevated). Purely descriptive — this is a catalog, not a live feed and not a
+scrape of LiveUAMap or any other source. Severity and descriptions are
+editorial judgement kept short and factual.
+
+Each zone carries an internal ``bbox`` (``min_lat, min_lon, max_lat, max_lon``)
+used only to count pre-existing geocoded news/GDELT/``/api/news/map`` rows that
+fall inside it. The bbox is not part of the API response; callers get the
+``eventCount`` roll-up instead.
+
+Never call an upstream API from here — event counts come from rows already in
+the local database (``events`` with geocoords + ``news_items`` map pins).
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+# id → zone. ``lat``/``lon`` is the fly-to anchor; ``bbox`` is the internal
+# count window in ``min_lat, min_lon, max_lat, max_lon`` order.
+_ZONES: tuple[dict, ...] = (
+    {
+        "id": "ukraine",
+        "label": "Ukraine",
+        "severity": "war",
+        "lat": 48.5,
+        "lon": 31.0,
+        "description": "Full-scale Russian invasion since 2022; active front lines in the east and south.",
+        "bbox": (44.3, 22.1, 52.4, 40.2),
+    },
+    {
+        "id": "gaza",
+        "label": "Gaza",
+        "severity": "war",
+        "lat": 31.4,
+        "lon": 34.4,
+        "description": "Israel–Hamas war; sustained fighting and a severe humanitarian crisis in the Gaza Strip.",
+        "bbox": (31.0, 34.1, 31.8, 34.7),
+    },
+    {
+        "id": "sudan",
+        "label": "Sudan",
+        "severity": "war",
+        "lat": 15.5,
+        "lon": 30.0,
+        "description": "Civil war between the SAF and RSF since 2023, with mass displacement across the country.",
+        "bbox": (8.7, 21.8, 22.0, 38.6),
+    },
+    {
+        "id": "myanmar",
+        "label": "Myanmar",
+        "severity": "war",
+        "lat": 21.5,
+        "lon": 96.0,
+        "description": "Post-2021 coup conflict pitting the junta against resistance and ethnic armed groups.",
+        "bbox": (9.5, 92.2, 28.5, 101.2),
+    },
+    {
+        "id": "drc",
+        "label": "DR Congo",
+        "severity": "war",
+        "lat": -1.5,
+        "lon": 28.0,
+        "description": "Eastern DRC conflict involving M23 and other armed groups; heavy displacement around Goma.",
+        "bbox": (-5.0, 26.0, 3.0, 31.0),
+    },
+    {
+        "id": "yemen",
+        "label": "Yemen",
+        "severity": "war",
+        "lat": 15.5,
+        "lon": 47.5,
+        "description": "Protracted Houthi–government/coalition war with one of the world's worst humanitarian emergencies.",
+        "bbox": (12.6, 42.5, 19.0, 54.0),
+    },
+    {
+        "id": "syria",
+        "label": "Syria",
+        "severity": "war",
+        "lat": 34.5,
+        "lon": 38.5,
+        "description": "Multi-sided civil war; government, opposition, and external actors continue to engage.",
+        "bbox": (32.3, 35.7, 37.3, 42.4),
+    },
+    {
+        "id": "lebanon",
+        "label": "Lebanon",
+        "severity": "high",
+        "lat": 33.9,
+        "lon": 35.9,
+        "description": "Israel–Hezbollah hostilities with periodic escalation along the southern border.",
+        "bbox": (33.0, 35.0, 34.7, 36.6),
+    },
+    {
+        "id": "sahel",
+        "label": "Sahel",
+        "severity": "high",
+        "lat": 14.5,
+        "lon": 0.0,
+        "description": "Jihadist insurgencies across Mali, Burkina Faso, and Niger destabilising the central Sahel.",
+        "bbox": (10.0, -10.0, 20.0, 12.0),
+    },
+    {
+        "id": "somalia",
+        "label": "Somalia",
+        "severity": "high",
+        "lat": 6.0,
+        "lon": 45.0,
+        "description": "Al-Shabaab insurgency against the federal government and security forces.",
+        "bbox": (-2.0, 41.0, 12.0, 51.5),
+    },
+    {
+        "id": "red_sea",
+        "label": "Red Sea",
+        "severity": "high",
+        "lat": 18.0,
+        "lon": 40.0,
+        "description": "Houthi attacks on commercial shipping transiting the Red Sea corridor.",
+        "bbox": (12.0, 34.0, 22.0, 44.0),
+    },
+    {
+        "id": "taiwan_strait",
+        "label": "Taiwan Strait",
+        "severity": "elevated",
+        "lat": 24.5,
+        "lon": 119.5,
+        "description": "Heightened military standoff between China and Taiwan, including deterrence patrols.",
+        "bbox": (21.9, 117.0, 26.5, 122.0),
+    },
+    {
+        "id": "korean_dmz",
+        "label": "Korean DMZ",
+        "severity": "elevated",
+        "lat": 38.3,
+        "lon": 127.0,
+        "description": "Heavily fortified inter-Korean border with periodic tensions and military drills.",
+        "bbox": (37.5, 126.0, 39.0, 128.5),
+    },
+)
+
+SEVERITIES: frozenset[str] = frozenset({"war", "high", "elevated"})
+
+
+def conflict_zones() -> list[dict]:
+    """Return a fresh shallow copy of the catalog (callers must not mutate)."""
+    return [dict(z) for z in _ZONES]
+
+
+def zone_event_stats(
+    points: list[tuple[float, float, datetime | None]],
+    bbox: tuple[float, float, float, float],
+) -> tuple[int, datetime | None]:
+    """Count points inside ``bbox`` and return (count, latest timestamp).
+
+    ``points`` is an iterable of ``(lat, lon, ts)``; ``ts`` may be ``None``.
+    ``bbox`` is ``(min_lat, min_lon, max_lat, max_lon)``.
+    """
+    min_lat, min_lon, max_lat, max_lon = bbox
+    count = 0
+    latest: datetime | None = None
+    for lat, lon, ts in points:
+        if min_lat <= lat <= max_lat and min_lon <= lon <= max_lon:
+            count += 1
+            if ts is not None and (latest is None or ts > latest):
+                latest = ts
+    return count, latest

app/fire_sources.py

@@ -22,6 +22,7 @@ UTC date (YYYY-MM-DD).
 from __future__ import annotations
 
 import csv
+import hashlib
 import io
 import json
 import logging
@@ -44,6 +45,11 @@ from upstream_cache import firms_cache
 
 logger = logging.getLogger("osint.firms")
 
+# In-process poll state: skip byte-identical CSVs, persist only new hotspots.
+# Survives the 15-minute loop; one full ON CONFLICT after process start.
+_csv_digest: dict[tuple, bytes] = {}
+_seen_ids: dict[tuple, set[int]] = {}
+
 # ── FIRMS API ─────────────────────────────────────────────────────────────
 
 FIRMS_AREA_CSV = (
@@ -86,33 +92,34 @@ def normalize_acq_time(acq_date: object, acq_time: object) -> datetime | None:
         return None
 
 
-def parse_firms_csv(text: str) -> list[dict]:
-    """Parse a FIRMS area CSV payload into normalized fire messages.
+def _hotspot_id(lat: float, lon: float, acq_iso: str, satellite: str) -> int:
+    return hash((round(lat, 5), round(lon, 5), acq_iso, satellite))
 
-    Returns one dict per hotspot with the fields stored in the ``fires`` table
-    (acq_time already combined into a UTC ISO timestamp). Rows that don't look
-    like valid VIIRS detections are skipped rather than failing the whole poll.
+
+def parse_firms_csv_delta(
+    text: str, skip_ids: set[int] | None = None,
+) -> tuple[list[dict], set[int]]:
+    """Parse FIRMS CSV; optionally drop hotspots already seen this process.
+
+    Returns (new_or_all_points, ids_for_every_valid_row). Streaming — does not
+    materialize the raw CSV as a list of lists.
     """
-    rows = list(csv.reader(io.StringIO(text)))
-    if not rows:
-        return []
-
-    # Locate the real header row. FIRMS normally returns the CSV header first,
-    # but occasionally prepends a legend/info line, so scan until we see the
-    # canonical header.
-    header_idx = 0
-    for i, row in enumerate(rows):
+    reader = csv.reader(io.StringIO(text))
+    header = None
+    for row in reader:
         if row and row[0].strip().lower() == "latitude" and len(row) >= 4:
-            header_idx = i
+            header = [c.strip().lower() for c in row]
             break
-    header = [c.strip().lower() for c in rows[header_idx]]
-    # Guard against a header that isn't actually the FIRMS one.
-    if "latitude" not in header or "longitude" not in header:
-        logger.warning("FIRMS payload does not look like a hotspot CSV (first row: %r)", header[:6])
-        return []
+    if not header or "latitude" not in header or "longitude" not in header:
+        logger.warning(
+            "FIRMS payload does not look like a hotspot CSV (first row: %r)",
+            (header or [])[:6],
+        )
+        return [], set()
 
     points: list[dict] = []
-    for row in rows[header_idx + 1:]:
+    ids: set[int] = set()
+    for row in reader:
         if len(row) < len(header):
             continue
         rec = dict(zip(header, row))
@@ -123,13 +130,19 @@ def parse_firms_csv(text: str) -> list[dict]:
         acq_time = normalize_acq_time(rec.get("acq_date"), rec.get("acq_time"))
         if acq_time is None:
             continue
+        sat = str(rec.get("satellite") or "").strip()
+        acq_iso = acq_time.isoformat()
+        hid = _hotspot_id(lat, lon, acq_iso, sat)
+        ids.add(hid)
+        if skip_ids is not None and hid in skip_ids:
+            continue
         points.append({
             "latitude": lat,
             "longitude": lon,
             "brightness": _to_float(rec.get("bright_ti4")),
             "confidence": str(rec.get("confidence") or "").strip(),
-            "acq_time": acq_time.isoformat(),
-            "satellite": str(rec.get("satellite") or "").strip(),
+            "acq_time": acq_iso,
+            "satellite": sat,
             "instrument": str(rec.get("instrument") or "").strip(),
             "bright_ti5": _to_float(rec.get("bright_ti5")),
             "frp": _to_float(rec.get("frp")),
@@ -138,6 +151,17 @@ def parse_firms_csv(text: str) -> list[dict]:
             "track": _to_float(rec.get("track")),
             "version": str(rec.get("version") or "").strip(),
         })
+    return points, ids
+
+
+def parse_firms_csv(text: str) -> list[dict]:
+    """Parse a FIRMS area CSV payload into normalized fire messages.
+
+    Returns one dict per hotspot with the fields stored in the ``fires`` table
+    (acq_time already combined into a UTC ISO timestamp). Rows that don't look
+    like valid VIIRS detections are skipped rather than failing the whole poll.
+    """
+    points, _ids = parse_firms_csv_delta(text)
     return points
 
 
@@ -162,6 +186,16 @@ async def publish_fire_batch(points: list[dict]) -> int:
     return len(points)
 
 
+async def persist_hotspots(points: list[dict]) -> int:
+    """Write a FIRMS poll to Postgres in one ON CONFLICT batch.
+
+    NATS-per-row was 93k commits + geofence/correlation per hotspot.
+    """
+    from ingestor import ingest_fire_rows
+
+    return await ingest_fire_rows(points)
+
+
 async def ingest_fires(bbox: str | None = None) -> int:
     """Fetch the FIRMS hotspot CSV for an area and publish it to NATS.
 
@@ -205,11 +239,18 @@ async def ingest_fires(bbox: str | None = None) -> int:
                     dataset, first_line,
                 )
                 continue
-            points = parse_firms_csv(text)
-            published = await publish_fire_batch(points)
+            poll_key = (dataset, area, FIRMS_DAYS)
+            digest = hashlib.sha256(text.encode("utf-8", "surrogatepass")).digest()
+            if _csv_digest.get(poll_key) == digest:
+                logger.info("FIRMS %s CSV unchanged, skip parse/insert", dataset)
+                continue
+            points, ids = parse_firms_csv_delta(text, skip_ids=_seen_ids.get(poll_key))
+            published = await persist_hotspots(points) if points else 0
+            _csv_digest[poll_key] = digest
+            _seen_ids[poll_key] = ids
             total_published += published
             logger.info(
                 "FIRMS: fetched %d hotspot(s) for bbox=%s (%s), published %d",
-                len(points), area, dataset, published,
+                len(ids), area, dataset, published,
             )
     return total_published

app/geofence.py

@@ -319,3 +319,154 @@ async def record_and_notify(
         except Exception:
             pass
     return sent
+
+
+async def list_alerts(
+    *,
+    geofence_id: str | None = None,
+    since: datetime | None = None,
+    until: datetime | None = None,
+    source_kind: str | None = None,
+    limit: int = 100,
+) -> list[dict]:
+    """Filterable hit log. Empty list if the DB is down — never raises."""
+    where = ["TRUE"]
+    params: dict[str, Any] = {"limit": int(limit)}
+    if geofence_id:
+        where.append("geofence_id = CAST(:geofence_id AS uuid)")
+        params["geofence_id"] = geofence_id
+    if since is not None:
+        where.append("created_at >= :since")
+        params["since"] = since
+    if until is not None:
+        where.append("created_at <= :until")
+        params["until"] = until
+    if source_kind:
+        where.append("source_kind = :source_kind")
+        params["source_kind"] = source_kind
+    sql = f"""
+        SELECT id::text, geofence_id::text, source_kind, entity_id,
+               lat, lon, payload, created_at
+        FROM geofence_alerts
+        WHERE {' AND '.join(where)}
+        ORDER BY created_at DESC
+        LIMIT :limit
+    """
+    try:
+        async with async_session() as session:
+            rows = (await session.execute(text(sql), params)).mappings().all()
+        out = []
+        for r in rows:
+            item = dict(r)
+            if item.get("created_at") is not None:
+                item["created_at"] = item["created_at"].isoformat()
+            out.append(item)
+        return out
+    except Exception:
+        return []
+
+
+async def get_geofence(gid: str) -> dict | None:
+    current = next((f for f in _cache if f["id"] == gid), None)
+    if current is not None:
+        return current
+    try:
+        await refresh_cache()
+    except Exception:
+        return None
+    return next((f for f in _cache if f["id"] == gid), None)
+
+
+def _marker_from_track(row) -> dict:
+    from live_layers import to_marker
+
+    extra = {"bucket": row["bucket"].isoformat() if row.get("bucket") else None, "dvr": True}
+    return to_marker(
+        row["id"], row["lat"], row["lon"],
+        heading=row.get("heading"), speed=row.get("speed"),
+        label=row.get("label") or row["id"],
+        extra=extra,
+    )
+
+
+async def _cagg_inside(gid: str, kind: str, bucket: datetime, limit: int = 2000) -> list[dict]:
+    table = "aircraft_tracks_1min" if kind == "aircraft" else "vessel_tracks_1min"
+    id_col = "hex" if kind == "aircraft" else "mmsi"
+    sql = f"""
+        SELECT {id_col} AS id, lat, lon, heading, speed, label, bucket
+        FROM {table}
+        WHERE bucket = :bucket
+          AND ST_Intersects(
+                (SELECT geom FROM geofences WHERE id = CAST(:gid AS uuid)),
+                ST_SetSRID(ST_MakePoint(lon, lat), 4326)
+              )
+        LIMIT :limit
+    """
+    try:
+        async with async_session() as session:
+            rows = (await session.execute(
+                text(sql), {"bucket": bucket, "gid": gid, "limit": limit},
+            )).mappings().all()
+        return [
+            _marker_from_track(r)
+            for r in rows
+            if r["lat"] is not None and r["lon"] is not None
+        ]
+    except Exception:
+        return []
+
+
+async def _fires_inside(gid: str, ts: datetime, limit: int = 2000) -> list[dict]:
+    from tracks import minute_bucket
+
+    bucket = minute_bucket(ts)
+    t1 = bucket + timedelta(minutes=1)
+    sql = """
+        SELECT latitude, longitude, brightness, confidence, acq_time, satellite,
+               instrument, bright_ti5, frp, daynight
+        FROM fires
+        WHERE acq_time >= :t0 AND acq_time < :t1
+          AND ST_Intersects(
+                (SELECT geom FROM geofences WHERE id = CAST(:gid AS uuid)),
+                ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)
+              )
+        LIMIT :limit
+    """
+    try:
+        async with async_session() as session:
+            rows = (await session.execute(
+                text(sql),
+                {"t0": bucket, "t1": t1, "gid": gid, "limit": limit},
+            )).mappings().all()
+        out = []
+        for r in rows:
+            item = dict(r)
+            if item.get("acq_time") is not None:
+                item["acq_time"] = item["acq_time"].isoformat()
+            out.append(item)
+        return out
+    except Exception:
+        return []
+
+
+async def snapshot_at(gid: str, ts: datetime) -> dict | None:
+    """Positions inside the fence at time T. None if the fence is missing.
+
+    Does not persist or notify. Empty lists if track/fire queries fail.
+    """
+    fence = await get_geofence(gid)
+    if fence is None:
+        return None
+    from tracks import minute_bucket
+
+    bucket = minute_bucket(ts)
+    aircraft = await _cagg_inside(gid, "aircraft", bucket)
+    vessels = await _cagg_inside(gid, "vessel", bucket)
+    fires = await _fires_inside(gid, ts)
+    return {
+        "geofence_id": gid,
+        "timestamp": ts.isoformat(),
+        "aircraft": aircraft,
+        "vessels": vessels,
+        "fires": fires,
+    }

app/ingestor.py

@@ -14,10 +14,17 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
 from database import async_session
 from models import events as events_table
 from models import fires as fires_table
+from models import event_dedup as event_dedup_table
 from config import NATS_URL
+from sources import event_dedup_key
 
 logger = logging.getLogger("osint.ingestor")
 
+# asyncpg rejects statements with >32767 bind params. A FIRMS poll is ~90k
+# rows × 14 columns. Chunk inserts; still one transaction / one commit.
+FIRE_ROW_BIND_PARAMS = 14
+FIRE_INSERT_CHUNK = 2000
+
 # NATS connection settings
 NATS_URLS = NATS_URL
 NATS_STREAM = "events"
@@ -115,6 +122,45 @@ async def ingest_fire_row(msg: dict) -> bool:
     return inserted
 
 
+async def ingest_fire_rows(msgs: list[dict]) -> int:
+    """Bulk-insert FIRMS hotspots: one INSERT, one ON CONFLICT, one commit."""
+    rows = []
+    for msg in msgs:
+        row = _fire_row_from_msg(msg)
+        if row is not None:
+            rows.append(row)
+    if not rows:
+        return 0
+    inserted = 0
+    async with async_session() as session:
+        for i in range(0, len(rows), FIRE_INSERT_CHUNK):
+            chunk = rows[i:i + FIRE_INSERT_CHUNK]
+            stmt = (
+                pg_insert(fires_table)
+                .values(chunk)
+                .on_conflict_do_nothing(constraint="pk_fires_natural_key")
+            )
+            result = await session.execute(stmt)
+            inserted += int(result.rowcount or 0)
+        await session.commit()
+    if inserted:
+        logger.info("bulk ingested %d/%d FIRMS hotspots", inserted, len(rows))
+        from live_layers import aircraft_last_known
+        from fire_aircraft import correlate_and_notify
+        from tracks import recent_markers
+        acs = list(aircraft_last_known.values()) or await recent_markers("aircraft")
+        if acs:
+            fires = [
+                {
+                    "id": f"firms:{r['latitude']:.4f},{r['longitude']:.4f}",
+                    "lat": r["latitude"], "lon": r["longitude"], "label": "FIRMS",
+                }
+                for r in rows[:500]
+            ]
+            await correlate_and_notify(fires, acs)
+    return inserted
+
+
 async def ingest_event(msg: dict):
     """Ingest a single event from NATS into PostgreSQL."""
     # Active fire/hotspot messages carry a dedicated schema and land in the
@@ -149,10 +195,26 @@ async def ingest_event(msg: dict):
     }
 
     # Parse timestamp if string
-    if isinstance(event_row["source_timestamp"], str):
-        event_row["source_timestamp"] = datetime.fromisoformat(event_row["source_timestamp"])
+    ts = event_row["source_timestamp"]
+    if isinstance(ts, str):
+        ts = datetime.fromisoformat(ts.replace("Z", "+00:00"))
+    if isinstance(ts, datetime) and ts.tzinfo is None:
+        ts = ts.replace(tzinfo=timezone.utc)
+    event_row["source_timestamp"] = ts
 
+    key = event_dedup_key(event_row)
     async with async_session() as session:
+        if key:
+            dedup = (
+                pg_insert(event_dedup_table)
+                .values(url=key)
+                .on_conflict_do_nothing(index_elements=["url"])
+            )
+            claimed = await session.execute(dedup)
+            if not claimed.rowcount:
+                await session.commit()
+                logger.debug("skip duplicate event url=%s", key)
+                return None
         result = await session.execute(events_table.insert().values(**event_row))
         await session.commit()
         event_id = result.inserted_primary_key[0]  # type: ignore[union-attr]

app/keystore.py

@@ -58,7 +58,7 @@ KEY_REGISTRY: dict[str, dict] = {
         "example": "32-char hex string (e.g. 5f3c…9a02)",
     },
     "NOUS_API_KEY": {
-        "description": "Nous Portal API key — hourly news summarizer (inference-api.nousresearch.com).",
+        "description": "Nous Portal API key — 15-min news summarizer (inference-api.nousresearch.com).",
         "pattern": r"^.{16,}$",
         "example": "key from https://portal.nousresearch.com (API keys page)",
     },
@@ -68,10 +68,15 @@ KEY_REGISTRY: dict[str, dict] = {
         "example": "123456789:AA… (bot token from @BotFather)",
     },
     "AISSTREAM_API_KEY": {
-        "description": "AISStream WebSocket key — live vessel positions (server-side only).",
+        "description": "AISStream (open/shared) — live US-coast AIS. Server-side WebSocket only.",
         "pattern": r"^.{8,}$",
         "example": "key from https://aisstream.io/account (GitHub login)",
     },
+    "VESSELAPI_API_KEY": {
+        "description": "VesselAPI (commercial) — Strait of Hormuz AIS, 5×/day cache. Paste the Bearer token from dashboard.vesselapi.com. Not a US-coast feed.",
+        "pattern": r"^.{8,}$",
+        "example": "Bearer token from https://dashboard.vesselapi.com/",
+    },
     "OPENSKY_CLIENT_ID": {
         "description": "OpenSky OAuth client id — optional ADS-B fallback (unused until enabled).",
         "example": "client id from opensky-network.org account",

app/live_layers.py

@@ -12,18 +12,25 @@ or AIS poll.
 from __future__ import annotations
 
 import asyncio
+import logging
 import math
 import time
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from typing import Any, Awaitable, Callable
 
 import httpx
+from urllib.parse import quote, urlencode
 
-from config import OSINT_USER_AGENT
+from config import OSINT_USER_AGENT, TITILER_PUBLIC_BASE
+
+logger = logging.getLogger("osint.live_layers")
 
 MARKER_FIELDS = ("id", "lat", "lon", "heading", "speed", "label", "extra")
 
 ADSB_LOL_BASE = "https://api.adsb.lol"
+PLANESPOTTERS_PHOTO = "https://api.planespotters.net/pub/photos"
+# Planespotters ToS cap server-side JSON caching at 24 hours.
+PLANESPOTTERS_CACHE_TTL = 24 * 3600
 AMTRAKER_TRAINS = "https://api.amtraker.com/v3/trains"
 RAINVIEWER_MAPS = "https://api.rainviewer.com/public/weather-maps.json"
 NWS_ALERTS = "https://api.weather.gov/alerts/active"
@@ -38,6 +45,24 @@ WFIGS_PERIMETERS = (
 )
 NHC_STORMS = "https://www.nhc.noaa.gov/CurrentStorms.json"
 
+PC_STAC_SEARCH = "https://planetarycomputer.microsoft.com/api/stac/v1/search"
+PC_SAS_TOKEN = "https://planetarycomputer.microsoft.com/api/sas/v1/token/sentinel-1-grd"
+# Self-hosted TiTiler on the Pi, exposed same-origin through the osint.rpi.local
+# nginx vhost. Relative template — Leaflet resolves it against the page origin,
+# so the browser never touches a raw loopback port or titiler.xyz.
+TITILER_COG_TILES = f"{TITILER_PUBLIC_BASE}/cog/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}"
+SENTINEL1_TTL = 6 * 3600  # S-1 revisit is 6–12 days; cache the COG all afternoon
+SENTINEL1_CELL = 2.0  # degrees — pan/zoom inside a cell reuses the same scene
+SENTINEL1_STAC_LIMIT = 8
+SENTINEL1_ATTRIBUTION = "Copernicus Sentinel-1 / Microsoft Planetary Computer"
+
+# GPSJAM (John Wiseman / ADS-B Exchange): daily H3 hexes of aircraft nav
+# accuracy. Hexes are published as a gzip CSV at a stable per-date URL, soon
+# after midnight UTC. Red/yellow != proven jamming.
+GPSJAM_BASE = "https://gpsjam.org"
+GPSJAM_RES = 4
+GPSJAM_TTL = 3600.0  # 1h — whole-world layer, fetched once per day effectively
+
 IEM_NEXRAD = "https://mesonet.agron.iastate.edu/cache/tile.py/1.0.0/nexrad-n0q/{z}/{x}/{y}.png"
 GIBS_THERMAL = (
     "https://gibs.earthdata.nasa.gov/wmts/epsg3857/best/"
@@ -72,6 +97,11 @@ _MAX_VESSELS = 6000
 # Last ADS-B snapshot + WFIGS points for fire↔tanker correlation.
 aircraft_last_known: dict[str, dict] = {}
 fire_last_known: list[dict] = []
+# Last-known counts for the cheap GET /api/stats HUD counter. Updated by the
+# upstream fetchers so the stats endpoint never does its own network/SQL fan-out
+# for these layers; reads are O(1) in-process.
+train_count: int = 0
+nws_alert_count: int = 0
 
 
 def overlay_catalog() -> dict:
@@ -103,6 +133,15 @@ def overlay_catalog() -> dict:
             "maxZoom": 9,
             "attribution": "NASA GIBS / EOSDIS",
         },
+        "sentinel1_sar": {
+            "id": "sentinel1_sar",
+            "title": "Sentinel-1 SAR (Cloud-Penetrating)",
+            "kind": "raster",
+            "tileUrl": None,  # filled from /api/map/sentinel1 (signed COG)
+            "endpoint": "/api/map/sentinel1",
+            "maxZoom": 14,
+            "attribution": SENTINEL1_ATTRIBUTION,
+        },
         "nws_alerts": {"id": "nws_alerts", "kind": "geojson", "endpoint": "/api/weather-alerts"},
         "wfigs_incidents": {"id": "wfigs_incidents", "kind": "points", "endpoint": "/api/fire-incidents"},
         "wfigs_perimeters": {"id": "wfigs_perimeters", "kind": "geojson", "endpoint": "/api/fire-perimeters"},
@@ -110,6 +149,30 @@ def overlay_catalog() -> dict:
         "vessels": {"id": "vessels", "kind": "points", "endpoint": "/api/vessels"},
         "trains": {"id": "trains", "kind": "points", "endpoint": "/api/trains"},
         "storms": {"id": "storms", "kind": "points", "endpoint": "/api/storms"},
+        "gpsjam": {
+            "id": "gpsjam",
+            "kind": "geojson",
+            "endpoint": "/api/map/gpsjam",
+            "attribution": "GPSJAM / John Wiseman / ADS-B Exchange",
+        },
+        "satellites": {
+            "id": "satellites",
+            "kind": "points",
+            "endpoint": "/api/satellites",
+            "attribution": "CelesTrak (GP JSON / SGP4)",
+        },
+        "infra_nuclear": {
+            "id": "infra_nuclear",
+            "kind": "points",
+            "endpoint": "/api/infrastructure?types=nuclear",
+            "attribution": "OpenStreetMap contributors / Overpass API",
+        },
+        "conflicts": {
+            "id": "conflicts",
+            "kind": "points",
+            "endpoint": "/api/conflicts",
+            "attribution": "Curated OSINT conflict catalog",
+        },
     }
 
 
@@ -309,6 +372,204 @@ def _heading(value: object) -> float | None:
     return num
 
 
+def _s(value: object) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None
+
+
+# ADS-B emitter category (DO-260B). A3 airliners, A5 heavies, A7 helicopters.
+_EMITTER = {
+    "A0": "unknown", "A1": "light", "A2": "small", "A3": "large",
+    "A4": "high vortex", "A5": "heavy", "A6": "high performance", "A7": "rotorcraft",
+    "B0": "unknown", "B1": "glider", "B2": "airship", "B3": "parachute",
+    "B4": "ultralight", "B6": "UAV", "B7": "space",
+    "C0": "ground unknown", "C1": "emergency vehicle", "C2": "service vehicle",
+    "D0": "unknown", "D1": "emergency",
+}
+
+# Combat / dedicated-military ICAO types. C-130/C-17 omitted — those also fly
+# as fire tankers and civil contractors; dbFlags/hex catch the real mil ones.
+_MIL_ICAO = frozenset({
+    "F15", "F16", "F18", "FA18", "F22", "F35", "F117", "A10", "A10A",
+    "B1", "B1B", "B2", "B52", "AV8B", "F4", "F5", "F14",
+    "SU27", "SU30", "SU34", "SU35", "SU57",
+    "MG29", "MIG29", "MG31", "MIG31", "J10", "J11", "J15", "J16", "J20",
+    "EUFI", "RFAL", "TOR", "E3TF", "E3CF", "E6", "E8", "P8",
+    "MQ9", "MQ1", "RQ4", "V22", "AH64", "H64",
+})
+
+# US DoD Mode-S block AE0000–AEFFFF.
+_US_DOD_HEX_LO, _US_DOD_HEX_HI = 0xAE0000, 0xAEFFFF
+
+_MIL_CS_PREFIX = ("RCH", "NAVY", "ARMY", "MARINE", "GOTOF", "REACH")
+
+
+def classify_adsb(ac: dict) -> tuple[str, str]:
+    """Return (role, role_src). Prefer readsb dbFlags bit0, then type/hex/cs."""
+    flags = ac.get("dbFlags")
+    try:
+        flags_i = int(flags) if flags is not None else 0
+    except (TypeError, ValueError):
+        flags_i = 0
+    if flags_i & 1:
+        return "military", "dbFlags"
+    icao = str(ac.get("t") or "").strip().upper()
+    if icao in _MIL_ICAO:
+        return "military", "type"
+    try:
+        hx = int(str(ac.get("hex") or "").strip(), 16)
+    except ValueError:
+        hx = -1
+    if _US_DOD_HEX_LO <= hx <= _US_DOD_HEX_HI:
+        return "military", "hex"
+    cs = str(ac.get("flight") or "").strip().upper()
+    if cs.startswith(_MIL_CS_PREFIX):
+        return "military", "callsign"
+    return "civilian", "default"
+
+
+def _adsb_extra(ac: dict, hex_id: str) -> dict[str, Any]:
+    role, src = classify_adsb(ac)
+    cat = str(ac.get("category") or "").strip().upper()
+    extra: dict[str, Any] = {
+        "hex": hex_id,
+        "reg": _s(ac.get("r")),
+        "type": _s(ac.get("t")),
+        "alt_baro": ac.get("alt_baro"),
+        "squawk": _s(ac.get("squawk")),
+        "emergency": _s(ac.get("emergency")),
+        "category": cat or None,
+        "emitter": _EMITTER.get(cat),
+        "seen_pos": ac.get("seen_pos"),
+        "role": role,
+        "role_src": src,
+        "src": "adsb.lol",
+    }
+    desc = _s(ac.get("desc"))
+    if desc:
+        extra["desc"] = desc
+    own = _s(ac.get("ownOp") or ac.get("ownOpName") or ac.get("ownop"))
+    if own:
+        extra["ownOp"] = own
+    if ac.get("alt_geom") is not None:
+        extra["alt_geom"] = ac.get("alt_geom")
+    vs = ac.get("baro_rate")
+    if vs is None:
+        vs = ac.get("geom_rate")
+    if vs is not None:
+        extra["vs"] = vs
+    try:
+        raw_flags = ac.get("dbFlags")
+        flags_i = int(raw_flags) if raw_flags is not None else 0
+    except (TypeError, ValueError):
+        flags_i = 0
+    if flags_i:
+        extra["dbFlags"] = flags_i
+        extra["interesting"] = bool(flags_i & 2)
+    return extra
+
+
+_NAVSTAT = {
+    0: "underway",
+    1: "at anchor",
+    2: "not under command",
+    3: "restricted manoeuvre",
+    4: "constrained by draught",
+    5: "moored",
+    6: "aground",
+    7: "fishing",
+    8: "sailing",
+    14: "AIS-SART",
+    15: "undefined",
+}
+
+# Compact MID → country for the flags that actually show up on AIS.
+_MID_COUNTRY = {
+    211: "Germany", 218: "Germany",
+    219: "Denmark", 220: "Denmark",
+    224: "Spain", 225: "Spain",
+    226: "France", 227: "France", 228: "France",
+    232: "United Kingdom", 233: "United Kingdom", 234: "United Kingdom", 235: "United Kingdom",
+    236: "Gibraltar", 237: "Greece", 239: "Greece", 240: "Greece", 241: "Greece",
+    244: "Netherlands", 245: "Netherlands", 246: "Netherlands",
+    247: "Italy", 249: "Malta", 250: "Ireland", 251: "Iceland",
+    255: "Portugal", 256: "Malta",
+    257: "Norway", 258: "Norway", 259: "Norway",
+    261: "Poland", 263: "Portugal", 265: "Sweden", 266: "Sweden",
+    271: "Turkey", 273: "Russia", 276: "Estonia", 277: "Lithuania",
+    301: "Anguilla", 303: "United States", 310: "Bermuda", 316: "Canada",
+    319: "Cayman Islands", 338: "United States", 339: "Jamaica",
+    345: "Mexico", 352: "Panama", 353: "Panama", 354: "Panama",
+    355: "Panama", 356: "Panama", 357: "Panama",
+    366: "United States", 367: "United States", 368: "United States", 369: "United States",
+    370: "Panama", 371: "Panama", 372: "Panama", 373: "Panama", 374: "Panama",
+    375: "St Vincent", 376: "St Vincent", 377: "St Vincent",
+    412: "China", 413: "China", 414: "China", 416: "Taiwan",
+    419: "India", 431: "Japan", 432: "Japan", 440: "South Korea", 441: "South Korea",
+    477: "Hong Kong", 503: "Australia", 525: "Indonesia", 533: "Malaysia",
+    538: "Marshall Islands", 548: "Philippines", 563: "Singapore",
+    564: "Singapore", 565: "Singapore", 566: "Singapore", 567: "Thailand",
+    574: "Vietnam", 636: "Liberia", 637: "Liberia",
+    710: "Brazil", 725: "Chile", 730: "Colombia", 760: "Peru",
+}
+
+
+def _mmsi_country(mmsi: object) -> str | None:
+    digits = "".join(ch for ch in str(mmsi or "") if ch.isdigit())
+    if len(digits) < 3:
+        return None
+    try:
+        mid = int(digits[:3])
+    except ValueError:
+        return None
+    return _MID_COUNTRY.get(mid)
+
+
+def classify_ais_type(type_code: int | None) -> tuple[str, str]:
+    """Return (role, kind) from ITU-R M.1371 ship-and-cargo type."""
+    if type_code is None:
+        return "civilian", "unknown"
+    t = int(type_code)
+    tens = t // 10
+    if t == 35:
+        return "military", "military"
+    if t == 30:
+        return "civilian", "fishing"
+    if t in (31, 32, 52):
+        return "civilian", "tug"
+    if t == 33:
+        return "civilian", "dredger"
+    if t == 34:
+        return "civilian", "diving"
+    if t == 36:
+        return "civilian", "sailing"
+    if t == 37:
+        return "civilian", "pleasure"
+    if t == 50:
+        return "government", "pilot"
+    if t == 51:
+        return "government", "SAR"
+    if t == 55:
+        return "government", "law"
+    if t == 54:
+        return "government", "anti-pollution"
+    if t == 58:
+        return "government", "medical"
+    if tens == 4:
+        return "civilian", "HSC"
+    if tens == 6:
+        return "civilian", "passenger"
+    if tens == 7:
+        return "civilian", "cargo"
+    if tens == 8:
+        return "civilian", "tanker"
+    if tens in (5, 9) or t in (53, 56, 57, 59):
+        return "civilian", "special"
+    return "civilian", "other"
+
+
 def transform_adsb_lol(payload: dict | list | None) -> list[dict]:
     """Map ADSB.lol v2 aircraft list to shared markers. Dedup on hex."""
     if payload is None:
@@ -333,17 +594,7 @@ def transform_adsb_lol(payload: dict | list | None) -> list[dict]:
             heading=_heading(ac.get("track")),
             speed=_f(ac.get("gs")),
             label=flight,
-            extra={
-                "hex": hex_id,
-                "reg": ac.get("r"),
-                "type": ac.get("t"),
-                "alt_baro": ac.get("alt_baro"),
-                "squawk": ac.get("squawk"),
-                "emergency": ac.get("emergency"),
-                "category": ac.get("category"),
-                "seen_pos": ac.get("seen_pos"),
-                "src": "adsb.lol",
-            },
+            extra=_adsb_extra(ac, hex_id),
         ))
     return out
 
@@ -405,6 +656,9 @@ def transform_ais_frame(frame: dict | None) -> dict | None:
         or {}
     )
     extra: dict[str, Any] = {"src": "aisstream", "mmsi": mmsi}
+    country = _mmsi_country(mmsi)
+    if country:
+        extra["country"] = country
     if frame.get("MessageType") == "ShipStaticData":
         static = msg.get("ShipStaticData") or {}
         dest = str(static.get("Destination") or static.get("destination") or "").strip()
@@ -412,6 +666,42 @@ def transform_ais_frame(frame: dict | None) -> dict | None:
         extra["static"] = True
         if not name:
             name = str(static.get("Name") or static.get("name") or "").strip()
+        cs = _s(static.get("CallSign") or static.get("callSign"))
+        if cs:
+            extra["callsign"] = cs
+        try:
+            imo = int(static.get("ImoNumber") or static.get("imoNumber") or 0)
+        except (TypeError, ValueError):
+            imo = 0
+        if imo:
+            extra["imo"] = imo
+        type_code = static.get("Type") if "Type" in static else static.get("type")
+        try:
+            type_i = int(type_code) if type_code is not None else None
+        except (TypeError, ValueError):
+            type_i = None
+        if type_i is not None:
+            extra["type_code"] = type_i
+            role, kind = classify_ais_type(type_i)
+            extra["role"] = role
+            extra["kind"] = kind
+        dim = static.get("Dimension") or static.get("dimension") or {}
+        if isinstance(dim, dict):
+            a, b = _f(dim.get("A")), _f(dim.get("B"))
+            c, d = _f(dim.get("C")), _f(dim.get("D"))
+            if a is not None and b is not None:
+                extra["length"] = int(round(a + b))
+            if c is not None and d is not None:
+                extra["beam"] = int(round(c + d))
+        draught = _f(static.get("MaximumStaticDraught") or static.get("maximumStaticDraught"))
+        if draught is not None:
+            extra["draught"] = draught
+        eta = static.get("Eta") or static.get("eta") or {}
+        if isinstance(eta, dict) and eta.get("Month"):
+            extra["eta"] = (
+                f"{int(eta.get('Month') or 0):02d}-{int(eta.get('Day') or 0):02d} "
+                f"{int(eta.get('Hour') or 0):02d}:{int(eta.get('Minute') or 0):02d}"
+            )
         if lat is None or lon is None:
             # Static-only update: caller merges onto last-known by MMSI.
             return to_marker(str(mmsi), None, None, label=name or str(mmsi), extra=extra)
@@ -421,8 +711,13 @@ def transform_ais_frame(frame: dict | None) -> dict | None:
     sog = pos.get("Sog")
     navstat = pos.get("NavigationalStatus")
     extra["navstat"] = navstat
+    try:
+        extra["nav"] = _NAVSTAT.get(int(navstat)) if navstat is not None else None
+    except (TypeError, ValueError):
+        extra["nav"] = None
     extra["cog"] = pos.get("Cog")
-    extra["dest"] = extra.get("dest")
+    if not extra.get("dest"):
+        extra.pop("dest", None)
     if lat is None or lon is None:
         return None
     return to_marker(
@@ -512,6 +807,18 @@ def _headers() -> dict[str, str]:
     return {"User-Agent": OSINT_USER_AGENT, "Accept": "application/json"}
 
 
+def _planespotters_headers() -> dict[str, str]:
+    """Planespotters 403s server clients whose UA has no contact URL/email."""
+    ua = (OSINT_USER_AGENT or "").strip()
+    if ua and ("@" in ua or "http://" in ua.lower() or "https://" in ua.lower()):
+        return {"User-Agent": ua, "Accept": "application/json"}
+    base = ua or "osint-dashboard/1.0"
+    return {
+        "User-Agent": f"{base} (lancewalters94@gmail.com)",
+        "Accept": "application/json",
+    }
+
+
 async def _lock_for(key: str) -> asyncio.Lock:
     async with _key_locks_guard:
         lock = _key_locks.get(key)
@@ -555,21 +862,90 @@ async def close_http() -> None:
         _http = None
 
 
-async def _get_json(url: str, params: dict | None = None) -> Any:
+async def _get_json(
+    url: str, params: dict | None = None, headers: dict | None = None,
+) -> Any:
     if _http is None:
         async with httpx.AsyncClient(
             timeout=_HTTP_TIMEOUT, follow_redirects=True, headers=_headers(),
         ) as client:
-            resp = await client.get(url, params=params)
+            resp = await client.get(url, params=params, headers=headers)
             resp.raise_for_status()
             return resp.json()
-    resp = await _http.get(url, params=params)
+    resp = await _http.get(url, params=params, headers=headers)
     resp.raise_for_status()
     return resp.json()
 
 
-async def fetch_aircraft(bbox: str, limit: int = DEFAULT_LIMIT) -> list[dict]:
+def _normalize_planespotter_photo(photo: dict) -> dict | None:
+    """Slim a planespotters.net photo object for the aircraft popup.
+
+    Image binaries are never proxied/re-hosted: we return the CDN ``src``
+    and the browser loads it directly (per ToS).
+    """
+    if not isinstance(photo, dict):
+        return None
+    large = photo.get("thumbnail_large") or {}
+    small = photo.get("thumbnail") or {}
+    src = large.get("src") or small.get("src")
+    if not src:
+        return None
+    size = large.get("size") or small.get("size") or {}
+    return {
+        "id": str(photo.get("id") or ""),
+        "src": src,
+        "width": size.get("width"),
+        "height": size.get("height"),
+        "link": photo.get("link") or "",
+        "photographer": photo.get("photographer") or "",
+    }
+
+
+async def fetch_planespotters_photo(
+    hex_code: str | None = None, reg: str | None = None,
+) -> dict | None:
+    """Latest photo for an aircraft from planespotters.net (hex preferred).
+
+    Server-side only: planespotters 403s any request carrying an ``Origin``
+    header (browser fetch() always sends one), so the browser can never reach
+    it directly. We fetch here with the identifying UA, cache the JSON ≤24h,
+    and hand the CDN image URL back for the browser to load.
+    """
+    if hex_code:
+        key = f"psp-hex:{hex_code.strip().lower()}"
+        url = f"{PLANESPOTTERS_PHOTO}/hex/{quote(hex_code.strip())}"
+    elif reg:
+        key = f"psp-reg:{reg.strip().lower()}"
+        url = f"{PLANESPOTTERS_PHOTO}/reg/{quote(reg.strip())}"
+    else:
+        return None
+
+    async def _load() -> dict | None:
+        data = await _get_json(url, headers=_planespotters_headers())
+        photos = data.get("photos") or []
+        return _normalize_planespotter_photo(photos[0]) if photos else None
+
+    return await _ttl_get(key, PLANESPOTTERS_CACHE_TTL, _load)
+
+
+async def fetch_aircraft(
+    bbox: str, limit: int = DEFAULT_LIMIT, persist: bool = False,
+) -> list[dict]:
+    """Viewport ADS-B.
+
+    GET path (persist=False) serves in-memory last-known and never writes
+    tracks/geofences. Background refresh (persist=True) hits ADSB.lol and
+    then persist_aircraft_snapshot.
+    """
     minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
+    if not persist:
+        cached = [
+            dict(v) for v in aircraft_last_known.values()
+            if v.get("lat") is not None and v.get("lon") is not None
+        ]
+        filtered = filter_points_bbox(cached, minlon, minlat, maxlon, maxlat, limit)
+        if filtered:
+            return filtered
     qminlon, qminlat, qmaxlon, qmaxlat = quantize_bbox(minlon, minlat, maxlon, maxlat)
     lat, lon, radius = bbox_center_radius_nm(qminlon, qminlat, qmaxlon, qmaxlat)
     cache_key = f"adsb:{lat:.2f}:{lon:.2f}:{radius}"
@@ -579,12 +955,20 @@ async def fetch_aircraft(bbox: str, limit: int = DEFAULT_LIMIT) -> list[dict]:
         return transform_adsb_lol(await _get_json(url))
 
     rows = await _ttl_get(cache_key, 8.0, _load)
+    for m in rows:
+        aircraft_last_known[str(m.get("id"))] = m
+    if persist:
+        await persist_aircraft_snapshot(rows)
+    return filter_points_bbox(rows, minlon, minlat, maxlon, maxlat, limit)
+
+
+async def persist_aircraft_snapshot(rows: list[dict]) -> None:
+    """Track / geofence / WS / fire correlation — never on the GET path."""
     from ws_manager import manager
     from tracks import record_position
     from geofence import record_and_notify
-    aircraft_last_known.clear()
+
     for m in rows:
-        aircraft_last_known[str(m.get("id"))] = m
         mlat, mlon = m.get("lat"), m.get("lon")
         if mlat is None or mlon is None:
             continue
@@ -598,7 +982,6 @@ async def fetch_aircraft(bbox: str, limit: int = DEFAULT_LIMIT) -> list[dict]:
     if fire_last_known:
         from fire_aircraft import correlate_and_notify
         await correlate_and_notify(fire_last_known, rows)
-    return filter_points_bbox(rows, minlon, minlat, maxlon, maxlat, limit)
 
 
 async def fetch_trains(bbox: str | None, limit: int = DEFAULT_LIMIT) -> list[dict]:
@@ -606,16 +989,24 @@ async def fetch_trains(bbox: str | None, limit: int = DEFAULT_LIMIT) -> list[dic
         return transform_amtraker(await _get_json(AMTRAKER_TRAINS))
 
     rows = await _ttl_get("amtraker:trains", 20.0, _load)
+    global train_count
+    train_count = len(rows)
     if bbox:
         minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
         return filter_points_bbox(rows, minlon, minlat, maxlon, maxlat, limit)
     return rows[:limit]
 
 
-async def fetch_vessels(bbox: str | None, limit: int = DEFAULT_LIMIT) -> list[dict]:
+async def fetch_vessels(
+    bbox: str | None,
+    limit: int = DEFAULT_LIMIT,
+    src: str | None = None,
+) -> list[dict]:
     async with vessel_lock:
         rows = [dict(v) for v in vessel_last_known.values()
                 if v.get("lat") is not None and v.get("lon") is not None]
+    if src and src != "all":
+        rows = [r for r in rows if (r.get("extra") or {}).get("src") == src]
     if bbox:
         minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
         return filter_points_bbox(rows, minlon, minlat, maxlon, maxlat, limit)
@@ -648,8 +1039,18 @@ async def upsert_vessel(marker: dict) -> None:
         vessel_last_known[vid] = stored
         if len(vessel_last_known) > _MAX_VESSELS:
             excess = len(vessel_last_known) - int(_MAX_VESSELS * 0.9)
+            # AISStream's live US-coast feed can crowd the shared store past
+            # _MAX_VESSELS and evict the Hormuz VesselAPI rows (refreshed only
+            # every VESSELAPI_INTERVAL) as "oldest". Keep src=vesselapi rows
+            # resident so the blind-spot box stays visible between polls.
+            evictable = [
+                k for k, v in vessel_last_known.items()
+                if (v.get("extra") or {}).get("src") != "vesselapi"
+            ]
+            if len(evictable) < excess:
+                excess = len(evictable)
             oldest = sorted(
-                vessel_last_known,
+                evictable,
                 key=lambda k: vessel_last_known[k].get("seen_at", ""),
             )[:excess]
             for k in oldest:
@@ -732,14 +1133,19 @@ async def fetch_weather_alerts(area: str | None, bbox: str | None) -> dict:
         clip_box = None
         if area:
             nws_params["area"] = area.upper()
-        elif bbox:
+        if bbox:
             clip_box = quantize_bbox(*parse_bbox(bbox))
-            nws_params["bbox"] = f"{clip_box[0]},{clip_box[1]},{clip_box[2]},{clip_box[3]}"
+        # NWS /alerts/active 400s on bbox= — never send it; clip locally.
         nws_fc: dict = {"features": []}
+        nws_ok = True
         try:
             nws_fc = await _get_json(NWS_ALERTS, nws_params)
-        except Exception:
+        except Exception as exc:
+            logger.warning("NWS alerts fetch failed: %s", exc)
+            nws_ok = False
             nws_fc = {"features": []}
+        global nws_alert_count
+        nws_alert_count = len(nws_fc.get("features") or [])
         sbw_fc = await _ttl_get("iem:sbw", 45.0, _load_iem)
         features = []
         for feat in nws_fc.get("features") or []:
@@ -756,11 +1162,15 @@ async def fetch_weather_alerts(area: str | None, bbox: str | None) -> dict:
             if "event" not in props:
                 props["event"] = props.get("ps") or "Storm-based warning"
             features.append({**feat, "properties": slim_alert_properties(props)})
-        merged = {"type": "FeatureCollection", "features": features}
+        merged = {"type": "FeatureCollection", "features": features, "nws_ok": nws_ok}
         if clip_box:
-            merged = clip_fc_to_bbox(merged, *clip_box)
+            clipped = clip_fc_to_bbox(merged, *clip_box)
+            clipped["nws_ok"] = nws_ok
+            merged = clipped
         elif bbox:
-            merged = clip_fc_to_bbox(merged, *parse_bbox(bbox))
+            clipped = clip_fc_to_bbox(merged, *parse_bbox(bbox))
+            clipped["nws_ok"] = nws_ok
+            merged = clipped
         return merged
 
     key = f"alerts:{area or ''}:{bbox_cell_key(bbox) if bbox else ''}"
@@ -802,3 +1212,348 @@ async def fetch_storms() -> list[dict]:
         return transform_nhc_storms(await _get_json(NHC_STORMS))
 
     return await _ttl_get("nhc:storms", 300.0, _load)
+
+
+# ── Sentinel-1 SAR (Planetary Computer STAC → signed COG tile template) ────
+
+class UpstreamRateLimited(Exception):
+    """Planetary Computer returned 429. Carries Retry-After for the client."""
+
+    def __init__(self, retry_after: str | None = None):
+        self.retry_after = retry_after
+        super().__init__("planetary computer rate limited")
+
+
+async def _post_json(
+    url: str, json: dict | None = None, headers: dict | None = None,
+) -> Any:
+    if _http is None:
+        async with httpx.AsyncClient(
+            timeout=_HTTP_TIMEOUT, follow_redirects=True, headers=_headers(),
+        ) as client:
+            resp = await client.post(url, json=json, headers=headers)
+            resp.raise_for_status()
+            return resp.json()
+    resp = await _http.post(url, json=json, headers=headers)
+    resp.raise_for_status()
+    return resp.json()
+
+
+async def _pc_call(coro: Awaitable[Any]) -> Any:
+    """Run a Planetary Computer call, mapping 429 → UpstreamRateLimited."""
+    try:
+        return await coro
+    except httpx.HTTPStatusError as exc:
+        if exc.response.status_code == 429:
+            raise UpstreamRateLimited(
+                exc.response.headers.get("Retry-After")
+            ) from exc
+        raise
+
+
+def sign_cog_url(href: str, token: str) -> str:
+    """Append a SAS token to a PC blob URL (respect existing query string).
+
+    PC's SAS endpoint returns the token as an already-percent-encoded query
+    string (``st=…&se=…&sp=rl&…&sig=…``). Azure only honours those parameters
+    when they sit top-level on the blob URL — wrapping them under a single
+    ``token=`` param yields 403/409, so we append the token verbatim.
+    """
+    sep = "&" if "?" in href else "?"
+    return f"{href}{sep}{token}"
+
+
+def sentinel1_tile_url(signed_cog: str) -> str:
+    """TiTiler XYZ template for a signed COG (Leaflet substitutes {z}/{x}/{y})."""
+    params = urlencode({
+        "url": signed_cog,
+        "rescale": "0,500",
+        "colormap_name": "cfastie",
+    })
+    return f"{TITILER_COG_TILES}?{params}"
+
+
+def feature_bbox(feat: dict) -> list[float] | None:
+    """STAC Feature bbox as [minlon, minlat, maxlon, maxlat], or None."""
+    raw = feat.get("bbox") if isinstance(feat, dict) else None
+    if isinstance(raw, (list, tuple)) and len(raw) >= 4:
+        try:
+            return [float(raw[0]), float(raw[1]), float(raw[2]), float(raw[3])]
+        except (TypeError, ValueError):
+            pass
+    geom = (feat or {}).get("geometry") or {}
+    coords = geom.get("coordinates") if isinstance(geom, dict) else None
+    if not coords:
+        return None
+    lons: list[float] = []
+    lats: list[float] = []
+
+    def _walk(node: Any) -> None:
+        if isinstance(node, (list, tuple)) and node and isinstance(node[0], (int, float)):
+            lons.append(float(node[0]))
+            lats.append(float(node[1]))
+        elif isinstance(node, (list, tuple)):
+            for child in node:
+                _walk(child)
+
+    _walk(coords)
+    if not lons:
+        return None
+    return [min(lons), min(lats), max(lons), max(lats)]
+
+
+def pick_sentinel_feature(features: list, lon: float, lat: float) -> dict | None:
+    """Prefer the scene whose bbox covers the viewport center; else first."""
+    if not features:
+        return None
+    for feat in features:
+        bb = feature_bbox(feat)
+        if bb and bb[0] <= lon <= bb[2] and bb[1] <= lat <= bb[3]:
+            return feat
+    return features[0]
+
+
+async def fetch_sentinel1(bbox: str) -> dict | None:
+    """Most recent Sentinel-1 GRD COG for a viewport, signed and TiTiler-ready.
+
+    Returns the overlay tile-template dict, or ``None`` when no GRD imagery
+    covers the bbox in the last 7 days (caller maps to 404). Queries Planetary
+    Computer only when called; cached per 2° cell + UTC day so pan/zoom inside
+    the same region reuses the COG instead of picking a neighbouring swath.
+    """
+    minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
+    qminlon, qminlat, qmaxlon, qmaxlat = quantize_bbox(
+        minlon, minlat, maxlon, maxlat, step=SENTINEL1_CELL,
+    )
+    day = datetime.now(timezone.utc).date().isoformat()
+    key = f"sentinel1:{day}:{qminlon:.4f},{qminlat:.4f},{qmaxlon:.4f},{qmaxlat:.4f}"
+    clon = (qminlon + qmaxlon) / 2.0
+    clat = (qminlat + qmaxlat) / 2.0
+
+    async def _load() -> dict | None:
+        now = datetime.now(timezone.utc)
+        week_ago = now - timedelta(days=7)
+        payload = {
+            "collections": ["sentinel-1-grd"],
+            "bbox": [qminlon, qminlat, qmaxlon, qmaxlat],
+            "datetime": f"{week_ago.isoformat()}/{now.isoformat()}",
+            "limit": SENTINEL1_STAC_LIMIT,
+            "sortby": [{"field": "datetime", "direction": "desc"}],
+        }
+        data = await _pc_call(_post_json(PC_STAC_SEARCH, json=payload))
+        features = data.get("features") or []
+        if not features:
+            return None
+
+        feat = pick_sentinel_feature(features, clon, clat)
+        if not feat:
+            return None
+        assets = feat.get("assets") or {}
+        chosen_href: str | None = None
+        polarization: str | None = None
+        for pol in ("vv", "hh"):
+            href = (assets.get(pol) or {}).get("href")
+            if href:
+                chosen_href = href
+                polarization = pol
+                break
+        if not chosen_href:
+            return None
+
+        sas = await _pc_call(_get_json(PC_SAS_TOKEN))
+        token = (sas or {}).get("token")
+        if not token:
+            raise RuntimeError("planetarycomputer SAS token missing")
+        signed = sign_cog_url(chosen_href, token)
+
+        props = feat.get("properties") or {}
+        return {
+            "id": "sentinel-1-sar",
+            "kind": "raster",
+            "tileUrl": sentinel1_tile_url(signed),
+            "opacity": 0.8,
+            "itemId": feat.get("id"),
+            "datetime": props.get("datetime") or feat.get("datetime"),
+            "polarization": polarization,
+            "attribution": SENTINEL1_ATTRIBUTION,
+            "bbox": feature_bbox(feat),
+        }
+
+    return await _ttl_get(key, float(SENTINEL1_TTL), _load)
+# ── GPSJAM ──────────────────────────────────────────────────────────────────
+
+
+def gpsjam_level(percent_bad: float) -> str:
+    """Map a denoised bad-aircraft percentage to GPSJAM's three tiers."""
+    if percent_bad > 10.0:
+        return "high"
+    if percent_bad > 2.0:
+        return "medium"
+    return "low"
+
+
+def gpsjam_csv_to_geojson(text: str) -> dict:
+    """Convert a GPSJAM daily CSV to a world FeatureCollection of hex polygons.
+
+    Rows are ``hex,count_good_aircraft,count_bad_aircraft``. The interference
+    percentage uses GPSJAM's published denoise formula
+    ``100 * (bad - 1) / (good + bad)``; hexes with zero bad aircraft are the
+    "normal" background and are dropped (the base map already shows nothing).
+    """
+    import csv
+    import io
+
+    import h3
+
+    features = []
+    for row in csv.DictReader(io.StringIO(text)):
+        hex_id = (row.get("hex") or "").strip()
+        if not hex_id:
+            continue
+        try:
+            good = int(row.get("count_good_aircraft") or 0)
+            bad = int(row.get("count_bad_aircraft") or 0)
+        except (TypeError, ValueError):
+            continue
+        if bad < 1:
+            continue
+        denom = good + bad
+        percent = 100.0 * (bad - 1) / denom if denom > 0 else 0.0
+        try:
+            # h3 returns (lat, lng); GeoJSON needs (lng, lat) closed rings.
+            ring = [[lng, lat] for lat, lng in h3.cell_to_boundary(hex_id)]
+        except Exception:  # malformed/unknown cell id — skip
+            continue
+        ring.append(ring[0])
+        features.append({
+            "type": "Feature",
+            "geometry": {"type": "Polygon", "coordinates": [ring]},
+            "properties": {
+                "level": gpsjam_level(percent),
+                "percent_bad": round(percent, 2),
+                "good": good,
+                "bad": bad,
+                "hex": hex_id,
+            },
+        })
+    return {"type": "FeatureCollection", "features": features}
+
+
+async def fetch_gpsjam(date: str) -> dict:
+    """Fetch + convert one GPSJAM daily hex layer (whole world, 1h TTL)."""
+    url = f"{GPSJAM_BASE}/data/{date}-h3_{GPSJAM_RES}.csv"
+
+    async def _load():
+        if _http is None:
+            async with httpx.AsyncClient(
+                timeout=_HTTP_TIMEOUT, follow_redirects=True, headers=_headers(),
+            ) as client:
+                resp = await client.get(url)
+                resp.raise_for_status()
+                return gpsjam_csv_to_geojson(resp.text)
+        resp = await _http.get(url)
+        resp.raise_for_status()
+        return gpsjam_csv_to_geojson(resp.text)
+
+    return await _ttl_get(f"gpsjam:{date}", GPSJAM_TTL, _load)
+# ── Infrastructure (Overpass) ───────────────────────────────────────────────
+
+
+OVERPASS_INTERPRETER = "https://overpass-api.de/api/interpreter"
+# One in-flight query per quantized bbox (the per-key lock in _ttl_get). Overpass
+# asks for a 25s server timeout in-band; the client gives it 30s of headroom.
+OVERPASS_TIMEOUT = httpx.Timeout(30.0, connect=5.0)
+INFRA_TTL = 24 * 3600  # 24h per quantized bbox — static infrastructure
+
+# `types=` enum. Nuclear ships first; military/hospital slot in behind the same
+# query template without touching the transport. Overpass bbox is
+# (south, west, north, east), i.e. (minlat, minlon, maxlat, maxlon).
+_INFRA_QUERIES: dict[str, str] = {
+    "nuclear": (
+        '[out:json][timeout:25];\n'
+        'nwr["power"="plant"]["plant:source"="nuclear"]({bbox});\n'
+        'out center;'
+    ),
+}
+
+
+def infra_query(type_: str, minlon: float, minlat: float, maxlon: float, maxlat: float) -> str:
+    """Render one Overpass query with the bbox substituted in south,west,north,east."""
+    bbox = f"{minlat},{minlon},{maxlat},{maxlon}"
+    return _INFRA_QUERIES[type_].replace("{bbox}", bbox)
+
+
+def normalize_infra_element(elem: dict, type_: str) -> dict | None:
+    """Map one Overpass element to ``{id, name, lat, lon, type, extra}``.
+
+    ``out center`` gives nodes their own ``lat``/``lon`` and ways/relations a
+    ``center``. Elements with no usable coordinate are dropped.
+    """
+    etype = elem.get("type")
+    eid = elem.get("id")
+    if eid is None:
+        return None
+    if etype == "node":
+        lat, lon = elem.get("lat"), elem.get("lon")
+    else:
+        center = elem.get("center") or {}
+        lat, lon = center.get("lat"), center.get("lon")
+    if lat is None or lon is None:
+        return None
+    tags = elem.get("tags") or {}
+    name = tags.get("name") or tags.get("ref") or f"{etype}/{eid}"
+    extra = {k: v for k, v in tags.items() if k != "name"}
+    return {
+        "id": f"{etype}/{eid}",
+        "name": name,
+        "lat": lat,
+        "lon": lon,
+        "type": type_,
+        "extra": extra,
+    }
+
+
+def overpass_nuclear_to_markers(data: dict) -> list[dict]:
+    """Convert an Overpass JSON response to normalized nuclear markers."""
+    markers = []
+    for elem in data.get("elements") or []:
+        marker = normalize_infra_element(elem, "nuclear")
+        if marker is not None:
+            markers.append(marker)
+    return markers
+
+
+async def fetch_infrastructure(types: str, bbox: str) -> list[dict]:
+    """Fetch Overpass infrastructure markers, cached 24h per quantized bbox.
+
+    ``types`` is a single supported enum value (``nuclear`` for now). ``bbox``
+    is ``minlon,minlat,maxlon,maxlat``.
+    """
+    requested = [t.strip() for t in types.split(",") if t.strip()]
+    minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
+    key = f"infra:{','.join(requested)}:{bbox_cell_key(bbox)}"
+
+    async def _load() -> list[dict]:
+        # One query per requested type, concatenated. Nuclear is the only type
+        # today; the loop keeps the shape ready for military/hospital.
+        out: list[dict] = []
+        for type_ in requested:
+            query = infra_query(type_, minlon, minlat, maxlon, maxlat)
+            if _http is None:
+                async with httpx.AsyncClient(
+                    timeout=OVERPASS_TIMEOUT, follow_redirects=True,
+                    headers=_headers(),
+                ) as client:
+                    resp = await client.post(OVERPASS_INTERPRETER, data={"data": query})
+                    resp.raise_for_status()
+                    out.extend(overpass_nuclear_to_markers(resp.json()))
+            else:
+                resp = await _http.post(
+                    OVERPASS_INTERPRETER, data={"data": query},
+                    timeout=OVERPASS_TIMEOUT,
+                )
+                resp.raise_for_status()
+                out.extend(overpass_nuclear_to_markers(resp.json()))
+        return out
+
+    return await _ttl_get(key, float(INFRA_TTL), _load)

app/main.py

@@ -14,14 +14,19 @@ from __future__ import annotations
 import asyncio
 import json
 import logging
+import re
+import time
 from contextlib import asynccontextmanager
 from datetime import datetime, timedelta, timezone
 from decimal import Decimal
 from pathlib import Path
+from typing import NoReturn
+from urllib.parse import urlparse
 from uuid import UUID
 
+import httpx
 import structlog
-from fastapi import BackgroundTasks, FastAPI, HTTPException, Query, WebSocket, WebSocketDisconnect
+from fastapi import FastAPI, HTTPException, Query, WebSocket, WebSocketDisconnect
 from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
 from fastapi.staticfiles import StaticFiles
@@ -35,10 +40,11 @@ from models import (
 )
 from schemas import (
     AlertCreate, AlertOut, AlertSeverity, AlertType, AlertUpdate,
+    ConflictZoneOut, ConflictsOut,
     DashboardSummary, EntityCreate, EntityKind, EntityOut,
     EventCreate, EventOut, FireOut, NewsArticleOut, NewsMapItemOut,
     NewsSummaryOut, NewsTickerItemOut,
-    FeedSourceCreate, FeedSourceOut,
+    FeedSourceCreate, FeedSourceOut, FeedSourceUpdate,
     KeyOut, KeyValueIn,
     NewsModelsOut, SettingsIn, SettingsOut,
     SearchResult, SentimentSummary, SourceType,
@@ -46,15 +52,19 @@ from schemas import (
     GeofenceCreate, GeofenceUpdate,
 )
 from ingestor import ingest_event, fetch_and_process
-from sources import ingest_rss_feed, ingest_gdelt, ingest_earthquakes, ingest_social_signals
+from camera_scraper import is_public_url
+from sources import GDELT_API, ingest_rss_feed, ingest_gdelt, ingest_earthquakes, ingest_social_signals
 from fire_sources import ingest_fires
 from keystore import KeyFormatError, delete_key, list_keys, set_key
 from settings_store import SettingsError, get_app_settings, list_models, set_summary_model
 from live_layers import (
     fetch_aircraft, fetch_fire_incidents, fetch_fire_perimeters,
-    fetch_radar_meta, fetch_storms, fetch_trains, fetch_vessels,
-    fetch_weather_alerts, overlay_catalog, parse_bbox,
+    fetch_gpsjam, fetch_planespotters_photo, fetch_radar_meta, fetch_sentinel1,
+    fetch_storms, fetch_trains, fetch_vessels, fetch_weather_alerts,
+    fetch_infrastructure, overlay_catalog, parse_bbox, UpstreamRateLimited,
 )
+from satellites import fetch_satellites, parse_groups, DEFAULT_GROUPS
+from place import reverse_geocode
 
 logging.basicConfig(level=logging.INFO)
 logger = structlog.get_logger("osint.dashboard")
@@ -70,14 +80,31 @@ async def _lifespan(app: FastAPI):
         await refresh_cache()
     except Exception:
         pass
-    from config import AISSTREAM_IN_APP
+    try:
+        from vesselapi import hydrate_last_known
+        await hydrate_last_known()
+    except Exception:
+        pass
+    from config import AISSTREAM_IN_APP, VESSELAPI_IN_APP
     ais_task = None
+    vesselapi_task = None
+    adsb_task = None
     if AISSTREAM_IN_APP:
         from ais_stream import run_ais_worker
         ais_task = asyncio.create_task(run_ais_worker())
+    if VESSELAPI_IN_APP:
+        from vesselapi import run_vesselapi_worker
+        vesselapi_task = asyncio.create_task(run_vesselapi_worker())
+    adsb_task = asyncio.create_task(_adsb_refresh_loop())
     yield
     if ais_task is not None:
         ais_task.cancel()
+    if vesselapi_task is not None:
+        vesselapi_task.cancel()
+    if adsb_task is not None:
+        adsb_task.cancel()
+    from vesselapi import close_client
+    await close_client()
     await close_http()
 
 
@@ -92,6 +119,35 @@ app.add_middleware(GZipMiddleware, minimum_size=1024)
 STATIC_DIR = Path(__file__).parent / "static"
 
 
+async def _adsb_refresh_loop() -> None:
+    """Poll ADSB.lol for connected viewports — never from GET /api/aircraft."""
+    from ws_manager import manager
+    from live_layers import fetch_aircraft
+
+    while True:
+        try:
+            boxes = manager.viewports()
+            for box in boxes:
+                bbox = f"{box[0]},{box[1]},{box[2]},{box[3]}"
+                try:
+                    await fetch_aircraft(bbox, persist=True)
+                except Exception as exc:  # noqa: BLE001
+                    logger.warning("adsb_refresh_failed", error=str(exc))
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("adsb_refresh_loop", error=str(exc))
+        await asyncio.sleep(8)
+
+
+class CachedStaticFiles(StaticFiles):
+    """Long-cache hashed/vendor Leaflet assets; index.html is served separately."""
+
+    async def get_response(self, path: str, scope):
+        resp = await super().get_response(path, scope)
+        if path.startswith("vendor/") or path.endswith((".js", ".css", ".woff2", ".png", ".svg")):
+            resp.headers["Cache-Control"] = "public, max-age=31536000, immutable"
+        return resp
+
+
 # ── Helpers ───────────────────────────────────────────────────────────────
 
 def event_to_out(row: dict) -> EventOut:
@@ -154,11 +210,65 @@ def alert_to_out(row: dict) -> AlertOut:
 
 @app.get("/api/health")
 async def health():
-    """Health check with database connectivity."""
-    async with async_session() as session:
-        result = await session.execute(select(func.now()))
-        db_time = result.scalar()
-    return {"status": "ok", "db_time": db_time.isoformat() if db_time else None}
+    """Liveness: process + DB. Pipeline freshness is in ``checks`` (HTTP 200
+    unless DB is down, so docker healthcheck does not restart a working HUD).
+    """
+    try:
+        async with async_session() as session:
+            result = await session.execute(select(func.now()))
+            db_time = result.scalar()
+            checks = await _pipeline_checks(session)
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("health_db_failed", error=str(exc))
+        return JSONResponse(
+            {"status": "down", "db_time": None, "checks": {"db": False}},
+            status_code=503,
+        )
+    checks["db"] = True
+    stale = [
+        name for name, val in checks.items()
+        if isinstance(val, dict) and val.get("ok") is False
+    ]
+    status = "degraded" if stale else "ok"
+    return {
+        "status": status,
+        "db_time": db_time.isoformat() if db_time else None,
+        "checks": checks,
+    }
+
+
+@app.get("/api/ready")
+async def ready():
+    """Readiness: 503 only when the database is unreachable."""
+    payload = await health()
+    if isinstance(payload, JSONResponse):
+        return payload
+    return JSONResponse(payload, status_code=200)
+
+
+async def _pipeline_checks(session) -> dict:
+    now = datetime.now(timezone.utc)
+    checks: dict = {}
+
+    async def _age(sql: str, name: str, max_age_s: int) -> None:
+        try:
+            ts = (await session.execute(text(sql))).scalar()
+        except Exception:
+            checks[name] = {"ok": False, "age_s": None}
+            return
+        if ts is None:
+            checks[name] = {"ok": False, "age_s": None}
+            return
+        if getattr(ts, "tzinfo", None) is None:
+            ts = ts.replace(tzinfo=timezone.utc)
+        age = (now - ts).total_seconds()
+        checks[name] = {"ok": age <= max_age_s, "age_s": int(age)}
+
+    await _age("SELECT max(ingested_at) FROM events", "events", 20 * 60)
+    await _age("SELECT max(ingested_at) FROM fires", "fires", 30 * 60)
+    await _age("SELECT max(timestamp) FROM articles", "articles", 2 * 3600)
+    await _age("SELECT max(batch_timestamp) FROM article_summaries", "summaries", 2 * 3600)
+    return checks
 
 
 def overlay_json(data, max_age: int) -> JSONResponse:
@@ -168,6 +278,67 @@ def overlay_json(data, max_age: int) -> JSONResponse:
     return resp
 
 
+# ── HUD counters ─────────────────────────────────────────────────────────
+
+# Cheap ~100 B–2 KB counts for the layer rail. Cached in-process so the HUD
+# can poll every second without re-hitting SQL or upstream feeds.
+_STATS_TTL = 20.0
+_stats_cache: dict[str, tuple[float, dict]] = {}
+
+
+async def _stats_counts() -> dict:
+    """Fan out to in-memory last-known / cheap SQL counts. Never raises."""
+    from live_layers import (
+        aircraft_last_known, vessel_last_known, train_count, nws_alert_count,
+    )
+
+    counts: dict[str, int | str] = {
+        "aircraft": len(aircraft_last_known),
+        "vessels": len(vessel_last_known),
+        "trains": train_count,
+        "cameras": 0,
+        "fires": 0,
+        "quakes": 0,
+        "alerts": nws_alert_count,
+    }
+
+    # SQL counts are best-effort: a down DB or missing table must not 500 the
+    # rail — the frontend still renders with zeros.
+    try:
+        from camera_models import cameras as cam_table
+        async with async_session() as session:
+            counts["cameras"] = int(
+                (await session.execute(select(func.count()).select_from(cam_table))).scalar() or 0
+            )
+            counts["fires"] = int(
+                (await session.execute(select(func.count()).select_from(fires))).scalar() or 0
+            )
+            counts["quakes"] = int(
+                (await session.execute(
+                    select(func.count()).select_from(events).where(
+                        events.c.source_type == "earthquake"
+                    )
+                )).scalar() or 0
+            )
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("stats_db_failed", error=str(exc))
+
+    counts["timestamp"] = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+    return counts
+
+
+@app.get("/api/stats")
+async def api_stats():
+    """Cheap HUD counters (counts only — no GeoJSON). Cached ~20 s."""
+    now = time.monotonic()
+    cached = _stats_cache.get("stats")
+    if cached and now - cached[0] < _STATS_TTL:
+        return cached[1]
+    payload = await _stats_counts()
+    _stats_cache["stats"] = (now, payload)
+    return overlay_json(payload, 15)
+
+
 # ── Feed Sources ──────────────────────────────────────────────────────────
 
 @app.get("/api/sources", response_model=list[FeedSourceOut])
@@ -197,20 +368,22 @@ async def create_source(payload: FeedSourceCreate):
 
 
 @app.patch("/api/sources/{source_id}")
-async def update_source(source_id: UUID, payload: dict):
-    """Update a feed source (e.g., toggle enabled)."""
+async def update_source(source_id: UUID, payload: FeedSourceUpdate):
+    """Update a feed source (name/url/config/enabled only)."""
+    values = payload.model_dump(exclude_unset=True)
     async with async_session() as session:
         row = (await session.execute(
             select(feed_sources).where(feed_sources.c.id == source_id)
         )).mappings().one_or_none()
         if not row:
             raise HTTPException(404, "Source not found")
-        await session.execute(
-            feed_sources.update()
-            .where(feed_sources.c.id == source_id)
-            .values(**payload)
-        )
-        await session.commit()
+        if values:
+            await session.execute(
+                feed_sources.update()
+                .where(feed_sources.c.id == source_id)
+                .values(**values)
+            )
+            await session.commit()
     return {"ok": True}
 
 
@@ -653,9 +826,15 @@ async def put_settings(payload: SettingsIn):
 
 # ── Ingestion Triggers ───────────────────────────────────────────────────
 
+def _require_public_url(url: str, field: str) -> None:
+    if not is_public_url(url):
+        raise HTTPException(400, f"{field} is not a public URL")
+
+
 @app.post("/api/ingest/rss")
 async def trigger_rss_ingest(feed_url: str, source_id: str | None = None):
     """Trigger RSS feed ingestion."""
+    _require_public_url(feed_url, "feed_url")
     count = await ingest_rss_feed(feed_url, source_id)
     return {"status": "ok", "items_ingested": count}
 
@@ -663,6 +842,10 @@ async def trigger_rss_ingest(feed_url: str, source_id: str | None = None):
 @app.post("/api/ingest/gdelt")
 async def trigger_gdelt_ingest(query: str = "", max_articles: int = 50):
     """Trigger GDELT data ingestion."""
+    _require_public_url(GDELT_API, "GDELT target")
+    parsed = urlparse(query)
+    if parsed.scheme in ("http", "https") and parsed.hostname:
+        _require_public_url(query, "query")
     count = await ingest_gdelt(query, max_articles)
     return {"status": "ok", "articles_ingested": count}
 
@@ -688,24 +871,16 @@ async def trigger_social_ingest(query: str = "", max_items: int = 50):
     return {"status": "ok", "signals_ingested": count}
 
 
-@app.post("/api/ingest/masscan")
-async def trigger_masscan(background_tasks: BackgroundTasks):
-    """Queue one masscan pass at ≤200 pps. Does not block the request on the scan."""
-    from bg_jobs import MASSCAN_PPS_CAP, schedule_masscan_pass
-
-    async def _kick() -> None:
-        schedule_masscan_pass()
-
-    background_tasks.add_task(_kick)
-    return JSONResponse(
-        {"status": "accepted", "rate_pps": MASSCAN_PPS_CAP},
-        status_code=202,
-    )
-
-
 @app.websocket("/ws/live")
 async def live_ws(ws: WebSocket):
-    """Viewport-filtered AIS/ADS-B fan-out. Client sends {type:viewport,bbox}."""
+    """Viewport-filtered AIS/ADS-B fan-out.
+
+    Client JSON:
+      {"type":"viewport","bbox":"minlon,minlat,maxlon,maxlat"}
+      {"type":"watch_geofences","ids":["<uuid>", ...]}  — empty list = none
+    geofence_alert delivers if the point is in-viewport OR geofence_id is watched.
+    AIS/ADS-B/fire_aircraft stay viewport-only.
+    """
     from ws_manager import manager
 
     client_id = str(id(ws))
@@ -731,6 +906,10 @@ async def live_ws(ws: WebSocket):
                     manager.set_viewport(client_id, parse_bbox(str(data["bbox"])))
                 except ValueError:
                     continue
+            elif data.get("type") == "watch_geofences":
+                ids = data.get("ids") or []
+                if isinstance(ids, list):
+                    manager.set_watched_geofences(client_id, [str(x) for x in ids])
     except WebSocketDisconnect:
         pass
     finally:
@@ -821,17 +1000,19 @@ async def get_timeline(
     """Event timeline: counts and avg sentiment per time bucket."""
     async with async_session() as session:
         cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
-        # Use date_trunc for bucketing
-        buckets = await session.execute(text(f"""
+        bucket_s = bucket_hours * 3600
+        buckets = await session.execute(text("""
             SELECT
-                date_trunc('hour', source_timestamp) AS ts,
+                to_timestamp(
+                    floor(extract(epoch FROM source_timestamp) / :bucket_s) * :bucket_s
+                ) AT TIME ZONE 'UTC' AS ts,
                 COUNT(*) AS event_count,
                 COALESCE(AVG(sentiment_score), 0) AS avg_sentiment
             FROM events
             WHERE source_timestamp >= :cutoff
             GROUP BY ts
             ORDER BY ts
-        """), {"cutoff": cutoff})
+        """), {"cutoff": cutoff, "bucket_s": bucket_s})
         rows = buckets.mappings().all()
 
     return [TimelinePoint(timestamp=r["ts"], event_count=r["event_count"],
@@ -880,7 +1061,7 @@ async def list_cameras(
         True,
         description="Only cameras with a verified HTTP/MJPEG snapshot_url "
                     "(the ones that actually preview). Set false to include "
-                    "unverified masscan port-554 hits.",
+                    "rows without a snapshot_url.",
     ),
     limit: int = Query(500, ge=1, le=5000),
 ):
@@ -957,7 +1138,7 @@ async def get_camera(camera_id: UUID):
 async def camera_snapshot(camera_id: UUID):
     """Still image for one camera.
 
-    HTTP cameras go through the TTL cache. masscan/RTSP finds have no HTTP
+    HTTP cameras go through the TTL cache. RTSP finds have no HTTP
     snapshot_url — we probe common still-image paths and, failing that, grab
     one JPEG frame from RTSP via ffmpeg. No credentials are tried.
     """
@@ -1084,7 +1265,7 @@ async def camera_hls_segment(camera_id: UUID, u: str = Query(..., min_length=8))
 
 # ── News pipeline (scraper + summarizer) ──────────────────────────────────
 # Backing data for the frontend news panel. Written by the vendored
-# news-scraper (hourly Scrapy crawl) and news-summarizer (hourly Gemini
+# news-scraper (continuous Scrapy crawl) and news-summarizer (15-min Nous
 # map-reduce) services into the shared osint-db.
 
 @app.get("/api/news", response_model=list[NewsArticleOut])
@@ -1131,28 +1312,38 @@ async def list_news_summaries(
         None,
         description="Only summaries generated at/after this UTC instant.",
     ),
+    kind: str | None = Query(
+        None,
+        description="Filter: interval or daily_recap. Omit for all.",
+    ),
     limit: int = Query(20, ge=1, le=100),
     offset: int = Query(0, ge=0),
 ):
     """Most recent master LLM summaries (newest first)."""
+    if kind is not None and kind not in ("interval", "daily_recap"):
+        raise HTTPException(422, "kind must be interval or daily_recap")
     async with async_session() as session:
         stmt = select(article_summaries).order_by(
             article_summaries.c.batch_timestamp.desc().nullslast()
         )
         if since:
             stmt = stmt.where(article_summaries.c.batch_timestamp >= since)
+        if kind is not None:
+            stmt = stmt.where(article_summaries.c.kind == kind)
         stmt = stmt.limit(limit).offset(offset)
         rows = (await session.execute(stmt)).mappings().all()
     return [
         NewsSummaryOut(
             id=r["id"], summary_text=r["summary_text"],
             batch_timestamp=r["batch_timestamp"], model=r["model"],
+            kind=r["kind"],
         )
         for r in rows
     ]
 
 
 _FLAGGED = ("critical", "high")
+_LESSER = ("medium", "low")
 
 
 @app.get("/api/news/ticker", response_model=list[NewsTickerItemOut])
@@ -1163,7 +1354,7 @@ async def list_news_ticker(
     ),
     limit: int = Query(20, ge=1, le=50),
 ):
-    """Flagged ticker rows (critical/high), newest first. No LLM required."""
+    """Ticker rows: critical/high first; medium/low if nothing is flagged."""
     async with async_session() as session:
         stmt = (
             select(news_items)
@@ -1177,6 +1368,19 @@ async def list_news_ticker(
             stmt = stmt.where(news_items.c.created_at >= since)
         stmt = stmt.limit(limit)
         rows = (await session.execute(stmt)).mappings().all()
+        if not rows:
+            stmt = (
+                select(news_items)
+                .where(
+                    news_items.c.kind == "ticker",
+                    news_items.c.importance.in_(_LESSER),
+                )
+                .order_by(news_items.c.created_at.desc())
+            )
+            if since:
+                stmt = stmt.where(news_items.c.created_at >= since)
+            stmt = stmt.limit(limit)
+            rows = (await session.execute(stmt)).mappings().all()
     return [
         NewsTickerItemOut(
             id=r["id"], headline=r["headline"], importance=r["importance"],
@@ -1292,7 +1496,70 @@ async def map_layers():
     return {"layers": MAP_LAYERS, "overlays": overlay_catalog()}
 
 
-def _upstream_or_502(exc: Exception, name: str):
+@app.get("/api/map/chokepoints")
+async def map_chokepoints():
+    """Static one-tap fly-to presets (Strait of Hormuz, Bab el-Mandeb, …).
+
+    Pure catalog — no upstream calls and no VesselAPI quota spend. ``vesselapi``
+    is True only for Hormuz (the box the VesselAPI poller already covers).
+    """
+    from chokepoints import chokepoints
+    return {"chokepoints": chokepoints()}
+
+
+async def _fetch_geocoded_points() -> list[tuple[float, float, datetime | None]]:
+    """Collect geocoded ``(lat, lon, ts)`` rows from the local DB.
+
+    Sources are the flagged map pins (``news_items`` kind=map) and geocoded
+    news/GDELT events (``events`` with ``location_lat/lon``). This is the
+    pre-existing geocoded corpus the conflict-zone counters roll up — no
+    upstream scraping and no generated/jittered coordinates.
+    """
+    async with async_session() as session:
+        map_rows = (
+            await session.execute(
+                select(news_items.c.lat, news_items.c.lon, news_items.c.created_at)
+                .where(
+                    news_items.c.kind == "map",
+                    news_items.c.lat.isnot(None),
+                    news_items.c.lon.isnot(None),
+                )
+            )
+        ).all()
+        event_rows = (
+            await session.execute(
+                select(events.c.location_lat, events.c.location_lon, events.c.source_timestamp)
+                .where(
+                    events.c.source_type.in_(["rss", "gdel-t2"]),
+                    events.c.location_lat.isnot(None),
+                    events.c.location_lon.isnot(None),
+                )
+            )
+        ).all()
+    return [tuple(r) for r in map_rows] + [tuple(r) for r in event_rows]
+
+
+@app.get("/api/conflicts", response_model=ConflictsOut)
+async def list_conflicts():
+    """Curated conflict-zone catalog with per-zone event counts.
+
+    Static catalogue (severity + short factual description) merged with a live
+    ``eventCount`` roll-up of pre-existing geocoded news/GDELT//api/news/map
+    rows inside each zone bbox. Empty DB → ``eventCount=0`` (never 502).
+    """
+    from conflicts import conflict_zones, zone_event_stats
+
+    points = await _fetch_geocoded_points()
+    timestamp = datetime.now(timezone.utc)
+    zones = []
+    for z in conflict_zones():
+        bbox = z.pop("bbox")
+        count, latest = zone_event_stats(points, bbox)
+        zones.append({**z, "eventCount": count, "lastUpdated": latest})
+    return {"zones": zones, "timestamp": timestamp}
+
+
+def _upstream_or_502(exc: Exception, name: str) -> NoReturn:
     logger.warning("live_layer_upstream_failed", layer=name, error=str(exc))
     raise HTTPException(502, f"{name} upstream unavailable: {exc}") from exc
 
@@ -1306,6 +1573,33 @@ async def map_radar():
         _upstream_or_502(exc, "radar")
 
 
+@app.get("/api/map/sentinel1")
+async def map_sentinel1(bbox: str = Query(..., description="minlon,minlat,maxlon,maxlat")):
+    """Most recent Sentinel-1 GRD as a signed COG tile template (TiTiler).
+
+    Queries Planetary Computer only on demand; no tiles proxied through the Pi.
+    """
+    _parse_bbox_query(bbox)
+    try:
+        result = await fetch_sentinel1(bbox)
+    except UpstreamRateLimited as exc:
+        headers = {"Retry-After": exc.retry_after} if exc.retry_after else None
+        raise HTTPException(
+            429, "Planetary Computer rate limit", headers=headers,
+        ) from exc
+    except Exception as exc:
+        _upstream_or_502(exc, "sentinel1")
+    if result is None:
+        return JSONResponse(
+            status_code=404,
+            content={
+                "error": "no_imagery",
+                "message": "No Sentinel-1 GRD in the last 7 days for this bbox",
+            },
+        )
+    return overlay_json(result, 300)
+
+
 @app.get("/api/aircraft")
 async def list_aircraft(
     bbox: str = Query(..., description="minlon,minlat,maxlon,maxlat"),
@@ -1319,13 +1613,30 @@ async def list_aircraft(
         ts = parse_timestamp(timestamp)
         if ts is not None:
             return overlay_json(await fetch_positions_at("aircraft", ts, bbox, limit), 5)
-        return overlay_json(await fetch_aircraft(bbox, limit), 5)
+        return overlay_json(await fetch_aircraft(bbox, limit, persist=False), 5)
     except ValueError as exc:
         raise HTTPException(422, str(exc)) from exc
     except Exception as exc:
         _upstream_or_502(exc, "aircraft")
 
 
+@app.get("/api/aircraft/photo")
+async def aircraft_photo(
+    hex_code: str | None = Query(None, alias="hex", pattern="^[0-9a-fA-F]{6}$"),
+    reg: str | None = Query(None, min_length=1, max_length=12),
+):
+    """Latest planespotters.net photo for an aircraft (hex preferred, reg fallback)."""
+    if not hex_code and not reg:
+        raise HTTPException(422, "hex or reg required")
+    try:
+        photo = await fetch_planespotters_photo(hex_code=hex_code, reg=reg)
+    except Exception as exc:
+        _upstream_or_502(exc, "planespotters")
+    if photo is None:
+        raise HTTPException(404, "no photo")
+    return overlay_json(photo, 86400)
+
+
 @app.get("/api/trains")
 async def list_trains(
     bbox: str | None = Query(None, description="minlon,minlat,maxlon,maxlat"),
@@ -1347,16 +1658,36 @@ async def list_vessels(
     bbox: str | None = Query(None, description="minlon,minlat,maxlon,maxlat"),
     limit: int = Query(2000, ge=1, le=5000),
     timestamp: str | None = Query(None, description="ISO time — DVR 1-min tracks instead of live"),
+    src: str | None = Query(None, description="aisstream|vesselapi|all (default all)"),
 ):
-    """AIS last-known from the server-side AISStream worker. Empty without a key."""
+    """AIS last-known — union of two independent providers.
+
+    AISStream (extra.src="aisstream", live US-coast WebSocket) and VesselAPI
+    (extra.src="vesselapi", Strait of Hormuz 5×/day poll) both upsert into the
+    same store. Empty without either key / until the first successful poll.
+    ``src`` filters the union to one provider (default ``all``) so a Hormuz
+    view can skip the ~5k CONUS AISStream rows.
+    """
+    if src is not None and src not in ("aisstream", "vesselapi", "all"):
+        raise HTTPException(422, "src must be one of: aisstream, vesselapi, all")
     if bbox:
         _parse_bbox_query(bbox)
     try:
         from tracks import fetch_positions_at, parse_timestamp
         ts = parse_timestamp(timestamp)
         if ts is not None:
-            return overlay_json(await fetch_positions_at("vessel", ts, bbox, limit), 5)
-        return overlay_json(await fetch_vessels(bbox, limit), 5)
+            from vesselapi import fetch_vessels_as_of
+            if src == "vesselapi":
+                return overlay_json(await fetch_vessels_as_of(ts, bbox, limit), 5)
+            ais = await fetch_positions_at("vessel", ts, bbox, limit)
+            if src == "aisstream":
+                return overlay_json(ais, 5)
+            va = await fetch_vessels_as_of(ts, bbox, limit)
+            by_id = {m["id"]: m for m in ais}
+            for m in va:
+                by_id[m["id"]] = m
+            return overlay_json(list(by_id.values())[:limit], 5)
+        return overlay_json(await fetch_vessels(bbox, limit, src=src), 5)
     except ValueError as exc:
         raise HTTPException(422, str(exc)) from exc
 
@@ -1437,33 +1768,68 @@ async def api_update_geofence(gid: str, payload: GeofenceUpdate):
 @app.delete("/api/geofences/{gid}", status_code=204)
 async def api_delete_geofence(gid: str):
     from geofence import delete_geofence
-    await delete_geofence(gid)
+    ok = await delete_geofence(gid)
+    if not ok:
+        raise HTTPException(404, "geofence not found")
     return None
 
 
-@app.get("/api/geofence-alerts")
-async def api_geofence_alerts(limit: int = Query(100, ge=1, le=500)):
-    from sqlalchemy import text as sql_text
+@app.get("/api/geofences/{gid}/at")
+async def api_geofence_at(
+    gid: str,
+    timestamp: str = Query(..., description="ISO-8601 instant for the 1-minute DVR bucket"),
+):
+    """Aircraft/vessels/fires inside this fence at time T. Never writes."""
+    from geofence import snapshot_at
+    from tracks import parse_timestamp
+
     try:
-        async with async_session() as session:
-            rows = (await session.execute(sql_text(
-                """
-                SELECT id::text, geofence_id::text, source_kind, entity_id,
-                       lat, lon, payload, created_at
-                FROM geofence_alerts
-                ORDER BY created_at DESC
-                LIMIT :limit
-                """
-            ), {"limit": limit})).mappings().all()
-        out = []
-        for r in rows:
-            item = dict(r)
-            if item.get("created_at") is not None:
-                item["created_at"] = item["created_at"].isoformat()
-            out.append(item)
-        return out
+        ts = parse_timestamp(timestamp)
+    except ValueError as exc:
+        raise HTTPException(422, str(exc)) from exc
+    if ts is None:
+        raise HTTPException(422, "timestamp required")
+    try:
+        body = await snapshot_at(gid, ts)
     except Exception:
-        return []
+        body = {
+            "geofence_id": gid,
+            "timestamp": ts.isoformat(),
+            "aircraft": [],
+            "vessels": [],
+            "fires": [],
+        }
+    if body is None:
+        raise HTTPException(404, "geofence not found")
+    return body
+
+
+@app.get("/api/geofence-alerts")
+async def api_geofence_alerts(
+    geofence_id: UUID | None = Query(None),
+    since: str | None = Query(None, description="ISO-8601 inclusive lower bound"),
+    until: str | None = Query(None, description="ISO-8601 inclusive upper bound"),
+    source_kind: str | None = Query(None, description="firms|ais|adsb"),
+    limit: int = Query(100, ge=1, le=500),
+):
+    """Hit log for drawn fences. Not /api/alerts (entity/keyword)."""
+    from geofence import list_alerts
+    from tracks import parse_timestamp
+
+    if source_kind is not None and source_kind not in ("firms", "ais", "adsb"):
+        raise HTTPException(422, "source_kind must be one of: firms, ais, adsb")
+    try:
+        since_ts = parse_timestamp(since) if since else None
+        until_ts = parse_timestamp(until) if until else None
+    except ValueError as exc:
+        raise HTTPException(422, str(exc)) from exc
+    return await list_alerts(
+        geofence_id=str(geofence_id) if geofence_id else None,
+        since=since_ts,
+        until=until_ts,
+        source_kind=source_kind,
+        limit=limit,
+    )
 
 
 @app.get("/api/fire-aircraft")
@@ -1525,6 +1891,122 @@ async def list_storms():
         _upstream_or_502(exc, "storms")
 
 
+@app.get("/api/place")
+async def get_place(
+    lat: float = Query(..., ge=-90, le=90),
+    lon: float = Query(..., ge=-180, le=180),
+):
+    """Nominatim reverse geocode for the map \"What's here?\" dossier.
+
+    Identifying ``OSINT_USER_AGENT``, 1 req/s, 60s cache, 500 keys. The HUD
+    lists already-loaded overlay entities client-side — this route does not
+    refetch aircraft/vessels/cameras/fires.
+    """
+    try:
+        return overlay_json(await reverse_geocode(lat, lon), 60)
+    except ValueError as exc:
+        raise HTTPException(422, str(exc)) from exc
+    except Exception as exc:
+        _upstream_or_502(exc, "nominatim")
+
+
+_GPSJAM_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
+
+
+@app.get("/api/map/gpsjam")
+async def map_gpsjam(
+    date: str | None = Query(None, description="YYYY-MM-DD (default: yesterday UTC)"),
+):
+    """GPSJAM daily GPS-interference hex layer (whole world, GeoJSON).
+
+    Red/yellow hexes correlate with suspected jamming but are NOT proof of it.
+    Fetched once per day from gpsjam.org (ADS-B Exchange data) and cached 1h.
+    """
+    target = date
+    if target is None:
+        target = (datetime.now(timezone.utc) - timedelta(days=1)).strftime("%Y-%m-%d")
+    if not _GPSJAM_DATE.match(target):
+        raise HTTPException(422, "date must be YYYY-MM-DD")
+    try:
+        fc = await fetch_gpsjam(target)
+    except httpx.HTTPStatusError as exc:
+        if exc.response.status_code == 404:
+            return JSONResponse(
+                {"error": "unavailable", "href": "https://gpsjam.org/",
+                 "date": target},
+            )
+        _upstream_or_502(exc, "gpsjam")
+    except Exception as exc:
+        _upstream_or_502(exc, "gpsjam")
+    if not fc.get("features"):
+        return JSONResponse(
+            {"error": "unavailable", "href": "https://gpsjam.org/", "date": target},
+        )
+    return overlay_json(fc, 3600)
+
+
+@app.get("/api/satellites")
+async def list_satellites(
+    groups: str | None = Query(None, description="Comma-separated CelesTrak groups"),
+    bbox: str | None = Query(None, description="minlon,minlat,maxlon,maxlat"),
+    limit: int = Query(2000, ge=1, le=5000),
+):
+    """Last-known satellite positions from CelesTrak GP JSON, SGP4-propagated.
+
+    Default groups are ``stations,weather`` (tens of objects). The GP element
+    blob is fetched at most once per 2 hours per group and cached; positions
+    are re-propagated on every request. Falls back to the last good blob on a
+    CelesTrak 403 / stale response, and to SatNOGS TLE only when the cache is
+    empty. Unknown groups 400.
+    """
+    try:
+        group_list = parse_groups(groups if groups is not None else ",".join(DEFAULT_GROUPS))
+    except ValueError as exc:
+        raise HTTPException(400, str(exc)) from exc
+    if bbox:
+        _parse_bbox_query(bbox)
+    try:
+        payload = await fetch_satellites(group_list, bbox=bbox, limit=limit)
+    except ValueError as exc:
+        raise HTTPException(422, str(exc)) from exc
+    except Exception as exc:
+        _upstream_or_502(exc, "satellites")
+    return overlay_json(payload, 30)
+
+
+_INFRA_TYPES = frozenset({"nuclear"})
+
+
+@app.get("/api/infrastructure")
+async def api_infrastructure(
+    types: str = Query(..., description="comma-separated enum (nuclear)"),
+    bbox: str | None = Query(None, description="minlon,minlat,maxlon,maxlat"),
+):
+    """Overpass-derived static infrastructure markers (nuclear power plants).
+
+    ``bbox`` is required; ``types`` is a comma-separated subset of ``nuclear``.
+    Fetched from Overpass (identifying UA, 25s query) and cached 24h per
+    quantized bbox. Markers are ``{id, name, lat, lon, type, extra}``.
+    """
+    if not bbox:
+        raise HTTPException(400, "bbox required (minlon,minlat,maxlon,maxlat)")
+    requested = [t.strip() for t in (types or "").split(",") if t.strip()]
+    if not requested:
+        raise HTTPException(422, "types required (e.g. nuclear)")
+    unknown = [t for t in requested if t not in _INFRA_TYPES]
+    if unknown:
+        raise HTTPException(
+            422, f"unsupported types: {', '.join(unknown)} (supported: nuclear)"
+        )
+    try:
+        markers = await fetch_infrastructure(",".join(requested), bbox)
+    except ValueError as exc:
+        raise HTTPException(422, str(exc)) from exc
+    except Exception as exc:
+        _upstream_or_502(exc, "infrastructure")
+    return overlay_json(markers, 86400)
+
+
 @app.get("/api/map/times")
 async def map_layer_times(
     layer: str = Query(..., description="GIBS layer identifier, e.g. VIIRS_SNPP_CorrectedReflectance_TrueColor"),
@@ -1550,9 +2032,9 @@ async def map_layer_times(
     return {"layer": layer, **domain}
 
 
-app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
+app.mount("/static", CachedStaticFiles(directory=str(STATIC_DIR)), name="static")
 
 
 if __name__ == "__main__":
     import uvicorn
-    uvicorn.run(app, host="0.0.0.0", port=8000)
+    uvicorn.run(app, host="0.0.0.0", port=8000, workers=1)  # single worker: in-memory WS/pubsub + layer caches

app/masscan_config.py

@@ -1,65 +0,0 @@
-"""Active camera-discovery configuration (masscan-based, env-driven).
-
-All knobs read from the environment with safe defaults. The scanner targets
-open TCP port 554 (RTSP — the typical IP-camera port) across a configured
-range and feeds results into the same `cameras` table as the passive scraper
-(discovery_source='masscan'), deduped by URL hash.
-
-ETHICS / SCOPE (mirrors camera_scraper.py):
-  * Detection only — a SYN port scan for OPEN hosts. No credential guessing,
-    no login attempts, no banner grabbing, and no access to camera feeds.
-  * Private / reserved ranges are excluded via MASSCAN_EXCLUDEFILE so the
-    scanner never probes RFC1918, loopback, link-local, multicast, or the
-    bogons. Fail closed if the excludefile is missing.
-
-TIMING REALITY: at the residential-safe default of 200 pps a full IPv4
-sweep (0.0.0.0/0, ~4.29B addresses) takes ~8 months. This is therefore a
-CONTINUOUS ROLLING SWEEP, not a "finish in a day" job: masscan streams
-open hosts to stdout and the runner ingests them incrementally, then
-restarts the sweep when a pass completes. New cameras are detected as they
-appear on each pass. 1k/10k pps saturated a home uplink — do not raise the
-rate unless you are on a VPS / unmetered link.
-"""
-
-from __future__ import annotations
-
-import os
-
-# Path to the masscan binary (installed on the Pi host).
-MASSCAN_BIN = os.getenv("MASSCAN_BIN", "masscan")
-
-# CIDR(s) to sweep. Default = the whole public IPv4 space.
-MASSCAN_RANGE = os.getenv("MASSCAN_RANGE", "0.0.0.0/0")
-
-# Port(s) to probe. Default 554 = RTSP, the typical IP-camera port.
-MASSCAN_PORTS = os.getenv("MASSCAN_PORTS", "554")
-
-# Packets/sec. 200 is the residential-safe default — 1k/10k pps saturated
-# a home uplink. Raise only on a VPS / unmetered link.
-MASSCAN_RATE = int(os.getenv("MASSCAN_RATE", "200"))
-
-# Retransmission count. 1 maximizes unique-host coverage at low rate; the
-# default (10) spends most of the budget re-probing the same hosts.
-MASSCAN_RETRIES = int(os.getenv("MASSCAN_RETRIES", "1"))
-
-# Seconds to keep listening for straggler responses after the last probe.
-# 0 avoids a 10s tail per pass; tiny loss of the very last hosts is fine
-# since the sweep repeats.
-MASSCAN_WAIT = int(os.getenv("MASSCAN_WAIT", "0"))
-
-# Excludefile path on the Pi host. Must contain RFC1918/loopback/link-local/
-# multicast/bogons so the scanner never probes private ranges. Fail closed if
-# the file is absent (the runner refuses to start rather than scan wide).
-MASSCAN_EXCLUDEFILE = os.getenv(
-    "MASSCAN_EXCLUDEFILE", "/etc/osint-dashboard/masscan-excludes.txt"
-)
-
-# Ingest batch size — flush this many newly-seen hosts to the DB per round.
-MASSCAN_FLUSH_EVERY = int(os.getenv("MASSCAN_FLUSH_EVERY", "250"))
-
-# NATS subject newly-found cameras are published on (same feed as the
-# passive scraper so the shared ingester persists them).
-MASSCAN_NATS_SUBJECT = os.getenv("MASSCAN_NATS_SUBJECT", "events.camera")
-
-# discovery_source tag written into the cameras table.
-MASSCAN_DISCOVERY_SOURCE = os.getenv("MASSCAN_DISCOVERY_SOURCE", "masscan")

app/masscan_scanner.py

@@ -1,226 +0,0 @@
-"""masscan result parsing + ingestion for the OSINT dashboard.
-
-Turns a stream of masscan JSON-lines (open port 554 hosts) into rows in the
-`cameras` table with discovery_source='masscan', deduped by URL hash against
-whatever the passive scraper already found. Newly discovered hosts are also
-published to NATS (`events.camera`) so the shared ingester pipeline persists
-them exactly like scraper finds.
-
-Scope: detection of OPEN hosts only. No credentials, no banners, no feed
-access. Private/reserved ranges never enter masscan (see excludefile).
-"""
-
-from __future__ import annotations
-
-import asyncio
-import json
-import logging
-from datetime import datetime, timezone
-
-from camera_models import cameras
-from camera_scraper import url_hash, geolocate_ips
-from database import async_session
-
-from masscan_config import (
-    MASSCAN_NATS_SUBJECT, MASSCAN_DISCOVERY_SOURCE,
-)
-
-logger = logging.getLogger("osint.masscan_scanner")
-
-
-# ── URL building ──────────────────────────────────────────────────────────
-
-def build_rtsp_url(ip: str) -> str:
-    """Canonical URL for an open-RTSP host. Used as the dedupe key."""
-    return f"rtsp://{ip}/"
-
-
-# ── masscan JSON parsing ──────────────────────────────────────────────────
-# masscan --output-format=json --output-file=- emits line-delimited JSON on a
-# pipe (a bare object per open host), not the array form used for seekable
-# files. We parse per-line and tolerate an accidental leading '['.
-
-def parse_masscan_line(line: str) -> list[dict]:
-    """Parse one masscan stdout line into a list of host records.
-
-    A line may contain one JSON object or, defensively, be wrapped in an
-    array. Returns [] on anything unparseable (harmless — the sweep repeats).
-    """
-    s = line.strip()
-    if not s:
-        return []
-    s = s.lstrip("[").rstrip("]").strip()
-    if not s:
-        return []
-    # Multiple records may share a line separated by '},{'.
-    if s.endswith(","):
-        s = s[:-1].rstrip()
-    out: list[dict] = []
-    for cand in _split_records(s):
-        try:
-            obj = json.loads(cand)
-        except (json.JSONDecodeError, ValueError):
-            continue
-        if isinstance(obj, dict) and obj.get("ip"):
-            out.append(obj)
-    return out
-
-
-def _split_records(s: str) -> list[str]:
-    """Split a buffer into individual JSON object strings, honoring nesting."""
-    records, depth, start = [], 0, 0
-    for i, ch in enumerate(s):
-        if ch == "{":
-            if depth == 0:
-                start = i
-            depth += 1
-        elif ch == "}":
-            depth -= 1
-            if depth == 0:
-                records.append(s[start:i + 1])
-    return records
-
-
-def extract_open_ips(records: list[dict], port: int) -> list[str]:
-    """Return the list of IPs from records that have `port` open."""
-    ips: list[str] = []
-    for rec in records:
-        for p in rec.get("ports", []):
-            if p.get("port") == port and p.get("status") == "open":
-                ips.append(rec["ip"])
-                break
-    return ips
-
-
-# ── Persistence ───────────────────────────────────────────────────────────
-
-async def ingest_open_hosts(ips: list[str]) -> tuple[int, list[str]]:
-    """Insert-or-refresh camera rows for open RTSP hosts that have a public feed.
-
-    A host only lands in the table (and therefore on the map) if an
-    unauthenticated HTTP still or MJPEG URL responds. Port-554-only hosts
-    are skipped. Returns (newly_inserted, hosts_with_working_feed).
-    """
-    if not ips:
-        return 0, []
-    from camera_preview import probe_public_feed
-
-    unique = list(dict.fromkeys(ips))
-    sem = asyncio.Semaphore(20)
-
-    async def _probe(ip: str) -> tuple[str, str | None]:
-        async with sem:
-            return ip, await probe_public_feed(ip)
-
-    probed = await asyncio.gather(*(_probe(ip) for ip in unique))
-    live = [(ip, feed) for ip, feed in probed if feed]
-    if not live:
-        logger.info("masscan ingest: 0 working feeds of %d open-554 hosts",
-                    len(unique))
-        return 0, []
-
-    now = datetime.now(timezone.utc)
-    coords = await geolocate_ips([ip for ip, _ in live])
-    new = 0
-    async with async_session() as session:
-        for ip, feed in live:
-            url = build_rtsp_url(ip)
-            h = url_hash(url)
-            lat, lon = coords.get(ip, (None, None))
-            existing = (await session.execute(
-                cameras.select().where(cameras.c.url_hash == h)
-            )).one_or_none()
-            if existing is None:
-                await session.execute(cameras.insert().values(
-                    url_hash=h,
-                    source_url=url,
-                    snapshot_url=feed,
-                    discovery_source=MASSCAN_DISCOVERY_SOURCE,
-                    location_lat=lat,
-                    location_lon=lon,
-                    location_name=f"{ip} (IP-geo)" if lat is not None else None,
-                    vendor=None,
-                    device_type="rtsp",
-                    first_seen=now,
-                    last_seen=now,
-                    raw={"discovered_via": "masscan", "port": 554,
-                         "public_feed": feed},
-                ))
-                new += 1
-            else:
-                await session.execute(cameras.update().where(
-                    cameras.c.url_hash == h
-                ).values(
-                    last_seen=now,
-                    snapshot_url=feed,
-                    location_lat=lat,
-                    location_lon=lon,
-                    location_name=f"{ip} (IP-geo)" if lat is not None else None,
-                ))
-        await session.commit()
-    logger.info("masscan ingest: %d new working feeds (%d probed, %d open-554)",
-                new, len(live), len(unique))
-    return new, [ip for ip, _ in live]
-
-
-# ── NATS publish ──────────────────────────────────────────────────────────
-
-async def publish_new_hosts(ips: list[str]) -> int:
-    """Publish newly-found open hosts to NATS for the shared ingester.
-
-    Returns the number of messages published (0 if NATS is down).
-    """
-    import json as _json
-    import nats
-    from config import NATS_URL
-
-    if not ips:
-        return 0
-    try:
-        nc = await nats.connect(NATS_URL)
-    except Exception:  # noqa: BLE001
-        logger.warning("NATS unavailable — skipping publish pass")
-        return 0
-    published = 0
-    try:
-        js = nc.jetstream()
-        for ip in dict.fromkeys(ips):
-            url = build_rtsp_url(ip)
-            msg = {
-                "source_type": "camera",
-                "title": f"Open RTSP camera ({ip})",
-                "url": url,
-                "location_lat": None,
-                "location_lon": None,
-                "location_name": None,
-                "tags": ["osint", "camera", MASSCAN_DISCOVERY_SOURCE],
-                "raw": {
-                    "url_hash": url_hash(url),
-                    "source_url": url,
-                    "snapshot_url": None,
-                    "vendor": None,
-                    "device_type": "rtsp",
-                    "discovered_via": "masscan",
-                    "port": 554,
-                },
-                "source_timestamp": datetime.now(timezone.utc).isoformat(),
-            }
-            await js.publish(MASSCAN_NATS_SUBJECT, _json.dumps(msg).encode())
-            published += 1
-    finally:
-        await nc.close()
-    logger.info("published %d masscan finds to %s", published, MASSCAN_NATS_SUBJECT)
-    return published
-
-
-# ── Batch drain helper used by the runner ─────────────────────────────────
-
-async def flush(seen: set[str], new_accum: int) -> tuple[int, int]:
-    """Ingest + publish the accumulated host set; return (new, published)."""
-    if not seen:
-        return 0, 0
-    ips = list(seen)
-    new, live = await ingest_open_hosts(ips)
-    published = await publish_new_hosts(live)
-    seen.clear()
-    return new, published

app/models.py

@@ -68,6 +68,15 @@ Index("ix_events_search_vector", events.c.search_vector, postgresql_using="gin")
 # Spatial index on location
 Index("ix_events_location", events.c.location_lat, events.c.location_lon)
 
+# Timescale unique indexes must include the partition column, so URL
+# idempotency lives on a regular table — not the events hypertable.
+event_dedup = Table(
+    "event_dedup",
+    metadata,
+    Column("url", Text, primary_key=True),
+    Column("created_at", DateTime(timezone=True), server_default=func.now(), nullable=False),
+)
+
 
 # ── Entities (people, organizations, locations of interest) ──────────────
 
@@ -210,6 +219,7 @@ article_summaries = Table(
     Column("batch_timestamp", DateTime(timezone=True),
            server_default=func.now(), nullable=False),
     Column("model", Text),  # LLM id used for this batch; nullable for old rows
+    Column("kind", Text),  # interval | daily_recap; nullable for old rows
 )
 
 Index("ix_article_summaries_batch_timestamp", article_summaries.c.batch_timestamp)

app/place.py

@@ -0,0 +1,99 @@
+"""Nominatim reverse-geocode proxy for the map place dossier.
+
+Browser clients cannot set an identifying User-Agent, and Nominatim typically
+blocks CORS — so the HUD calls GET /api/place instead of talking to OSM
+directly. Cache 60s / 500 keys; never exceed 1 req/s upstream.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+
+import httpx
+from cachetools import TTLCache
+
+from config import NOMINATIM_MIN_INTERVAL, NOMINATIM_URL, OSINT_USER_AGENT
+
+_NOMINATIM = NOMINATIM_URL.rstrip("/")
+
+place_cache: TTLCache = TTLCache(maxsize=500, ttl=60)
+
+_lock = asyncio.Lock()
+_last_req = 0.0
+
+_ADDR_KEEP = (
+    "house_number", "road", "neighbourhood", "suburb", "city", "town",
+    "village", "hamlet", "county", "state", "postcode", "country", "country_code",
+)
+
+
+def cache_key(lat: float, lon: float) -> str:
+    return f"{lat:.4f},{lon:.4f}"
+
+
+def slim_place(lat: float, lon: float, data: dict | None) -> dict:
+    data = data or {}
+    raw_addr = data.get("address")
+    addr_in: dict = raw_addr if isinstance(raw_addr, dict) else {}
+    address = {k: addr_in[k] for k in _ADDR_KEEP if addr_in.get(k)}
+    err = data.get("error")
+    display = None if err else (data.get("display_name") or None)
+    name = None if err else (data.get("name") or address.get("city")
+                             or address.get("town") or address.get("village") or None)
+    return {
+        "lat": lat,
+        "lon": lon,
+        "display_name": display,
+        "name": name,
+        "address": address,
+        "osm_type": None if err else data.get("osm_type"),
+        "osm_id": None if err else data.get("osm_id"),
+        "attribution": "© OpenStreetMap contributors",
+    }
+
+
+async def reverse_geocode(lat: float, lon: float) -> dict:
+    """Reverse-geocode a point. Cache hits skip Nominatim entirely."""
+    if not (-90.0 <= lat <= 90.0 and -180.0 <= lon <= 180.0):
+        raise ValueError("lat/lon out of range")
+    key = cache_key(lat, lon)
+    qlat, qlon = (float(p) for p in key.split(","))
+    async with _lock:
+        hit = place_cache.get(key)
+        if hit is not None:
+            return hit
+        global _last_req
+        wait = _last_req + NOMINATIM_MIN_INTERVAL - time.monotonic()
+        if wait > 0:
+            await asyncio.sleep(wait)
+        body = await _fetch_nominatim(qlat, qlon)
+        _last_req = time.monotonic()
+        place_cache[key] = body
+        return body
+
+
+async def _fetch_nominatim(lat: float, lon: float) -> dict:
+    headers = {
+        "User-Agent": OSINT_USER_AGENT,
+        "Accept": "application/json",
+    }
+    url = f"{_NOMINATIM}/reverse"
+    params = {
+        "lat": f"{lat:.6f}",
+        "lon": f"{lon:.6f}",
+        "format": "jsonv2",
+        "addressdetails": "1",
+        "zoom": "18",
+    }
+    async with _http_client(timeout=10.0, follow_redirects=True) as client:
+        r = await client.get(url, params=params, headers=headers)
+        r.raise_for_status()
+        data = r.json()
+    if not isinstance(data, dict):
+        data = {}
+    return slim_place(lat, lon, data)
+
+
+def _http_client(**kwargs):
+    return httpx.AsyncClient(**kwargs)

app/requirements.txt

@@ -12,3 +12,5 @@ python-dateutil>=2.9
 structlog>=24.4
 websockets>=14
 cachetools>=5.5
+h3>=4.0
+sgp4>=2.23

app/run_ingester.py

@@ -24,8 +24,8 @@ import sys
 
 sys.path.insert(0, sys_path)
 
-from config import NATS_URL, FIRMS_INTERVAL, FIRMS_DATASET, AISSTREAM_IN_INGEST  # noqa: E402
-from sources import ingest_rss_feed, ingest_gdelt, ingest_earthquakes  # noqa: E402
+from config import NATS_URL, FIRMS_INTERVAL, FIRMS_DATASET, AISSTREAM_IN_INGEST, VESSELAPI_IN_INGEST  # noqa: E402
+from sources import ingest_rss_feed, ingest_gdelt, ingest_earthquakes, ingest_eonet, ingest_cisa_kev  # noqa: E402
 from fire_sources import ingest_fires  # noqa: E402
 from ingestor import ingest_event, start_nats_consumer  # noqa: E402
 
@@ -37,6 +37,8 @@ INTERVAL = int(os.getenv("INGEST_INTERVAL", "300"))
 GDELT_QUERY = os.getenv("GDELT_QUERY", "")
 ENABLE_QUAKES = os.getenv("INGEST_EARTHQUAKES", "1").lower() in ("1", "true", "yes")
 ENABLE_FIRES = os.getenv("INGEST_FIRES", "1").lower() in ("1", "true", "yes")
+ENABLE_EONET = os.getenv("INGEST_EONET", "1").lower() in ("1", "true", "yes")
+ENABLE_KEV = os.getenv("INGEST_KEV", "1").lower() in ("1", "true", "yes")
 
 NATS_STREAM = "events"
 
@@ -62,6 +64,18 @@ async def producer_loop() -> None:
                     logger.info("USGS -> %d events", q)
                 except Exception:  # noqa: BLE001
                     logger.exception("USGS fetch failed")
+            if ENABLE_EONET:
+                try:
+                    n = await ingest_eonet()
+                    logger.info("EONET -> %d events", n)
+                except Exception:  # noqa: BLE001
+                    logger.exception("EONET fetch failed")
+            if ENABLE_KEV:
+                try:
+                    k = await ingest_cisa_kev()
+                    logger.info("CISA KEV -> %d events", k)
+                except Exception:  # noqa: BLE001
+                    logger.exception("CISA KEV fetch failed")
         except Exception:  # noqa: BLE001
             logger.exception("producer cycle error")
         await asyncio.sleep(INTERVAL)
@@ -120,6 +134,9 @@ async def main() -> None:
     if AISSTREAM_IN_INGEST:
         from ais_stream import run_ais_worker  # noqa: E402
         tasks.append(asyncio.create_task(run_ais_worker()))
+    if VESSELAPI_IN_INGEST:
+        from vesselapi import run_vesselapi_worker  # noqa: E402
+        tasks.append(asyncio.create_task(run_vesselapi_worker()))
     await asyncio.gather(producer_loop(), consumer_loop(), *tasks)
 
 

app/run_masscan_service.py

@@ -1,149 +0,0 @@
-"""Continuous masscan rolling-sweep service for the OSINT dashboard.
-
-Runs masscan against the configured range for open port 554 (RTSP), streams
-the JSON-lines output, and ingests open hosts into the `cameras` table (new
-finds only) plus publishes them to NATS — exactly like the passive scraper.
-
-Because a full IPv4 sweep at a conservative rate takes days, this runs
-masscan CONTINUOUSLY: each pass streams results in as they're found, and when
-a pass completes the sweep restarts from the top. New cameras are picked up
-on every pass.
-
-Ethics: detection-only (open-port SYN scan). Private/reserved ranges are
-excluded and the service REFUSES to start if the excludefile is missing, so
-we never probe private space by accident.
-
-Run once (for a manual/test pass):  python app/run_masscan_service.py --once
-Run forever (systemd):            python app/run_masscan_service.py
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-import os
-import sys
-from pathlib import Path
-
-sys_path = str(Path(__file__).parent)
-sys.path.insert(0, sys_path)
-
-import masscan_config as cfg  # noqa: E402
-from database import init_extensions  # noqa: E402
-from masscan_scanner import (  # noqa: E402
-    parse_masscan_line, extract_open_ips, flush,
-)
-
-logging.basicConfig(level=logging.INFO,
-                    format="%(asctime)s %(levelname)s %(name)s: %(message)s")
-logger = logging.getLogger("osint.masscan_service")
-
-ONCE = "--once" in sys.argv[1:]
-
-
-def _verify_excludefile() -> None:
-    """Fail closed: refuse to sweep the wide range without an excludefile."""
-    if not cfg.MASSCAN_EXCLUDEFILE:
-        raise SystemExit("MASSCAN_EXCLUDEFILE is empty — refusing to run")
-    if not Path(cfg.MASSCAN_EXCLUDEFILE).is_file():
-        raise SystemExit(
-            f"excludefile {cfg.MASSCAN_EXCLUDEFILE!r} missing — refusing to "
-            f"run (would risk probing private ranges). Install the excludefile "
-            f"first (see deploy/masscan-excludes.txt)."
-        )
-
-
-def build_command() -> list[str]:
-    cmd = [
-        cfg.MASSCAN_BIN,
-        cfg.MASSCAN_RANGE,
-        f"-p{cfg.MASSCAN_PORTS}",
-        f"--rate={cfg.MASSCAN_RATE}",
-        f"--retries={cfg.MASSCAN_RETRIES}",
-        f"--wait={cfg.MASSCAN_WAIT}",
-        "--output-format=json",
-        "--output-file=-",
-    ]
-    if cfg.MASSCAN_EXCLUDEFILE:
-        cmd.append(f"--excludefile={cfg.MASSCAN_EXCLUDEFILE}")
-    return cmd
-
-
-async def _drain_stderr(stream: asyncio.StreamReader) -> None:
-    """Consume masscan's progress chatter so its stderr pipe never fills."""
-    while True:
-        line = await stream.readline()
-        if not line:
-            break
-        text = line.decode(errors="ignore").strip()
-        if text and not text.startswith("rate:"):
-            logger.debug("masscan: %s", text)
-
-
-async def run_pass() -> tuple[int, int]:
-    """Run one full sweep pass, ingesting incrementally.
-
-    Returns (new_hosts, total_hosts_seen) for the whole pass.
-    """
-    cmd = build_command()
-    logger.info("starting masscan pass: %s", " ".join(cmd))
-    proc = await asyncio.create_subprocess_exec(
-        *cmd,
-        stdout=asyncio.subprocess.PIPE,
-        stderr=asyncio.subprocess.PIPE,
-    )
-    if proc.stderr is not None:
-        asyncio.ensure_future(_drain_stderr(proc.stderr))
-
-    seen: set[str] = set()
-    total_seen = 0
-    total_new = 0
-    try:
-        while True:
-            raw = await proc.stdout.readline()
-            if not raw:
-                break
-            records = parse_masscan_line(raw.decode(errors="ignore"))
-            for ip in extract_open_ips(records, 554):
-                if ip in seen:
-                    continue
-                seen.add(ip)
-                if len(seen) >= cfg.MASSCAN_FLUSH_EVERY:
-                    new, _published = await flush(seen, total_new)
-                    total_new += new
-                    total_seen += new
-        # Drain the final partial batch.
-        if seen:
-            new, _published = await flush(seen, total_new)
-            total_new += new
-        rc = await proc.wait()
-    except asyncio.CancelledError:
-        proc.kill()
-        raise
-    logger.info("masscan pass finished (rc=%s): %d new hosts ingested",
-                rc, total_new)
-    return total_new, total_seen
-
-
-async def main() -> None:
-    _verify_excludefile()
-    await init_extensions()
-    logger.info(
-        "masscan service starting: range=%s ports=%s rate=%s pps (full sweep "
-        "~%.0fh at this rate)",
-        cfg.MASSCAN_RANGE, cfg.MASSCAN_PORTS, cfg.MASSCAN_RATE,
-        4.29e9 / cfg.MASSCAN_RATE / 3600,
-    )
-    while True:
-        try:
-            await run_pass()
-        except Exception:  # noqa: BLE001
-            logger.exception("masscan pass error")
-        if ONCE:
-            return
-        # Small gap between passes so the restart is visible in logs.
-        await asyncio.sleep(5)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

app/satellites.py

@@ -0,0 +1,289 @@
+"""CelesTrak satellites last-known overlay.
+
+Fetches GP **JSON** (OMM mean elements — not TLE) per group at most once per
+2 hours, caches the element blob, and propagates positions with a real SGP4
+library on every request. Positions move every second; the *element set* is
+what we cache, not the derived lat/lon.
+
+Catalog numbers >= 100000 only fit OMM/JSON, never a 5-column TLE field, so
+elements are initialized through :func:`sgp4.omm.initialize` (which consumes
+the CelesTrak GP JSON fields verbatim) rather than round-tripping to TLE.
+
+CelesTrak usage policy is non-negotiable: fetch the GP JSON blob at most once
+per 2 hours per group, never fan out every GROUP, never also fetch
+``GROUP=active`` plus subsets, and identify with ``OSINT_USER_AGENT``.
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+from datetime import datetime, timezone
+from urllib.parse import quote
+
+logger = logging.getLogger("osint.satellites")
+
+CELESTRAK_GP = "https://celestrak.org/NORAD/elements/gp.php"
+SATNOGS_TLE = "https://db.satnogs.org/api/tle/"
+DEFAULT_GROUPS = ("stations", "weather")
+ALLOWED_GROUPS = ("stations", "weather", "gps-ops", "starlink")
+# CelesTrak policy: do not hit gp.php more than once per 2 hours per group.
+SATELLITE_TTL = 2 * 3600.0
+SOURCE_CELESTRAK = "celestrak"
+SOURCE_SATNOGS = "satnogs"
+DEFAULT_LIMIT = 2000
+
+# WGS-84 ellipsoid for TEME -> geodetic.
+_WGS84_A = 6378.137
+_WGS84_F = 1.0 / 298.257223563
+
+# Last-good element blob per group, kept past TTL so a 403 / "has not updated
+# since ..." still serves the previous set instead of failing the overlay.
+_last_good: dict[str, list[dict]] = {}
+
+
+def parse_groups(raw: str | None) -> list[str]:
+    """Validate + normalize a comma-separated group list. Raises ValueError.
+
+    Starlink is allowed only when explicitly requested (never in the default);
+    it is a large supplemental feed, not part of the stations/weather default.
+    """
+    groups = [g.strip().lower() for g in (raw or "").split(",") if g.strip()]
+    if not groups:
+        raise ValueError("groups must be a non-empty comma-separated list")
+    bad = [g for g in groups if g not in ALLOWED_GROUPS]
+    if bad:
+        raise ValueError(f"unknown group(s): {', '.join(bad)}")
+    # Dedup, preserve order.
+    seen: set[str] = set()
+    out: list[str] = []
+    for g in groups:
+        if g not in seen:
+            seen.add(g)
+            out.append(g)
+    return out
+
+
+def _teme_to_geodetic(
+    r: tuple[float, float, float],
+    jd: float,
+    fr: float,
+) -> tuple[float, float, float]:
+    """SGP4 TEME position (km) -> geodetic (lat_deg, lon_deg, alt_km).
+
+    Rotate TEME into an Earth-fixed frame via GMST, then iterate the WGS-84
+    geodetic conversion. Good to well under a km for a ground-track overlay.
+    """
+    # GMST (radians) from UT1 ~= UTC here (sub-second error is negligible).
+    d = (jd + fr) - 2451545.0
+    t = d / 36525.0
+    gmst_s = (
+        67310.54841
+        + (876600.0 * 3600.0 + 8640184.812866) * t
+        + 0.093104 * t * t
+        - 6.2e-6 * t * t * t
+    )
+    theta = math.radians((gmst_s % 86400.0) / 240.0)
+
+    x, y, z = r
+    xe = x * math.cos(theta) + y * math.sin(theta)
+    ye = -x * math.sin(theta) + y * math.cos(theta)
+    ze = z
+
+    e2 = _WGS84_F * (2.0 - _WGS84_F)
+    p = math.sqrt(xe * xe + ye * ye)
+    lon = math.atan2(ye, xe)
+    lat = math.atan2(ze, p * (1.0 - e2))
+    alt = 0.0
+    for _ in range(10):
+        n = _WGS84_A / math.sqrt(1.0 - e2 * math.sin(lat) ** 2)
+        alt = p / math.cos(lat) - n
+        lat = math.atan2(ze, p * (1.0 - e2 * n / (n + alt)))
+    n = _WGS84_A / math.sqrt(1.0 - e2 * math.sin(lat) ** 2)
+    alt = p / math.cos(lat) - n
+    return math.degrees(lat), math.degrees(lon), alt
+
+
+def propagate_gp(
+    elements: list[dict],
+    group: str,
+    now: datetime,
+) -> list[dict]:
+    """Propagate CelesTrak GP JSON elements to geodetic positions at ``now``.
+
+    Pure and deterministic given ``now``. Returns ``[{id, name, lat, lon,
+    alt_km, group}]``; malformed elements and propagation errors are skipped.
+    """
+    from sgp4.api import Satrec, jday
+    import sgp4.omm as omm
+
+    jd, fr = jday(
+        now.year, now.month, now.day,
+        now.hour, now.minute, now.second + now.microsecond / 1e6,
+    )
+    out: list[dict] = []
+    for rec in elements:
+        if not isinstance(rec, dict):
+            continue
+        sat = Satrec()
+        try:
+            omm.initialize(sat, rec)
+        except (KeyError, ValueError, TypeError):
+            continue
+        err, r, _v = sat.sgp4(jd, fr)
+        if err != 0:
+            continue
+        lat, lon, alt = _teme_to_geodetic(r, jd, fr)
+        norad = rec.get("NORAD_CAT_ID")
+        out.append({
+            "id": str(norad) if norad is not None else "",
+            "name": rec.get("OBJECT_NAME") or str(norad or ""),
+            "lat": round(lat, 5),
+            "lon": round(lon, 5),
+            "alt_km": round(alt, 2),
+            "group": group,
+        })
+    return out
+
+
+def _max_epoch(elements: list[dict]) -> str | None:
+    """Most recent EPOCH across an element set (ISO-8601 lexical max)."""
+    epochs = [
+        str(e["EPOCH"]) for e in elements
+        if isinstance(e, dict) and e.get("EPOCH")
+    ]
+    return max(epochs) if epochs else None
+
+
+def propagate_satnogs_tle(
+    payload: list[dict],
+    group: str,
+    now: datetime,
+) -> tuple[list[dict], str | None]:
+    """Fallback parser for SatNOGS TLE JSON (``[{tle0,tle1,tle2,updated}]``).
+
+    Returns ``(satellites, epoch)`` where epoch is the max ``updated`` time.
+    Only used when the CelesTrak cache is completely empty.
+    """
+    from sgp4.api import Satrec, jday
+
+    jd, fr = jday(
+        now.year, now.month, now.day,
+        now.hour, now.minute, now.second + now.microsecond / 1e6,
+    )
+    out: list[dict] = []
+    epochs: list[str] = []
+    for rec in payload or []:
+        if not isinstance(rec, dict):
+            continue
+        line1 = rec.get("tle1")
+        line2 = rec.get("tle2")
+        if not line1 or not line2:
+            continue
+        try:
+            sat = Satrec.twoline2rv(line1, line2)
+        except (ValueError, TypeError):
+            continue
+        e, r, _v = sat.sgp4(jd, fr)
+        if e != 0:
+            continue
+        lat, lon, alt = _teme_to_geodetic(r, jd, fr)
+        satnum = getattr(sat, "satnum_str", None) or rec.get("norad_cat_id")
+        name = (rec.get("tle0") or "").strip().lstrip("0").strip() or str(satnum or "")
+        out.append({
+            "id": str(satnum).strip() or "",
+            "name": name,
+            "lat": round(lat, 5),
+            "lon": round(lon, 5),
+            "alt_km": round(alt, 2),
+            "group": group,
+        })
+        if rec.get("updated"):
+            epochs.append(str(rec["updated"]))
+    return out, (max(epochs) if epochs else None)
+
+
+async def _group_elements(group: str) -> tuple[list[dict], str | None]:
+    """CelesTrak GP blob for one group, TTL-cached with a last-good fallback.
+
+    Returns ``(elements, epoch)``. On a fetch failure (403 / "has not updated
+    since ...") falls back to the previous successful blob for that group.
+    """
+    from live_layers import _get_json, _ttl_get
+
+    url = f"{CELESTRAK_GP}?GROUP={quote(group)}&FORMAT=JSON"
+
+    async def _load() -> list[dict]:
+        data = await _get_json(url)
+        if not isinstance(data, list):
+            raise ValueError(f"unexpected CelesTrak payload for {group}")
+        if data:
+            _last_good[group] = data
+        return data
+
+    key = f"celestrak:gp:{group}"
+    try:
+        elements = await _ttl_get(key, SATELLITE_TTL, _load)
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("celestrak_fetch_failed group=%s: %s", group, exc)
+        elements = _last_good.get(group, [])
+        if not elements:
+            return [], None
+    return elements, _max_epoch(elements)
+
+
+async def fetch_satellites(
+    groups: list[str],
+    bbox: str | None = None,
+    limit: int = DEFAULT_LIMIT,
+) -> dict:
+    """Assemble the ``/api/satellites`` payload for the requested groups."""
+    from live_layers import _get_json, _ttl_get, filter_points_bbox, parse_bbox
+
+    now = datetime.now(timezone.utc)
+    satellites: list[dict] = []
+    epoch: str | None = None
+    source = SOURCE_CELESTRAK
+
+    for group in groups:
+        elements, group_epoch = await _group_elements(group)
+        if not elements:
+            continue
+        if group_epoch and (epoch is None or group_epoch > epoch):
+            epoch = group_epoch
+        satellites.extend(propagate_gp(elements, group, now))
+
+    if not satellites:
+        # Fallback only when the CelesTrak cache is entirely empty — never
+        # poll both providers every cycle.
+        async def _load_satnogs() -> list[dict]:
+            data = await _get_json(SATNOGS_TLE, params={"format": "json"})
+            return data if isinstance(data, list) else []
+
+        try:
+            satnogs = await _ttl_get("satnogs:tle", SATELLITE_TTL, _load_satnogs)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("satnogs_fetch_failed: %s", exc)
+            satnogs = []
+        if satnogs:
+            source = SOURCE_SATNOGS
+            for group in groups:
+                rows, sn_epoch = propagate_satnogs_tle(satnogs, group, now)
+                if sn_epoch and (epoch is None or sn_epoch > epoch):
+                    epoch = sn_epoch
+                satellites.extend(rows)
+
+    if bbox:
+        minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
+        satellites = filter_points_bbox(
+            satellites, minlon, minlat, maxlon, maxlat, limit,
+        )
+    else:
+        satellites = satellites[:limit]
+
+    return {
+        "satellites": satellites,
+        "source": source,
+        "tle_epoch": epoch,
+        "timestamp": now.isoformat(),
+    }

app/schemas.py

@@ -4,10 +4,10 @@ from __future__ import annotations
 
 from datetime import datetime
 from enum import Enum
-from typing import Optional
+from typing import Literal, Optional
 from uuid import UUID
 
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 
 # ─── Enums ───────────────────────────────────────────────────────────────
@@ -63,6 +63,17 @@ class FeedSourceCreate(BaseModel):
     config: Optional[dict] = None
 
 
+class FeedSourceUpdate(BaseModel):
+    """PATCH /api/sources/{id} — only these keys may be set."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    name: Optional[str] = None
+    url: Optional[str] = None
+    config: Optional[dict] = None
+    enabled: Optional[bool] = None
+
+
 class FeedSourceOut(BaseModel):
     id: UUID
     name: str
@@ -274,6 +285,7 @@ class NewsSummaryOut(BaseModel):
     summary_text: str
     batch_timestamp: datetime
     model: Optional[str] = None
+    kind: Optional[str] = None
 
 
 class NewsTickerItemOut(BaseModel):
@@ -385,3 +397,24 @@ class GeofenceUpdate(BaseModel):
     geojson: Optional[dict] = None
     active: Optional[bool] = None
 
+
+class ConflictZoneOut(BaseModel):
+    """One curated conflict theatre as exposed by GET /api/conflicts."""
+
+    id: str
+    label: str
+    severity: Literal["war", "high", "elevated"]
+    lat: float
+    lon: float
+    description: str
+    eventCount: int
+    lastUpdated: Optional[datetime] = None
+
+
+class ConflictsOut(BaseModel):
+    """Response envelope for GET /api/conflicts."""
+
+    zones: list[ConflictZoneOut]
+    timestamp: datetime
+
+

app/sources.py

@@ -11,7 +11,7 @@ import httpx
 import feedparser
 import nats
 
-from config import NATS_URL
+from config import NATS_URL, OSINT_USER_AGENT
 from upstream_cache import rss_cache
 
 logger = logging.getLogger("osint.sources")
@@ -26,26 +26,75 @@ def _parse_rfc822(date_str: object) -> str | None:
     except (ValueError, TypeError):
         return None
 
-# NATS connection
+
 NATS_URLS = NATS_URL
+_nc = None
+
+
+async def _jetstream():
+    """Reuse one NATS connection across publishes (no connect/close per event)."""
+    global _nc
+    if _nc is None or _nc.is_closed:
+        _nc = await nats.connect(NATS_URLS)
+    return _nc.jetstream()
 
 
 async def publish_event(subject: str, event: dict):
     """Publish an event to NATS JetStream."""
-    nc = await nats.connect(NATS_URLS)
-    js = nc.jetstream()
+    js = await _jetstream()
     await js.publish(subject, json.dumps(event).encode())
-    await nc.close()
     logger.debug("Published event to %s", subject)
 
 
+def event_dedup_key(msg: dict) -> str | None:
+    """Natural key for generic events. URL when present; else None (always insert)."""
+    url = msg.get("url")
+    if not isinstance(url, str):
+        return None
+    url = url.strip()
+    return url or None
+
+
+async def existing_event_urls(urls: list[str]) -> set[str]:
+    """URLs already claimed in event_dedup. Empty input -> empty set."""
+    if not urls:
+        return set()
+    from sqlalchemy import select
+
+    from database import async_session
+    from models import event_dedup as event_dedup_table
+
+    async with async_session() as session:
+        result = await session.execute(
+            select(event_dedup_table.c.url).where(event_dedup_table.c.url.in_(urls))
+        )
+        return {row[0] for row in result}
+
+
+async def _publish_unknown(subject: str, events: list[dict]) -> int:
+    """Publish only events whose URL is not already in event_dedup."""
+    keys = [event_dedup_key(e) for e in events]
+    known = await existing_event_urls([k for k in keys if k])
+    published = 0
+    for event, key in zip(events, keys):
+        if key and key in known:
+            continue
+        await publish_event(subject, event)
+        published += 1
+    return published
+
+
+def _ua_headers() -> dict[str, str]:
+    return {"User-Agent": OSINT_USER_AGENT}
+
+
 # ─── RSS Feed Ingestor ──────────────────────────────────────────────────
 
 async def ingest_rss_feed(feed_url: str, source_id: str | None = None):
     """Fetch and parse an RSS feed, publish items to NATS."""
     text = rss_cache.get(feed_url)
     if text is None:
-        async with httpx.AsyncClient(timeout=30) as client:
+        async with httpx.AsyncClient(timeout=30, headers=_ua_headers()) as client:
             resp = await client.get(feed_url)
             resp.raise_for_status()
             text = resp.text
@@ -76,51 +125,80 @@ async def ingest_rss_feed(feed_url: str, source_id: str | None = None):
     return count
 
 
-# ─── GDELT 2.0 Ingestor ─────────────────────────────────────────────────
+# ─── GDELT 2.0 DOC API ──────────────────────────────────────────────────
 
-GDELT_API = "https://api.gdeltproject.org/gdeltv2"
+GDELT_API = "https://api.gdeltproject.org/api/v2/doc/doc"
+GDELT_DEFAULT_QUERY = '(unrest OR protest OR outage OR cyber OR "power outage")'
+
+
+def gdelt_params(query: str = "", max_articles: int = 50) -> dict[str, str]:
+    """DOC 2.0 query string (not the retired gdeltv2 ``search`` param)."""
+    q = (query or "").strip() or GDELT_DEFAULT_QUERY
+    return {
+        "query": q,
+        "mode": "ArtList",
+        "format": "json",
+        "maxrecords": str(int(max_articles)),
+        "timespan": "1d",
+    }
+
+
+def _parse_gdelt_seendate(value: object) -> str:
+    if isinstance(value, str) and len(value) >= 15:
+        try:
+            return datetime.strptime(value[:15], "%Y%m%dT%H%M%S").replace(
+                tzinfo=timezone.utc
+            ).isoformat()
+        except ValueError:
+            pass
+    return datetime.now(timezone.utc).isoformat()
+
+
+def parse_gdelt_articles(data: dict) -> list[dict]:
+    events = []
+    for article in data.get("articles") or []:
+        if not isinstance(article, dict):
+            continue
+        url = article.get("url")
+        if not url:
+            continue
+        events.append({
+            "source_type": "gdel-t2",
+            "title": article.get("title"),
+            "body": article.get("domain") or article.get("language"),
+            "url": url,
+            "location_name": article.get("sourcecountry"),
+            "source_timestamp": _parse_gdelt_seendate(article.get("seendate")),
+            "tags": [t for t in (article.get("language"), article.get("sourcecountry")) if t],
+            "raw": article,
+        })
+    return events
 
 
 async def ingest_gdelt(query: str = "", max_articles: int = 50):
-    """Fetch articles from GDELT 2.0 API."""
-    params = {
-        "mode": "artlist",
-        "format": "json",
-        "maxrecords": max_articles,
-        "mode": "artlist",
-    }
-    if query:
-        params["search"] = query
+    """Fetch articles from the GDELT DOC 2.0 API."""
+    params = gdelt_params(query=query, max_articles=max_articles)
+    data: dict = {"articles": []}
+    async with httpx.AsyncClient(
+        timeout=60, headers=_ua_headers(), follow_redirects=True,
+    ) as client:
+        try:
+            resp = await client.get(GDELT_API, params=params)
+            resp.raise_for_status()
+            data = resp.json()
+        except (httpx.TransportError, httpx.HTTPStatusError) as exc:
+            # gdeltproject.org certs have expired in the wild; HTTP fallback.
+            logger.warning("GDELT HTTPS failed (%s); retrying HTTP", exc)
+            http_url = GDELT_API.replace("https://", "http://", 1)
+            resp = await client.get(http_url, params=params)
+            resp.raise_for_status()
+            data = resp.json()
 
-    async with httpx.AsyncClient(timeout=60) as client:
-        resp = await client.get(GDELT_API, params=params)
-        resp.raise_for_status()
-        data = resp.json()
-
-    count = 0
-    for article in data.get("articles", []):
-        event = {
-            "source_type": "gdel-t2",
-            "title": article.get("title"),
-            "body": article.get("articleBody"),
-            "url": article.get("url"),
-            "sentiment_score": _parse_gdelt_tone(article.get("Tone", "0")),
-            "location_lat": article.get("Latitude"),
-            "location_lon": article.get("Longitude"),
-            "location_name": article.get("Location"),
-            "source_timestamp": article.get("FirstCreated"),
-            "entities": [
-                {"name": e.get("Topic"), "type": "topic"}
-                for e in article.get("Mentions", [])
-                if e.get("Topic")
-            ],
-            "raw": article,
-        }
+    events = parse_gdelt_articles(data if isinstance(data, dict) else {})
+    for event in events:
         await publish_event("events.gdelt", event)
-        count += 1
-
-    logger.info("Ingested %d articles from GDELT", count)
-    return count
+    logger.info("Ingested %d articles from GDELT", len(events))
+    return len(events)
 
 
 def _parse_gdelt_tone(tone: str) -> float | None:
@@ -137,34 +215,45 @@ def _parse_gdelt_tone(tone: str) -> float | None:
 USGS_API = "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_hour.geojson"
 
 
+def parse_usgs_feature(feature: dict) -> dict:
+    """Map one USGS GeoJSON feature, keeping the stable event id."""
+    props = feature.get("properties") or {}
+    geometry = (feature.get("geometry") or {}).get("coordinates") or []
+    usgs_id = feature.get("id")
+    url = props.get("url") or (
+        f"https://earthquake.usgs.gov/earthquakes/eventpage/{usgs_id}" if usgs_id else None
+    )
+    raw = dict(props)
+    raw["usgs_id"] = usgs_id
+    return {
+        "source_type": "earthquake",
+        "title": props.get("title"),
+        "body": props.get("description"),
+        "url": url,
+        "location_lat": geometry[1] if len(geometry) > 1 else None,
+        "location_lon": geometry[0] if len(geometry) > 0 else None,
+        "location_name": props.get("place"),
+        "sentiment_label": "neutral",
+        "tags": [f"magnitude:{props.get('mag')}"] if props.get("mag") else [],
+        "source_timestamp": (
+            datetime.utcfromtimestamp(props.get("time", 0) / 1000)
+            .replace(tzinfo=timezone.utc)
+            .isoformat()
+        ),
+        "raw": raw,
+    }
+
+
 async def ingest_earthquakes():
     """Fetch recent earthquakes from USGS."""
-    async with httpx.AsyncClient(timeout=30) as client:
+    async with httpx.AsyncClient(timeout=30, headers=_ua_headers()) as client:
         resp = await client.get(USGS_API)
         resp.raise_for_status()
         data = resp.json()
 
     count = 0
     for feature in data.get("features", []):
-        props = feature.get("properties", {})
-        geometry = feature.get("geometry", {}).get("coordinates", [])
-        event = {
-            "source_type": "earthquake",
-            "title": props.get("title"),
-            "body": props.get("description"),
-            "url": props.get("url"),
-            "location_lat": geometry[1] if len(geometry) > 1 else None,
-            "location_lon": geometry[0] if len(geometry) > 0 else None,
-            "location_name": props.get("place"),
-            "sentiment_label": "neutral",
-            "tags": [f"magnitude:{props.get('mag')}"] if props.get("mag") else [],
-            "source_timestamp": (
-                datetime.utcfromtimestamp(props.get("time", 0) / 1000)
-                .replace(tzinfo=timezone.utc)
-                .isoformat()
-            ),
-            "raw": props,
-        }
+        event = parse_usgs_feature(feature)
         await publish_event("events.earthquake", event)
         count += 1
 
@@ -172,6 +261,108 @@ async def ingest_earthquakes():
     return count
 
 
+# ─── NASA EONET v3 ──────────────────────────────────────────────────────
+
+EONET_API = "https://eonet.gsfc.nasa.gov/api/v3/events"
+
+
+def parse_eonet_events(payload: dict) -> list[dict]:
+    events = []
+    for item in payload.get("events") or []:
+        if not isinstance(item, dict):
+            continue
+        eid = item.get("id")
+        geoms = item.get("geometry") or []
+        point = None
+        for g in geoms:
+            if isinstance(g, dict) and g.get("type") == "Point":
+                point = g
+        if point is None:
+            continue
+        coords = point.get("coordinates") or []
+        if len(coords) < 2:
+            continue
+        lon, lat = float(coords[0]), float(coords[1])
+        cats = item.get("categories") or []
+        tags = []
+        for c in cats:
+            if isinstance(c, dict) and c.get("id"):
+                tags.append(str(c["id"]))
+        url = item.get("link") or (f"https://eonet.gsfc.nasa.gov/api/v3/events/{eid}" if eid else None)
+        ts = point.get("date") or datetime.now(timezone.utc).isoformat()
+        events.append({
+            "source_type": "disaster",
+            "title": item.get("title"),
+            "body": ", ".join(tags) if tags else None,
+            "url": url,
+            "location_lat": lat,
+            "location_lon": lon,
+            "location_name": item.get("title"),
+            "tags": tags,
+            "source_timestamp": ts,
+            "raw": {**item, "eonet_id": eid},
+        })
+    return events
+
+
+async def ingest_eonet():
+    """Volcanoes, storms, floods, drought — gaps USGS/FIRMS don't cover."""
+    async with httpx.AsyncClient(timeout=30, headers=_ua_headers()) as client:
+        resp = await client.get(EONET_API, params={"status": "open", "limit": 100})
+        resp.raise_for_status()
+        data = resp.json()
+    events = parse_eonet_events(data if isinstance(data, dict) else {})
+    published = await _publish_unknown("events.disaster", events)
+    logger.info("Ingested %d EONET events (%d already known)", published, len(events) - published)
+    return published
+
+
+# ─── CISA KEV ───────────────────────────────────────────────────────────
+
+CISA_KEV_API = (
+    "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
+)
+
+
+def parse_cisa_kev(payload: dict) -> list[dict]:
+    events = []
+    for row in payload.get("vulnerabilities") or []:
+        if not isinstance(row, dict):
+            continue
+        cve = row.get("cveID")
+        if not cve:
+            continue
+        title = row.get("vulnerabilityName") or cve
+        vendor = row.get("vendorProject") or ""
+        product = row.get("product") or ""
+        events.append({
+            "source_type": "disaster",
+            "title": f"{cve}: {title}",
+            "body": row.get("shortDescription") or f"{vendor} {product}".strip(),
+            "url": f"https://nvd.nist.gov/vuln/detail/{cve}",
+            "location_lat": None,
+            "location_lon": None,
+            "location_name": None,
+            "tags": ["cisa-kev", cve, "ransomware" if row.get("knownRansomwareCampaignUse") == "Known" else None],
+            "source_timestamp": row.get("dateAdded") or datetime.now(timezone.utc).isoformat(),
+            "raw": {**row, "cveID": cve},
+        })
+        events[-1]["tags"] = [t for t in events[-1]["tags"] if t]
+    return events
+
+
+async def ingest_cisa_kev():
+    """Exploited-in-the-wild CVEs. No fake map coords — ticker/events only."""
+    async with httpx.AsyncClient(timeout=30, headers=_ua_headers(), follow_redirects=True) as client:
+        resp = await client.get(CISA_KEV_API)
+        resp.raise_for_status()
+        data = resp.json()
+    events = parse_cisa_kev(data if isinstance(data, dict) else {})
+    published = await _publish_unknown("events.disaster", events)
+    logger.info("Ingested %d CISA KEV rows (%d already known)", published, len(events) - published)
+    return published
+
+
 # ─── Social Signals (Twitter/X-like placeholder) ────────────────────────
 
 async def ingest_social_signals(query: str = "", max_items: int = 50):

app/tracks.py

@@ -186,15 +186,14 @@ async def track_range() -> dict:
         async with async_session() as session:
             row = (await session.execute(text(
                 """
-                SELECT
-                  LEAST(
-                    (SELECT min(bucket) FROM vessel_tracks_1min),
-                    (SELECT min(bucket) FROM aircraft_tracks_1min)
-                  ) AS tmin,
-                  GREATEST(
-                    (SELECT max(bucket) FROM vessel_tracks_1min),
-                    (SELECT max(bucket) FROM aircraft_tracks_1min)
-                  ) AS tmax
+                SELECT min(t) AS tmin, max(t) AS tmax FROM (
+                  SELECT min(bucket) AS t FROM vessel_tracks_1min
+                  UNION ALL SELECT max(bucket) FROM vessel_tracks_1min
+                  UNION ALL SELECT min(bucket) FROM aircraft_tracks_1min
+                  UNION ALL SELECT max(bucket) FROM aircraft_tracks_1min
+                  UNION ALL SELECT min(poll_at) FROM vessels
+                  UNION ALL SELECT max(poll_at) FROM vessels
+                ) s
                 """
             ))).mappings().first()
         if not row or row["tmin"] is None:

app/vesselapi.py

@@ -0,0 +1,671 @@
+"""VesselAPI REST poller — quota-capped AIS for the Middle East (free tier 150 calls/mo).
+
+VesselAPI and AISStream are two independent, first-class vessel providers —
+not a primary/fallback pair. AISStream (WebSocket) owns live US-coast AIS;
+VesselAPI (REST) covers the Strait of Hormuz (default box) where AISStream
+has no coverage. Missing one key never disables the other. This worker polls
+the REST ``GET /v1/location/vessels/bounding-box`` endpoint at most
+``VESSELAPI_MAX_CALLS_PER_DAY`` (default 5) *successful 2xx* calls per UTC day
+and upserts the results into the shared ``vessel_last_known`` store.
+
+Idle (no crash) when VESSELAPI_API_KEY is unset. Never called from the GET
+/api/vessels path — map pans must not hit upstream. One request per poll,
+``pagination.limit=50``, never follow ``nextToken``, never send ``filter.sat``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import calendar
+import json
+import logging
+import os
+from datetime import date, datetime, timezone
+
+import httpx
+from sqlalchemy import Column, Date, DateTime, Integer, Table, func, select, text
+
+from config import (
+    OSINT_USER_AGENT,
+    VESSELAPI_API_KEY,
+    VESSELAPI_BBOX,
+    VESSELAPI_INTERVAL,
+    VESSELAPI_MAX_CALLS_PER_DAY,
+)
+from database import async_session, engine, metadata
+from live_layers import parse_bbox, to_marker, upsert_vessel, vessel_last_known, vessel_lock
+
+logger = logging.getLogger("osint.vesselapi")
+
+BASE_URL = "https://api.vesselapi.com/v1"
+ENDPOINT = f"{BASE_URL}/location/vessels/bounding-box"
+MAX_SPAN_DEG = 4.0  # |dLat| + |dLon| — VesselAPI 400s above this.
+PAGE_LIMIT = 50  # pagination.limit; never follow nextToken on the free tier.
+
+_client: httpx.AsyncClient | None = None
+_client_lock = asyncio.Lock()
+
+
+# ── Box parsing / span validation ─────────────────────────────────────────
+
+class BboxError(ValueError):
+    """A configured VesselAPI box violates the 4° span rule or is malformed."""
+
+
+def validate_bbox_span(
+    minlat: float, minlon: float, maxlat: float, maxlon: float,
+) -> None:
+    """Reject boxes VesselAPI would 400 on (span > 4°, bad order, bad range)."""
+    if not (-90 <= minlat <= 90 and -90 <= maxlat <= 90
+            and -180 <= minlon <= 180 and -180 <= maxlon <= 180):
+        raise BboxError("coordinates out of range")
+    if minlat >= maxlat or minlon >= maxlon:
+        raise BboxError("bbox must have min < max on both axes")
+    dlat = abs(maxlat - minlat)
+    dlon = abs(maxlon - minlon)
+    if dlat + dlon > MAX_SPAN_DEG:
+        raise BboxError(
+            f"span |dLat|+|dLon| = {dlat + dlon:.2f}° exceeds {MAX_SPAN_DEG}° cap"
+        )
+
+
+def parse_boxes(raw: str) -> list[tuple[float, float, float, float]]:
+    """Env format: ``minlat,minlon,maxlat,maxlon[; ...]`` (lat/lon order)."""
+    out: list[tuple[float, float, float, float]] = []
+    for chunk in (raw or "").split(";"):
+        parts = [p.strip() for p in chunk.split(",") if p.strip()]
+        if len(parts) != 4:
+            continue
+        try:
+            minlat = float(parts[0])
+            minlon = float(parts[1])
+            maxlat = float(parts[2])
+            maxlon = float(parts[3])
+        except ValueError:
+            continue
+        out.append((minlat, minlon, maxlat, maxlon))
+    return out
+
+
+def parse_boxes_validated(raw: str) -> list[tuple[float, float, float, float]]:
+    """Parse boxes, log + skip any that violate the span/order/range rules."""
+    valid: list[tuple[float, float, float, float]] = []
+    for box in parse_boxes(raw):
+        try:
+            validate_bbox_span(*box)
+            valid.append(box)
+        except BboxError as exc:
+            logger.warning("VesselAPI bbox %r skipped: %s", box, exc)
+    return valid
+
+
+# ── Position → marker transform ───────────────────────────────────────────
+
+def _f(value: object) -> float | None:
+    if value is None or value == "":
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _s(value: object) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None
+
+
+def transform_vesselapi_position(obj: dict | None) -> dict | None:
+    """Map one VesselAPI position object to the shared marker contract.
+
+    Returns None for glitch rows, missing MMSI, or missing coordinates.
+    """
+    if not obj or not isinstance(obj, dict):
+        return None
+    if obj.get("suspected_glitch") is True:
+        return None
+    mmsi = obj.get("mmsi")
+    if mmsi is None:
+        return None
+    lat = _f(obj.get("latitude"))
+    lon = _f(obj.get("longitude"))
+    if lat is None or lon is None:
+        return None
+    mmsi_s = str(mmsi)
+    name = _s(obj.get("vessel_name") or obj.get("name"))
+    heading = _f(obj.get("heading"))
+    if heading is None:
+        heading = _f(obj.get("cog"))
+    sog = _f(obj.get("sog"))
+    extra: dict = {
+        "src": "vesselapi",
+        "mmsi": mmsi_s,
+        "cog": obj.get("cog"),
+        "sog": obj.get("sog"),
+        "navstat": obj.get("nav_status"),
+    }
+    imo = obj.get("imo")
+    if imo:
+        extra["imo"] = imo
+    dest = _s(obj.get("dest") or obj.get("destination"))
+    if dest:
+        extra["dest"] = dest
+    ts = obj.get("timestamp") or obj.get("processed_timestamp")
+    if ts:
+        extra["timestamp"] = ts
+    return to_marker(
+        mmsi_s, lat, lon,
+        heading=heading,
+        speed=sog,
+        label=name or mmsi_s,
+        extra=extra,
+    )
+
+
+def transform_vesselapi_payload(payload: dict | None) -> list[dict]:
+    """Flatten a bounding-box response ``{vessels: [...]}`` to markers."""
+    if not payload or not isinstance(payload, dict):
+        return []
+    rows = payload.get("vessels") or []
+    out = []
+    for row in rows:
+        marker = transform_vesselapi_position(row)
+        if marker:
+            out.append(marker)
+    return out
+
+
+def utc_day_start(now: datetime) -> datetime:
+    """Floor ``now`` to 00:00:00 UTC."""
+    if now.tzinfo is None:
+        now = now.replace(tzinfo=timezone.utc)
+    now = now.astimezone(timezone.utc)
+    return now.replace(hour=0, minute=0, second=0, microsecond=0)
+
+
+def pick_poll_at(poll_times: list[datetime], as_of: datetime) -> datetime | None:
+    """Latest poll timestamp at or before ``as_of`` (DVR as-of)."""
+    if as_of.tzinfo is None:
+        as_of = as_of.replace(tzinfo=timezone.utc)
+    else:
+        as_of = as_of.astimezone(timezone.utc)
+    eligible: list[datetime] = []
+    for raw in poll_times:
+        ts = raw if raw.tzinfo else raw.replace(tzinfo=timezone.utc)
+        ts = ts.astimezone(timezone.utc)
+        if ts <= as_of:
+            eligible.append(ts)
+    return max(eligible) if eligible else None
+
+
+def snapshot_as_of(rows: list[dict], as_of: datetime) -> list[dict]:
+    """Keep only rows from the latest poll_at ≤ ``as_of``."""
+    chosen = pick_poll_at(
+        [r["poll_at"] for r in rows if r.get("poll_at") is not None],
+        as_of,
+    )
+    if chosen is None:
+        return []
+    out = []
+    for row in rows:
+        ts = row.get("poll_at")
+        if ts is None:
+            continue
+        if ts.tzinfo is None:
+            ts = ts.replace(tzinfo=timezone.utc)
+        if ts.astimezone(timezone.utc) == chosen:
+            out.append(row)
+    return out
+
+
+# ── Durable daily quota (Postgres, survives restarts) ─────────────────────
+# Mirrors keystore.api_keys: lazy CREATE TABLE IF NOT EXISTS, no alembic fork.
+
+vesselapi_quota = Table(
+    "vesselapi_quota",
+    metadata,
+    Column("day", Date, primary_key=True),
+    Column("calls", Integer, nullable=False, server_default="0"),
+    Column("remaining", Integer, nullable=True),
+    Column("updated_at", DateTime(timezone=True), server_default=func.now(), nullable=False),
+)
+
+_CREATE_QUOTA_SQL = text(
+    """
+    CREATE TABLE IF NOT EXISTS vesselapi_quota (
+        day        DATE PRIMARY KEY,
+        calls      INTEGER NOT NULL DEFAULT 0,
+        remaining  INTEGER,
+        updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+    )
+    """
+)
+
+_quota_lock = asyncio.Lock()
+_quota_ensured = False
+
+
+async def ensure_quota_table() -> None:
+    global _quota_ensured
+    if _quota_ensured:
+        return
+    async with _quota_lock:
+        if _quota_ensured:
+            return
+        async with engine.begin() as conn:
+            await conn.execute(_CREATE_QUOTA_SQL)
+        _quota_ensured = True
+
+
+class PgQuotaStore:
+    """Postgres-backed daily call counter. Injected for tests."""
+
+    async def calls_today(self, day: date) -> int:
+        await ensure_quota_table()
+        async with async_session() as session:
+            row = (await session.execute(
+                select(vesselapi_quota.c.calls).where(vesselapi_quota.c.day == day)
+            )).scalar()
+        return int(row) if row else 0
+
+    async def remaining_today(self, day: date) -> int | None:
+        await ensure_quota_table()
+        async with async_session() as session:
+            row = (await session.execute(
+                select(vesselapi_quota.c.remaining).where(vesselapi_quota.c.day == day)
+            )).scalar()
+        return int(row) if row is not None else None
+
+    async def bump(self, day: date, remaining: int | None) -> int:
+        await ensure_quota_table()
+        now = datetime.now(timezone.utc)
+        async with async_session() as session:
+            existing = (await session.execute(
+                select(vesselapi_quota.c.calls).where(vesselapi_quota.c.day == day)
+            )).scalar()
+            if existing is None:
+                await session.execute(
+                    vesselapi_quota.insert().values(
+                        day=day, calls=1, remaining=remaining, updated_at=now,
+                    )
+                )
+            else:
+                await session.execute(
+                    vesselapi_quota.update()
+                    .where(vesselapi_quota.c.day == day)
+                    .values(
+                        calls=vesselapi_quota.c.calls + 1,
+                        remaining=remaining,
+                        updated_at=now,
+                    )
+                )
+            await session.commit()
+        return (int(existing) if existing else 0) + 1
+
+
+# ── Daily VesselAPI snapshots (DVR as-of + survive restarts) ──────────────
+# Cleared at the UTC day boundary so the table holds today's 5 polls only.
+
+_CREATE_VESSELS_SQL = text(
+    """
+    CREATE TABLE IF NOT EXISTS vessels (
+        mmsi     TEXT NOT NULL,
+        poll_at  TIMESTAMPTZ NOT NULL,
+        lat      DOUBLE PRECISION NOT NULL,
+        lon      DOUBLE PRECISION NOT NULL,
+        heading  DOUBLE PRECISION,
+        speed    DOUBLE PRECISION,
+        label    TEXT,
+        extra    JSONB,
+        PRIMARY KEY (mmsi, poll_at)
+    )
+    """
+)
+_CREATE_VESSELS_POLL_IDX = text(
+    "CREATE INDEX IF NOT EXISTS ix_vessels_poll_at ON vessels (poll_at DESC)"
+)
+_CREATE_VESSELS_BBOX_IDX = text(
+    "CREATE INDEX IF NOT EXISTS ix_vessels_bbox ON vessels (lon, lat)"
+)
+
+_vessels_lock = asyncio.Lock()
+_vessels_ensured = False
+
+
+async def ensure_vessels_table() -> None:
+    global _vessels_ensured
+    if _vessels_ensured:
+        return
+    async with _vessels_lock:
+        if _vessels_ensured:
+            return
+        async with engine.begin() as conn:
+            await conn.execute(_CREATE_VESSELS_SQL)
+            await conn.execute(_CREATE_VESSELS_POLL_IDX)
+            await conn.execute(_CREATE_VESSELS_BBOX_IDX)
+        _vessels_ensured = True
+
+
+def _marker_from_vessel_row(r) -> dict:
+    extra = r.get("extra") or {}
+    if isinstance(extra, str):
+        try:
+            extra = json.loads(extra)
+        except (TypeError, ValueError):
+            extra = {}
+    if not isinstance(extra, dict):
+        extra = {}
+    extra.setdefault("src", "vesselapi")
+    poll_at = r.get("poll_at")
+    if poll_at is not None and hasattr(poll_at, "isoformat"):
+        extra["poll_at"] = poll_at.isoformat()
+    marker = to_marker(
+        str(r["id"]), r["lat"], r["lon"],
+        heading=r.get("heading"), speed=r.get("speed"),
+        label=r.get("label") or str(r["id"]),
+        extra=extra,
+    )
+    marker["seen_at"] = extra.get("poll_at") or datetime.now(timezone.utc).isoformat()
+    return marker
+
+
+async def persist_vessel_snapshot(markers: list[dict], poll_at: datetime) -> None:
+    """Write one VesselAPI poll into ``vessels`` (today's snapshots)."""
+    await ensure_vessels_table()
+    if not markers:
+        return
+    async with async_session() as session:
+        for m in markers:
+            vid = str(m.get("id") or "")
+            lat, lon = m.get("lat"), m.get("lon")
+            if not vid or lat is None or lon is None:
+                continue
+            extra = dict(m.get("extra") or {})
+            extra.setdefault("src", "vesselapi")
+            await session.execute(
+                text(
+                    """
+                    INSERT INTO vessels
+                        (mmsi, poll_at, lat, lon, heading, speed, label, extra)
+                    VALUES
+                        (:mmsi, :poll_at, :lat, :lon, :heading, :speed, :label,
+                         CAST(:extra AS jsonb))
+                    ON CONFLICT (mmsi, poll_at) DO UPDATE SET
+                        lat = EXCLUDED.lat,
+                        lon = EXCLUDED.lon,
+                        heading = EXCLUDED.heading,
+                        speed = EXCLUDED.speed,
+                        label = EXCLUDED.label,
+                        extra = EXCLUDED.extra
+                    """
+                ),
+                {
+                    "mmsi": vid,
+                    "poll_at": poll_at,
+                    "lat": float(lat),
+                    "lon": float(lon),
+                    "heading": m.get("heading"),
+                    "speed": m.get("speed"),
+                    "label": m.get("label") or vid,
+                    "extra": json.dumps(extra),
+                },
+            )
+        await session.commit()
+
+
+async def purge_old_vessels(before: datetime | None = None) -> None:
+    """Drop snapshots from before the current UTC day (or ``before``)."""
+    await ensure_vessels_table()
+    cutoff = before or utc_day_start(datetime.now(timezone.utc))
+    async with async_session() as session:
+        await session.execute(
+            text("DELETE FROM vessels WHERE poll_at < :cutoff"),
+            {"cutoff": cutoff},
+        )
+        await session.commit()
+
+
+async def fetch_vessels_as_of(
+    ts: datetime,
+    bbox: str | None = None,
+    limit: int = 2000,
+) -> list[dict]:
+    """Latest VesselAPI poll at or before ``ts`` (DVR as-of, not exact minute)."""
+    try:
+        await ensure_vessels_table()
+        async with async_session() as session:
+            poll = (await session.execute(
+                text("SELECT max(poll_at) FROM vessels WHERE poll_at <= :ts"),
+                {"ts": ts},
+            )).scalar()
+            if poll is None:
+                return []
+            sql = """
+                SELECT mmsi AS id, lat, lon, heading, speed, label, extra, poll_at
+                FROM vessels
+                WHERE poll_at = :poll
+            """
+            params: dict = {"poll": poll, "limit": limit}
+            if bbox:
+                minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
+                sql += (
+                    " AND lon BETWEEN :minlon AND :maxlon"
+                    " AND lat BETWEEN :minlat AND :maxlat"
+                )
+                params.update(
+                    minlon=minlon, minlat=minlat, maxlon=maxlon, maxlat=maxlat,
+                )
+            sql += " LIMIT :limit"
+            rows = (await session.execute(text(sql), params)).mappings().all()
+        return [_marker_from_vessel_row(r) for r in rows]
+    except Exception:
+        logger.exception("VesselAPI snapshot fetch failed")
+        return []
+
+
+async def hydrate_last_known() -> int:
+    """Seed in-memory last-known from today's latest poll (app boot)."""
+    try:
+        rows = await fetch_vessels_as_of(datetime.now(timezone.utc))
+    except Exception:
+        logger.exception("VesselAPI hydrate failed")
+        return 0
+    if not rows:
+        return 0
+    async with vessel_lock:
+        for m in rows:
+            vid = str(m.get("id") or "")
+            if vid:
+                vessel_last_known[vid] = m
+    return len(rows)
+
+
+# ── Budget / scheduling (pure, unit-testable) ─────────────────────────────
+
+def days_left_in_month(now: datetime) -> int:
+    """UTC days remaining in the current month, inclusive of today."""
+    _, last = calendar.monthrange(now.year, now.month)
+    return last - now.day + 1
+
+
+def budget_allows(
+    calls_today: int,
+    remaining: int | None,
+    days_left: int,
+    max_per_day: int,
+) -> bool:
+    """True if another poll is permitted today.
+
+    Local hard cap: fewer than ``max_per_day`` successful calls today.
+    Monthly floor: if ``X-RateLimit-Remaining`` is known, keep at least
+    ``max_per_day * days_left`` in reserve for the rest of the month.
+    """
+    if calls_today >= max_per_day:
+        return False
+    if remaining is not None and remaining <= max_per_day * days_left:
+        return False
+    return True
+
+
+def choose_box(
+    boxes: list[tuple[float, float, float, float]],
+    calls_today: int,
+    max_per_day: int,
+) -> int:
+    """Index into ``boxes`` for the next poll.
+
+    Prefer refreshing the first (primary) box rather than spraying one call
+    across every region — round-robin only when the remaining daily budget is
+    enough to cover all boxes.
+    """
+    if len(boxes) <= 1:
+        return 0
+    budget_left = max_per_day - calls_today
+    if budget_left >= len(boxes):
+        return calls_today % len(boxes)
+    return 0
+
+
+# ── HTTP / poll ───────────────────────────────────────────────────────────
+
+async def _resolve_key() -> str:
+    from keystore import get_api_key
+    return (
+        os.getenv("VESSELAPI_API_KEY")
+        or VESSELAPI_API_KEY
+        or (await get_api_key("VESSELAPI_API_KEY"))
+        or ""
+    ).strip()
+
+
+async def _get_client() -> httpx.AsyncClient:
+    global _client
+    if _client is None:
+        async with _client_lock:
+            if _client is None:
+                _client = httpx.AsyncClient(
+                    timeout=httpx.Timeout(15.0, connect=5.0),
+                    follow_redirects=True,
+                    headers={"User-Agent": OSINT_USER_AGENT, "Accept": "application/json"},
+                    limits=httpx.Limits(max_connections=1, max_keepalive_connections=1),
+                )
+    return _client
+
+
+async def close_client() -> None:
+    global _client
+    if _client is not None:
+        await _client.aclose()
+        _client = None
+
+
+def _int_header(value: str | None) -> int | None:
+    if value is None:
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+async def poll_once(store, boxes: list[tuple[float, float, float, float]], key: str) -> bool:
+    """One quota-checked poll. Returns True if a successful 2xx was made.
+
+    Only successful 2xx responses count against the monthly quota; 4xx/5xx/429
+    are skipped without retry-storming (Retry-After respected by simply
+    sleeping the interval).
+    """
+    now = datetime.now(timezone.utc)
+    today = now.date()
+    calls = await store.calls_today(today)
+    remaining = await store.remaining_today(today)
+    days_left = days_left_in_month(now)
+    if not budget_allows(calls, remaining, days_left, VESSELAPI_MAX_CALLS_PER_DAY):
+        logger.info(
+            "VesselAPI quota reached (calls_today=%d, remaining=%s, days_left=%d) — skip poll",
+            calls, remaining, days_left,
+        )
+        return False
+
+    idx = choose_box(boxes, calls, VESSELAPI_MAX_CALLS_PER_DAY)
+    minlat, minlon, maxlat, maxlon = boxes[idx]
+    client = await _get_client()
+    params = {
+        "filter.latBottom": str(minlat),
+        "filter.latTop": str(maxlat),
+        "filter.lonLeft": str(minlon),
+        "filter.lonRight": str(maxlon),
+        "pagination.limit": str(PAGE_LIMIT),
+    }
+    headers = {"Authorization": f"Bearer {key}"}
+    try:
+        resp = await client.get(ENDPOINT, params=params, headers=headers)
+    except httpx.HTTPError as exc:
+        logger.warning("VesselAPI request failed: %s", exc)
+        return False
+
+    if resp.status_code == 429:
+        logger.warning(
+            "VesselAPI rate-limited (Retry-After=%s) — skip poll",
+            resp.headers.get("Retry-After"),
+        )
+        return False
+    if resp.status_code >= 400:
+        logger.warning("VesselAPI HTTP %d — not counted against quota", resp.status_code)
+        return False
+
+    # 2xx success — counts against the monthly quota.
+    remaining = _int_header(resp.headers.get("X-RateLimit-Remaining"))
+    calls = await store.bump(today, remaining)
+    try:
+        data = resp.json()
+    except ValueError:
+        logger.warning("VesselAPI 2xx with non-JSON body — counted but ignored")
+        return True
+    markers = transform_vesselapi_payload(data)
+    for m in markers:
+        await upsert_vessel(m)
+    try:
+        await persist_vessel_snapshot(markers, now)
+        await purge_old_vessels(utc_day_start(now))
+    except Exception:  # noqa: BLE001 — live overlay must not die on persist
+        logger.exception("VesselAPI snapshot persist failed")
+    logger.info(
+        "VesselAPI poll OK: %d vessels (remaining=%s, calls_today=%d)",
+        len(markers), remaining, calls,
+    )
+    return True
+
+
+# ── Worker loop ───────────────────────────────────────────────────────────
+
+async def run_vesselapi_worker(store: PgQuotaStore | None = None) -> None:
+    """Long-lived poll loop. Idle when the key is unset; never crashes the app."""
+    if store is None:
+        store = PgQuotaStore()
+    boxes = parse_boxes_validated(VESSELAPI_BBOX)
+    if not boxes:
+        logger.warning(
+            "VESSELAPI_BBOX has no valid boxes (span ≤ %.1f°) — poller idle", MAX_SPAN_DEG,
+        )
+    while True:
+        try:
+            if not boxes:
+                await asyncio.sleep(VESSELAPI_INTERVAL)
+                continue
+            key = await _resolve_key()
+            if not key:
+                logger.warning(
+                    "VESSELAPI_API_KEY not set — VesselAPI poller idle. "
+                    "Create a free key at https://dashboard.vesselapi.com/"
+                )
+                await asyncio.sleep(VESSELAPI_INTERVAL)
+                continue
+            await poll_once(store, boxes, key)
+        except asyncio.CancelledError:
+            raise
+        except Exception:  # noqa: BLE001 — keep the loop alive across transient failures
+            logger.exception("VesselAPI poll error")
+        await asyncio.sleep(VESSELAPI_INTERVAL)

app/ws_manager.py

@@ -8,10 +8,18 @@ from __future__ import annotations
 
 import asyncio
 from typing import Any
+from uuid import UUID
 
 BBox = tuple[float, float, float, float]  # minlon, minlat, maxlon, maxlat
 
 
+def _uuid_str(value: object) -> str | None:
+    try:
+        return str(UUID(str(value)))
+    except (ValueError, TypeError, AttributeError):
+        return None
+
+
 def point_in_bbox(lon: float, lat: float, bbox: BBox | None) -> bool:
     """True if (lon, lat) sits inside an axis-aligned viewport."""
     if bbox is None:
@@ -26,6 +34,7 @@ class ConnectionManager:
     def __init__(self) -> None:
         self._queues: dict[str, asyncio.Queue] = {}
         self._viewports: dict[str, BBox] = {}
+        self._watched: dict[str, set[str]] = {}
 
     def register(self, client_id: str, maxsize: int = 256) -> asyncio.Queue:
         q: asyncio.Queue = asyncio.Queue(maxsize=maxsize)
@@ -35,6 +44,21 @@ class ConnectionManager:
     def unregister(self, client_id: str) -> None:
         self._queues.pop(client_id, None)
         self._viewports.pop(client_id, None)
+        self._watched.pop(client_id, None)
+
+    def set_watched_geofences(self, client_id: str, ids: list[str]) -> None:
+        """Watch these fence UUIDs so geofence_alert delivers off-viewport.
+
+        Invalid UUIDs are ignored. Empty list = watch none (viewport-only).
+        """
+        if client_id not in self._queues:
+            return
+        watched: set[str] = set()
+        for raw in ids:
+            uid = _uuid_str(raw)
+            if uid is not None:
+                watched.add(uid)
+        self._watched[client_id] = watched
 
     def set_viewport(self, client_id: str, bbox: BBox) -> None:
         if client_id in self._queues:
@@ -43,6 +67,9 @@ class ConnectionManager:
     def viewport_of(self, client_id: str) -> BBox | None:
         return self._viewports.get(client_id)
 
+    def viewports(self) -> list[BBox]:
+        return list(self._viewports.values())
+
     def has_clients(self) -> bool:
         return bool(self._queues)
 
@@ -56,13 +83,21 @@ class ConnectionManager:
     ) -> int:
         """Enqueue `{type, payload}` for clients whose viewport contains the point.
 
-        Drops the oldest queued message if a client's buffer is full so a slow
-        tab cannot stall ingest. Returns the number of clients that got a copy.
+        kind=geofence_alert also delivers when payload.geofence_id is in the
+        client's watch set (even if the point is off-viewport). Other kinds
+        stay viewport-only. Drops the oldest queued message if a client's
+        buffer is full. Returns the number of clients that got a copy.
         """
         msg = {"type": kind, "payload": payload}
         sent = 0
+        gid = _uuid_str(payload.get("geofence_id")) if kind == "geofence_alert" else None
         for client_id, queue in list(self._queues.items()):
-            if not point_in_bbox(lon, lat, self._viewports.get(client_id)):
+            in_view = point_in_bbox(lon, lat, self._viewports.get(client_id))
+            if kind == "geofence_alert":
+                watching = gid is not None and gid in self._watched.get(client_id, set())
+                if not in_view and not watching:
+                    continue
+            elif not in_view:
                 continue
             if queue.full():
                 try:

deploy/README.md

@@ -1,31 +0,0 @@
-# systemd unit template — copy to /etc/systemd/system/osint-masscan.service
-#
-# The masscan service is a CONTINUOUS rolling sweep (a full IPv4 pass at a
-# conservative rate takes ~5 days), so it runs as a long-lived service, NOT a
-# daily timer. The [Install] WantedBy means it starts at boot and Restart=always
-# keeps it up. Install steps (run once on the Pi, as root):
-#
-#   apt install -y masscan      # or: apt-get install masscan
-#   mkdir -p /etc/osint-dashboard /opt/siriusdevops
-#   cp deploy/masscan-excludes.txt /etc/osint-dashboard/masscan-excludes.txt
-#
-#   # Optional tuning (override env in this file; the DB_* values in the unit
-#   # already point at the host-published Postgres on 127.0.0.1:5432):
-#   cat > /etc/osint-dashboard/masscan.env <<'EOF'
-#   MASSCAN_RANGE=0.0.0.0/0
-#   MASSCAN_PORTS=554
-#   MASSCAN_RATE=1000
-#   EOF
-#
-#   # Venv for the scanner (host-level, not the compose image):
-#   cd /opt/siriusdevops/osint-dashboard
-#   python3 -m venv .venv-masscan
-#   .venv-masscan/bin/pip install -r app/requirements.txt
-#
-#   install -m 644 deploy/osint-masscan.service /etc/systemd/system/
-#   systemctl daemon-reload
-#   systemctl enable --now osint-masscan
-#
-# Watch:  journalctl -u osint-masscan -f
-# DB:     writes into the same Postgres the compose stack uses (127.0.0.1:5432)
-#         so findings appear on the dashboard camera map automatically.

deploy/masscan-excludes.txt

@@ -1,33 +0,0 @@
-# masscan excludefile — never probe these ranges.
-# RFC1918 private + loopback + link-local + multicast + documentation/bogons.
-# The service refuses to start if this file is missing (fail closed).
-
-# Loopback
-127.0.0.0/8
-
-# RFC1918 private
-10.0.0.0/8
-172.16.0.0/12
-192.168.0.0/16
-
-# Link-local
-169.254.0.0/16
-
-# CGNAT (RFC 6598)
-100.64.0.0/10
-
-# Multicast + reserved
-224.0.0.0/4
-240.0.0.0/4
-
-# Documentation / benchmark / example ranges (never real hosts)
-0.0.0.0/8
-192.0.2.0/24
-198.51.100.0/24
-203.0.113.0/24
-192.0.0.0/24
-198.18.0.0/15
-255.255.255.255/32
-
-# Carrier NAT / TEST-NET leftovers
-233.252.0.0/24

deploy/osint-masscan.service

@@ -1,29 +0,0 @@
-[Unit]
-Description=OSINT dashboard — masscan rolling sweep (open RTSP port 554)
-Documentation=https://forgejo.siriusdevops.com/sirius/osint-dashboard
-After=network-online.target
-Wants=network-online.target
-
-[Service]
-Type=simple
-# masscan needs raw sockets (CAP_NET_RAW) — run as root on the Pi host.
-User=root
-WorkingDirectory=/opt/siriusdevops/osint-dashboard
-EnvironmentFile=-/etc/osint-dashboard/masscan.env
-# Point at the compose-published Postgres on the HOST (127.0.0.1:5432), not the
-# docker service name 'postgres' which doesn't resolve outside the compose net.
-Environment=DB_HOST=127.0.0.1
-Environment=DB_PORT=5432
-Environment=DB_USER=osint
-Environment=DB_PASSWORD=osint
-Environment=DB_NAME=osint_data
-Environment=MASSCAN_EXCLUDEFILE=/etc/osint-dashboard/masscan-excludes.txt
-ExecStart=/opt/siriusdevops/osint-dashboard/.venv-masscan/bin/python app/run_masscan_service.py
-Restart=always
-RestartSec=10
-# Log the sweep to journald (read with: journalctl -u osint-masscan -f)
-StandardOutput=journal
-StandardError=journal
-
-[Install]
-WantedBy=multi-user.target

deploy/osint-titiler.nginx.conf

@@ -0,0 +1,21 @@
+# osint.rpi.local — Sentinel-1 SAR tile proxy (/titiler/)
+#
+# GitOps: this file is the source of truth. On the Pi:
+#   sudo cp deploy/osint-titiler.nginx.conf /etc/nginx/snippets/osint-titiler.conf
+# then `include snippets/osint-titiler.conf;` inside the osint.rpi.local server
+# block (before `location /`), `nginx -t && systemctl reload nginx`.
+#
+# The browser hits /titiler/cog/tiles/... (same-origin). We strip the /titiler
+# prefix so self-hosted TiTiler (127.0.0.1:8001) sees /cog/tiles/... and proxy
+# its response straight back. Tiles are heavy PNGs — disable buffering so a
+# slow client doesn't hold a worker open.
+
+location /titiler/ {
+    proxy_pass http://127.0.0.1:8001/;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_buffering off;
+    proxy_read_timeout 300s;
+}

deploy/osint-ws.nginx.conf

@@ -0,0 +1,23 @@
+# osint.rpi.local — WebSocket upgrade for /ws/live
+#
+# GitOps: this file is the source of truth. On the Pi:
+#   sudo cp deploy/osint-ws.nginx.conf /etc/nginx/snippets/osint-ws.conf
+# then `include snippets/osint-ws.conf;` inside the osint.rpi.local server
+# block (before `location /`), `nginx -t && systemctl reload nginx`.
+#
+# Without these headers nginx proxies GET /ws/live as HTTP/1.0 → FastAPI 404
+# and the HUD reconnects every few seconds.
+
+location /ws/ {
+    proxy_pass http://127.0.0.1:8000;
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_read_timeout 3600s;
+    proxy_send_timeout 3600s;
+    proxy_buffering off;
+}

docker-compose.yml

@@ -20,8 +20,10 @@ services:
     # networks (and ISP abuse-mitigation blackholes) block, so rebuilding it
     # on every CI deploy made the pipeline flaky. Rebuild manually when the
     # base image or extensions need bumping:
-    #   docker compose build db && docker compose up -d db
+    #   docker build -f Dockerfile.pg -t localhost/osint-dashboard-pg:latest .
+    #   FORCE_RECREATE_DB=1 scripts/compose-reup.sh db
     image: localhost/osint-dashboard-pg:latest
+    pull_policy: never
     container_name: osint-db
     restart: unless-stopped
     environment:
@@ -52,6 +54,7 @@ services:
 
   nats:
     image: nats:2.10
+    pull_policy: missing
     platform: linux/arm64
     container_name: osint-nats
     restart: unless-stopped
@@ -67,6 +70,7 @@ services:
       dockerfile: Dockerfile
       platforms: ["linux/arm64"]
     image: localhost/osint-dashboard:latest
+    pull_policy: never
     container_name: osint-ingester
     restart: unless-stopped
     profiles: ["ingest"]
@@ -93,10 +97,15 @@ services:
       FIRMS_DATASETS: ${FIRMS_DATASETS:-VIIRS_NOAA20_NRT,VIIRS_NOAA21_NRT}
       FIRMS_BBOX: ${FIRMS_BBOX:--180,-60,180,75}
       FIRMS_INTERVAL: ${FIRMS_INTERVAL:-900}
-      OSINT_USER_AGENT: ${OSINT_USER_AGENT:-osint-dashboard/1.0 (self-hosted)}
+      OSINT_USER_AGENT: ${OSINT_USER_AGENT:-osint-dashboard/1.0 (self-hosted; lancewalters94@gmail.com)}
       AISSTREAM_API_KEY: ${AISSTREAM_API_KEY:-}
       AISSTREAM_BBOX: ${AISSTREAM_BBOX:-24,-125,50,-66}
       AISSTREAM_IN_INGEST: ${AISSTREAM_IN_INGEST:-0}
+      VESSELAPI_API_KEY: ${VESSELAPI_API_KEY:-}
+      VESSELAPI_BBOX: ${VESSELAPI_BBOX:-25.5,55.4,27.3,57.2}
+      VESSELAPI_INTERVAL: ${VESSELAPI_INTERVAL:-17280}
+      VESSELAPI_MAX_CALLS_PER_DAY: ${VESSELAPI_MAX_CALLS_PER_DAY:-5}
+      VESSELAPI_IN_INGEST: ${VESSELAPI_IN_INGEST:-0}
     command: ["python", "app/run_ingester.py"]
     entrypoint: ["python", "app/run_ingester.py"]
 
@@ -106,6 +115,7 @@ services:
       dockerfile: Dockerfile
       platforms: ["linux/arm64"]
     image: localhost/osint-dashboard:latest
+    pull_policy: never
     container_name: osint-dashboard
     restart: unless-stopped
     depends_on:
@@ -127,10 +137,20 @@ services:
       FIRMS_DATASET: ${FIRMS_DATASET:-VIIRS_NOAA20_NRT}
       FIRMS_DATASETS: ${FIRMS_DATASETS:-VIIRS_NOAA20_NRT,VIIRS_NOAA21_NRT}
       FIRMS_BBOX: ${FIRMS_BBOX:--180,-60,180,75}
-      OSINT_USER_AGENT: ${OSINT_USER_AGENT:-osint-dashboard/1.0 (self-hosted)}
+      OSINT_USER_AGENT: ${OSINT_USER_AGENT:-osint-dashboard/1.0 (self-hosted; lancewalters94@gmail.com)}
+      NOMINATIM_URL: ${NOMINATIM_URL:-https://nominatim.openstreetmap.org}
+      NOMINATIM_MIN_INTERVAL: ${NOMINATIM_MIN_INTERVAL:-1.0}
       AISSTREAM_API_KEY: ${AISSTREAM_API_KEY:-}
       AISSTREAM_BBOX: ${AISSTREAM_BBOX:-24,-125,50,-66}
       AISSTREAM_IN_APP: ${AISSTREAM_IN_APP:-1}
+      VESSELAPI_API_KEY: ${VESSELAPI_API_KEY:-}
+      VESSELAPI_BBOX: ${VESSELAPI_BBOX:-25.5,55.4,27.3,57.2}
+      VESSELAPI_INTERVAL: ${VESSELAPI_INTERVAL:-17280}
+      VESSELAPI_MAX_CALLS_PER_DAY: ${VESSELAPI_MAX_CALLS_PER_DAY:-5}
+      VESSELAPI_IN_APP: ${VESSELAPI_IN_APP:-1}
+      # ── Self-hosted TiTiler (Sentinel-1 SAR tiles) ──
+      TITILER_PUBLIC_BASE: ${TITILER_PUBLIC_BASE:-/titiler}
+      TITILER_INTERNAL_URL: ${TITILER_INTERNAL_URL:-http://titiler:8000}
     ports:
       - "127.0.0.1:8000:8000"
     deploy:
@@ -143,12 +163,35 @@ services:
       timeout: 5s
       retries: 5
 
+  # ── Self-hosted TiTiler (Sentinel-1 SAR COG → XYZ tiles) ────────────────
+  # Warps the signed Planetary Computer COG into WebMercator XYZ tiles so the
+  # browser never loads a multi-GB GeoTIFF. The FastAPI app signs the COG URL
+  # and returns a /titiler/... template; nginx routes /titiler/ here.
+  # Listens on 8000 INSIDE the container (the app already owns host 8000);
+  # published on host loopback 127.0.0.1:8001 only.
+  titiler:
+    image: ghcr.io/developmentseed/titiler:latest@sha256:1809958d063543e3ec858259536002b2de78e9f8f09a22a8d9591bdc2b550b14
+    pull_policy: missing
+    container_name: osint-titiler
+    platform: linux/arm64
+    restart: unless-stopped
+    environment:
+      - PORT=8000
+      - WORKERS_PER_CORE=1
+    ports:
+      - "127.0.0.1:8001:8000"
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+
   camera-service:
     build:
       context: .
       dockerfile: Dockerfile
       platforms: ["linux/arm64"]
     image: localhost/osint-dashboard:latest
+    pull_policy: never
     container_name: osint-camera-scraper
     restart: unless-stopped
     profiles: ["ingest"]
@@ -176,7 +219,7 @@ services:
     volumes:
       - camera-snapshots:/data/snapshots
 
-  # ── News pipeline: hourly scraper (:00) + summarizer (:05) ───────────────
+  # ── News pipeline: continuous scraper + 15-min summarizer ───────────────
   # Both services point at the EXISTING osint-db (tables articles +
   # article_summaries, created by idempotent alembic migration 003_news).
   # Scheduling replaces the upstream k8s CronJobs with in-compose wall-clock
@@ -187,6 +230,7 @@ services:
       dockerfile: Dockerfile
       platforms: ["linux/arm64"]
     image: localhost/osint-news-scraper:latest
+    pull_policy: never
     container_name: osint-news-scraper
     restart: unless-stopped
     profiles: ["ingest"]
@@ -200,7 +244,7 @@ services:
       DB_PORT: ${DB_PORT:-5432}
       DB_NAME: ${DB_NAME:-osint_data}
       LOG_LEVEL: ${NEWS_LOG_LEVEL:-INFO}
-      NEWS_SCRAPE_MINUTE: ${NEWS_SCRAPE_MINUTE:-0}
+      NEWS_SCRAPE_INTERVAL_S: ${NEWS_SCRAPE_INTERVAL_S:-10}
       NEWS_SCRAPE_RUN_ON_START: ${NEWS_SCRAPE_RUN_ON_START:-1}
     # Override the image ENTRYPOINT ["scrapy"] with the scheduler loop.
     entrypoint: []
@@ -212,6 +256,7 @@ services:
       dockerfile: Dockerfile
       platforms: ["linux/arm64"]
     image: localhost/osint-news-summarizer:latest
+    pull_policy: never
     container_name: osint-news-summarizer
     restart: unless-stopped
     profiles: ["ingest"]
@@ -229,11 +274,14 @@ services:
       SUMMARY_MODEL: ${SUMMARY_MODEL:-}
       OSINT_USER_AGENT: ${OSINT_USER_AGENT:-osint-dashboard-news-summarizer}
       BATCH_SIZE: ${NEWS_BATCH_SIZE:-50}
-      SUMMARY_WINDOW_HOURS: ${SUMMARY_WINDOW_HOURS:-1}
+      SUMMARY_WINDOW_MINUTES: ${SUMMARY_WINDOW_MINUTES:-15}
       INCLUDE_FUTURES: ${INCLUDE_FUTURES:-0}
-      NEWS_SUMMARIZE_MINUTE: ${NEWS_SUMMARIZE_MINUTE:-5}
+      NEWS_SUMMARIZE_INTERVAL_S: ${NEWS_SUMMARIZE_INTERVAL_S:-900}
       NEWS_SUMMARIZE_RUN_ON_START: ${NEWS_SUMMARIZE_RUN_ON_START:-1}
       NEWS_SUMMARIZE_FORCE: ${NEWS_SUMMARIZE_FORCE:-0}
+      TZ: ${TZ:-America/New_York}
+      NEWS_RECAP_HOUR: ${NEWS_RECAP_HOUR:-23}
+      NEWS_RECAP_MINUTE: ${NEWS_RECAP_MINUTE:-0}
     command: ["python", "run_news_summarizer.py"]
 
 volumes:

docs/free-data-streams.md

@@ -2,7 +2,7 @@
 
 Builder brief for backend + frontend. Researched 2026-08-27. Every endpoint below was either live-probed from this machine or taken from the provider’s current docs. Prefer **free, no-key, CORS-open** sources first. Keys are called out explicitly.
 
-This is **not** a camera-discovery / masscan change. Existing camera rules still apply: never emit `rtsp://` hrefs; masscan pins go through `/api/cameras/{id}/snapshot`; HTTP directory cams use `/stream` MJPEG.
+This is **not** a camera-discovery change. Existing camera rules still apply: never emit `rtsp://` hrefs; camera pins go through `/api/cameras/{id}/snapshot`; HTTP directory cams use `/stream` MJPEG.
 
 ---
 
@@ -13,7 +13,7 @@ This is **not** a camera-discovery / masscan change. Existing camera rules still
 | NASA FIRMS VIIRS hotspots | Ingested (`app/fire_sources.py` → NATS `events.fire` → `fires` hypertable → `GET /api/fires`) | Needs free `FIRMS_MAP_KEY`. See `docs/firms.md`. |
 | NASA GIBS basemaps | Frontend tiles via `app/gibs_map.py` | No key. CORS `*`. |
 | GIBS VIIRS thermal tiles | Documented, not wired as overlay | Same GIBS stack; no key. |
-| Cameras | Scraper + masscan → `cameras` table | Defaults already include ALERTWest JPEGs + Live-Environment-Streams HLS/YouTube GeoJSON. |
+| Cameras | Scraper → `cameras` table | Defaults already include ALERTWest JPEGs + Live-Environment-Streams HLS/YouTube GeoJSON. |
 | News / RSS / GDELT / USGS quakes | Ingest | Out of scope for this brief. |
 
 **Action for existing fire ingest:** NASA will stop Suomi NPP product delivery on **2026-11-01**. Switch `FIRMS_DATASET` from `VIIRS_SNPP_NRT` to `VIIRS_NOAA20_NRT` and/or `VIIRS_NOAA21_NRT` before then.[20]
@@ -261,7 +261,7 @@ Use later if you want commuter rail / subway vehicle positions (LA Metro, MTA, e
 
 ## 6. Open video / camera feeds (official public only)
 
-Do **not** add Insecam-style random IP cams as a new source. The scraper already has a public list + masscan; this section is **agency-published** JPEG/HLS.
+Do **not** add Insecam-style random IP cams as a new source. The scraper already has a public list; this section is **agency-published** JPEG/HLS.
 
 ### 6.1 Already wired
 
@@ -304,7 +304,7 @@ Do not call the YouTube Data API unless you want search. Embedding existing stre
 
 ### 6.5 Skip
 
-- Insecam / random “public IP cam” aggregators — ToS / privacy / already covered by masscan ethics.
+- Insecam / random “public IP cam” aggregators — ToS / privacy.
 - TrafficLand, EarthCam commercial APIs.
 - SkylineWebcams — scraping, not an API.
 
@@ -523,7 +523,7 @@ Attribution bar (required): OpenSky / ADSB.lol ODbL / Amtraker / RainViewer / IE
 
 ## 12. Legal / ethics (non-negotiable)
 
-- Masscan / RTSP policy unchanged.
+- RTSP policy unchanged (never emit `rtsp://` hrefs).
 - AISStream: server-side only; do not put the key in JS.[5]
 - OpenSky: non-commercial unless licensed; cite if you publish.[2]
 - ADSB.lol: ODbL share-alike on derived databases.[4]

docs/news.md

@@ -1,22 +1,22 @@
 # News pipeline — scraper + Nous Portal summarizer
 
-The OSINT dashboard ingests ~257 global news RSS sources hourly and produces
-an English LLM brief plus flagged ticker/map rows. Both services were vendored
-from the upstream `~/Projects/newsPipeline` project and re-integrated here to
-replace the old k8s CronJob choreography with in-compose scheduling against
-the EXISTING osint-db — **no second Postgres**. The LLM is **Nous Portal**
+The OSINT dashboard ingests a large curated feed list (`news/scraper/urls.txt`)
+continuously and produces an English LLM brief plus flagged ticker/map rows
+every 15 minutes. Both services were vendored from the upstream
+`~/Projects/newsPipeline` project and re-integrated here against the EXISTING
+osint-db — **no second Postgres**. The LLM is **Nous Portal**
 (`inference-api.nousresearch.com`) — not Gemini.
 
 ## Architecture
 
 ```
-257 RSS feeds (news/scraper/urls.txt)
+urls.txt (RSS + homepages)
         │
         ▼
-news-scraper   (Scrapy, hourly :00) ──► articles table (osint-db)
+news-scraper   (Scrapy, continuous) ──► articles table (osint-db)
         │                                      │
         │                                      ▼
-news-summarizer (Nous Portal map-reduce, :05) ──► article_summaries + news_items
+news-summarizer (Nous Portal, every 15m + 23:00 recap) ──► article_summaries + news_items
                                                       │
                                                       ▼
      GET /api/news · /api/news/summaries · /api/news/ticker · /api/news/map
@@ -25,8 +25,8 @@ news-summarizer (Nous Portal map-reduce, :05) ──► article_summaries + news
 
 | Component | Image | Container | Scheduling |
 |---|---|---|---|
-| Scraper | `localhost/osint-news-scraper` | `osint-news-scraper` | wall-clock loop, minute `NEWS_SCRAPE_MINUTE` (default :00) |
-| Summarizer | `localhost/osint-news-summarizer` | `osint-news-summarizer` | wall-clock loop, minute `NEWS_SUMMARIZE_MINUTE` (default :05) |
+| Scraper | `localhost/osint-news-scraper` | `osint-news-scraper` | loop, `NEWS_SCRAPE_INTERVAL_S` (default 10s after each crawl) |
+| Summarizer | `localhost/osint-news-summarizer` | `osint-news-summarizer` | loop, `NEWS_SUMMARIZE_INTERVAL_S` (default 900s) |
 
 Both services live under the `ingest` compose profile (same as the ingester
 and camera-scraper): `docker compose --profile ingest up -d`.
@@ -39,26 +39,24 @@ feeds (`GET /api/news` exact key set is unchanged on purpose).
 ## Data flow
 
 1. **Scraper** — `news/scraper/run_news_scraper.py` runs
-   `scrapy crawl articles` (spider `news/scraper/newsScraper/spiders/news_spider.py`)
-   at the top of each hour. The spider reads the RSS feed URLs from `urls.txt`,
-   follows each `<item>` link, extracts the main article body, and the
-   `PostgresPipeline` writes to `articles` with URL-based dedup
+   `scrapy crawl articles` back-to-back (default 10s pause). The spider reads
+   URLs from `urls.txt` (homepages autodiscover RSS; feed URLs are parsed
+   directly), follows each `<item>` link, extracts the main article body, and
+   the `PostgresPipeline` writes to `articles` with URL-based dedup
    (`ON CONFLICT (url) DO NOTHING`).
 2. **Summarizer** — `news/summerizer/run_news_summarizer.py` runs
-   `summarizer.py` at :05 past each hour. It reads articles from the last
-   `SUMMARY_WINDOW_HOURS`, map-reduces them through Nous Portal
-   (`SUMMARY_MODEL` / Settings, default `Hermes-4.3-36B`), writes the English
-   brief to `article_summaries` (column `model` is the LLM id), and flagged
-   ticker/map rows to `news_items`.
+   `summarizer.py` every `NEWS_SUMMARIZE_INTERVAL_S` (default 900) over the
+   last `SUMMARY_WINDOW_MINUTES` (default 15), and again at 23:00
+   `America/New_York` (`TZ`) over the last 24 hours as a daily recap
+   (`kind=daily_recap`). Both map-reduce through Nous Portal (`SUMMARY_MODEL`
+   / Settings, default `Hermes-4.3-36B`), write the English brief to
+   `article_summaries` (column `model` is the LLM id; `kind` is
+   `interval` or `daily_recap`), and flagged ticker/map rows to `news_items`.
 
-Scheduling is done with small in-compose wall-clock loops (not host cron): each
-loop runs once on boot (`*_RUN_ON_START=1`, seeds data fast) then sleeps until
-the next scheduled minute. The loop is serial, so a run that overruns its slot
-simply shifts to the next boundary — two crawls/summaries never overlap.
-
-Hour-truncation idempotency: if `article_summaries` already has a row for the
-current UTC hour, the summarizer **skips** (prevents double-pins on
-`RUN_ON_START` recreate). Set `NEWS_SUMMARIZE_FORCE=1` to ignore that skip.
+Loops are serial (two crawls/summaries never overlap). Interval idempotency:
+if `article_summaries` already has a row in the last interval, the summarizer
+**skips** (prevents double-pins on `RUN_ON_START` recreate). Set
+`NEWS_SUMMARIZE_FORCE=1` to ignore that skip.
 
 The `articles` and `article_summaries` tables are created by the idempotent
 alembic migration `003_news` (also created by the scraper's own
@@ -120,18 +118,22 @@ Key set **unchanged** (no `lat`/`lon` on articles; geo lives on `/api/news/map`)
     "id": 1,
     "summary_text": "English markdown brief…",
     "batch_timestamp": "2026-08-24T18:10:00Z",
-    "model": "Hermes-4.3-36B"
+    "model": "Hermes-4.3-36B",
+    "kind": "daily_recap"
   }
 ]
 ```
 
-`model` is additive. Empty DB → `[]` (no crash).
+`model` and `kind` are additive (`interval` | `daily_recap` | `null` for old rows).
+`?kind=daily_recap` pins the nightly 24h recap. Empty DB → `[]` (no crash).
+Malformed `kind` → `422`.
 
-### GET /api/news/ticker — flagged HUD headlines
+### GET /api/news/ticker — HUD headlines
 
-Critical/high `news_items` with `kind=ticker` only. Do **not** reuse
-`GET /api/alerts`. Bottom HUD `#nt-track` scrolls these rows, not a dump of
-the whole brief.
+Critical/high `news_items` with `kind=ticker` first. If none are flagged,
+medium/low ticker rows fill the tape so the dock is not blank. Do **not**
+reuse `GET /api/alerts`. Bottom HUD `#nt-track` scrolls these rows, not a
+dump of the whole brief.
 
 | Query param | Meaning | Default |
 |---|---|---|
@@ -225,10 +227,10 @@ markdown json fences, then brace-slices:
 }
 ```
 
-Persist ticker/map only for `importance` in `critical`/`high`. Map rows also
-need valid coords; Unknown / invented places are dropped. Caps: 12 ticker
-(≤140 chars, no markdown), 20 map. Empty ticker is allowed. `summary_en`
-lands in `article_summaries.summary_text`.
+Persist ticker for critical/high first; if none, persist medium/low so the
+tape is not empty. Map rows stay critical/high with valid coords; Unknown /
+invented places are dropped. Caps: 12 ticker (≤140 chars, no markdown), 20
+map. `summary_en` lands in `article_summaries.summary_text`.
 
 ## Configuration (all via env / `.env`)
 
@@ -238,13 +240,16 @@ lands in `article_summaries.summary_text`.
 | `NOUS_BASE_URL` | `https://inference-api.nousresearch.com/v1` | Read-only in Settings. |
 | `SUMMARY_MODEL` | `Hermes-4.3-36B` | Compose default. Operator-facing choice is Settings → `app_settings.SUMMARY_MODEL`. |
 | `NEWS_BATCH_SIZE` | `50` | Articles per map-phase batch (compose maps to container `BATCH_SIZE`). |
-| `SUMMARY_WINDOW_HOURS` | `1` | How far back the summarizer looks for new articles. |
-| `INCLUDE_FUTURES` | `0` | Legacy futures-prices coupling (upstream pipeline). OFF for OSINT; set `1` + install `yfinance` to enable. |
-| `NEWS_SCRAPE_MINUTE` | `0` | Wall-clock minute the scraper fires. |
-| `NEWS_SUMMARIZE_MINUTE` | `5` | Wall-clock minute the summarizer fires. |
+| `SUMMARY_WINDOW_MINUTES` | `15` | How far back the summarizer looks for new articles. |
+| `NEWS_SCRAPE_INTERVAL_S` | `10` | Pause after each crawl before the next (scraper is otherwise continuous). |
+| `NEWS_SUMMARIZE_INTERVAL_S` | `900` | Seconds between analyst runs (default 15 min). |
+| `TZ` | `America/New_York` | Timezone for the 23:00 daily recap. |
+| `NEWS_RECAP_HOUR` | `23` | Local hour of the daily 24h recap. |
+| `NEWS_RECAP_MINUTE` | `0` | Local minute of the daily recap. |
 | `NEWS_SCRAPE_RUN_ON_START` | `1` | Run one scrape immediately on container start. |
 | `NEWS_SUMMARIZE_RUN_ON_START` | `1` | Run one summarize immediately on container start. |
-| `NEWS_SUMMARIZE_FORCE` | `0` | `1` ignores the current-UTC-hour idempotency skip (double-pins on recreate). |
+| `NEWS_SUMMARIZE_FORCE` | `0` | `1` ignores the interval/recap idempotency skip (double-pins on recreate). |
+| `INCLUDE_FUTURES` | `0` | Legacy. Ignored — prompts never inject futures/market tape. |
 | `NEWS_LOG_LEVEL` | `INFO` | Scrapy log level. |
 | `OSINT_USER_AGENT` | `osint-dashboard-news-summarizer` | Sent on every outbound Nous call. |
 | `TELEGRAM_TOKEN` / `TELEGRAM_CHAT_ID` | *(blank)* | Reserved for the (out-of-scope) Telegram delivery bot. |
@@ -258,11 +263,11 @@ No Hermes-4 reasoning system prompt. Reduce uses `json_mode=True`.
 
 ## Prompts
 
-Both prompts are env-overridable — the default `MAP_PROMPT` is OSINT-neutral
-(facts, locations, entities, category, OSINT signal per article; English) and
-the default `SUMMARY_PROMPT` demands the reduce JSON above (with a
-"no qualifying events" escape hatch). Upstream's futures/markets prompt
-language is gated behind `INCLUDE_FUTURES=1`.
+Both prompts are env-overridable. Defaults recap the articles actually
+provided, ranked by breaking important news, and ignore futures / commodity
+tape. ticker/map may be empty; `summary_en` must still be a real brief.
+`RECAP_PROMPT` (23:00, 24h window) is the daily recap; `SUMMARY_PROMPT` is the
+15-min analyst. `INCLUDE_FUTURES` is ignored.
 
 ## Tests
 

docs/satellite-feeds.md

@@ -0,0 +1,347 @@
+# Free satellite feeds for the OSINT map
+
+Builder inventory (research profile). Probed **2026-08-29** from this machine. Do **not** treat search snippets as live — every row below had a `curl`/GET (tile, GetCapabilities, STAC, or GetMap). 404 tile rows are omitted unless Capabilities/DescribeDomains still prove the layer exists (sparse fire overlays 404 on empty tiles).
+
+**Pi rules:** browser `L.tileLayer` when CORS `*`; do not proxy multi-GB COGs through the Pi; STAC+SAS like existing Sentinel-1 is “backend same as S-1”; no Redis; home uplink is small.
+
+GIBS Web Mercator REST template (no key):[2]
+
+```
+https://gibs.earthdata.nasa.gov/wmts/epsg3857/best/{layer}/default/{time}/{TileMatrixSet}/{z}/{y}/{x}.{jpg|png}
+```
+
+Omit `{time}` for static layers. Sub-daily GOES/Himawari accept `YYYY-MM-DD` **or** `YYYY-MM-DDTHH:MI:SSZ` (GIBS snaps to nearest).[2] Attribution: NASA asks clients to acknowledge GIBS/ESDIS.[1]
+
+Live GetCapabilities `epsg3857/best` on 2026-08-29: **1315** `Layer` entries, **all** with a `GoogleMapsCompatible_LevelN` matrix, `access-control-allow-origin: *`.[4] GIBS documents **1000+** visualizations; many LANCE layers appear within **3.5 hours** of observation.[3]
+
+Worldview is the interactive catalog of the same tiles.[5] GIBS developer portal: Earthdata GIBS API page (HTTP 403 from this host at probe time; docs site above is the working copy).[27]
+
+---
+
+## Already in the product (do not rediscover)
+
+| id | status |
+|---|---|
+| `BlueMarble_ShadedRelief_Bathymetry` | GIBS basemap (`app/gibs_map.py`) |
+| `VIIRS_SNPP_CorrectedReflectance_TrueColor` | GIBS basemap |
+| `MODIS_Terra_CorrectedReflectance_TrueColor` | GIBS basemap |
+| `MODIS_Aqua_CorrectedReflectance_TrueColor` | GIBS basemap |
+| `VIIRS_SNPP_DayNightBand_ENCC` | GIBS night lights |
+| FIRMS VIIRS hotspot CSV | ingest + `FIRMS_MAP_KEY` |
+| `VIIRS_SNPP_Thermal_Anomalies_375m_All` | overlay in `app/live_layers.py` (`gibs_thermal`). **Caps now say TMS `GoogleMapsCompatible_Level8`**, not Level9 — the wired URL uses Level9 (will 400). |
+| Sentinel-1 GRD | Planetary Computer STAC + SAS + TiTiler `GET /api/map/sentinel1` |
+| IEM NEXRAD / RainViewer | weather radar, not satellite |
+
+Repo docs already flag **Suomi NPP product stop 2026-11-01** — swap SNPP true color / DNB / thermal / FIRMS `VIIRS_SNPP_NRT` to NOAA-20/21 before then.
+
+---
+
+## Ranked “add tomorrow” (sections 1–2)
+
+Most new OSINT signal per **zero dollars**, browser tiles only:
+
+1. **VIIRS NOAA-20 + NOAA-21 true color** — SNPP replacement, same dropdown pattern.
+2. **VIIRS false-color SWIR** (`BandsM11-I2-I1`, `BandsM3-I3-M11`, MODIS 7-2-1) — burn scars, flood, bare soil.
+3. **GIBS GOES-East/West GeoColor + Band13 IR** — 10-minute weather-sat, Hormuz + CONUS.
+4. **GIBS Himawari AHI vis + IR** — same for IO/WestPac.
+5. **HLS S30/L30** — 30 m Landsat/Sentinel-2 look without TiTiler.
+6. **OPERA RTC Sentinel-1 + DIST-ALERT + DSWx** — SAR / disturbance / flood as GIBS tiles (not COGs).
+7. **NOAA-20/21 DNB** — night lights after SNPP.
+8. **IEM GOES XYZ** — “latest” tiles, no time in the URL, already CORS `*` like NEXRAD.[6]
+9. **EUMETView WMS** — Meteosat/MTG for Europe–Africa–IO, CORS `*`.[18]
+10. **GFW GLAD-S2 / integrated deforestation alerts** — raster tiles, CORS `*` when `Origin` is sent.[15]
+11. **MUR SST + VIIRS/PACE/OLCI chlorophyll** — ocean.
+12. **MODIS NDVI 8-day + IMERG rain** — veg / flood context.
+13. **SRTM / ASTER GDEM color index** — satellite-derived DEM, static.
+14. **NOAA-20/21 thermal anomalies** — FIRMS-shaped overlay after SNPP; empty tiles 404.
+
+---
+
+## 1. Drop-in GIBS WMTS
+
+All rows: **key? no**. **CORS `*`**. **Pi fit: browser `L.tileLayer`**. Same time-domain helper as `gibs_map.py` (`…/1.0.0/{id}/default/{tms}/all/all.xml`).
+
+Format of URL column: layer id + TMS + ext. Date used in probes: `2026-08-27` unless noted.
+
+### 1.1 Optical (true / false / SWIR)
+
+| id | what you see | tile pattern | cadence | max zoom | license | already have? | probe |
+|---|---|---|---|---|---|---|---|
+| `VIIRS_NOAA20_CorrectedReflectance_TrueColor` | Daily true color, JPSS-1 | `…/{id}/default/{time}/GoogleMapsCompatible_Level9/{z}/{y}/{x}.jpg` | daily | 9 (~250 m) | NASA GIBS ack[1] | **no** (SNPP only) | 200 `*` jpeg |
+| `VIIRS_NOAA21_CorrectedReflectance_TrueColor` | Daily true color, JPSS-2 | same Level9 jpg | daily | 9 | NASA GIBS[1] | no | 200 |
+| `VIIRS_SNPP_CorrectedReflectance_BandsM11-I2-I1` | False color SWIR (burns, flood) | Level9 jpg | daily | 9 | NASA GIBS[1] | no | 200 |
+| `VIIRS_SNPP_CorrectedReflectance_BandsM3-I3-M11` | False color (snow/ice/desert) | Level9 jpg | daily | 9 | NASA GIBS[1] | no | 200 |
+| `VIIRS_NOAA20_CorrectedReflectance_BandsM11-I2-I1` | NOAA-20 SWIR false color | Level9 jpg | daily | 9 | NASA GIBS[1] | no | 200 |
+| `VIIRS_NOAA21_CorrectedReflectance_BandsM11-I2-I1` | NOAA-21 SWIR false color | Level9 jpg | daily | 9 | NASA GIBS[1] | no | 200 |
+| `MODIS_Terra_CorrectedReflectance_Bands721` | Classic 7-2-1 burn/SWIR | Level9 jpg | daily | 9 | NASA GIBS[1] | no | 200 |
+| `MODIS_Terra_CorrectedReflectance_Bands367` | 3-6-7 false color | Level9 jpg | daily | 9 | NASA GIBS[1] | no | 200 |
+| `MODIS_Aqua_CorrectedReflectance_Bands721` | Aqua 7-2-1 | Level9 jpg | daily | 9 | NASA GIBS[1] | no | 200 |
+| `HLS_S30_Nadir_BRDF_Adjusted_Reflectance` | Harmonized Sentinel-2 30 m RGB | Level12 png | ~2–3 d when a granule exists | 12 (~30 m) | NASA GIBS[1] | no | 200 at z=5 NC; 404 on empty granules is normal. Domain from 2015–present |
+| `HLS_L30_Nadir_BRDF_Adjusted_Reflectance` | Harmonized Landsat 30 m | Level12 png | 8–16 d | 12 | NASA GIBS[1] | no | in caps; tile 404 on empty scene |
+| `Landsat_WELD_CorrectedReflectance_TrueColor_Global_Monthly` | Landsat WELD monthly mosaic | Level12 jpg | monthly, **not NRT** | 12 | NASA GIBS[1] | no | 200 |
+| `Landsat_WELD_CorrectedReflectance_TrueColor_Global_Annual` | WELD annual | Level12 jpg | yearly | 12 | NASA GIBS[1] | no | 200 |
+
+### 1.2 Weather satellites (imagery, not NEXRAD)
+
+Sub-daily. Probe with `2026-08-27` **and** `2026-08-27T18:00:00Z` both 200 (nearestValue).[2]
+
+| id | what you see | TMS / ext | cadence | max zoom | already have? | probe |
+|---|---|---|---|---|---|---|
+| `GOES-East_ABI_GeoColor` | GeoColor full disk (Americas, Atlantic, Hormuz west edge) | Level7 png | ~10 min | 7 | no | 200 |
+| `GOES-West_ABI_GeoColor` | GeoColor Pacific / CONUS west | Level7 png | ~10 min | 7 | no | 200 (also 200 over CA with ISO time) |
+| `GOES-East_ABI_Band2_Red_Visible_1km` | ABI vis | Level7 png | ~10 min | 7 | no | 200 |
+| `GOES-West_ABI_Band2_Red_Visible_1km` | ABI vis | Level7 png | ~10 min | 7 | no | 200 |
+| `GOES-East_ABI_Band13_Clean_Infrared` | Clean IR window | Level6 png | ~10 min | 6 | no | 200 |
+| `GOES-West_ABI_Band13_Clean_Infrared` | Clean IR | Level6 png | ~10 min | 6 | no | 200 |
+| `GOES-East_ABI_FireTemp` | Fire temperature RGB | Level7 png | ~10 min | 7 | no | 200 |
+| `GOES-West_ABI_FireTemp` | Fire temperature RGB | Level7 png | ~10 min | 7 | no | 404 on NC tile (wrong disk); use west longitudes |
+| `GOES-East_ABI_Dust` | Dust RGB | Level7 png | ~10 min | 7 | no | 200 |
+| `GOES-East_ABI_Air_Mass` | Air mass RGB | Level6 png | ~10 min | 6 | no | 200 |
+| `GOES-West_ABI_Air_Mass` | Air mass RGB | Level6 png | ~10 min | 6 | no | 200 |
+| `Himawari_AHI_Band3_Red_Visible_1km` | Himawari vis (IO / WestPac / Aus) | Level7 png | ~10 min | 7 | no | 200 (ISO time over Japan) |
+| `Himawari_AHI_Band13_Clean_Infrared` | Himawari IR | Level6 png | ~10 min | 6 | no | 200 |
+| `Himawari_AHI_Air_Mass` | Himawari air mass | Level6 png | ~10 min | 6 | no | 200 |
+
+**Meteosat is not in GIBS.** Use section 2 EUMETView.
+
+### 1.3 SAR / flood / disturbance (GIBS tiles — skip TiTiler)
+
+| id | what you see | TMS | cadence | max zoom | already have? | probe |
+|---|---|---|---|---|---|---|
+| `OPERA_L2_Radiometric_Terrain_Corrected_SAR_Sentinel-1` | S-1 RTC browse (better than GRD for terrain) | Level12 png | scene-based from 2025-01 | 12 | **no** (you have GRD COGs, not RTC tiles) | 200 at z=5; domain 2025-01-10/… |
+| `OPERA_L3_DIST-ALERT-HLS_Color_Index` | Vegetation disturbance / clearing alert | Level12 png | ~2–3 d | 12 | no | 200 |
+| `OPERA_L3_DIST-ANN-HLS_Color_Index` | Annual DIST | Level12 png | yearly | 12 | no | in caps |
+| `OPERA_L3_Dynamic_Surface_Water_Extent-HLS` | Surface water / flood (HLS, 30 m) | Level12 png | ~2–3 d | 12 | no | 200 at z=5 |
+| `OPERA_L3_Dynamic_Surface_Water_Extent-Sentinel-1` | Surface water from S-1 (clouds irrelevant) | Level12 png | S-1 revisit | 12 | no | 200 at z=5 |
+| `NISAR_L2_Geocoded_Polarimetric_Covariance` | NISAR early browse | Level13 png | when downlinked | 13 | no | 200 (layer exists; coverage still sparse) |
+| `SMAP_L4_Analyzed_Surface_Soil_Moisture` | Soil moisture | Level6 png | daily | 6 | no | 200 |
+| `SMAP_L3_Active_Sigma0_VV` | SMAP radar σ0 | Level6 png | 2–3 d | 6 | no | in caps (SMAP radar died 2015 — historical) |
+
+No ICEYE / Capella / Umbra / ALOS PALSAR **daily** layers in this GIBS 3857 dump.[4] ALOS shows up as mosaics on Planetary Computer (section 3).
+
+### 1.4 Thermal / fire / volcano
+
+Sparse PNG overlays: **empty tiles 404**. Capabilities + DescribeDomains still 200. Frontend must tolerate 404 (Leaflet does).
+
+| id | what you see | TMS | cadence | max zoom | already have? | probe |
+|---|---|---|---|---|---|---|
+| `VIIRS_SNPP_Thermal_Anomalies_375m_All` | 375 m hotspots | **Level8** png (not 9) | daily | 8 | **yes**, but wired as Level9 | Domain 200; many tiles 404 |
+| `VIIRS_NOAA20_Thermal_Anomalies_375m_All` | NOAA-20 hotspots | Level8 png | daily | 8 | no | Domain 200 (`2020-01-01/…` through at least 2025-09); tiles 404 if no fire in tile |
+| `VIIRS_NOAA21_Thermal_Anomalies_375m_All` | NOAA-21 hotspots | Level8 png | daily | 8 | no | same |
+| `VIIRS_*_Thermal_Anomalies_375m_{Day,Night}` | day/night split | Level8 png | daily | 8 | no | in caps |
+| `MODIS_{Terra,Aqua,Combined}_Thermal_Anomalies_All` | 1 km MODIS fire | Level7 png | daily | 7 | no | in caps |
+| `GOES-East_ABI_FireTemp` | geostationary fire RGB | Level7 png | ~10 min | 7 | no | 200 |
+
+Also keep FIRMS CSV — points beat raster for click/query.
+
+### 1.5 Night lights (beyond current DNB ENCC)
+
+| id | what you see | TMS / ext | cadence | max zoom | already have? | probe |
+|---|---|---|---|---|---|---|
+| `VIIRS_NOAA20_DayNightBand` | NOAA-20 DNB | Level7 png | daily | 7 | no | 200 |
+| `VIIRS_NOAA21_DayNightBand` | NOAA-21 DNB | Level7 png | daily | 7 | no | 200 |
+| `VIIRS_NOAA20_DayNightBand_At_Sensor_Radiance` | radiance, not ENCC | Level8 png | daily | 8 | no | 200 |
+| `VIIRS_SNPP_DayNightBand_At_Sensor_Radiance` | SNPP radiance | Level8 png | daily | 8 | no | 200 |
+| `VIIRS_NOAA20_DayNightBand_AtSensor_M15` | DNB+M15 composite jpg | Level8 jpg | daily | 8 | no | 200 |
+| `VIIRS_Night_Lights` | Black-marble-style annual-ish | Level8 png | time-dim | 8 | no | 200 on 2026-08-27 mosaic date |
+| `VIIRS_CityLights_2012` | Static 2012 city lights | Level8 jpg | **static** (`has_time=false`) | 8 | no | 200 |
+
+`VIIRS_Black_Marble` and `VIIRS_NOAA20_DayNightBand_ENCC` are in caps; ENCC-NOAA20 returned HTTP 400 on the Level8 template we tried — do not ship until DescribeDomains + a known-good date are wired. SNPP ENCC stays as the current layer.
+
+### 1.6 Ocean
+
+| id | what you see | TMS | cadence | max zoom | probe |
+|---|---|---|---|---|---|
+| `GHRSST_L4_MUR_Sea_Surface_Temperature` | 1 km MUR SST | Level7 png | daily | 7 | 200 |
+| `GHRSST_L4_MUR_Sea_Surface_Temperature_Anomalies` | SST anomaly | Level7 png | daily | 7 | in caps |
+| `MODIS_Aqua_L3_SST_MidIR_4km_Night_Daily` | MODIS SST | Level6 png | daily | 6 | 200 |
+| `MODIS_Aqua_L2_Chlorophyll_A` | Aqua chl-a | Level7 png | daily | 7 | 200 |
+| `VIIRS_SNPP_L2_Chlorophyll_A` | VIIRS chl-a | Level7 png | daily | 7 | 200 |
+| `VIIRS_NOAA20_Chlorophyll_a` | NOAA-20 chl-a | Level7 png | daily | 7 | 200 |
+| `OCI_PACE_Chlorophyll_a` | PACE OCI chl-a | Level7 png | daily | 7 | 200 |
+| `S3A_OLCI_Chlorophyll_a` | Sentinel-3A OLCI chl-a | Level7 png | daily | 7 | 200 |
+| `S3B_OLCI_Chlorophyll_a` | Sentinel-3B OLCI | Level7 png | daily | 7 | in caps |
+| `MODIS_Terra_Sea_Ice` | sea ice | Level7 png | daily | 7 | 200 |
+| `GHRSST_L4_MUR_Sea_Ice_Concentration` | MUR ice | Level7 png | daily | 7 | in caps |
+
+No dedicated “SAR oil slick” GIBS layer in the 3857 dump. Closest: OPERA RTC / DSWx-S1 + existing S-1 GRD TiTiler.
+
+### 1.7 Vegetation / burn / flood / precip / atm
+
+| id | what you see | TMS | cadence | max zoom | probe |
+|---|---|---|---|---|---|
+| `MODIS_Terra_NDVI_8Day` | NDVI | Level9 png | 8-day | 9 | 200 |
+| `MODIS_Terra_L3_NDVI_16Day` | NDVI 16-day | Level9 png | 16-day | 9 | 200 |
+| `IMERG_Precipitation_Rate` | GPM IMERG rain | Level6 png | sub-daily | 6 | 200 |
+| `MODIS_Terra_Aerosol` | AOD | Level6 png | daily | 6 | 200 |
+| `MODIS_Terra_Land_Surface_Temp_Day` | LST | Level7 png | daily | 7 | 200 |
+| `VIIRS_SNPP_Land_Surface_Temp_Day` | VIIRS LST | Level7 png | daily | 7 | 200 |
+| `AIRS_L3_Carbon_Monoxide_500hPa_Volume_Mixing_Ratio_Daily_Night` | CO (fires, industry) | Level6 png | daily | 6 | 200 |
+| `OMI_NO2` / `OMI_Aerosol_Index` | NO2 / smoke index | Level6 png | daily | 6 | OMI AI 200; several OMPS 200 |
+| `MODIS_Water_Mask` | static water mask | Level9 png | static | 9 | 200 |
+
+MODIS burned-area monthly (`MCD64` / `MODIS_Combined_L3_Burned_Area_Monthly`) is in caps; our dated tile 400’d — wire only after a DescribeDomains date hits 200.
+
+### 1.8 DEM (satellite-derived, tileable)
+
+| id | what you see | TMS / ext | cadence | max zoom | probe |
+|---|---|---|---|---|---|
+| `SRTM_Color_Index` | SRTM elevation color | Level12 png | static | 12 | 200 |
+| `ASTER_GDEM_Color_Index` | ASTER GDEM color | Level12 png | static | 12 | 200 |
+| `ASTER_GDEM_Color_Shaded_Relief` | ASTER hillshade | Level12 jpg | static | 12 | 200 |
+| `ASTER_GDEM_Greyscale_Shaded_Relief` | grey hillshade | Level12 jpg | static | 12 | in caps |
+| `GEDI_ISS_L3_Canopy_Height_Mean_RH100_201904-202303` | GEDI canopy height | Level7 png | static epoch | 7 | related GEDI biomass 200 |
+
+Blue Marble shaded relief is **already** the basemap — these are extra.
+
+---
+
+## 2. Other XYZ / WMTS / WMS (no key)
+
+Ranked after GIBS for signal/$; still free.
+
+| id | what you see | provider | URL pattern | key? | CORS | cadence | max zoom / res | license / attribution | Pi fit | already have? | probe 2026-08-29 |
+|---|---|---|---|---|---|---|---|---|---|---|---|
+| `iem_goes_east_conus_ch02` | GOES-East CONUS ABI ch02 vis, **latest** | Iowa State IEM | `https://mesonet.agron.iastate.edu/cache/tile.py/1.0.0/goes_east_conus_ch02/{z}/{x}/{y}.png` | no | `*` | ~5 min cache header | TMS; vis ~1 km | Cite IEM / NOAA GOES[6][7] | **browser** (same stack as NEXRAD) | no | 200 image/png `*` |
+| `iem_goes_east_conus_ch13` | GOES-East CONUS IR ch13 | IEM | `…/goes_east_conus_ch13/{z}/{x}/{y}.png` | no | `*` | ~5 min | IR ~2 km | IEM[6] | browser | no | 200 |
+| `iem_goes_east_fulldisk_ch02` | GOES-East full disk vis | IEM | `…/goes_east_fulldisk_ch02/{z}/{x}/{y}.png` | no | `*` | ~5 min | full disk | IEM[6] | browser | no | 200 |
+| `iem_goes_west_conus_ch02` | GOES-West CONUS vis | IEM | `…/goes_west_conus_ch02/{z}/{x}/{y}.png` | no | `*` | ~5 min | | IEM[6] | browser | no | 200 |
+| `iem_goes_vis_1km` | Legacy name → GOES-East vis | IEM | `…/goes-vis-1km/{z}/{x}/{y}.png` | no | `*` | ~5 min | | IEM[6] | browser | no | 200 |
+| IEM GOES template | Any bird/sector/channel | IEM | `goes_{east\|west}_{fulldisk\|conus\|mesoscale-1\|mesoscale-2\|alaska\|puertorico}_ch{01–16}`[6] | no | `*` | NRT | 16 ABI bands | IEM[6] | browser | no | template documented; ch02/ch13 probed |
+| `eumet_msg_natural` | Meteosat natural color | EUMETSAT EUMETView GeoServer | WMS `https://view.eumetsat.int/geoserver/ows` layer `msg_fes:rgb_natural` EPSG:3857 GetMap | no | `*` | NRT | SEVIRI ~3 km | EUMETSAT viz; cite EUMETSAT[18] | **browser `L.tileLayer.wms`** (not XYZ). Caps 200, 165 layer names | no | GetMap 200 image/png `*` |
+| `eumet_msg_ir108` | Meteosat IR 10.8 | EUMETView | WMS `msg_fes:ir108` | no | `*` | NRT | | EUMETSAT | browser WMS | no | name in caps |
+| `eumet_msg_fire` | Meteosat fire | EUMETView | WMS `msg_fes:fire` | no | `*` | NRT | | EUMETSAT | browser WMS | no | name in caps |
+| `eumet_mtg_ir105` | MTG-I IR | EUMETView | WMS `mtg_fd:ir105_hrfi` | no | `*` | NRT | FCI | EUMETSAT | browser WMS | no | name in caps |
+| `eumet_s3_olci_rgb` | S3 OLCI RGB mosaic | EUMETView | WMS `copernicus:daily_sentinel3ab_olci_l1_rgb_fulres` | no | `*` | daily | OLCI | Copernicus/EUMETSAT | browser WMS | no | name in caps |
+| `eumet_s3_chl` | S3 chl-a | EUMETView | WMS `copernicus:daily_sentinel3ab_olci_l2_chl_fullres` | no | `*` | daily | | Copernicus | browser WMS | no | name in caps |
+| `gfw_glad_s2` | GLAD Sentinel-2 deforestation alerts | GFW tile cache | `https://tiles.globalforestwatch.org/umd_glad_sentinel2_alerts/latest/default/{z}/{x}/{y}.png` | no | `*` **if `Origin` header** (null without it) | ~daily | raster z 0–22 documented[15] | WRI/UMD; cite GFW | **browser** (Leaflet sends Origin) | no | 200 image/png; with Origin → CORS `*` |
+| `gfw_integrated` | Integrated deforestation alerts | GFW | `https://tiles.globalforestwatch.org/gfw_integrated_alerts/latest/default/{z}/{x}/{y}.png` | no | `*` + Origin | ~daily | | GFW | browser | no | 200 |
+| `gfw_tcl` | UMD tree-cover loss | GFW | `https://tiles.globalforestwatch.org/umd_tree_cover_loss/latest/tcd_30/{z}/{x}/{y}.png` | no | (same host) | annual | | GFW/UMD | browser | no | 200 |
+| `star_goes19_fd_geocolor` | GOES-19 full-disk GeoColor **JPEG** (not XYZ) | NOAA NESDIS STAR CDN | `https://cdn.star.nesdis.noaa.gov/GOES19/ABI/FD/GEOCOLOR/latest.jpg` also `…/CONUS/GEOCOLOR/latest.jpg` | no | `*` | minutes | full-disk / CONUS image | NOAA | **not a map layer** — optional lightbox. Do not tile-proxy | no | 200 jpeg `*`[21] |
+| `star_goes18_fd_geocolor` | GOES-18 FD GeoColor JPEG | STAR | `https://cdn.star.nesdis.noaa.gov/GOES18/ABI/FD/GEOCOLOR/latest.jpg` | no | `*` | minutes | | NOAA | lightbox only | no | 200 |
+
+IEM JSON `…/GOES/conus/channel02/GOES-16_C02.json` is **stale** (`generated_at` 2025-04-07) but the **tile names still 200**. Prefer GIBS GeoColor when you need a time slider; prefer IEM when you want “whatever is latest” with zero time plumbing.[6][7]
+
+### 2.x Works but **not** browser-direct (no CORS)
+
+| id | what you see | URL | CORS | Pi fit | probe |
+|---|---|---|---|---|---|
+| RAMMB/CIRA SLIDER GeoColor tiles | GOES-19 / Himawari / JPSS loops, ~10 min | Times: `https://rammb-slider.cira.colostate.edu/data/json/goes-19/full_disk/geocolor/latest_times.json` (`timestamps_int`). Tile: `https://rammb-slider.cira.colostate.edu/data/imagery/{YYYY}/{MM}/{DD}/goes-19---full_disk/geocolor/{ts}/{zz}/{yyy}_{xxx}.png` e.g. `…/2026/08/28/goes-19---full_disk/geocolor/20260828225021/00/000_000.png`. Himawari times JSON also 200. | **none** | **Do not proxy tiles through the Pi.** Bookmark / deep-link SLIDER instead.[19][26] | times 200; tile 200 png; CORS null |
+| NICT Himawari-8 Real-time Web | 10-min full disk PNG grid | `https://himawari8.nict.go.jp/img/D531106/latest.json` then `https://himawari8.nict.go.jp/img/D531106/2d/550/{YYYY}/{MM}/{DD}/{HHMMSS}_{x}_{y}.png` | **none** | same — no Pi proxy | latest.json 200; tile 200 png; CORS null[22] |
+| OpenAerialMap | Per-scene TMS of open UAV/sat | `https://api.openaerialmap.org/meta` → `properties.tms` | CORS **only** `https://map.openaerialmap.org` | not usable from the dashboard origin without a proxy; opportunistic, not a global basemap[23][25] | meta 200 |
+| USGS LandsatLook STAC | Landsat C2 STAC | `https://landsatlook.usgs.gov/stac-server` | CORS locked to `https://landsatlook.usgs.gov/stac-server` | backend-only if ever; prefer Earth Search / PC / GIBS HLS[24] | collections + search 200 |
+| NOAA CoastWatch ERDDAP WMS (`jplMURSST41`) | MUR SST WMS | `https://coastwatch.pfeg.noaa.gov/erddap/wms/jplMURSST41/request` | mixed | **flaky**: GetCapabilities 200 earlier, **503** on later GetMap/GetCapabilities. Prefer GIBS MUR | 503 on 2nd pass |
+| RainViewer `satellite.infrared` | would be IR sat frames | `https://api.rainviewer.com/public/weather-maps.json` | `*` | **empty list** (`"infrared": []`) at probe time — do not ship. Radar path already in product[20] | JSON 200, satellite IR empty |
+
+---
+
+## 3. STAC / COG (TiTiler, same pattern as Sentinel-1)
+
+Do **not** stream COGs through the Pi for a basemap. Viewport bbox + short datetime window + SAS/public HTTPS + existing TiTiler. Prefer GIBS HLS / OPERA tiles (section 1) when a browse PNG is enough.
+
+| id | what you see | provider | STAC | key? | CORS | cadence | res | license | Pi fit | already have? | probe |
+|---|---|---|---|---|---|---|---|---|---|---|---|
+| `sentinel-2-l2a` (Earth Search) | S2 L2A COGs, public HTTPS | Element 84 / AWS Open Data | `https://earth-search.aws.element84.com/v1` collections: `sentinel-2-l2a`, `sentinel-2-c1-l2a`, `sentinel-2-l1c`, `sentinel-2-pre-c1-l2a`, `sentinel-1-grd`, `landsat-c2-l2`, `naip`, `cop-dem-glo-30`, `cop-dem-glo-90`[8][9] | no | STAC `*` | S2 ~5 d | 10 m | Copernicus open; AWS public bucket HTTPS (not requester-pays for these COGs)[9] | **backend same as S-1**: search → TCI/visual COG → TiTiler. Live item `S2A_40RCP_20260827_0_L2A` href `https://sentinel-cogs.s3.us-west-2.amazonaws.com/…/TCI.tif` | no | collections + search 200 `*` |
+| `sentinel-2-l2a` (Planetary Computer) | same S2 on Azure | Microsoft PC | `https://planetarycomputer.microsoft.com/api/stac/v1/collections/sentinel-2-l2a` | SAS token (unsigned search works) | STAC `*` | ~5 d | 10 m | Copernicus; Azure blob needs SAS like current S-1 | backend same as S-1 | no | collection + search 200 `*` (136 collections listed)[10][11] |
+| `sentinel-1-rtc` | S-1 IW RTC γ0 COGs | PC / Catalyst | `/collections/sentinel-1-rtc` | **PC account required to retrieve SAS** for RTC blobs[12] | STAC `*` | IW land | ~10 m pixels | **CC BY 4.0**[12] | backend same as S-1 **plus** PC login for SAS. Prefer GIBS OPERA RTC tiles if browse is enough | no | collection 200; search item `S1D_IW_GRDH_…_rtc` assets `vv,vh,tilejson,rendered_preview` |
+| `sentinel-1-grd` (PC) | GRD | PC | `/collections/sentinel-1-grd` | SAS | `*` | 6–12 d | | Copernicus | **already have** | yes | 200 |
+| `sentinel-1-grd` (Earth Search) | GRD on AWS | E84 | `/collections/sentinel-1-grd` | requester-pays **s3://** URLs per E84 README[9] | `*` | | | Copernicus | worse than PC for the Pi (AWS creds) | no | collection 200 |
+| `landsat-c2-l2` | Landsat 8/9 SR | E84 + PC | both catalogs | no / SAS | `*` | 8–16 d | 30 m | USGS public | backend TiTiler; or just use GIBS HLS | no | both 200 |
+| `hls2-s30` / `hls2-l30` | HLS v2 COGs | PC | `/collections/hls2-s30`, `hls2-l30` | SAS | `*` | 2–3 d | 30 m | NASA | prefer GIBS HLS tiles | no | collections 200 |
+| `goes-cmi` | GOES Cloud & Moisture Imagery COGs | PC | `/collections/goes-cmi` | SAS | `*` | 5–10 min | ABI | NOAA | **overkill vs GIBS/IEM tiles** | no | collection 200 |
+| `modis-14A1-061` / `modis-64A1-061` | MODIS fire / burned area | PC | `/collections/modis-14A1-061`, `modis-64A1-061` | SAS | `*` | daily / monthly | 1 km / 500 m | NASA | prefer GIBS fire tiles + FIRMS | no | 200 |
+| `alos-palsar-mosaic` / `alos-fnf-mosaic` | ALOS PALSAR yearly mosaic / forest-nonforest | PC | those collection ids | SAS | `*` | **annual** | 25 m | JAXA (check collection) | backend mosaic, not live SAR | no | 200 |
+| `nasadem` / `cop-dem-glo-30` | DEM COGs | PC + E84 | `nasadem`, `cop-dem-glo-30` | public / SAS | `*` | static | 30 m | NASA / Copernicus | prefer GIBS SRTM/ASTER tiles | no | 200 |
+| `naip` | USDA NAIP aerial (CONUS) | E84 + PC | `naip` | no | `*` | leaf-on, not NRT | ~0.6 m | USDA | CONUS only; huge. Optional TiTiler | no | 200 |
+| `io-lulc-annual-v02` | 10 m land cover | PC | `io-lulc-annual-v02` | SAS | `*` | annual | 10 m | various | overlay, not sat photo | no | 200 |
+| CDSE `sentinel-2-l2a` / `sentinel-1-grd` | Copernicus Dataspace STAC | ESA CDSE | `https://stac.dataspace.copernicus.eu/v1/collections/sentinel-2-l2a` (lowercase ids work; `SENTINEL-2` 404) | **free account** for many assets | **CORS none** | same as ESA | | Copernicus | backend only; Earth Search/PC easier on a Pi | no | collection 200, CORS null. List endpoint is paginated (first page was CLMS burned-area COGs)[14] |
+
+PC catalog also has Sentinel-3 OLCI/SLSTR NetCDF, Sentinel-5P, GOES-GLM — NetCDF is a bad TiTiler citizen; use GIBS/EUMETView for those.
+
+---
+
+## 4. Free-account / license-gated (no card this week)
+
+| id | note | why not a dropdown tomorrow |
+|---|---|---|
+| Microsoft PC SAS for RTC (and some blobs) | “A Planetary Computer account is required to retrieve SAS tokens to read the RTC data.”[12] | Search is open; **read** needs an account. GRD path you already have may not need this. |
+| Copernicus Data Space (`stac.dataspace.copernicus.eu`) | STAC search 200 without cookie; **no CORS**; downloads often need a free CDSE login | Use Earth Search/PC unless you want official ESA provenance |
+| JAXA P-Tree / Himawari Monitor | Himawari standard data, account | NICT/GIBS already cover browse |
+| EUMETSAT Data Store | full MTG/MSG granules | EUMETView WMS is the browse path |
+| FIRMS map key | already in product | add `VIIRS_NOAA20_NRT` / `VIIRS_NOAA21_NRT` before SNPP sunset |
+| USGS ERS / EarthExplorer | Landsat/ASTER download login | GIBS HLS + Earth Search cover browse |
+| Planet Tropical Forest Observatory | paid successor after NICFI | see skip |
+
+---
+
+## 5. Skip / costs money / dead
+
+| id | why |
+|---|---|
+| **NICFI / Planet tropical mosaics (free)** | Free NICFI phase **ended 1 Apr 2025**. Removed from GFW and Collect Earth Online. Successor is Planet **Tropical Forest Observatory (subscription)** or a future NICFI re-compete.[16][17] PC collections `planet-nicfi-analytic` / `planet-nicfi-visual` still exist but assets are **RFP winners only** + proprietary PLA.[13] |
+| Sentinel Hub (paid tiers) | billed processing units |
+| Google Earth Engine | billing project |
+| Maxar / Planet commercial | $ |
+| ICEYE commercial | no free global tile/STAC found this pass |
+| Umbra / Capella / Maxar **open data** STAC | catalogs 200 (`maxar-opendata`, `umbra-open-data-catalog`) but **disaster events only**, not a standing layer |
+| Esri World Imagery / Clarity | tiles 200 CORS `*` — **ToS not a free basemap we should wrap** |
+| Mapbox / Google satellite | key + ToS |
+| GEE Dynamic World / NICFI in EE | EE billing |
+| `nowcoast.noaa.gov` | HTTP **403** |
+| FIRMS WMS (`firms.modaps.eosdis.nasa.gov/wms/…`) | HTTP **404** — use CSV + GIBS |
+| RainViewer satellite IR | payload empty[20] |
+| CoastWatch ERDDAP | 503 at probe; GIBS MUR replaces SST |
+| Proxying RAMMB or NICT tiles | no CORS; would soak the home uplink |
+
+---
+
+## Implementation notes for builders
+
+1. **GIBS dropdown:** reuse `MAP_LAYERS` in `app/gibs_map.py`. New rows are `{id, title, tms, format, has_time, max_zoom}`. Time-domain fetch already exists.
+2. **SNPP sunset:** NOAA-20/21 true color, DNB, thermal, FIRMS datasets first. SNPP true color can stay as fallback until 2026-11-01.
+3. **Fix thermal TMS:** caps say `GoogleMapsCompatible_Level8` for `VIIRS_*_Thermal_Anomalies_375m_*`. Level9 GetTile is HTTP 400 XML.
+4. **GOES time:** either GIBS `{time}` ISO + existing date slider, or IEM “latest” XYZ with no time (simpler, CONUS/FD only).
+5. **HLS / OPERA:** empty granules 404 — same as “today’s MODIS isn’t ingested yet”. Clamp latest date via DescribeDomains like current daily mosaics.
+6. **Do not add TiTiler S2 as a global basemap.** 10 m COGs will thrash the Pi. GIBS HLS Level12 is the browse path; Earth Search TCI is a “inspect this viewport” action like S-1.
+7. **EUMETView:** `L.tileLayer.wms` against `https://view.eumetsat.int/geoserver/ows`, layers `msg_fes:rgb_natural` / `msg_fes:ir108`. Caps CORS `*`.
+8. **GFW:** send browser Origin (Leaflet does). No key.
+9. **Attribution strings:** NASA GIBS acknowledgment[1]; IEM; EUMETSAT; GFW/UMD; NOAA STAR.
+
+### Probe stats (this run)
+
+- GIBS WMTS caps: 5 796 177 bytes, CORS `*`, 1315 layers.[4]
+- Curated GIBS GetTile: **78/90 HTTP 200** first batch; extra GOES/DNB/HLS/OPERA/ocean 200 as tabulated.
+- Earth Search collections (complete list): `sentinel-2-pre-c1-l2a`, `cop-dem-glo-30`, `naip`, `cop-dem-glo-90`, `landsat-c2-l2`, `sentinel-2-l2a`, `sentinel-2-l1c`, `sentinel-2-c1-l2a`, `sentinel-1-grd`.[8]
+- Planetary Computer: 136 collections; S-1 RTC CC-BY-4.0.[11][12]
+
+Raw probe JSON lives next to this file in the kanban workspace (`gibs_probes.json`, `wave2_probes.json`, `wave3_probes.json`, `gibs_all_layers.json`).
+
+## Sources
+
+[1] https://nasa-gibs.github.io/gibs-api-docs
+[2] https://nasa-gibs.github.io/gibs-api-docs/access-basics
+[3] https://nasa-gibs.github.io/gibs-api-docs/available-visualizations
+[4] https://gibs.earthdata.nasa.gov/wmts/epsg3857/best/1.0.0/WMTSCapabilities.xml
+[5] https://worldview.earthdata.nasa.gov
+[6] https://mesonet.agron.iastate.edu/ogc
+[7] https://mesonet.agron.iastate.edu/GIS/goes.phtml
+[8] https://earth-search.aws.element84.com/v1/collections
+[9] https://github.com/Element84/earth-search
+[10] https://planetarycomputer.microsoft.com/catalog
+[11] https://planetarycomputer.microsoft.com/api/stac/v1/collections
+[12] https://planetarycomputer.microsoft.com/dataset/sentinel-1-rtc
+[13] https://planetarycomputer.microsoft.com/dataset/planet-nicfi-analytic
+[14] https://stac.dataspace.copernicus.eu/v1/collections
+[15] https://tiles.globalforestwatch.org
+[16] https://www.collect.earth/planet-imagery-via-nicfi-is-no-longer-available-on-ceo
+[17] https://www.globalforestwatch.org/blog/data-and-tools/planet-imagery-changes-gfw
+[18] https://view.eumetsat.int/geoserver/ows?service=WMS&request=GetCapabilities
+[19] https://rammb-slider.cira.colostate.edu
+[20] https://api.rainviewer.com/public/weather-maps.json
+[21] https://cdn.star.nesdis.noaa.gov/GOES19/ABI/FD/GEOCOLOR/latest.jpg
+[22] https://himawari8.nict.go.jp
+[23] https://api.openaerialmap.org/meta?limit=1
+[24] https://landsatlook.usgs.gov/stac-server/collections
+[25] https://openaerialmap.org
+[26] https://bellingcat.gitbook.io/toolkit/more/all-tools/rammb-slider
+[27] https://www.earthdata.nasa.gov/engage/open-data-services-software/earthdata-developer-portal/gibs-api

news/scraper/newsScraper/feed_util.py

@@ -0,0 +1,32 @@
+"""Feed helpers shared by the news spider (no Scrapy import)."""
+
+from __future__ import annotations
+
+import datetime
+from email.utils import parsedate_to_datetime
+from urllib.parse import urlparse
+
+AUDIO_EXT = (".mp3", ".m4a", ".ogg", ".wav", ".aac", ".flac", ".opus")
+
+
+def is_audio_url(url: str) -> bool:
+    path = urlparse(url or "").path.lower()
+    return any(path.endswith(ext) for ext in AUDIO_EXT)
+
+
+def article_timestamp(pub_date: str | None) -> datetime.datetime:
+    """Prefer the feed's pubDate/published; fall back to now (UTC)."""
+    if pub_date:
+        try:
+            return parsedate_to_datetime(pub_date).astimezone(datetime.timezone.utc)
+        except (TypeError, ValueError, IndexError):
+            pass
+        try:
+            raw = pub_date.replace("Z", "+00:00")
+            ts = datetime.datetime.fromisoformat(raw)
+            if ts.tzinfo is None:
+                ts = ts.replace(tzinfo=datetime.timezone.utc)
+            return ts
+        except ValueError:
+            pass
+    return datetime.datetime.now(datetime.timezone.utc)

news/scraper/newsScraper/pipelines.py

@@ -3,7 +3,7 @@
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
 
-import logging 
+import logging
 import psycopg2
 import os
 from scrapy.exceptions import DropItem
@@ -53,8 +53,10 @@ class PostgresPipeline:
         self.connection.commit()
 
     def process_item(self, item, spider):
-        if item ['url'] in self.seen_urls:
-            raise DropItem()
+        url = item['url']
+        if url in self.seen_urls:
+            raise DropItem(f"Duplicate URL (in-memory): {url}")
+        self.seen_urls.add(url)
         try:
             self.cur.execute("""
                 INSERT INTO articles (title, url, content, domain, timestamp)
@@ -68,15 +70,16 @@ class PostgresPipeline:
                 item['timestamp']
             ))
             if self.cur.rowcount == 0:
-                e = DropItem("Duplicate URL (database conflict)")
-                e.log_level = logging.DEBUG
-                raise e 
+                raise DropItem(f"Duplicate URL (database): {url}")
             self.connection.commit()
-            return item 
+            return item
+        except DropItem:
+            self.connection.rollback()
+            raise
         except Exception as e:
             spider.logger.error(f"Error saving to Postgres: {e}")
             self.connection.rollback()
-            raise 
+            raise
 
     def close_spider(self, spider):
         self.cur.close()
@@ -88,5 +91,3 @@ from itemadapter import ItemAdapter
 class NewsscraperPipeline:
     def process_item(self, item, spider):
         return item
-
-

news/scraper/newsScraper/spiders/news_spider.py

@@ -4,11 +4,13 @@ from urllib.parse import urljoin, urlparse
 import datetime
 import re
 
+from newsScraper.feed_util import article_timestamp, is_audio_url
+
 
 class NewsRSSSpider(Spider):
     """Crawl the curated news sources in urls.txt and extract articles.
 
-    urls.txt contains 257 news HOMEPAGES (not feed URLs), so this spider
+    urls.txt contains curated news HOMEPAGES and RSS/Atom feeds, so this spider
     implements feed autodiscovery: it fetches each start URL, finds the
     RSS/Atom feed link (`<link rel="alternate" type="application/rss+xml">`
     or a visible /rss|/feed link), follows it, and then follows each feed
@@ -87,6 +89,8 @@ class NewsRSSSpider(Spider):
                 or node.xpath('updated/text()').get()
             )
             if link:
+                if is_audio_url(link):
+                    continue
                 yield scrapy.Request(
                     link,
                     callback=self.parse_article,
@@ -112,5 +116,5 @@ class NewsRSSSpider(Spider):
                 'url': response.url,
                 'text': pure_text,
                 'domain': urlparse(response.url).netloc,
-                'timestamp': datetime.datetime.now().isoformat()
+                'timestamp': article_timestamp(response.meta.get('date')).isoformat()
             }

news/scraper/run_news_scraper.py

@@ -1,18 +1,12 @@
 #!/usr/bin/env python3
-"""Scheduler loop for the news scraper — hourly scrape at minute :00.
+"""Scheduler loop for the news scraper — crawl continuously.
 
-Replaces the k8s CronJob (`0 * * * *`) with an in-compose loop so the whole
-news pipeline lives inside docker-compose. Each iteration:
-
-  1. (optionally, on first boot) runs the Scrapy crawl once to seed data fast
-  2. sleeps until the next :NEWS_SCRAPE_MINUTE wall-clock boundary
-
-Because the loop is serial, a crawl that overruns its hour simply delays the
-next run to the following boundary — two crawls never overlap.
+As soon as one Scrapy pass finishes, wait NEWS_SCRAPE_INTERVAL_S seconds
+and start the next. Two crawls never overlap (the loop is serial).
 
 Env (all optional, 12-factor):
-  NEWS_SCRAPE_MINUTE        minute of the hour to fire (default 0)
-  NEWS_SCRAPE_RUN_ON_START  "1" to crawl once immediately on boot (default 1)
+  NEWS_SCRAPE_INTERVAL_S    seconds between crawls (default 10)
+  NEWS_SCRAPE_RUN_ON_START  "1" to crawl immediately on boot (default 1)
 """
 
 from __future__ import annotations
@@ -27,19 +21,12 @@ import time
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
 logger = logging.getLogger("news.scraper")
 
-MINUTE = int(os.getenv("NEWS_SCRAPE_MINUTE", "0"))
+INTERVAL_S = max(0, int(os.getenv("NEWS_SCRAPE_INTERVAL_S", "10")))
 RUN_ON_START = os.getenv("NEWS_SCRAPE_RUN_ON_START", "1").lower() in ("1", "true", "yes")
 
 CRAWL_CMD = ["scrapy", "crawl", "articles"]
 
 
-def seconds_until_next(minute: int) -> float:
-    """Seconds until the next occurrence of ``minute`` past the hour (local time)."""
-    now = datetime.datetime.now()
-    nxt = now.replace(minute=minute, second=0, microsecond=0) + datetime.timedelta(hours=1)
-    return (nxt - now).total_seconds()
-
-
 def run_crawl() -> None:
     logger.info("scrape starting at %s", datetime.datetime.now().isoformat(timespec="seconds"))
     try:
@@ -51,15 +38,14 @@ def run_crawl() -> None:
 
 def main() -> None:
     logger.info(
-        "news scraper loop starting (minute=%s, run_on_start=%s)",
-        MINUTE, RUN_ON_START,
+        "news scraper loop starting (interval_s=%s, run_on_start=%s)",
+        INTERVAL_S, RUN_ON_START,
     )
     if RUN_ON_START:
         run_crawl()
     while True:
-        delay = seconds_until_next(MINUTE)
-        logger.info("next scrape at :%02d (in %.0fs)", MINUTE, delay)
-        time.sleep(delay)
+        logger.info("next scrape in %ss", INTERVAL_S)
+        time.sleep(INTERVAL_S)
         run_crawl()
 
 

news/scraper/urls.txt

@@ -1,257 +1,336 @@
-# --- NORTH AMERICA ---
-# USA
-https://www.npr.org
-https://www.pbs.org/newshour
-https://www.usatoday.com
-https://www.cbsnews.com
-https://www.nbcnews.com
-
-# Canada
-https://www.cbc.ca/news
-https://www.ctvnews.ca
-https://globalnews.ca
-https://nationalpost.com
-https://www.thestar.com
-
-# Mexico
-https://www.eluniversal.com.mx
-https://www.milenio.com
-https://www.jornada.com.mx
-https://www.excelsior.com.mx
-https://aristeguinoticias.com
-
-# --- SOUTH AMERICA ---
-# Brazil
-https://g1.globo.com
-https://www.uol.com.br
-https://agenciabrasil.ebc.com.br
-https://www.metropoles.com
-https://www.terra.com.br/noticias
-
-# Argentina
-https://www.infobae.com
-https://www.clarin.com
-https://www.lanacion.com.ar
-https://www.pagina12.com.ar
-https://www.cronista.com
-
-# Colombia
-https://www.eltiempo.com
-https://www.elespectador.com
-https://www.semana.com
-https://www.bluradio.com
-https://www.rcnradio.com
-
-# --- EUROPE ---
-# United Kingdom
-https://www.bbc.com/news
-https://www.theguardian.com/uk
-https://news.sky.com
-https://www.independent.co.uk
-https://metro.co.uk
-
-# France
-https://www.france24.com/en
-https://www.lefigaro.fr
-https://www.20minutes.fr
-https://www.francetvinfo.fr
-https://www.lemonde.fr
-
-# Germany
-https://www.dw.com/en
-https://www.tagesschau.de
-https://www.spiegel.de
-https://www.zeit.de
-https://www.bild.de
-
-# Spain
-https://elpais.com
-https://www.elmundo.es
-https://www.rtve.es/noticias
-https://www.20minutos.es
-https://www.elconfidencial.com
-
-# Italy
-https://www.ansa.it
-https://www.corriere.it
-https://www.repubblica.it
-https://www.lastampa.it
-https://tg24.sky.it
-
-# Russia (State & Independent mix)
-https://tass.com
-https://www.interfax.ru
-https://www.rt.com
-https://www.themoscowtimes.com
-https://meduza.io/en
-
-# --- ASIA ---
-# China
-https://www.xinhuanet.com/english
-https://www.chinadaily.com.cn
-https://www.globaltimes.cn
-https://www.cgtn.com
-https://www.scmp.com
-
-# India
-https://www.ndtv.com
-https://timesofindia.indiatimes.com
-https://indianexpress.com
-https://www.thehindu.com
-https://www.hindustantimes.com
-
-# Japan
-https://www3.nhk.or.jp/nhkworld
-https://www.japantimes.co.jp
-https://www.asahi.com/ajw
-https://mainichi.jp/english
-https://english.kyodonews.net
-
-# South Korea
-https://en.yna.co.kr
-https://www.koreaherald.com
-https://koreajoongangdaily.joins.com
-https://www.donga.com/en
-https://english.chosun.com
-
-# --- AFRICA ---
-# South Africa
-https://www.news24.com
-https://www.iol.co.za
-https://www.dailymaverick.co.za
-https://www.sabcnews.com
-https://www.timeslive.co.za
-
-# Nigeria
-https://www.vanguardngr.com
-https://punchng.com
-https://dailypost.ng
-https://saharareporters.com
-https://thenationonlineng.net
-
-# --- MIDDLE EAST ---
-# General Region
-https://www.aljazeera.com
-https://english.alarabiya.net
-https://www.timesofisrael.com
-https://www.tehrantimes.com
-https://www.middleeasteye.net
-
-# --- OCEANIA ---
-# Australia
-https://www.abc.net.au/news
-https://www.news.com.au
-https://www.9news.com.au
-https://www.smh.com.au
-https://www.theage.com.au
-# --- USA: MAJOR CITIES & LOCAL ---
-https://www.latimes.com
-https://www.chicagotribune.com
-https://www.sfchronicle.com
-https://www.bostonglobe.com
-https://www.seattletimes.com
-https://www.houstonchronicle.com
-https://www.inquirer.com
-https://www.denverpost.com
-https://www.miamiherald.com
-https://www.dallasnews.com
-https://www.startribune.com
-https://www.detroitnews.com
-https://www.ajc.com
-https://www.nydailynews.com
-https://nypost.com
-https://www.mercurynews.com
-https://www.baltimoresun.com
-https://www.oregonlive.com
-https://www.cleveland.com
-https://www.tampabay.com
-
-# --- EUROPE: LOCAL & INDEPENDENT ---
-https://www.manchestereveningnews.co.uk
-https://www.scotsman.com
-https://www.belfasttelegraph.co.uk
-https://www.irishtimes.com
-https://www.berliner-zeitung.de
-https://www.leparisien.fr
-https://www.corriere.it
-https://www.elperiodico.com
-https://kyivindependent.com
-https://www.pravda.com.ua/en
-https://balkaninsight.com
-https://www.ekathimerini.com
-https://www.swissinfo.ch
-https://www.thelocal.se
-https://www.thelocal.fr
-https://www.thelocal.de
-https://www.novinite.com
-https://www.romania-insider.com
-https://hungarytoday.hu
-https://polandin.com
-
-# --- MIDDLE EAST & CONFLICT ZONES ---
-https://www.haaretz.com
-https://www.jpost.com
-https://www.timesofisrael.com
-https://www.rudaw.net/english
-https://www.kurdistan24.net/en
-https://www.middleeasteye.net
-https://www.al-monitor.com
-https://www.dailysabah.com
-https://www.duvarenglish.com
-https://english.aawsat.com
-https://www.arabnews.com
-https://www.thenationalnews.com
-https://www.jordantimes.com
-https://www.naharnet.com
-https://www.tehrantimes.com
-
-# --- ASIA: HOTSPOTS & LOCAL ---
-https://www.taipeitimes.com
-https://focustaiwan.tw
-https://hongkongfp.com
-https://www.bangkokpost.com
-https://www.thejakartapost.com
-https://www.straitstimes.com
-https://www.khmertimeskh.com
-https://www.irrawaddy.com
-https://www.myanmarnow.org/en
-https://www.rappler.com
-https://www.philstar.com
-https://english.hani.co.kr
-https://www.japantoday.com
-https://www.caixinglobal.com
-https://thediplomat.com
-
-# --- LATIN AMERICA & AFRICA: LOCAL ---
-https://buenosairesherald.com
-https://riotimesonline.com
-https://mercopress.com
-https://www.elmostrador.cl
-https://www.jornada.com.mx
-https://www.theeastafrican.co.ke
-https://allafrica.com
-https://www.premiumtimesng.com
-https://www.dailytrust.com
-https://www.newtimes.co.rw
-https://www.herald.co.zw
-https://www.namibian.com.na
-https://www.graphic.com.gh
-https://www.thecitizen.co.tz
-https://www.monitor.co.ug
-
-# --- ALTERNATIVE, INVESTIGATIVE & "FRINGE" ---
-https://theintercept.com
-https://www.propublica.org
-https://www.democracynow.org
-https://reason.com
-https://www.motherjones.com
-https://www.vox.com
-https://slate.com
-https://www.axios.com
-https://www.politico.com
-https://www.vice.com
-https://www.bellingcat.com
-https://www.project-syndicate.org
-https://cryptonews.com
-https://www.coindesk.com
-https://techcrunch.com
+https://www.investing.com/rss/news.rss
+https://www.ftchinese.com/rss
+https://www.alwatan.com
+https://albiladpress.com
+https://www.aletihad.ae/
+https://www.albayan.ae
+https://www.aljazeera.com/xml/rss/all.xml
+https://english.alarabiya.net/.mrss/en.xml
+https://english.aawsat.com/home/rss
+https://www.newarab.com/rss
+https://www.skynewsarabia.com/rss/feeds/rss-1.xml
+https://www.thenationalnews.com/arc/outboundfeeds/rss/
+https://www.arabnews.com/rss.xml
+https://gulfnews.com/rss
+https://www.kuwaittimes.com/feed/
+https://www.omanobserver.om/feed/
+https://www.khaleejtimes.com/rss/news
+http://www.akhbar-alkhaleej.com/rss/all
+https://today.lorientleyour.com/rss
+https://www.annahar.com/english/rss
+https://english.almayadeen.net/rss
+https://english.ahram.org.eg/rss/0/Home.aspx
+https://www.dailynewsegypt.com/feed/
+http://www.jordantimes.com/rss
+https://www.alraimedia.com/rss
+https://alghad.com/feed/
+https://nypost.com/feed/
+https://gothamist.com/feed/
+https://www.cityandstateny.com/rss
+https://feeds.nytimes.com/nyt/rss/HomePage
+https://www.thecity.nyc/rss/index.xml
+https://brooklyneagle.com/feed/
+https://www.reutersagency.com/feed/
+https://newsatme.com/api/v1/rss/ap/world
+https://feeds.bbci.co.uk/news/world/rss.xml
+https://rss.dw.com/rdf/rss-en-all
+https://www.france24.com/en/rss
+https://www3.nhk.or.jp/rss/news/shakaitokushu.xml
+https://www.cbc.ca/cctoc/rss/topstories.north
+https://www.defensenews.com/arc/outboundfeeds/rss/
+https://therecord.media/feed
+https://www.cfr.org/rss/newsletters/daily-news-brief
+https://warontherocks.com/feed/
+https://www.thecipherbrief.com/feed
+https://www.foreignaffairs.com/rss.xml
+https://geopoliticalfutures.com/feed
+# --- TACTICAL CYBER & VULNERABILITIES ---
+https://www.bleepingcomputer.com/feed/
+https://www.cisa.gov/cybersecurity-advisory-feeds
+https://krebsonsecurity.com/feed/
+https://thehackernews.com/feeds/posts/default
+https://www.darkreading.com/rss.xml
+https://www.mandiant.com/resources/blog/rss.xml
+https://schneier.com/feed/atom/
+https://www.securityweek.com/feed/
+# --- REGIONAL THREAT LANDSCAPE ---
+https://www.thenationalnews.com/rss/
+https://www.scmp.com/rss/91/feed
+https://www.batimes.com.ar/rss
+https://brazilian.report/feed/
+https://www.khon2.com/feed/
+https://www.staradvertiser.com/feed/
+https://www.westhawaiitoday.com/feed/
+https://mauinow.com/feed/
+https://www.idahofallsidaho.gov/RSSFeed.aspx?ModID=1&CID=All-newsflash.xml
+https://www.eastidahonews.com/feed/
+https://localnews8.com/feed/
+https://www.boisestatepublicradio.org/news.rss
+https://www.illinoistimes.com/springfield/Rss.xml
+https://www.thecentersquare.com/search/?f=rss&t=article&l=20&s=start_time&fulltext=showtext&sd=desc&c%5B%5D=Illinois
+https://chicago.suntimes.com/rss/index.xml
+https://wgntv.com/feed/
+http://feeds.indiana.statenews.net/rss/7b3aa09cdd5d5eac
+https://fox59.com/feed/
+https://www.nwitimes.com/search/?f=rss&t=article&c=news/local&l=50&s=start_time&sd=desc
+https://www.wishtv.com/feed/
+https://www.kcci.com/topstories-rss
+https://www.myiowainfo.com/feed/
+https://feeds.feedburner.com/radioiowanews
+https://www.mississippivalleypublishing.com/search/?f=rss&t=article&c=the_hawk_eye&l=50&s=start_time&sd=desc
+https://www.ksn.com/feed/
+https://www.ksnt.com/feed/
+https://www.hdnews.net/feed/
+https://themercury.com/search/?f=rss&t=article&c=news&l=50&s=start_time&sd=desc
+https://www.wdrb.com/search/?f=rss&t=article&c=news&l=50&s=start_time&sd=desc
+https://www.wtvq.com/feed/
+https://www.wnky.com/feed/
+https://www.wlky.com/topstories-rss
+https://thehayride.com/feed/
+https://wgno.com/feed/
+https://feeds.feedburner.com/wbrz/news
+https://thelensnola.org/feed/
+https://www.pressherald.com/news/feed/
+https://www.centralmaine.com/feed/
+https://www.bangordailynews.com/feed/
+https://www.sunjournal.com/news/feed/
+https://www.wbaltv.com/topstories-rss
+https://www.manisteenews.com/news/feed/Latest-News-Feed-2564.php
+https://www.theoaklandpress.com/feed/
+https://www.macombdaily.com/feed/
+https://www.startribune.com/local/index.rss2
+https://www.wctrib.com/index.rss
+https://www.austindailyherald.com/feed/
+https://helenair.com/search/?f=rss&t=article&l=50&s=start_time&sd=desc
+https://www.ktvq.com/news.rss
+https://mtstandard.com/search/?f=rss&t=article&l=50&s=start_time&sd=desc
+https://www.ketv.com/topstories-rss
+https://nebraskaexaminer.com/feed/
+https://kearneyhub.com/rss
+https://www.wowt.com/rss
+https://thenevadaindependent.com/feed/
+https://www.8newsnow.com/feed/
+https://www.reviewjournal.com/feed/
+https://thisisreno.com/feed/
+https://www.conwaydailysun.com/search/?f=rss&t=article&c=berlin_sun/community/news&l=50&s=start_time&sd=desc
+https://newhampshirebulletin.com/feed/
+https://www.nhgazette.com/feed/
+https://www.nhbr.com/feed/
+https://www.nj.com/arc/outboundfeeds/rss/?outputType=xml
+https://www.njspotlightnews.org/feed/
+https://njmonthly.com/feed/
+https://www.trentonian.com/feed/
+https://www.krqe.com/feed/
+https://www.santafenewmexican.com/search/?f=rss&t=article&l=50&s=start_time&sd=desc
+https://www.easternnewmexiconews.com/rss
+https://www.koat.com/topstories-rss
+https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml
+https://www.thecity.nyc/feed/
+https://www.nbcnewyork.com/?rss=y
+https://www.wral.com/news/rss/48/
+https://www.cbs17.com/news/north-carolina-news/feed/
+https://abc11.com/feed/
+https://myfox8.com/news/feed/
+https://www.kxnet.com/feed/
+https://www.wday.com/feed/
+https://www.jamestownsun.com/index.rss
+https://www.inforum.com/index.rss
+http://rssfeeds.wkyc.com/wkyc/news
+https://theohiostar.com/feed/
+https://feeds.feedblitz.com/wtol/news
+https://www.wcpo.com/news.rss
+https://kfor.com/feed/
+https://oklahomawatch.org/feed/
+https://freepressokc.com/feed/
+https://osagenews.org/feed/
+http://rssfeeds.kgw.com/kgw/local
+https://www.koin.com/feed/
+https://www.bendsource.com/bend/Rss.xml/feed
+https://eugeneweekly.com/feed/
+https://www.wtae.com/topstories-rss
+https://www.montgomerycountypa.gov/RSSFeed.aspx?ModID=76&CID=All-0
+https://www.mainlinemedianews.com/feed/
+https://www.dailylocal.com/feed/
+https://www.wpri.com/feed/
+https://www.abc6.com/feed/
+https://whdh.com/regional/rhode-island/feed/
+https://warwickpost.com/feed/
+https://www.wyff4.com/topstories-rss
+https://www.wispolitics.com/feed/
+https://wiseye.org/feed/
+https://wisconsinexaminer.com/feed/
+https://trib.com/search/?f=rss&t=article&c=news/state-and-regional&l=50&s=start_time&sd=desc
+https://wyofile.com/feed/
+https://www.wyomingnews.com/search/?f=rss&t=article&c=news&l=50&s=start_time&sd=desc
+https://www.wyodaily.com/rss
+https://www.wnct.com/news/north-carolina/feed/
+https://www.usnews.com/rss/news/north-carolina
+https://indyweek.com/feed/
+https://portcitydaily.com/feed/
+https://www.theguardian.com/uk/rss
+https://feeds.bbci.co.uk/news/england/rss.xml
+https://www.lemonde.fr/rss/une.xml
+https://www.ansa.it/sito/notizie/rss.xml
+https://www.ilgiornale.it/feed
+https://www.larepublica.it/rss/homepage/rss2.xml
+https://www.sueddeutsche.de/rss
+https://www.welt.de/feeds/top-news.rss
+https://www.rfi.fr/en/rss
+https://www.bangkokpost.com/rss
+https://thephnompenhpost.com/rss
+https://www.thejakartapost.com/rss
+https://www.straitstimes.com/news/singapore/rss.xml
+https://www.channelnewsasia.com/rss
+https://www.antaranews.com/rss/
+https://www.irrawaddy.com/feed
+https://news.abs-cbn.com/rss
+https://www.hindustantimes.com/feeds/rss
+https://www.africanews.com/feed/rss
+https://www.clarin.com/rss
+https://www.lanacion.com.ar/rss
+https://www.eluniversal.com.mx/rss
+https://www.excelsior.com.mx/rss
+https://www.eltiempo.com/rss
+https://www.elespectador.com/rss
+https://www.larepublica.pe/rss
+https://www.elcomercio.com/rss
+https://www.abc.net.au/news/feed/
+https://www.smh.com.au/rss/world.xml
+https://www.theage.com.au/rss
+https://www.brisbanetimes.com.au/rss
+https://www.stuff.co.nz/rss
+https://www.nzherald.co.nz/arcio/rss/
+https://www.rnz.co.nz/rss
+https://globalvoices.org/regions/africa/feed/
+https://globalvoices.org/regions/asia/feed/
+https://globalvoices.org/regions/latin-america/feed/
+https://globalvoices.org/regions/eastern-europe/feed/
+https://globalvoices.org/regions/middle-east-north-africa/feed/
+https://globalvoices.org/regions/south-asia/feed/
+https://globalvoices.org/regions/sub-saharan-africa/feed/
+https://globalvoices.org/regions/west-africa/feed/
+https://globalvoices.org/regions/east-asia/feed/
+https://globalvoices.org/regions/southeast-asia/feed/
+https://globalvoices.org/regions/central-asia/feed/
+https://globalvoices.org/regions/pacific/feed/
+https://globalvoices.org/regions/caribbean/feed/
+https://www.townandcountry-mo.gov/rss.aspx
+https://feeds.smh.com.au/rssheadlines/national.xml
+https://www.abc.net.au/local/rss/sydney/
+https://www.voanews.com/rssfeeds
+https://rss.feedspot.com/southeast_asian_rss_feeds
+https://www.crisisgroup.org/rss
+https://news.panasonic.com/global/rss/area01/index.xml
+https://news.panasonic.com/global/rss/area04/index.xml
+https://allafrica.com/tools/headlines/rdf/latest/headlines.rdf
+https://www.afro.who.int/rss-feeds
+https://pressat.co.uk/rss-list
+https://www.monitor.co.ug/rss
+https://www.standardmedia.co.ke/rss
+https://www.ft.com/rss/home
+https://www.economist.com/rss/the-world-this-week
+https://feeds.bloomberg.com/economics/news.rss
+https://feeds.bloomberg.com/markets/news.rss
+https://www.reuters.com/arc/outboundfeeds/newsroom/business/
+https://www.cnbc.com/id/10000113/device/rss/rss.html
+https://feeds.a.dj.com/rss/RSSWorldBusiness.xml
+https://www.marketwatch.com/rss/topstories
+https://www.investing.com/rss/news_14.rss
+https://feeds.bbci.co.uk/news/business/rss.xml
+https://feeds.feedburner.com/TheHackersNews
+https://www.darkreading.com/rss/all.xml
+https://isc.sans.edu/rssfeed_full.xml
+https://securelist.com/feed/
+https://feeds.feedburner.com/eset/blog
+https://news.sophos.com/en-us/feed/
+https://www.schneier.com/feed/atom/
+https://www.securitymagazine.com/rss/topic/2236-cybersecurity-news
+https://www.reuters.com/arc/outboundfeeds/rss/?outputType=xml
+https://www.investing.com/rss/news_462.rss
+https://www.investing.com/rss/news_1.rss
+https://www.investing.com/rss/stock_Futures.rss
+https://www.ecb.europa.eu/rss/fxref-ecbpress.en.xml
+https://www.federalreserve.gov/feeds/news-events.xml
+https://www.boj.or.jp/en/rss/whatsnew.xml
+https://www.bankofengland.co.uk/rss/news
+https://www.centralbanking.com/feeds/rss
+https://oilprice.com/rss/
+https://www.spglobal.com/commodityinsights/en/rss
+https://www.eia.gov/tools/rssfeeds/
+https://www.cmegroup.com/rss
+https://globalvoices.org/-/topics/economics-business/feed/
+http://globalization.einnews.com/rss
+https://financefeeds.com/feed/
+https://newsquawk.com/blog/feed.rss
+https://www.coindesk.com/arc/outboundfeeds/rss/
+https://ishookfinance.com/feed/
+https://www.scmp.com/rss/92/feed
+https://www.scmp.com/rss/93/feed
+https://www.scmp.com/rss/94/feed
+https://www.scmp.com/rss/317/feed
+https://asia.nikkei.com/rss
+https://www.caixin.com/rss/index_EN.xml
+https://www.straitstimes.com/news/asia/rss.xml
+https://www.straitstimes.com/business/rss.xml
+https://www.reuters.com/arc/outboundfeeds/rss/?outputType=xml&section=asia
+https://www.reuters.com/arc/outboundfeeds/rss/?outputType=xml&section=china
+https://www.bloomberg.com/feeds/asia.rss
+https://www.bloomberg.com/feeds/markets.rss
+https://www.ft.com/asia-pacific?format=rss
+https://www.ft.com/china?format=rss
+https://english.kyodonews.net/rss/news.xml
+https://en.yna.co.kr/RSS/news.xml
+https://www.thejakartapost.com/rss/business
+https://www.nationthailand.com/rss/business
+https://www.aramco.com/api/v1/com/rss/news?sc_lang=en
+https://www.worldoil.com/rss?feed=topic:saudi+arabia
+https://www.worldoil.com/rss?feed=topic:iraq
+https://www.worldoil.com/rss?feed=topic:uae
+https://www.worldoil.com/rss?feed=topic:russia
+https://www.worldoil.com/rss?feed=topic:canada
+https://www.worldoil.com/rss?feed=topic:oil+sands
+https://www.rigzone.com/news/europe_russia/production/rss/
+https://www.argusmedia.com/en/news-and-insights/latest-market-news/rss
+https://www.eia.gov/rss/
+https://www.opec.org/opec_web/en/pressreleases.rss
+https://www.opec.org
+https://www.rosneft.com/press/news/rss/
+https://feeds.content.dowjones.io/public/rss/RSSMarketsMain
+https://feeds.content.dowjones.io/public/rss/socialeconomyfeed
+https://feeds.content.dowjones.io/public/rss/WSJcomUSBusiness
+https://feeds.content.dowjones.io/public/rss/RSSWorldNews
+http://feeds.feedburner.com/EconomicEventsAgriculture
+http://feeds.feedburner.com/EconomicEventsEnergy
+http://feeds.feedburner.com/EconomicEventsInterestRates
+http://feeds.feedburner.com/mediaroom/CMsF
+http://feeds.feedburner.com/CMEClearPortNoticesRss
+http://feeds.feedburner.com/GlobexAdvisories
+https://feeds.content.dowjones.io/public/rss/mw_topstories
+https://feeds.content.dowjones.io/public/rss/mw_realtimeheadlines
+http://feeds.marketwatch.com/marketwatch/bulletins
+https://feeds.content.dowjones.io/public/rss/mw_marketpulse
+https://www.nasdaqtrader.com/rss.aspx?feed=currentheadlines&categorylist=51
+https://www.nasdaqtrader.com/rss.aspx?feed=currentheadlines&categorylist=11
+https://www.investing.com/rss/stock_Options.rss
+https://www.investing.com/rss/news_11.rss
+https://www.investing.com/rss/news_25.rss
+https://www.nasdaq.com/feed/rssoutbound?category=Markets
+https://www.nasdaq.com/feed/rssoutbound?category=Commodities
+https://www.barchart.com/news/rss/financials/options-news
+https://www.barchart.com/news/rss/commodities/futures-news
+https://www.spglobal.com/spdji/en/rss
+https://www.litefinance.org/rss/analytics/
+https://www.mrt.com/arc/outboundfeeds/rss/category/business/oil/?outputType=xml
+https://www.oaoa.com/category/local-news/inthepipeline/rss
+https://pboilandgasmagazine.com/feed/
+https://www.rigzone.com/news/rss.asp
+https://rbnenergy.com/blogcast.rss
+https://www.eia.gov/rss/todayinenergy.xml
+https://www.firstalert7.com/news/energy
+https://www.energyvoice.com/feed/?category=oilandgas/north-sea
+https://www.rigzone.com/news/rss/north_sea
+https://www.oedigital.com/feeds/rss
+https://www.sodir.no/en/whats-new/news/rss
+https://www.worldoil.com/rss?feed=topic:offshore
+https://oilandgas.einnews.com/rss/north-sea-offshore
+https://www.energyvoice.com/feed/

news/summerizer/Dockerfile

@@ -7,13 +7,13 @@ WORKDIR /app
 
 # libpq-dev + gcc for psycopg2 build/adapters; keep the image lean.
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    libpq-dev gcc \
+    libpq-dev gcc tzdata \
     && rm -rf /var/lib/apt/lists/*
 
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
-COPY summarizer.py run_news_summarizer.py ./
+COPY summarizer.py run_news_summarizer.py intel.py nous_client.py ./
 
 # Security: run as a non-privileged user.
 RUN useradd -m summarizer_user

news/summerizer/intel.py

@@ -7,6 +7,7 @@ import re
 
 _EMPTY = {"summary_en": "", "ticker": [], "map_items": []}
 _KEEP = frozenset({"critical", "high"})
+_RANK = {"critical": 0, "high": 1, "medium": 2, "low": 3}
 _THINK_RE = re.compile(r"<think>.*?</think>", re.DOTALL)
 _FENCE_RE = re.compile(r"```(?:json)?", re.IGNORECASE)
 
@@ -58,19 +59,29 @@ def _trimmed_headline(row: dict, limit: int) -> str:
 
 
 def select_ticker(rows: list) -> list:
-    out = []
+    flagged = []
+    medium = []
+    low = []
     for row in rows:
-        if row.get("importance") not in _KEEP:
+        imp = row.get("importance")
+        if imp not in _RANK:
             continue
         headline = _trimmed_headline(row, TICKER_HEADLINE_MAX)
         if not headline:
             continue
         item = dict(row)
         item["headline"] = headline
-        out.append(item)
-        if len(out) >= TICKER_CAP:
+        if imp in _KEEP:
+            flagged.append(item)
+        elif imp == "medium":
+            medium.append(item)
+        else:
+            low.append(item)
+        if len(flagged) >= TICKER_CAP:
             break
-    return out
+    if flagged:
+        return flagged[:TICKER_CAP]
+    return (medium + low)[:TICKER_CAP]
 
 
 def select_map(items: list) -> list:

news/summerizer/nous_client.py

@@ -8,6 +8,10 @@ import httpx
 
 _DEFAULT_UA = "osint-dashboard-news-summarizer"
 _DEFAULT_BASE = "https://inference-api.nousresearch.com/v1"
+_JSON_SYSTEM = (
+    "You are an OSINT executive briefer. Reply with a single complete JSON object. "
+    "Never truncate mid-sentence. If you run out of room, drop the lowest-priority item."
+)
 
 
 def chat(prompt, *, api_key, model, base_url, json_mode=False) -> str:
@@ -17,20 +21,37 @@ def chat(prompt, *, api_key, model, base_url, json_mode=False) -> str:
         "Authorization": f"Bearer {api_key}",
         "User-Agent": os.environ.get("OSINT_USER_AGENT") or _DEFAULT_UA,
     }
+    max_tokens = 8192 if json_mode else 4096
+    timeout = 120.0 if json_mode else 60.0
+    messages = [{"role": "user", "content": prompt}]
+    if json_mode:
+        messages = [
+            {"role": "system", "content": _JSON_SYSTEM},
+            {"role": "user", "content": prompt},
+        ]
     payload = {
         "model": model,
-        "messages": [{"role": "user", "content": prompt}],
+        "messages": messages,
         "temperature": 0.2,
-        "max_tokens": 4096,
+        "max_tokens": max_tokens,
     }
     if json_mode:
         payload["response_format"] = {"type": "json_object"}
+    last_content = ""
     try:
-        with httpx.Client(timeout=60.0) as client:
-            resp = client.post(url, headers=headers, json=payload)
-        if resp.status_code == 401 or resp.status_code >= 500:
-            return ""
-        data = resp.json()
-        return data["choices"][0]["message"]["content"]
+        for attempt in range(2):
+            with httpx.Client(timeout=timeout) as client:
+                resp = client.post(url, headers=headers, json=payload)
+            if resp.status_code == 401 or resp.status_code >= 500:
+                return ""
+            data = resp.json()
+            choice = (data.get("choices") or [{}])[0]
+            last_content = (choice.get("message") or {}).get("content") or ""
+            finish = choice.get("finish_reason")
+            if finish == "length" and attempt == 0:
+                payload["max_tokens"] = min(int(payload["max_tokens"]) * 2, 16384)
+                continue
+            return last_content
+        return last_content
     except Exception:
         return ""

news/summerizer/run_news_summarizer.py

@@ -1,48 +1,97 @@
 #!/usr/bin/env python3
-"""Scheduler loop for the news summarizer — hourly summarize at minute :05.
+"""Scheduler loop for the news summarizer.
 
-Replaces the k8s CronJob (`5 * * * *`) with an in-compose loop. Runs once on
-boot (catches up on any articles scraped since the last summary), then fires
-at each :NEWS_SUMMARIZE_MINUTE wall-clock boundary.
+15-minute analyst (NEWS_SUMMARIZE_INTERVAL_S, default 900s) plus a daily
+recap at 23:00 in TZ (default America/New_York) over the last 24 hours.
 
-The loop is serial, so a slow LLM pass never overlaps the next run.
+Serial: a slow LLM pass never overlaps the next.
 
 Env (all optional, 12-factor):
-  NEWS_SUMMARIZE_MINUTE        minute of the hour to fire (default 5)
+  NEWS_SUMMARIZE_INTERVAL_S    seconds between analyst runs (default 900)
   NEWS_SUMMARIZE_RUN_ON_START  "1" to summarize once immediately on boot (default 1)
+  NEWS_RECAP_HOUR / MINUTE     wall-clock recap time (default 23:00)
+  TZ                           IANA tz (default America/New_York)
   NOUS_API_KEY                 optional in env; Keys UI / api_keys also works
 """
 
 from __future__ import annotations
 
-import datetime
 import logging
 import os
 import subprocess
 import sys
 import time
+from datetime import datetime, timedelta
+from zoneinfo import ZoneInfo
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
 logger = logging.getLogger("news.summarizer.scheduler")
 
-MINUTE = int(os.getenv("NEWS_SUMMARIZE_MINUTE", "5"))
+INTERVAL_S = max(1, int(os.getenv("NEWS_SUMMARIZE_INTERVAL_S", "900")))
 RUN_ON_START = os.getenv("NEWS_SUMMARIZE_RUN_ON_START", "1").lower() in ("1", "true", "yes")
+DEFAULT_TZ = "America/New_York"
+DEFAULT_RECAP_HOUR = 23
+DEFAULT_RECAP_MINUTE = 0
 
 
-def seconds_until_next(minute: int) -> float:
-    """Seconds until the next occurrence of ``minute`` past the hour (local time)."""
-    now = datetime.datetime.now()
-    nxt = now.replace(minute=minute, second=0, microsecond=0) + datetime.timedelta(hours=1)
-    return (nxt - now).total_seconds()
+def _tz() -> ZoneInfo:
+    name = (os.getenv("TZ") or DEFAULT_TZ).strip() or DEFAULT_TZ
+    return ZoneInfo(name)
 
 
-def run_summarize() -> None:
-    logger.info("summarize starting at %s", datetime.datetime.now().isoformat(timespec="seconds"))
+def recap_hour_minute() -> tuple[int, int]:
+    hour = int(os.getenv("NEWS_RECAP_HOUR", str(DEFAULT_RECAP_HOUR)))
+    minute = int(os.getenv("NEWS_RECAP_MINUTE", str(DEFAULT_RECAP_MINUTE)))
+    return hour, minute
+
+
+def next_recap_datetime(
+    now: datetime, hour: int | None = None, minute: int | None = None
+) -> datetime:
+    """Next 23:00 (or hour/minute) strictly after *now* in now's timezone."""
+    if now.tzinfo is None:
+        now = now.replace(tzinfo=_tz())
+    env_h, env_m = recap_hour_minute()
+    hour = env_h if hour is None else hour
+    minute = env_m if minute is None else minute
+    candidate = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
+    if now >= candidate:
+        candidate += timedelta(days=1)
+    return candidate
+
+
+def next_event(
+    now: datetime,
+    last_periodic: datetime | None,
+    interval_s: int,
+    hour: int = DEFAULT_RECAP_HOUR,
+    minute: int = DEFAULT_RECAP_MINUTE,
+) -> tuple[datetime, str]:
+    """Return (when, 'recap'|'interval') for the sooner of recap vs interval."""
+    recap_at = next_recap_datetime(now, hour=hour, minute=minute)
+    periodic_at = now if last_periodic is None else last_periodic + timedelta(seconds=interval_s)
+    if recap_at <= periodic_at:
+        return recap_at, "recap"
+    return periodic_at, "interval"
+
+
+def run_summarize(*, recap: bool = False) -> None:
+    kind = "recap" if recap else "interval"
+    logger.info(
+        "%s starting at %s", kind, datetime.now().isoformat(timespec="seconds")
+    )
+    env = os.environ.copy()
+    if recap:
+        env["NEWS_RECAP"] = "1"
+    else:
+        env.pop("NEWS_RECAP", None)
     try:
-        proc = subprocess.run([sys.executable, "summarizer.py"], cwd="/app")
-        logger.info("summarize finished rc=%s", proc.returncode)
+        proc = subprocess.run(
+            [sys.executable, "summarizer.py"], cwd="/app", env=env
+        )
+        logger.info("%s finished rc=%s", kind, proc.returncode)
     except Exception:  # noqa: BLE001 — keep the loop alive across failures
-        logger.exception("summarize failed")
+        logger.exception("%s failed", kind)
 
 
 def main() -> None:
@@ -50,17 +99,32 @@ def main() -> None:
         logger.warning(
             "NOUS_API_KEY unset in env — will read api_keys on each run; idle if both empty"
         )
+    tz = _tz()
+    hour, minute = recap_hour_minute()
     logger.info(
-        "news summarizer loop starting (minute=%s, run_on_start=%s)",
-        MINUTE, RUN_ON_START,
+        "news summarizer loop starting (interval_s=%s, run_on_start=%s, recap=%02d:%02d %s)",
+        INTERVAL_S, RUN_ON_START, hour, minute, tz,
     )
+    last_periodic: datetime | None = None
     if RUN_ON_START:
-        run_summarize()
+        run_summarize(recap=False)
+        last_periodic = datetime.now(tz)
     while True:
-        delay = seconds_until_next(MINUTE)
-        logger.info("next summarize at :%02d (in %.0fs)", MINUTE, delay)
-        time.sleep(delay)
-        run_summarize()
+        now = datetime.now(tz)
+        when, kind = next_event(
+            now, last_periodic, INTERVAL_S, hour=hour, minute=minute
+        )
+        sleep_s = max(1, (when - now).total_seconds())
+        logger.info("next %s in %ss", kind, int(sleep_s))
+        time.sleep(sleep_s)
+        now = datetime.now(tz)
+        if kind == "recap":
+            run_summarize(recap=True)
+            # Recap covers the 15-min window; don't immediately fire interval.
+            last_periodic = now
+        else:
+            run_summarize(recap=False)
+            last_periodic = now
 
 
 if __name__ == "__main__":

news/summerizer/summarizer.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 """News summarizer — Nous map-reduce of scraped articles into brief/ticker/map.
 
-Reads articles scraped within the last hour from the shared `articles` table,
+Reads articles scraped within the last SUMMARY_WINDOW_MINUTES from the shared `articles` table,
 maps them with Nous (per-article English fact blocks), reduces to one JSON
 object (summary_en + ticker + map_items), and stores the brief in
 `article_summaries` plus flagged rows in `news_items`. Tables live in the
@@ -15,17 +15,11 @@ of each summarize_news() — env wins, else api_keys / app_settings:
   NOUS_BASE_URL            default https://inference-api.nousresearch.com/v1
   SUMMARY_MODEL            default Hermes-4.3-36B (else app_settings)
   BATCH_SIZE               articles per map-phase batch (default 50)
-  SUMMARY_WINDOW_HOURS     look-back window in hours (default 1)
-  OSINT_USER_AGENT         default osint-dashboard-news-summarizer
-  MAP_PROMPT               override map-phase prompt (uses {batch_text})
-  SUMMARY_PROMPT           override reduce-phase prompt (uses {final_input})
-  NEWS_SUMMARIZE_FORCE     "1" to ignore the current-UTC-hour idempotency skip
-  INCLUDE_FUTURES          "1" to prepend live futures prices (default 0)
-
-The futures/markets coupling from the original pipeline is gated behind
-INCLUDE_FUTURES and OFF by default — it is irrelevant to the OSINT dashboard
-and pulled yfinance into the image. Re-enable by installing yfinance and
-setting INCLUDE_FUTURES=1.
+  SUMMARY_WINDOW_MINUTES   look-back window (default 15; SUMMARY_WINDOW_HOURS wins if set)
+  NEWS_RECAP               "1" for the 23:00 daily recap (24h window, recap prompt)
+  NEWS_SUMMARIZE_FORCE     "1" to ignore the interval/recap idempotency skip
+  TZ                       IANA tz for recap-day bounds (default America/New_York)
+  INCLUDE_FUTURES          legacy; ignored — prompts never inject futures data
 """
 
 from __future__ import annotations
@@ -33,6 +27,7 @@ from __future__ import annotations
 import logging
 import os
 from datetime import datetime
+from zoneinfo import ZoneInfo
 
 import psycopg2
 
@@ -54,7 +49,35 @@ DB_CONFIG = {
 DEFAULT_NOUS_BASE_URL = "https://inference-api.nousresearch.com/v1"
 DEFAULT_SUMMARY_MODEL = "Hermes-4.3-36B"
 BATCH_SIZE = int(os.getenv("BATCH_SIZE", "50"))
-SUMMARY_WINDOW_HOURS = int(os.getenv("SUMMARY_WINDOW_HOURS", "1"))
+
+
+def _summary_window_minutes() -> int:
+    hours = (os.getenv("SUMMARY_WINDOW_HOURS") or "").strip()
+    if hours:
+        return max(1, int(hours) * 60)
+    mins = (os.getenv("SUMMARY_WINDOW_MINUTES") or "").strip()
+    if mins:
+        return max(1, int(mins))
+    return 15
+
+
+def _summarize_interval_seconds() -> int:
+    return max(1, int(os.getenv("NEWS_SUMMARIZE_INTERVAL_S", "900")))
+
+
+def is_recap_run() -> bool:
+    return os.getenv("NEWS_RECAP", "0").lower() in ("1", "true", "yes")
+
+
+def effective_window_minutes(*, recap: bool | None = None) -> int:
+    if recap is None:
+        recap = is_recap_run()
+    if recap:
+        return 24 * 60
+    return _summary_window_minutes()
+
+
+SUMMARY_WINDOW_MINUTES = _summary_window_minutes()
 INCLUDE_FUTURES = os.getenv("INCLUDE_FUTURES", "0").lower() in ("1", "true", "yes")
 
 # Only touched when INCLUDE_FUTURES=1 (legacy markets coupling, OSINT-off).
@@ -71,6 +94,8 @@ FUTURES_TICKERS = {
 MAP_PROMPT_DEFAULT = """\
 You are a precise, factual OSINT news processor. Your ONLY source of information is the articles provided below. Do NOT add external knowledge, assumptions, training data, or invented facts.
 
+Focus on breaking important news (geopolitical, military/conflict, security, disasters, major political developments). Ignore futures prices, commodity tape, ticker chatter, and routine market moves unless they themselves are the breaking event. If the batch has no critical/high stories, still extract minor incidents and crime reports.
+
 Write every field in English. Translate if the article is not English.
 
 For EACH article in the batch:
@@ -78,7 +103,7 @@ For EACH article in the batch:
 2. Location: country/city/region or Unknown. If you can estimate coordinates, emit them as numbers; otherwise omit.
 3. Entities: list the key people, organizations, or governments mentioned (comma-separated, only names present in the text), else "None".
 4. Category: pick one — politics, military/conflict, economy, technology, environment/disaster, health, crime, society, sport, other.
-5. OSINT signal: if the article describes an event with geopolitical, security, military, economic, or disaster significance, say so in one short sentence. Otherwise write: "No notable OSINT signal."
+5. OSINT signal: if the article describes a breaking event with geopolitical, security, military, or disaster significance, say so in one short sentence. Otherwise write: "No notable OSINT signal."
 6. Importance: critical (breaking geopolitical/military/disaster with immediate impact), high, medium, low, none.
 
 If several articles cover the same story, add one short batch-level note at the end: "Batch theme: [one sentence]".
@@ -104,21 +129,52 @@ Articles in this batch:
 """
 
 SUMMARY_PROMPT_DEFAULT = """\
-CRITICAL INSTRUCTION - REPEAT 3 TIMES: YOU MUST USE ONLY THE DATA PROVIDED BELOW. DO NOT INVENT, RECALL, OR ADD ANY EVENTS, NAMES, DATES, IMPLICATIONS, PROJECTS, OR DETAILS NOT EXPLICITLY PRESENT IN THE DATA. IF THE DATA HAS NO MAJOR GEOPOLITICAL/TECH/MILITARY/ECONOMIC/IMPACTFUL EVENTS OR UNUSUAL STORIES, set summary_en to exactly: "No qualifying impactful or unusual events in the recent hourly news data." and use empty ticker and map_items arrays. AND STOP. NO EXTERNAL KNOWLEDGE FROM TRAINING.
+You are writing an English operator HUD brief from the article facts in DATA below. Use ONLY that data. Do not invent events, names, dates, places, or implications.
 
-All text in English.
+Always write a real summary_en that recaps the most important stories present in DATA. Rank geopolitics, military/conflict, security, disasters, and major political developments first. Ignore futures prices, commodity tape, ticker chatter, and routine market data — do not treat price ticks as news.
+
+Lead with critical and high breaking events. If DATA has no critical/high stories, fill the brief with minor incidents and crime reports rather than writing an empty or unfinished brief. Never truncate mid-sentence; finish every sentence. If you run out of room, drop the lowest-priority item instead of cutting a line short.
+
+ticker: prefer critical and high. If nothing is critical or high, fill ticker with medium then low incidents and crime so the HUD is not blank.
+
+map_items may be empty if no located critical/high event is explicit in the data.
 
 Demand a single JSON object (no markdown fences) with this exact shape:
 
 {
-  "summary_en": "English markdown brief or the no-qualifying-events sentence",
+  "summary_en": "English markdown brief of the provided stories",
   "ticker": [{"headline": "", "importance": "critical", "url": "", "location_name": ""}],
   "map_items": [{"headline": "", "importance": "critical", "location_name": "", "lat": 0, "lon": 0, "location_confidence": "city", "category": "military/conflict", "url": ""}]
 }
 
-ticker: only critical and high, max 12, ≤140 chars, no markdown.
-map_items: only critical and high where a real-world location is explicit in the data. Estimate lat/lon. If location is Unknown or not in the data, omit the item. Never invent a place. Max 20.
-summary_en: English markdown brief for an operator HUD.
+ticker: max 12, ≤140 chars, no markdown. Rank critical > high > medium > low.
+map_items: only where a real-world location is explicit in the data. Estimate lat/lon. If location is Unknown or not in the data, omit the item. Never invent a place. Max 20.
+summary_en: English markdown executive brief for an operator HUD (4–8 complete bullets or short paragraphs). Cover the actual stories in DATA. Complete — never an unfinished sentence.
+
+DATA:
+{final_input}
+"""
+
+RECAP_PROMPT_DEFAULT = """\
+You are writing a daily recap of the last 24 hours of news for an OSINT operator HUD, using ONLY the article facts in DATA below. Do not invent events, names, dates, places, or implications.
+
+Always write a real summary_en daily recap of the most important stories in DATA. Rank geopolitics, military/conflict, security, disasters, and major political developments first. Ignore futures prices, commodity tape, ticker chatter, and routine market data — do not treat price ticks as news.
+
+Lead with critical and high breaking events. If DATA has no critical/high stories, fill the recap with minor incidents and crime reports rather than writing an empty or unfinished recap. Never truncate mid-sentence; finish every sentence.
+
+ticker: prefer critical and high. If nothing is critical or high, fill ticker with medium then low incidents and crime so the HUD is not blank.
+
+Demand a single JSON object (no markdown fences) with this exact shape:
+
+{
+  "summary_en": "English markdown daily recap of the provided stories",
+  "ticker": [{"headline": "", "importance": "critical", "url": "", "location_name": ""}],
+  "map_items": [{"headline": "", "importance": "critical", "location_name": "", "lat": 0, "lon": 0, "location_confidence": "city", "category": "military/conflict", "url": ""}]
+}
+
+ticker: max 12, ≤140 chars, no markdown. Rank critical > high > medium > low.
+map_items: only where a real-world location is explicit in the data. Estimate lat/lon. If location is Unknown or not in the data, omit the item. Never invent a place. Max 20.
+summary_en: English markdown daily recap of the last 24 hours. Complete sentences. Cover the actual stories in DATA.
 
 DATA:
 {final_input}
@@ -250,6 +306,7 @@ def ensure_tables() -> None:
             batch_timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW()
         );
         ALTER TABLE article_summaries ADD COLUMN IF NOT EXISTS model TEXT;
+        ALTER TABLE article_summaries ADD COLUMN IF NOT EXISTS kind TEXT;
         CREATE TABLE IF NOT EXISTS news_items (
             id              SERIAL PRIMARY KEY,
             summary_id      INTEGER REFERENCES article_summaries(id) ON DELETE CASCADE,
@@ -281,19 +338,20 @@ def ensure_tables() -> None:
         logger.error("Error ensuring news tables: %s", exc)
 
 
-def get_recent_news() -> list[dict]:
-    """Fetch articles from the last SUMMARY_WINDOW_HOURS (content > 100 chars)."""
+def get_recent_news(window_minutes: int | None = None) -> list[dict]:
+    """Fetch articles from the look-back window (content > 100 chars)."""
+    mins = window_minutes if window_minutes is not None else effective_window_minutes()
     query = """
         SELECT title, content, url, domain
         FROM articles
-        WHERE timestamp > NOW() - make_interval(hours => %s)
+        WHERE timestamp > NOW() - make_interval(mins => %s)
           AND content IS NOT NULL AND length(content) > 100
         ORDER BY timestamp DESC;
     """
     try:
         conn = psycopg2.connect(**DB_CONFIG)
         cur = conn.cursor()
-        cur.execute(query, (SUMMARY_WINDOW_HOURS,))
+        cur.execute(query, (mins,))
         rows = cur.fetchall()
         cur.close()
         conn.close()
@@ -306,28 +364,55 @@ def get_recent_news() -> list[dict]:
         return []
 
 
-def _already_summarized_this_hour() -> bool:
-    """True when article_summaries already has a row for the current UTC hour."""
+def _already_summarized_this_interval() -> bool:
+    """True when article_summaries already has a row in the last interval."""
     if os.getenv("NEWS_SUMMARIZE_FORCE", "") == "1":
         return False
     query = (
         "SELECT 1 FROM article_summaries "
-        "WHERE batch_timestamp >= date_trunc('hour', NOW() AT TIME ZONE 'utc')"
+        "WHERE batch_timestamp >= NOW() - make_interval(secs => %s)"
     )
     try:
         conn = psycopg2.connect(**DB_CONFIG)
         cur = conn.cursor()
-        cur.execute(query)
+        cur.execute(query, (_summarize_interval_seconds(),))
         row = cur.fetchone()
         cur.close()
         conn.close()
         return row is not None
     except Exception as exc:  # noqa: BLE001
-        logger.error("Error checking hourly idempotency: %s", exc)
+        logger.error("Error checking interval idempotency: %s", exc)
         return False
 
 
-def save_batch(summary_en: str, model: str, ticker: list, map_items: list) -> None:
+def _already_recapped_today() -> bool:
+    """True when a daily_recap row already exists for the local calendar day."""
+    if os.getenv("NEWS_SUMMARIZE_FORCE", "") == "1":
+        return False
+    tz_name = (os.getenv("TZ") or "America/New_York").strip() or "America/New_York"
+    start = datetime.now(ZoneInfo(tz_name)).replace(
+        hour=0, minute=0, second=0, microsecond=0
+    )
+    query = (
+        "SELECT 1 FROM article_summaries "
+        "WHERE kind = 'daily_recap' AND batch_timestamp >= %s"
+    )
+    try:
+        conn = psycopg2.connect(**DB_CONFIG)
+        cur = conn.cursor()
+        cur.execute(query, (start,))
+        row = cur.fetchone()
+        cur.close()
+        conn.close()
+        return row is not None
+    except Exception as exc:  # noqa: BLE001
+        logger.error("Error checking recap idempotency: %s", exc)
+        return False
+
+
+def save_batch(
+    summary_en: str, model: str, ticker: list, map_items: list, *, kind: str = "interval"
+) -> None:
     """Insert the master brief plus flagged ticker/map rows."""
     ticker_rows = select_ticker(ticker or [])
     map_rows = select_map(map_items or [])
@@ -345,8 +430,8 @@ def save_batch(summary_en: str, model: str, ticker: list, map_items: list) -> No
         conn = psycopg2.connect(**DB_CONFIG)
         cur = conn.cursor()
         cur.execute(
-            "INSERT INTO article_summaries (summary_text, model) VALUES (%s, %s) RETURNING id",
-            (text, model),
+            "INSERT INTO article_summaries (summary_text, model, kind) VALUES (%s, %s, %s) RETURNING id",
+            (text, model, kind),
         )
         summary_id = cur.fetchone()[0]
         for row in ticker_rows:
@@ -383,8 +468,8 @@ def save_batch(summary_en: str, model: str, ticker: list, map_items: list) -> No
             )
         conn.commit()
         logger.info(
-            "Master summary saved id=%s model=%s ticker=%d map=%d",
-            summary_id, model, len(ticker_rows), len(map_rows),
+            "Master summary saved id=%s model=%s kind=%s ticker=%d map=%d",
+            summary_id, model, kind, len(ticker_rows), len(map_rows),
         )
         cur.close()
         conn.close()
@@ -400,29 +485,39 @@ def build_map_prompt(batch: list[dict]) -> str:
         for a in batch
     )
     template = os.getenv("MAP_PROMPT", MAP_PROMPT_DEFAULT)
-    prefix = build_futures_context() + "\n" if INCLUDE_FUTURES else ""
     try:
-        return prefix + template.format(batch_text=batch_text)
+        return template.format(batch_text=batch_text)
     except KeyError:
-        return prefix + template
+        return template
 
 
-def build_master_prompt(final_input: str) -> str:
-    template = os.getenv("SUMMARY_PROMPT", SUMMARY_PROMPT_DEFAULT)
-    prefix = build_futures_context() + "\n" if INCLUDE_FUTURES else ""
-    try:
-        return prefix + template.format(final_input=final_input)
-    except KeyError:
-        return prefix + template
+def build_master_prompt(final_input: str, recap: bool = False) -> str:
+    if recap:
+        template = os.getenv("RECAP_PROMPT", RECAP_PROMPT_DEFAULT)
+    else:
+        template = os.getenv("SUMMARY_PROMPT", SUMMARY_PROMPT_DEFAULT)
+    if "{final_input}" in template:
+        return template.replace("{final_input}", final_input)
+    return template
 
 
 def summarize_news() -> None:
     """Map-reduce summarize recent articles and store brief + ticker + map."""
+    recap = is_recap_run()
+    window = effective_window_minutes(recap=recap)
     ensure_tables()
-    if _already_summarized_this_hour():
+    if recap:
+        if _already_recapped_today():
+            logger.info(
+                "Skipping recap: article_summaries already has daily_recap today "
+                "(set NEWS_SUMMARIZE_FORCE=1 to override)"
+            )
+            return
+    elif _already_summarized_this_interval():
         logger.info(
-            "Skipping summarize: article_summaries already has a row this UTC hour "
-            "(set NEWS_SUMMARIZE_FORCE=1 to override)"
+            "Skipping summarize: article_summaries already has a row in the last %ss "
+            "(set NEWS_SUMMARIZE_FORCE=1 to override)",
+            _summarize_interval_seconds(),
         )
         return
 
@@ -433,14 +528,14 @@ def summarize_news() -> None:
         logger.warning("NOUS_API_KEY unset in env and api_keys — idle this run")
         return
 
-    articles = get_recent_news()
+    articles = get_recent_news(window)
     if not articles:
-        logger.info("No new articles found in the last %sh.", SUMMARY_WINDOW_HOURS)
+        logger.info("No new articles found in the last %s min.", window)
         return
 
     logger.info(
-        "Processing %d articles with %s (batch_size=%d, futures=%s)...",
-        len(articles), model, BATCH_SIZE, INCLUDE_FUTURES,
+        "Processing %d articles with %s (batch_size=%d, recap=%s, window_min=%s)...",
+        len(articles), model, BATCH_SIZE, recap, window,
     )
 
     partial_summaries: list[str] = []
@@ -467,7 +562,7 @@ def summarize_news() -> None:
 
     logger.info("reduce phase over %d partial summaries", len(partial_summaries))
     master_raw = call_llm(
-        build_master_prompt(final_input),
+        build_master_prompt(final_input, recap=recap),
         api_key=api_key,
         model=model,
         base_url=base_url,
@@ -477,7 +572,13 @@ def summarize_news() -> None:
         logger.warning("Reduce phase returned empty — nothing to persist.")
         return
     parsed = parse_reduce_json(master_raw)
-    save_batch(parsed["summary_en"], model, parsed["ticker"], parsed["map_items"])
+    save_batch(
+        parsed["summary_en"],
+        model,
+        parsed["ticker"],
+        parsed["map_items"],
+        kind="daily_recap" if recap else "interval",
+    )
 
 
 if __name__ == "__main__":

news/summerizer/tests/conftest.py

@@ -0,0 +1,11 @@
+"""Keep summarizer unit tests importable without Postgres drivers."""
+
+from __future__ import annotations
+
+import sys
+from types import ModuleType
+
+if "psycopg2" not in sys.modules:
+    fake = ModuleType("psycopg2")
+    fake.connect = lambda **kwargs: None  # type: ignore[attr-defined]
+    sys.modules["psycopg2"] = fake

news/summerizer/tests/test_intel.py

@@ -32,6 +32,16 @@ def test_select_ticker_keeps_critical_high_caps_12():
     assert len(out) == 12
     assert all(r["importance"] in ("critical", "high") for r in out)
 
+
+def test_select_ticker_falls_back_to_medium_low_when_nothing_flagged():
+    rows = [
+        {"headline": "shop theft", "importance": "low"},
+        {"headline": "highway crash", "importance": "medium"},
+        {"headline": "none", "importance": "none"},
+    ]
+    out = select_ticker(rows)
+    assert [r["headline"] for r in out] == ["highway crash", "shop theft"]
+
 def test_select_map_requires_valid_coords_and_flag():
     items = [
         {"headline": "A", "importance": "critical", "lat": 50.45, "lon": 30.52, "location_name": "Kyiv"},

news/summerizer/tests/test_nous_client.py

@@ -67,6 +67,33 @@ def test_json_mode_sets_response_format(monkeypatch):
     captured = _install_fake(monkeypatch, lambda *a: _ok_response("{}"))
     chat("p", api_key="k", model="m", base_url=BASE, json_mode=True)
     assert captured["json"]["response_format"] == {"type": "json_object"}
+    assert captured["json"]["max_tokens"] >= 8192
+    roles = [m["role"] for m in captured["json"]["messages"]]
+    assert "system" in roles
+    assert "user" in roles
+
+
+def test_retries_once_when_finish_reason_is_length(monkeypatch):
+    calls = {"n": 0}
+
+    def post_impl(*a):
+        calls["n"] += 1
+        if calls["n"] == 1:
+            resp = MagicMock()
+            resp.status_code = 200
+            resp.json.return_value = {
+                "choices": [{
+                    "message": {"content": "{\"summary_en\": \"cut off"},
+                    "finish_reason": "length",
+                }]
+            }
+            return resp
+        return _ok_response('{"summary_en": "complete brief."}')
+
+    _install_fake(monkeypatch, post_impl)
+    out = chat("p", api_key="k", model="m", base_url=BASE, json_mode=True)
+    assert calls["n"] == 2
+    assert "complete brief" in out
 
 
 def test_401_returns_empty_string(monkeypatch):

news/summerizer/tests/test_prompts.py

@@ -0,0 +1,60 @@
+"""Default prompts: breaking news, ignore futures/market tape."""
+
+from summarizer import MAP_PROMPT_DEFAULT, RECAP_PROMPT_DEFAULT, SUMMARY_PROMPT_DEFAULT
+
+
+def _assert_breaking_not_futures(prompt: str) -> None:
+    p = prompt.lower()
+    assert "breaking" in p
+    assert "futures" in p
+    assert "ignore" in p or "do not" in p or "not" in p
+    assert "es=f" not in p
+    assert "yfinance" not in p
+
+
+def test_map_prompt_focuses_on_breaking_news_not_futures():
+    _assert_breaking_not_futures(MAP_PROMPT_DEFAULT)
+
+
+def test_summary_prompt_focuses_on_breaking_news_not_futures():
+    _assert_breaking_not_futures(SUMMARY_PROMPT_DEFAULT)
+    p = SUMMARY_PROMPT_DEFAULT.lower()
+    assert "commodity" in p or "market" in p
+
+
+def test_summary_prompt_covers_critical_then_incidents():
+    p = SUMMARY_PROMPT_DEFAULT.lower()
+    assert "critical" in p
+    assert "crime" in p
+    assert "incident" in p
+    assert "complete" in p or "truncat" in p or "unfinished" in p or "mid-sentence" in p
+
+
+def test_summary_prompt_does_not_bail_out_with_canned_empty_brief():
+    p = SUMMARY_PROMPT_DEFAULT
+    assert "AND STOP" not in p
+    assert "REPEAT 3 TIMES" not in p
+    assert "No qualifying" not in p
+    assert "no-qualifying" not in p.lower()
+    low = p.lower()
+    assert "always" in low
+    assert "recap" in low or "summar" in low
+
+
+def test_recap_prompt_does_not_bail_out_with_canned_empty_brief():
+    p = RECAP_PROMPT_DEFAULT
+    assert "AND STOP" not in p
+    assert "No qualifying" not in p
+    assert "no-qualifying" not in p.lower()
+    low = p.lower()
+    assert "always" in low
+    assert "daily" in low
+    assert "24" in low
+
+    p = RECAP_PROMPT_DEFAULT.lower()
+    _assert_breaking_not_futures(RECAP_PROMPT_DEFAULT)
+    assert "daily" in p
+    assert "24" in p
+    assert "summary_en" in p
+    assert "ticker" in p
+    assert "map_items" in p

news/summerizer/tests/test_scheduler.py

@@ -0,0 +1,42 @@
+"""Wall-clock scheduling: 15-min analyst + 23:00 America/New_York recap."""
+
+from datetime import datetime, timedelta
+from zoneinfo import ZoneInfo
+
+from run_news_summarizer import next_event, next_recap_datetime
+
+TZ = ZoneInfo("America/New_York")
+
+
+def test_next_recap_is_11pm_same_day_before_2300():
+    now = datetime(2026, 8, 28, 15, 4, tzinfo=TZ)
+    got = next_recap_datetime(now)
+    assert got == datetime(2026, 8, 28, 23, 0, tzinfo=TZ)
+
+
+def test_next_recap_is_11pm_next_day_at_or_after_2300():
+    now = datetime(2026, 8, 28, 23, 0, tzinfo=TZ)
+    got = next_recap_datetime(now)
+    assert got == datetime(2026, 8, 29, 23, 0, tzinfo=TZ)
+
+
+def test_next_recap_honors_custom_hour():
+    now = datetime(2026, 8, 28, 10, 0, tzinfo=TZ)
+    got = next_recap_datetime(now, hour=22, minute=30)
+    assert got == datetime(2026, 8, 28, 22, 30, tzinfo=TZ)
+
+
+def test_next_event_picks_recap_when_sooner_than_interval():
+    now = datetime(2026, 8, 28, 22, 50, tzinfo=TZ)
+    last_periodic = now - timedelta(seconds=100)
+    when, kind = next_event(now, last_periodic=last_periodic, interval_s=900)
+    assert kind == "recap"
+    assert when == datetime(2026, 8, 28, 23, 0, tzinfo=TZ)
+
+
+def test_next_event_picks_interval_when_recap_is_hours_away():
+    now = datetime(2026, 8, 28, 10, 0, tzinfo=TZ)
+    last_periodic = now
+    when, kind = next_event(now, last_periodic=last_periodic, interval_s=900)
+    assert kind == "interval"
+    assert when == now + timedelta(seconds=900)

news/summerizer/tests/test_summarizer.py

@@ -0,0 +1,45 @@
+"""Summarizer window, recap flag, and no futures injection into prompts."""
+
+from summarizer import (
+    build_map_prompt,
+    build_master_prompt,
+    effective_window_minutes,
+    is_recap_run,
+)
+
+
+def test_interval_window_defaults_to_15_minutes(monkeypatch):
+    monkeypatch.delenv("NEWS_RECAP", raising=False)
+    monkeypatch.delenv("SUMMARY_WINDOW_HOURS", raising=False)
+    monkeypatch.setenv("SUMMARY_WINDOW_MINUTES", "15")
+    assert is_recap_run() is False
+    assert effective_window_minutes() == 15
+
+
+def test_recap_window_is_24_hours(monkeypatch):
+    monkeypatch.setenv("NEWS_RECAP", "1")
+    monkeypatch.setenv("SUMMARY_WINDOW_MINUTES", "15")
+    assert is_recap_run() is True
+    assert effective_window_minutes() == 24 * 60
+
+
+def test_build_map_prompt_does_not_inject_futures():
+    prompt = build_map_prompt(
+        [{"title": "Blast", "domain": "ex.com", "url": "https://ex.com/1", "content": "x" * 120}]
+    )
+    assert "FUTURES PRICES" not in prompt
+    assert "ES=F" not in prompt
+    assert "Blast" in prompt
+
+
+def test_build_master_prompt_interval_uses_summary_not_recap():
+    prompt = build_master_prompt("partial facts", recap=False)
+    assert "partial facts" in prompt
+    assert "daily recap" not in prompt.lower()
+
+
+def test_build_master_prompt_recap_uses_daily_template():
+    prompt = build_master_prompt("partial facts", recap=True)
+    assert "partial facts" in prompt
+    assert "daily recap" in prompt.lower()
+    assert "24" in prompt

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py

scripts/compose-reup.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Recreate selected OSINT compose services WITHOUT bouncing Postgres.
+#
+# The old path was `compose down` + up, which stopped osint-db on every merge
+# even when Dockerfile.pg did not change. Name-pinned leftovers are still
+# removed, but only for the services we are actually replacing.
+#
+# Usage: scripts/compose-reup.sh [compose-service ...]
+#        (default: app ingester camera-service news-scraper news-summarizer)
+# Env:   COMPOSE_PROJECT_NAME (default osint-dashboard)
+#        COMPOSE_PROFILES     (default ingest)
+#        FORCE_RECREATE_DB=1  also recreate db
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+cd "$ROOT"
+
+export COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-osint-dashboard}"
+PROFILE="${COMPOSE_PROFILES:-ingest}"
+
+DEFAULT_SVCS=(app ingester camera-service news-scraper news-summarizer)
+if [ "$#" -gt 0 ]; then
+  SVCS=("$@")
+else
+  SVCS=("${DEFAULT_SVCS[@]}")
+fi
+
+if [ "${FORCE_RECREATE_DB:-0}" = "1" ]; then
+  SVCS+=(db)
+fi
+
+# Never recreate db unless it was requested.
+FILTERED=()
+for svc in "${SVCS[@]}"; do
+  if [ "$svc" = "db" ] && [ "${FORCE_RECREATE_DB:-0}" != "1" ]; then
+    echo "compose-reup: skipping db (set FORCE_RECREATE_DB=1 to bounce Postgres)"
+    continue
+  fi
+  FILTERED+=("$svc")
+done
+SVCS=("${FILTERED[@]}")
+
+declare -A CONTAINER_NAME=(
+  [app]=osint-dashboard
+  [ingester]=osint-ingester
+  [camera-service]=osint-camera-scraper
+  [news-scraper]=osint-news-scraper
+  [news-summarizer]=osint-news-summarizer
+  [db]=osint-db
+  [nats]=osint-nats
+  [titiler]=osint-titiler
+)
+
+echo "compose-reup: project=${COMPOSE_PROJECT_NAME} profile=${PROFILE} dir=${ROOT}"
+echo "compose-reup: recreate=${SVCS[*]:-none}"
+
+# Keep data-plane containers running (db / nats / titiler).
+docker compose --profile "${PROFILE}" up -d --no-build --no-recreate db nats titiler || true
+
+if [ "${#SVCS[@]}" -eq 0 ]; then
+  docker compose --profile "${PROFILE}" ps
+  exit 0
+fi
+
+for svc in "${SVCS[@]}"; do
+  c="${CONTAINER_NAME[$svc]:-}"
+  if [ -n "$c" ] && docker inspect "$c" >/dev/null 2>&1; then
+    echo "compose-reup: replacing ${c}"
+    docker rm -f "$c" >/dev/null
+  fi
+done
+
+docker compose --profile "${PROFILE}" up -d --no-build --no-deps "${SVCS[@]}"
+docker compose --profile "${PROFILE}" ps

tests/test_aircraft_popup_frontend.py

@@ -0,0 +1,57 @@
+"""Aircraft popup enrichment + emergency/MIL layer contract (static HTML)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+HTML = (ROOT / "app/static/index.html").read_text()
+
+
+def _fn(name: str, nxt: str) -> str:
+    return HTML.split(f"function {name}", 1)[1].split(f"function {nxt}", 1)[0]
+
+
+def test_popup_has_required_adsb_fields_and_photo():
+    js = _fn("pointPopup", "loadPlanePhoto")
+    for field in ("callsign", "hex", "registration", "type", "alt", "gs", "squawk"):
+        assert f"add('{field}'" in js
+    assert "class=\"ps-photo\"" in js or "class='ps-photo'" in js
+    assert "wikipedia" not in js.lower()
+    assert "ceo" not in js.lower()
+
+
+def test_emergency_badge_and_squawk_codes():
+    assert "role-badge emergency" in HTML
+    assert "hdg-emerg" in HTML
+    assert "EMERG_SQUAWK" in HTML
+    assert "['7700', '7600', '7500']" in HTML
+    emerg = HTML.split("function acIsEmergency", 1)[1].split("function acVisible", 1)[0]
+    assert "EMERG_SQUAWK.has(sq)" in emerg
+    color = HTML.split("function acColor", 1)[1].split("function connectLiveWs", 1)[0]
+    assert "acIsEmergency(p)" in color
+    assert "#ff5d5d" in color
+
+
+def test_mil_toggle_hidden_until_role_flag_and_never_hits_adsb_lol():
+    assert 'id="lp-ac-mil-row"' in HTML
+    assert 'id="lp-ac-mil-on"' in HTML
+    row = HTML.split('id="lp-ac-mil-row"', 1)[1].split(">", 1)[0]
+    assert "hidden" in row
+    on = HTML.split('id="lp-ac-mil-on"', 1)[1].split(">", 1)[0]
+    assert "checked" not in on
+    load = HTML.split("async function loadAircraft", 1)[1].split("async function toggleTrains", 1)[0]
+    assert "/api/aircraft?bbox=" in load
+    assert "api.adsb.lol" not in load
+    assert "noteMilSupport" in load
+    assert "acMilOn" in load
+    note = HTML.split("function noteMilSupport", 1)[1].split("function acColor", 1)[0]
+    assert "extra.role" in note
+    assert "lp-ac-mil-row" in note
+    assert "hidden = false" in note
+
+
+def test_planespotters_lazy_photo_still_wired():
+    assert "function loadPlanePhoto" in HTML
+    assert "/api/aircraft/photo?" in HTML
+    assert "map.on('popupopen', (e) => { loadPlanePhoto(e.popup); });" in HTML

tests/test_api_live_layers.py

@@ -75,3 +75,35 @@ def test_vessels_subscribe_rejects_bad_bbox():
         assert resp.status_code == 422, bad
     # Explicit null bbox is the "reset to env default" path (still 200).
     assert asyncio.run(_post("/api/vessels/subscribe", {"bbox": None})).status_code == 200
+
+
+def test_aircraft_photo_requires_hex_or_reg():
+    assert asyncio.run(_get("/api/aircraft/photo")).status_code == 422
+
+
+def test_aircraft_photo_rejects_bad_hex():
+    # hex must be exactly 6 hex chars
+    resp = asyncio.run(_get("/api/aircraft/photo?hex=xyz1234"))
+    assert resp.status_code == 422
+
+
+def test_aircraft_photo_returns_photo(monkeypatch):
+    async def fake(hex_code=None, reg=None):
+        return {"id": "1", "src": "https://t.plnspttrs.net/a_280.jpg",
+                "link": "https://www.planespotters.net/photo/1/x", "photographer": "A"}
+
+    monkeypatch.setattr("main.fetch_planespotters_photo", fake)
+    resp = asyncio.run(_get("/api/aircraft/photo?hex=e8027e"))
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["src"].startswith("https://t.plnspttrs.net/")
+    assert "max-age" in (resp.headers.get("cache-control") or "").lower()
+
+
+def test_aircraft_photo_404_when_no_photo(monkeypatch):
+    async def fake(hex_code=None, reg=None):
+        return None
+
+    monkeypatch.setattr("main.fetch_planespotters_photo", fake)
+    resp = asyncio.run(_get("/api/aircraft/photo?reg=D-ABCD"))
+    assert resp.status_code == 404

tests/test_api_news.py

@@ -68,14 +68,14 @@ def _seed_article(title: str, url: str, domain: str, ts: str, content: str = "bo
     asyncio.run(run())
 
 
-def _seed_summary(text: str, ts: str, model: str | None = None) -> int:
+def _seed_summary(text: str, ts: str, model: str | None = None, kind: str | None = None) -> int:
     async def run() -> int:
         conn = await asyncpg.connect(**_conn_kwargs())
         try:
             row = await conn.fetchrow(
-                "INSERT INTO article_summaries (summary_text, batch_timestamp, model) "
-                "VALUES ($1, $2, $3) RETURNING id",
-                text, datetime.fromisoformat(ts), model,
+                "INSERT INTO article_summaries (summary_text, batch_timestamp, model, kind) "
+                "VALUES ($1, $2, $3, $4) RETURNING id",
+                text, datetime.fromisoformat(ts), model, kind,
             )
             return int(row["id"])
         finally:
@@ -164,9 +164,21 @@ def test_api_news_summaries_contract(clean_news):
     assert isinstance(body, list)
     assert len(body) == 1
     s = body[0]
-    assert set(s.keys()) == {"id", "summary_text", "batch_timestamp", "model"}
+    assert set(s.keys()) == {"id", "summary_text", "batch_timestamp", "model", "kind"}
     assert s["summary_text"] == "master summary markdown…"
     assert s["batch_timestamp"].startswith("2026-08-24T18:05")
+    assert s["kind"] is None
+
+
+@requires_db
+def test_api_news_summaries_kind_filter(clean_news):
+    _seed_summary("interval brief", "2026-08-28T22:05:00+00:00", kind="interval")
+    _seed_summary("daily recap", "2026-08-28T03:00:00+00:00", kind="daily_recap")
+    recap = _get("/api/news/summaries?kind=daily_recap").json()
+    assert len(recap) == 1
+    assert recap[0]["summary_text"] == "daily recap"
+    assert recap[0]["kind"] == "daily_recap"
+    assert _get("/api/news/summaries?kind=nope").status_code == 422
 
 
 @requires_db
@@ -218,6 +230,21 @@ def test_api_news_ticker_returns_only_flagged(clean_news):
     assert item["url"] == "https://example.com/ticker"
 
 
+@requires_db
+def test_api_news_ticker_falls_back_to_lesser_when_nothing_flagged(clean_news):
+    sid = _seed_summary("quiet brief", "2026-08-27T18:05:00+00:00", "Hermes-4.3-36B")
+    _seed_news_item(
+        sid, "ticker", "Shop theft downtown", "low",
+        location_name="Raleigh", url="https://example.com/theft",
+    )
+    resp = _get("/api/news/ticker")
+    assert resp.status_code == 200
+    body = resp.json()
+    assert len(body) == 1
+    assert body[0]["headline"] == "Shop theft downtown"
+    assert body[0]["importance"] == "low"
+
+
 @requires_db
 def test_api_news_map_returns_only_flagged_with_coords(clean_news):
     _seed_flagged_items()

tests/test_api_place.py

@@ -0,0 +1,127 @@
+"""GET /api/place — Nominatim reverse proxy (60s cache, 500 keys, 1 req/s)."""
+
+from __future__ import annotations
+
+import asyncio
+
+import httpx
+import pytest
+
+from main import app
+from place import cache_key, place_cache, slim_place
+
+BASE = "http://test"
+
+SAMPLE = {
+    "display_name": "Raleigh, Wake County, North Carolina, United States",
+    "name": "Raleigh",
+    "osm_type": "relation",
+    "osm_id": 123,
+    "address": {
+        "city": "Raleigh",
+        "state": "North Carolina",
+        "country": "United States",
+        "country_code": "us",
+        "tourism": "ignore-me",
+    },
+}
+
+
+class _FakeResp:
+    def __init__(self, payload, status=200):
+        self._payload = payload
+        self.status_code = status
+
+    def raise_for_status(self):
+        if self.status_code >= 400:
+            req = httpx.Request("GET", "https://nominatim.openstreetmap.org/reverse")
+            raise httpx.HTTPStatusError(
+                "upstream", request=req,
+                response=httpx.Response(self.status_code, request=req),
+            )
+
+    def json(self):
+        return self._payload
+
+
+class _FakeNominatim:
+    calls: list[dict] = []
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *args):
+        return False
+
+    async def get(self, url, params=None, headers=None):
+        _FakeNominatim.calls.append({"url": url, "params": params, "headers": headers})
+        return _FakeResp(SAMPLE)
+
+
+def _nominatim_client(**kwargs):
+    return _FakeNominatim()
+
+
+async def _get(path: str) -> httpx.Response:
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
+        return await client.get(path)
+
+
+@pytest.fixture(autouse=True)
+def _reset_place(monkeypatch):
+    place_cache.clear()
+    _FakeNominatim.calls = []
+    monkeypatch.setattr("place._http_client", _nominatim_client)
+    monkeypatch.setattr("place.NOMINATIM_MIN_INTERVAL", 0.0)
+    monkeypatch.setattr("place._last_req", 0.0)
+    yield
+    place_cache.clear()
+
+
+def test_slim_place_keeps_address_subset():
+    body = slim_place(35.78, -78.64, SAMPLE)
+    assert body["display_name"].startswith("Raleigh")
+    assert body["name"] == "Raleigh"
+    assert body["address"]["city"] == "Raleigh"
+    assert "tourism" not in body["address"]
+    assert body["attribution"].startswith("© OpenStreetMap")
+
+
+def test_cache_key_quantizes_to_4_decimals():
+    assert cache_key(35.77961, -78.63821) == cache_key(35.77964, -78.63819)
+
+
+def test_place_requires_lat_lon():
+    resp = asyncio.run(_get("/api/place"))
+    assert resp.status_code == 422
+
+
+def test_place_rejects_out_of_range():
+    assert asyncio.run(_get("/api/place?lat=99&lon=0")).status_code == 422
+    assert asyncio.run(_get("/api/place?lat=0&lon=200")).status_code == 422
+
+
+def test_place_reverse_and_cache():
+    r1 = asyncio.run(_get("/api/place?lat=35.7796&lon=-78.6382"))
+    assert r1.status_code == 200
+    body = r1.json()
+    assert body["display_name"].startswith("Raleigh")
+    assert body["lat"] == pytest.approx(35.7796, abs=0.001)
+    assert "max-age=60" in (r1.headers.get("cache-control") or "").lower()
+    assert len(_FakeNominatim.calls) == 1
+    ua = _FakeNominatim.calls[0]["headers"]["User-Agent"]
+    assert "osint-dashboard" in ua.lower() or "@" in ua
+    r2 = asyncio.run(_get("/api/place?lat=35.77961&lon=-78.63821"))
+    assert r2.status_code == 200
+    assert len(_FakeNominatim.calls) == 1  # cache hit, same 4-decimal key
+
+
+def test_place_cache_cap_500():
+    from cachetools import TTLCache
+    assert isinstance(place_cache, TTLCache)
+    assert place_cache.maxsize == 500
+    assert place_cache.ttl == 60

tests/test_api_stats.py

@@ -0,0 +1,87 @@
+"""GET /api/stats HUD counter contract (counts only, small, never 500)."""
+
+from __future__ import annotations
+
+import asyncio
+import re
+from datetime import timezone
+
+import httpx
+
+from main import app, _stats_counts
+
+BASE = "http://test"
+
+EXPECTED_KEYS = ("aircraft", "vessels", "trains", "cameras",
+                 "fires", "quakes", "alerts", "timestamp")
+
+
+async def _get(path: str) -> httpx.Response:
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
+        return await client.get(path)
+
+
+def test_stats_200_all_keys_present():
+    resp = asyncio.run(_get("/api/stats"))
+    assert resp.status_code == 200
+    body = resp.json()
+    for key in EXPECTED_KEYS:
+        assert key in body, f"missing key {key}"
+    assert "max-age" in (resp.headers.get("cache-control") or "").lower()
+
+
+def test_stats_counters_are_ints():
+    body = asyncio.run(_get("/api/stats")).json()
+    for key in EXPECTED_KEYS:
+        if key == "timestamp":
+            continue
+        assert isinstance(body[key], int), f"{key} is not an int: {body[key]!r}"
+
+
+def test_stats_timestamp_is_iso8601_z():
+    body = asyncio.run(_get("/api/stats")).json()
+    ts = body["timestamp"]
+    # ISO8601 with a trailing Z (we normalize +00:00 -> Z).
+    assert isinstance(ts, str) and ts.endswith("Z")
+    assert re.match(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", ts)
+
+
+def test_stats_payload_is_tiny():
+    resp = asyncio.run(_get("/api/stats"))
+    assert len(resp.content) < 2048, "stats payload must be counts-only, not GeoJSON"
+
+
+def test_stats_counts_reflect_last_known(monkeypatch):
+    """aircraft/vessels/trains/alerts come from in-memory last-known state."""
+    import live_layers
+
+    monkeypatch.setattr(live_layers, "aircraft_last_known", {str(i): {} for i in range(7)})
+    monkeypatch.setattr(live_layers, "vessel_last_known", {str(i): {} for i in range(3)})
+    monkeypatch.setattr(live_layers, "train_count", 11)
+    monkeypatch.setattr(live_layers, "nws_alert_count", 5)
+
+    # _stats_counts imports the dicts/counters inside the function from live_layers,
+    # so monkeypatching the module attributes is what it observes.
+    from main import _stats_counts as fn
+
+    body = asyncio.run(fn())
+    assert body["aircraft"] == 7
+    assert body["vessels"] == 3
+    assert body["trains"] == 11
+    assert body["alerts"] == 5
+
+
+def test_stats_db_failure_degrades_to_zero(monkeypatch):
+    """A down DB yields zeros for the SQL-backed counters, never a 500."""
+    # Make the session factory raise synchronously so the try/except in
+    # _stats_counts degrades the SQL counters to zero (no dangling coroutine).
+    def _raise(*args, **kwargs):
+        raise RuntimeError("db down")
+
+    monkeypatch.setattr("main.async_session", _raise)
+    body = asyncio.run(_stats_counts())
+    assert body["cameras"] == 0
+    assert body["fires"] == 0
+    assert body["quakes"] == 0
+    assert isinstance(body["timestamp"], str)

tests/test_bg_jobs.py

@@ -1,4 +1,4 @@
-"""masscan/ffmpeg stay off the request path (asyncio.create_task)."""
+"""ffmpeg snapshots stay off the request path (asyncio.create_task)."""
 
 from __future__ import annotations
 
@@ -7,36 +7,27 @@ import asyncio
 import bg_jobs
 
 
-def test_schedule_masscan_pass_returns_without_awaiting_scan(monkeypatch):
-    started = {"n": 0}
-
-    async def slow_pass():
-        started["n"] += 1
-        await asyncio.sleep(30)
-
-    monkeypatch.setattr(bg_jobs, "_run_masscan_capped", slow_pass)
-    bg_jobs._masscan_task = None
-
-    async def run():
-        launched = bg_jobs.schedule_masscan_pass()
-        assert launched is True
-        # Must not have blocked for the 30s pass.
-        assert bg_jobs._masscan_task is not None
-        assert not bg_jobs._masscan_task.done()
-        launched2 = bg_jobs.schedule_masscan_pass()
-        assert launched2 is False  # already running
-        bg_jobs._masscan_task.cancel()
-        try:
-            await bg_jobs._masscan_task
-        except (asyncio.CancelledError, Exception):
-            pass
-        bg_jobs._masscan_task = None
-
-    asyncio.run(run())
+def test_bg_jobs_has_no_pps_cap():
+    assert not any(name.endswith("_PPS_CAP") for name in dir(bg_jobs))
 
 
-def test_masscan_rate_cap_is_200():
-    assert bg_jobs.MASSCAN_PPS_CAP == 200
+def test_camera_preview_has_no_public_feed_probe():
+    import camera_preview
+
+    assert not hasattr(camera_preview, "probe_public_feed")
+    assert not hasattr(camera_preview, "_http_feed_url")
+
+
+def test_ingest_routes_exclude_active_discovery():
+    from main import app
+
+    ingest = [
+        getattr(r, "path", "")
+        for r in app.routes
+        if getattr(r, "path", "").startswith("/api/ingest/")
+    ]
+    assert "/api/ingest/fires" in ingest
+    assert all("scan" not in path for path in ingest)
 
 
 def test_schedule_ffmpeg_snapshot_is_a_task_not_inline(monkeypatch):

tests/test_cache_and_timeline.py

@@ -0,0 +1,21 @@
+"""Timeline bucket_hours + static Cache-Control."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def test_timeline_uses_bucket_hours():
+    src = (ROOT / "app/main.py").read_text()
+    fn = src.split("async def get_timeline")[1].split("async def sentiment_by_source")[0]
+    assert "bucket_hours" in fn
+    assert "date_trunc('hour'" not in fn or "bucket" in fn.lower()
+    # Must not ignore the query param.
+    assert ":bucket" in fn or "bucket_hours" in fn.split("text(")[1][:800]
+
+
+def test_static_vendor_cache_control():
+    src = (ROOT / "app/main.py").read_text()
+    assert "max-age=31536000" in src or "immutable" in src.lower()

tests/test_camera_popup_contract.py

@@ -0,0 +1,15 @@
+"""Camera list is slim (no URLs). Popup must fetch GET /api/cameras/{id}."""
+
+from pathlib import Path
+
+HTML = Path(__file__).resolve().parents[1] / "app/static/index.html"
+
+
+def test_popupopen_fetches_camera_detail_row():
+    html = HTML.read_text()
+    start = html.index("map.on('popupopen'")
+    end = html.index("map.on('popupclose'")
+    block = html[start:end]
+    assert "fetch(" in block
+    assert "/api/cameras/" in block
+    assert "camPopupHtml(" in block

tests/test_chokepoints.py

@@ -0,0 +1,122 @@
+"""Tests for the chokepoint preset catalog + vessels ``src=`` filter.
+
+- Span: every catalog box passes VesselAPI's ``|dLat|+|dLon| <= 4`` validator.
+- Catalog: ``GET /api/map/chokepoints`` returns 200 with the documented shape.
+- Vessels filter: ``GET /api/vessels?src=`` narrows the union store by provider.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import httpx
+
+from chokepoints import chokepoints
+from live_layers import fetch_vessels, vessel_last_known
+from main import app
+from vesselapi import validate_bbox_span
+
+BASE = "http://test"
+
+
+async def _get(path: str) -> httpx.Response:
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
+        return await client.get(path)
+
+
+# ── Span validation (VesselAPI rule) ──────────────────────────────────────
+
+def test_all_catalog_boxes_within_span() -> None:
+    for preset in chokepoints():
+        minlat, minlon, maxlat, maxlon = (float(p) for p in preset["bbox"].split(","))
+        dlat = abs(maxlat - minlat)
+        dlon = abs(maxlon - minlon)
+        assert dlat + dlon <= 4.0, preset["id"]
+        validate_bbox_span(minlat, minlon, maxlat, maxlon)  # no raise
+
+
+# ── Catalog API contract ──────────────────────────────────────────────────
+
+def test_chokepoints_catalog_shape() -> None:
+    resp = asyncio.run(_get("/api/map/chokepoints"))
+    assert resp.status_code == 200
+    body = resp.json()
+    assert set(body) == {"chokepoints"}
+    rows = body["chokepoints"]
+    assert [r["id"] for r in rows] == [
+        "hormuz", "bab_el_mandeb", "suez", "malacca", "taiwan",
+    ]
+    for r in rows:
+        assert set(r) == {"id", "title", "bbox", "center", "zoom", "vesselapi"}
+        assert isinstance(r["center"], list) and len(r["center"]) == 2
+        assert r["zoom"] == 9
+        assert isinstance(r["vesselapi"], bool)
+        # bbox is minlat,minlon,maxlat,maxlon
+        minlat, minlon, maxlat, maxlon = (float(p) for p in r["bbox"].split(","))
+        assert minlat < maxlat and minlon < maxlon
+
+
+def test_only_hormuz_is_vesselapi() -> None:
+    rows = chokepoints()
+    by_id = {r["id"]: r for r in rows}
+    assert by_id["hormuz"]["vesselapi"] is True
+    for cid in ("bab_el_mandeb", "suez", "malacca", "taiwan"):
+        assert by_id[cid]["vesselapi"] is False
+
+
+# ── Vessels src= filter (mocked store) ────────────────────────────────────
+
+def _seed_store() -> None:
+    vessel_last_known.clear()
+    vessel_last_known["422050100"] = {
+        "id": "422050100", "lat": 26.5, "lon": 56.3, "label": "HORMUZ STAR",
+        "extra": {"src": "vesselapi", "mmsi": "422050100"},
+    }
+    vessel_last_known["366001230"] = {
+        "id": "366001230", "lat": 35.0, "lon": -79.0, "label": "CONUS SHIP",
+        "extra": {"src": "aisstream", "mmsi": "366001230"},
+    }
+    vessel_last_known["366001231"] = {
+        "id": "366001231", "lat": 36.0, "lon": -78.0, "label": "CONUS SHIP 2",
+        "extra": {"src": "aisstream", "mmsi": "366001231"},
+    }
+
+
+def test_fetch_vessels_src_filters() -> None:
+    _seed_store()
+    assert {v["id"] for v in asyncio.run(fetch_vessels(None, src="vesselapi"))} == {"422050100"}
+    assert {v["id"] for v in asyncio.run(fetch_vessels(None, src="aisstream"))} == {
+        "366001230", "366001231",
+    }
+    assert len(asyncio.run(fetch_vessels(None, src="all"))) == 3
+    assert len(asyncio.run(fetch_vessels(None))) == 3  # default all
+
+
+def test_vessels_src_query_param(monkeypatch) -> None:
+    _seed_store()
+
+    async def _fake_fetch(bbox, limit, src=None):
+        rows = [
+            {"id": k, **{kk: v[kk] for kk in ("lat", "lon", "label", "extra")}}
+            for k, v in vessel_last_known.items()
+        ]
+        if src and src != "all":
+            rows = [r for r in rows if (r.get("extra") or {}).get("src") == src]
+        return rows
+
+    monkeypatch.setattr("main.fetch_vessels", _fake_fetch)
+
+    body = asyncio.run(_get("/api/vessels?src=vesselapi")).json()
+    assert [r["id"] for r in body] == ["422050100"]
+
+    body = asyncio.run(_get("/api/vessels?src=aisstream")).json()
+    assert {r["id"] for r in body} == {"366001230", "366001231"}
+
+    body = asyncio.run(_get("/api/vessels?src=all")).json()
+    assert len(body) == 3
+
+
+def test_vessels_src_rejects_bad_value() -> None:
+    resp = asyncio.run(_get("/api/vessels?src=marine-traffic"))
+    assert resp.status_code == 422

tests/test_conflicts.py

@@ -0,0 +1,132 @@
+"""GET /api/conflicts — curated conflict-zone catalog + event-count roll-up.
+
+No outbound HTTP: event counts come from geocoded rows already (or not) in the
+DB, and the API tests monkeypatch ``main._fetch_geocoded_points`` so no database
+is required for the contract checks.
+"""
+
+from datetime import datetime, timezone
+
+import httpx
+
+from conflicts import SEVERITIES, conflict_zones, zone_event_stats
+from live_layers import overlay_catalog
+from main import app
+
+BASE = "http://test"
+
+
+def _get(path: str, monkeypatch=None, points=None) -> httpx.Response:
+    import asyncio
+
+    async def run() -> httpx.Response:
+        if monkeypatch is not None:
+            async def fake():
+                return points or []
+
+            monkeypatch.setattr("main._fetch_geocoded_points", fake)
+        transport = httpx.ASGITransport(app=app)
+        async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
+            return await client.get(path)
+
+    return asyncio.run(run())
+
+
+# ── Catalog shape ──────────────────────────────────────────────────────
+
+
+def test_catalog_length():
+    zones = conflict_zones()
+    assert len(zones) == 13
+
+
+def test_catalog_severity_enum():
+    zones = conflict_zones()
+    sevs = {z["severity"] for z in zones}
+    assert sevs.issubset(SEVERITIES)
+    # All three tiers are represented.
+    assert sevs == SEVERITIES
+
+
+def test_catalog_fields_factual_and_complete():
+    zones = conflict_zones()
+    ids = [z["id"] for z in zones]
+    assert len(set(ids)) == len(ids)  # unique ids
+    for z in zones:
+        assert z["label"]
+        assert z["description"].strip()
+        assert -90.0 <= z["lat"] <= 90.0
+        assert -180.0 <= z["lon"] <= 180.0
+        # internal-only bbox is well-formed: (min_lat, min_lon, max_lat, max_lon)
+        min_lat, min_lon, max_lat, max_lon = z["bbox"]
+        assert min_lat <= max_lat and min_lon <= max_lon
+        assert min_lat <= z["lat"] <= max_lat and min_lon <= z["lon"] <= max_lon
+
+
+def test_overlay_catalog_has_conflicts():
+    entry = overlay_catalog()["conflicts"]
+    assert entry["kind"] == "points"
+    assert entry["endpoint"] == "/api/conflicts"
+
+
+# ── Pure counting ──────────────────────────────────────────────────────
+
+TS1 = datetime(2026, 8, 30, 12, 0, tzinfo=timezone.utc)
+TS2 = datetime(2026, 8, 30, 13, 0, tzinfo=timezone.utc)
+
+
+def test_zone_event_stats_counts_and_picks_latest():
+    bbox = (40.0, 20.0, 52.0, 40.0)  # roughly Ukraine
+    points = [
+        (50.45, 30.52, TS1),  # inside
+        (48.0, 25.0, TS2),    # inside, later
+        (0.0, -60.0, TS1),    # outside
+        (15.0, 45.0, TS2),    # outside (lat ok, lon out)
+    ]
+    count, latest = zone_event_stats(points, bbox)
+    assert count == 2
+    assert latest == TS2
+
+
+def test_zone_event_stats_empty_bbox():
+    count, latest = zone_event_stats([], (0.0, 0.0, 1.0, 1.0))
+    assert count == 0
+    assert latest is None
+
+
+# ── API contract (mocked map items, no DB) ─────────────────────────────
+
+
+def test_conflicts_returns_catalog_with_mocked_counts(monkeypatch):
+    points = [
+        (50.45, 30.52, TS1),   # Ukraine
+        (25.03, 121.56, TS2),  # Taiwan Strait
+        (0.0, -60.0, TS1),     # nowhere
+    ]
+    resp = _get("/api/conflicts", monkeypatch=monkeypatch, points=points)
+    assert resp.status_code == 200
+    body = resp.json()
+    assert "zones" in body and "timestamp" in body
+    by_id = {z["id"]: z for z in body["zones"]}
+    assert len(body["zones"]) == 13
+
+    zone = by_id["ukraine"]
+    assert zone["eventCount"] == 1
+    assert zone["lastUpdated"] == TS1.isoformat().replace("+00:00", "Z")
+    assert zone["severity"] == "war"
+
+    assert by_id["taiwan_strait"]["eventCount"] == 1
+    assert by_id["gaza"]["eventCount"] == 0
+    # exact per-zone key contract the frontend consumes
+    assert set(zone.keys()) == {
+        "id", "label", "severity", "lat", "lon",
+        "description", "eventCount", "lastUpdated",
+    }
+
+
+def test_conflicts_empty_db_yields_zero_counts(monkeypatch):
+    resp = _get("/api/conflicts", monkeypatch=monkeypatch, points=[])
+    assert resp.status_code == 200
+    body = resp.json()
+    assert all(z["eventCount"] == 0 for z in body["zones"])
+    assert all(z["lastUpdated"] is None for z in body["zones"])

tests/test_conflicts_frontend.py

@@ -0,0 +1,66 @@
+"""Conflicts Leaflet overlay: default-off toggle, catalog fetch, no jitter."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+HTML = (ROOT / "app/static/index.html").read_text()
+
+
+def _fn(name: str, until: str | None = None) -> str:
+    chunk = HTML.split(f"function {name}", 1)[1]
+    if until:
+        chunk = chunk.split(until, 1)[0]
+    return chunk
+
+
+def test_conflicts_toggle_default_off():
+    assert 'id="lp-conflicts-on"' in HTML
+    assert 'id="conflicts-layer"' in HTML
+    assert "> Conflicts<" in HTML or "> Conflicts</" in HTML
+    on = HTML.split('id="lp-conflicts-on"', 1)[1].split(">", 1)[0]
+    assert "checked" not in on
+
+
+def test_conflicts_fetches_catalog_not_liveuamap():
+    js = _fn("loadConflicts", "/* ═══════════════ INITIAL LOAD")
+    assert "/api/conflicts" in js
+    assert "liveuamap.com" not in HTML.lower()
+    assert "Math.random" not in js
+    assert "jitter" not in js.lower()
+
+
+def test_conflicts_not_refetched_on_moveend():
+    refresh = HTML.split("function refreshLiveOverlays", 1)[1].split(
+        "function addExtraAttrib", 1
+    )[0]
+    assert "loadConflicts" not in refresh
+    assert "probeConflicts" not in refresh
+    init = HTML.split("function initMap", 1)[1].split("function readMapPrefs", 1)[0]
+    assert "probeConflicts()" in init
+    assert "loadConflicts(true)" not in init
+    assert "paintConflicts()" not in init
+
+
+def test_conflicts_hides_toggle_on_404():
+    js = _fn("loadConflicts", "/* ═══════════════ INITIAL LOAD")
+    assert "r.status === 404" in js
+    assert "hideConflictsToggle()" in js
+    hide = _fn("hideConflictsToggle", "function paintConflicts")
+    assert "row.hidden = true" in hide
+    assert "lp-conflicts-on" in hide
+
+
+def test_conflicts_popup_and_severity_colors():
+    paint = _fn("paintConflicts", "async function probeConflicts")
+    assert "z.label" in paint
+    assert "z.description" in paint
+    assert "eventCount" in paint
+    assert "L.circleMarker" in paint
+    assert "z.lat == null || z.lon == null" in paint
+    assert "Number.isFinite(lat)" in paint
+    color = _fn("conflictSeverityColor", "function hideConflictsToggle")
+    assert "war" in color and "#ff2a6d" in color
+    assert "high" in color and "#fb923c" in color
+    assert "elevated" in color and "#facc15" in color

tests/test_event_ingest.py

@@ -0,0 +1,133 @@
+"""Generic event ingest: idempotency, USGS ids, GDELT DOC URL."""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timezone
+
+
+def test_event_dedup_key_prefers_url():
+    from sources import event_dedup_key
+
+    assert event_dedup_key({"url": "https://earthquake.usgs.gov/earthquakes/eventpage/ci1"}) == (
+        "https://earthquake.usgs.gov/earthquakes/eventpage/ci1"
+    )
+    assert event_dedup_key({"url": "  "}) is None
+    assert event_dedup_key({}) is None
+
+
+def test_usgs_feature_keeps_id_and_url():
+    from sources import parse_usgs_feature
+
+    feature = {
+        "id": "ci39818991",
+        "properties": {
+            "title": "M 2.1 - 5 km W of",
+            "url": "https://earthquake.usgs.gov/earthquakes/eventpage/ci39818991",
+            "place": "5 km W of",
+            "mag": 2.1,
+            "time": 1_700_000_000_000,
+        },
+        "geometry": {"coordinates": [-118.5, 34.1, 10.0]},
+    }
+    event = parse_usgs_feature(feature)
+    assert event["url"] == "https://earthquake.usgs.gov/earthquakes/eventpage/ci39818991"
+    assert event["raw"]["usgs_id"] == "ci39818991"
+    assert event["location_lat"] == 34.1
+    assert event["location_lon"] == -118.5
+    assert event["source_type"] == "earthquake"
+
+
+def test_gdelt_uses_doc_api_and_query_param():
+    from sources import GDELT_API, gdelt_params
+
+    assert GDELT_API == "https://api.gdeltproject.org/api/v2/doc/doc"
+    params = gdelt_params(query="unrest", max_articles=50)
+    assert params["query"] == "unrest"
+    assert "search" not in params
+    assert params["mode"] == "ArtList"
+    assert params["format"] == "json"
+    assert int(params["maxrecords"]) == 50
+
+
+def test_gdelt_default_query_when_empty():
+    from sources import gdelt_params
+
+    params = gdelt_params(query="", max_articles=25)
+    assert params["query"]
+    assert "unrest" in params["query"].lower() or "cyber" in params["query"].lower()
+
+
+def test_parse_gdelt_articles_maps_doc_payload():
+    from sources import parse_gdelt_articles
+
+    payload = {
+        "articles": [
+            {
+                "url": "https://example.com/a",
+                "title": "Outage",
+                "seendate": "20240101T120000Z",
+                "domain": "example.com",
+                "language": "English",
+                "sourcecountry": "US",
+            }
+        ]
+    }
+    events = parse_gdelt_articles(payload)
+    assert len(events) == 1
+    assert events[0]["source_type"] == "gdel-t2"
+    assert events[0]["url"] == "https://example.com/a"
+    assert events[0]["title"] == "Outage"
+
+
+def test_ingest_event_skips_duplicate_url(monkeypatch):
+    """Second insert with the same url must not hit events_table.insert."""
+    from ingestor import ingest_event
+
+    calls = {"insert": 0, "dedup": 0}
+
+    class _Result:
+        rowcount = 1
+        inserted_primary_key = ["evt-1"]
+
+    class _Session:
+        async def execute(self, stmt):
+            sql = str(stmt).lower()
+            if "event_dedup" in sql or "on conflict" in sql:
+                calls["dedup"] += 1
+                self_result = _Result()
+                if calls["dedup"] > 1:
+                    self_result.rowcount = 0
+                return self_result
+            calls["insert"] += 1
+            return _Result()
+
+        async def commit(self):
+            return None
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *a):
+            return False
+
+    import ingestor
+
+    monkeypatch.setattr(ingestor, "async_session", lambda: _Session())
+
+    msg = {
+        "source_type": "earthquake",
+        "title": "M 2.1",
+        "url": "https://earthquake.usgs.gov/earthquakes/eventpage/ci1",
+        "source_timestamp": datetime(2026, 1, 1, tzinfo=timezone.utc).isoformat(),
+    }
+
+    async def run():
+        first = await ingest_event(msg)
+        second = await ingest_event(msg)
+        return first, second
+
+    first, second = asyncio.run(run())
+    assert first is not None
+    assert second is None
+    assert calls["insert"] == 1

tests/test_fire_ingest.py

@@ -169,3 +169,99 @@ def test_ingest_fire_row_correlates_from_hypertable_when_last_known_empty(monkey
     assert len(correlated) == 1
     assert correlated[0][1][0]["id"] == "acf001"
     assert correlated[0][0][0]["lat"] == 39.45678
+
+
+def test_ingest_fire_rows_one_execute_one_commit(monkeypatch):
+    """93k FIRMS points must not be 93k commits."""
+    from ingestor import ingest_fire_rows
+
+    class _Session:
+        def __init__(self):
+            self.executes = 0
+            self.commits = 0
+            self.rowcount = 3
+
+        async def execute(self, *a, **k):
+            self.executes += 1
+            return self
+
+        async def commit(self):
+            self.commits += 1
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *a):
+            return False
+
+    session = _Session()
+    import ingestor
+    monkeypatch.setattr(ingestor, "async_session", lambda: session)
+
+    msgs = [
+        make_fire_msg(latitude=39.1 + i * 0.01, longitude=-121.1)
+        for i in range(3)
+    ]
+
+    async def no_corr(*a, **k):
+        return []
+
+    monkeypatch.setattr("fire_aircraft.correlate_and_notify", no_corr)
+    monkeypatch.setattr("geofence.record_and_notify", no_corr)
+
+    inserted = asyncio.run(ingest_fire_rows(msgs))
+    assert inserted == 3
+    assert session.executes == 1
+    assert session.commits == 1
+
+
+def test_fire_insert_chunk_stays_under_asyncpg_bind_limit():
+    """asyncpg caps bind params at 32767 — a 93k-row INSERT dies."""
+    from ingestor import FIRE_INSERT_CHUNK, FIRE_ROW_BIND_PARAMS
+
+    assert FIRE_INSERT_CHUNK * FIRE_ROW_BIND_PARAMS < 32767
+    assert FIRE_INSERT_CHUNK >= 500
+
+
+def test_ingest_fire_rows_chunks_when_over_limit(monkeypatch):
+    from ingestor import ingest_fire_rows
+    import ingestor
+
+    class _Session:
+        def __init__(self):
+            self.executes = 0
+            self.commits = 0
+            self.rowcount = 0
+
+        async def execute(self, *a, **k):
+            self.executes += 1
+            self.rowcount = 2 if self.executes < 3 else 1
+            return self
+
+        async def commit(self):
+            self.commits += 1
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *a):
+            return False
+
+    session = _Session()
+    monkeypatch.setattr(ingestor, "async_session", lambda: session)
+    monkeypatch.setattr(ingestor, "FIRE_INSERT_CHUNK", 2)
+
+    async def no_corr(*a, **k):
+        return []
+
+    monkeypatch.setattr("fire_aircraft.correlate_and_notify", no_corr)
+    monkeypatch.setattr("geofence.record_and_notify", no_corr)
+
+    msgs = [
+        make_fire_msg(latitude=39.1 + i * 0.01, longitude=-121.1)
+        for i in range(5)
+    ]
+    inserted = asyncio.run(ingest_fire_rows(msgs))
+    assert inserted == 5
+    assert session.executes == 3
+    assert session.commits == 1

tests/test_fire_sources.py

@@ -123,7 +123,7 @@ def test_ingest_fires_uses_keystore_key(monkeypatch):
         published.extend(points)
         return len(points)
 
-    monkeypatch.setattr("fire_sources.publish_fire_batch", fake_publish)
+    monkeypatch.setattr("fire_sources.persist_hotspots", fake_publish)
 
     assert asyncio.run(ingest_fires()) == 10  # NOAA-20 + NOAA-21 dual-write
     assert "a" * 32 in captured["url"]
@@ -131,6 +131,100 @@ def test_ingest_fires_uses_keystore_key(monkeypatch):
     assert any("VIIRS_NOAA21_NRT" in u for u in captured["urls"])
 
 
+def _reset_firms_poll_state():
+    from upstream_cache import firms_cache
+    import fire_sources
+
+    firms_cache.clear()
+    if hasattr(fire_sources, "_csv_digest"):
+        fire_sources._csv_digest.clear()
+    if hasattr(fire_sources, "_seen_ids"):
+        fire_sources._seen_ids.clear()
+
+
+def _fake_firms_http(monkeypatch, bodies_by_call: list[str] | None = None, body: str = SAMPLE_CSV):
+    hits = {"n": 0}
+
+    class FakeResp:
+        def __init__(self, text):
+            self.text = text
+
+        def raise_for_status(self):
+            pass
+
+    class FakeClient:
+        def __init__(self, **kw):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *exc):
+            return False
+
+        async def get(self, url):
+            idx = hits["n"]
+            hits["n"] += 1
+            if bodies_by_call is not None:
+                text = bodies_by_call[min(idx, len(bodies_by_call) - 1)]
+            else:
+                text = body
+            return FakeResp(text)
+
+    monkeypatch.setenv("FIRMS_MAP_KEY", "k" * 32)
+    monkeypatch.setattr("fire_sources.FIRMS_DATASETS", ["VIIRS_NOAA20_NRT"])
+    monkeypatch.setattr("fire_sources.httpx.AsyncClient", FakeClient)
+    return hits
+
+
+def test_ingest_fires_skips_unchanged_csv(monkeypatch):
+    """Same FIRMS CSV must not be re-parsed into a 100k-row ON CONFLICT insert."""
+    _reset_firms_poll_state()
+    hits = _fake_firms_http(monkeypatch)
+    persisted = []
+
+    async def fake_persist(points):
+        persisted.append(len(points))
+        return len(points)
+
+    monkeypatch.setattr("fire_sources.persist_hotspots", fake_persist)
+
+    assert asyncio.run(ingest_fires()) == 5
+    assert persisted == [5]
+    firms_cache_hits = hits["n"]
+    persisted.clear()
+    assert asyncio.run(ingest_fires()) == 0
+    assert persisted == []
+    # TTL cache may skip HTTP; either way we must not persist again.
+    assert hits["n"] >= firms_cache_hits
+
+
+def test_ingest_fires_persists_only_new_hotspots(monkeypatch):
+    """When the CSV grows, persist the delta — not the whole 2-day dump."""
+    _reset_firms_poll_state()
+    extra = (
+        SAMPLE_CSV
+        + "16.00000,-12.00000,340.00,0.40,0.40,2025-06-06,1500,N20,VIIRS,h,2.0NRT,310.00,8.00,D\n"
+    )
+    hits = _fake_firms_http(monkeypatch, bodies_by_call=[SAMPLE_CSV, extra])
+    persisted = []
+
+    async def fake_persist(points):
+        persisted.append([p["latitude"] for p in points])
+        return len(points)
+
+    monkeypatch.setattr("fire_sources.persist_hotspots", fake_persist)
+
+    from upstream_cache import firms_cache
+
+    assert asyncio.run(ingest_fires()) == 5
+    firms_cache.clear()  # force the next poll to see the grown CSV
+    persisted.clear()
+    assert asyncio.run(ingest_fires()) == 1
+    assert persisted == [[16.0]]
+    assert hits["n"] == 2
+
+
 def _async_return(value):
     async def inner():
         return value

tests/test_frontend_reliability.py

@@ -0,0 +1,86 @@
+"""HUD load-time: no market 404 poll, deferred overlays, WS backoff, nginx snippet."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+HTML = (ROOT / "app/static/index.html").read_text()
+
+
+def test_summarizer_dockerfile_copies_intel_modules():
+    df = (ROOT / "news/summerizer/Dockerfile").read_text()
+    assert "intel.py" in df
+    assert "nous_client.py" in df
+
+
+def test_news_panel_pins_daily_recap():
+    assert "kind=daily_recap" in HTML
+    assert "DAILY RECAP" in HTML
+
+
+def test_nginx_ws_snippet_has_upgrade_headers():
+    conf = (ROOT / "deploy/osint-ws.nginx.conf").read_text()
+    assert "proxy_http_version 1.1" in conf
+    assert "Upgrade" in conf
+    assert "Connection" in conf
+    assert "/ws/" in conf
+
+
+def test_market_ticker_does_not_poll_unwired_endpoint():
+    assert "setInterval(probeMarket" not in HTML
+    assert "initMarketTicker()" not in HTML or "probeMarket();" not in HTML.split("function initMarketTicker")[1][:400]
+
+
+def test_startup_defers_nonessential_overlays():
+    init = HTML.split("function initMap")[1].split("function readMapPrefs")[0]
+    # Must not fire all four DB loads + live overlays in the same tick.
+    assert "setTimeout" in init or "requestAnimationFrame" in init
+
+
+def test_ws_reconnect_uses_backoff():
+    assert "setTimeout(connectLiveWs, 4000)" not in HTML
+    ws = HTML.split("function connectLiveWs")[1][:1200]
+    assert "backoff" in ws.lower() or "wsRetry" in ws or "wsDelay" in ws
+
+
+def test_check_health_treats_degraded_status():
+    fn = HTML.split("async function checkHealth")[1].split("/* ═══════════════ NAV")[0]
+    assert "degraded" in fn.lower() or "d.status" in fn
+
+
+def test_chokepoint_presets_in_toolbar():
+    assert 'id="chokepoint-btns"' in HTML
+    assert 'id="chokepoint-select"' in HTML
+    assert "loadChokepoints()" in HTML
+    assert "/api/map/chokepoints" in HTML
+    assert "function applyChokepoint" in HTML
+    for name in ("Hormuz", "Bab el-Mandeb", "Suez", "Malacca", "Taiwan"):
+        assert name in HTML
+
+
+def test_chokepoint_skips_aisstream_subscribe_outside_conus():
+    load = HTML.split("async function loadVessels")[1].split("async function toggleStorms")[0]
+    assert "intersectsConus()" in load
+    assert "api/vessels/subscribe" in load
+    assert "src=${encodeURIComponent(vesselSrcPref)}" in load or "&src=" in load
+    apply = HTML.split("function applyChokepoint")[1].split("function currentBBox")[0]
+    assert "vesselapi" in apply
+    assert "lp-vessels-on" in apply
+    assert "lp-sentinel-on" in apply
+    assert "map.setView" in apply
+    assert "minlat,minlon,maxlat,maxlon" in HTML.split("function chokepointLeafletBounds")[1][:400]
+
+
+def test_news_ticker_polls_more_often_than_summarizer_cycle():
+    assert "NEWS_REFRESH_MS" in HTML
+    # Summarizer is 15 min; ticker should refresh on a shorter cadence so
+    # lesser-news fills show up without waiting for the next brief.
+    line = [ln for ln in HTML.splitlines() if "NEWS_REFRESH_MS" in ln][0]
+    assert "900000" not in line
+
+
+def test_phone_chokepoints_use_select_not_buttons():
+    mobile = HTML.split("@media (max-width: 820px)")[1].split("@media (prefers-reduced-motion")[0]
+    assert "#chokepoint-select { display: block; }" in mobile
+    assert ".chokepoint-btns { display: none; }" in mobile or "#chokepoint-label, .chokepoint-btns { display: none; }" in mobile

tests/test_geofence.py

@@ -51,23 +51,33 @@ def test_matching_geofences_only_active_hits():
     assert matching_geofences(-122.4, 37.7, fences) == []
 
 
+FENCE_ID = "11111111-1111-1111-1111-111111111111"
+NC_VIEW = (-80.0, 35.0, -78.0, 36.0)
+SF_VIEW = (-123.0, 37.0, -121.0, 38.0)
+
+
+def _alert_payload(gid=FENCE_ID):
+    return {
+        "geofence_id": gid,
+        "geofence_name": "NC",
+        "source_kind": "ais",
+        "entity_id": "366123456",
+        "lat": 35.5,
+        "lon": -79.0,
+    }
+
+
 def test_geofence_alert_fans_out_only_to_viewport_clients():
     mgr = ConnectionManager()
     q_nc = mgr.register("nc")
     q_sf = mgr.register("sf")
-    mgr.set_viewport("nc", (-80.0, 35.0, -78.0, 36.0))
-    mgr.set_viewport("sf", (-123.0, 37.0, -121.0, 38.0))
+    mgr.set_viewport("nc", NC_VIEW)
+    mgr.set_viewport("sf", SF_VIEW)
 
     async def run():
-        payload = {
-            "geofence_id": "a",
-            "geofence_name": "NC",
-            "source_kind": "ais",
-            "entity_id": "366123456",
-            "lat": 35.5,
-            "lon": -79.0,
-        }
-        n = await mgr.publish_point("geofence_alert", payload, lat=35.5, lon=-79.0)
+        n = await mgr.publish_point(
+            "geofence_alert", _alert_payload(), lat=35.5, lon=-79.0,
+        )
         assert n == 1
         msg = q_nc.get_nowait()
         assert msg["type"] == "geofence_alert"
@@ -77,6 +87,109 @@ def test_geofence_alert_fans_out_only_to_viewport_clients():
     asyncio.run(run())
 
 
+def test_off_viewport_watch_receives_geofence_alert():
+    mgr = ConnectionManager()
+    q_sf = mgr.register("sf")
+    mgr.set_viewport("sf", SF_VIEW)
+    mgr.set_watched_geofences("sf", [FENCE_ID])
+
+    async def run():
+        n = await mgr.publish_point(
+            "geofence_alert", _alert_payload(), lat=35.5, lon=-79.0,
+        )
+        assert n == 1
+        msg = q_sf.get_nowait()
+        assert msg["type"] == "geofence_alert"
+        assert msg["payload"]["geofence_id"] == FENCE_ID
+
+    asyncio.run(run())
+
+
+def test_off_viewport_without_watch_does_not_receive_geofence_alert():
+    mgr = ConnectionManager()
+    q_sf = mgr.register("sf")
+    mgr.set_viewport("sf", SF_VIEW)
+
+    async def run():
+        n = await mgr.publish_point(
+            "geofence_alert", _alert_payload(), lat=35.5, lon=-79.0,
+        )
+        assert n == 0
+        assert q_sf.empty()
+
+    asyncio.run(run())
+
+
+def test_on_viewport_receives_geofence_alert_without_watch():
+    mgr = ConnectionManager()
+    q_nc = mgr.register("nc")
+    mgr.set_viewport("nc", NC_VIEW)
+
+    async def run():
+        n = await mgr.publish_point(
+            "geofence_alert", _alert_payload(), lat=35.5, lon=-79.0,
+        )
+        assert n == 1
+        assert q_nc.get_nowait()["type"] == "geofence_alert"
+
+    asyncio.run(run())
+
+
+def test_ais_stays_viewport_only_even_when_watching():
+    mgr = ConnectionManager()
+    q_sf = mgr.register("sf")
+    mgr.set_viewport("sf", SF_VIEW)
+    mgr.set_watched_geofences("sf", [FENCE_ID])
+
+    async def run():
+        n = await mgr.publish_point("ais", {"id": "366123456"}, lat=35.5, lon=-79.0)
+        assert n == 0
+        assert q_sf.empty()
+
+    asyncio.run(run())
+
+
+def test_invalid_watch_uuids_ignored_empty_list_clears():
+    mgr = ConnectionManager()
+    q = mgr.register("sf")
+    mgr.set_viewport("sf", SF_VIEW)
+    mgr.set_watched_geofences("sf", ["not-a-uuid", FENCE_ID, "also-bad"])
+
+    async def run():
+        n = await mgr.publish_point(
+            "geofence_alert", _alert_payload(), lat=35.5, lon=-79.0,
+        )
+        assert n == 1
+        q.get_nowait()
+        mgr.set_watched_geofences("sf", [])
+        n2 = await mgr.publish_point(
+            "geofence_alert", _alert_payload(), lat=35.5, lon=-79.0,
+        )
+        assert n2 == 0
+        assert q.empty()
+
+    asyncio.run(run())
+
+
+def test_unregister_clears_watched_geofences():
+    mgr = ConnectionManager()
+    q = mgr.register("sf")
+    mgr.set_viewport("sf", SF_VIEW)
+    mgr.set_watched_geofences("sf", [FENCE_ID])
+    mgr.unregister("sf")
+    q2 = mgr.register("sf")
+    mgr.set_viewport("sf", SF_VIEW)
+
+    async def run():
+        n = await mgr.publish_point(
+            "geofence_alert", _alert_payload(), lat=35.5, lon=-79.0,
+        )
+        assert n == 0
+        assert q2.empty()
+
+    asyncio.run(run())
+
+
 def test_record_and_notify_queries_postgis_when_cache_empty(monkeypatch):
     """FIRMS ingest in the ingester has an empty in-process cache — still ST_Intersects."""
     import geofence
@@ -134,3 +247,117 @@ def test_record_and_notify_queries_postgis_when_cache_empty(monkeypatch):
     inserts = [p for p in executed if isinstance(p, dict)]
     assert inserts and inserts[0]["source_kind"] == "firms"
     assert "commit" in executed
+
+
+def test_list_alerts_sql_filters(monkeypatch):
+    captured: dict = {}
+
+    class FakeResult:
+        def mappings(self):
+            return self
+
+        def all(self):
+            return []
+
+    class FakeSession:
+        async def execute(self, stmt, params=None):
+            captured["sql"] = str(stmt)
+            captured["params"] = params
+            return FakeResult()
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *a):
+            return False
+
+    monkeypatch.setattr(geofence, "async_session", FakeSession)
+    from datetime import datetime, timezone
+
+    since = datetime(2026, 8, 28, tzinfo=timezone.utc)
+    until = datetime(2026, 8, 29, tzinfo=timezone.utc)
+
+    async def run():
+        return await geofence.list_alerts(
+            geofence_id=FENCE_ID, since=since, until=until,
+            source_kind="firms", limit=5,
+        )
+
+    assert asyncio.run(run()) == []
+    sql = captured["sql"].lower()
+    assert "geofence_id" in sql
+    assert "created_at >=" in sql
+    assert "created_at <=" in sql
+    assert "source_kind" in sql
+    assert captured["params"]["geofence_id"] == FENCE_ID
+    assert captured["params"]["source_kind"] == "firms"
+    assert captured["params"]["limit"] == 5
+
+
+def test_alembic_fence_created_index_exists():
+    from pathlib import Path
+    text = Path(__file__).resolve().parent.parent.joinpath(
+        "alembic/versions/011_geofence_alerts_fence.py",
+    ).read_text()
+    assert "ix_geofence_alerts_fence_created" in text
+    assert "010_bbox_gist" in text
+
+
+def test_snapshot_at_404_when_fence_missing(monkeypatch):
+    geofence._cache.clear()
+
+    async def boom():
+        raise RuntimeError("db down")
+
+    monkeypatch.setattr(geofence, "refresh_cache", boom)
+
+    async def run():
+        from datetime import datetime, timezone
+        return await geofence.snapshot_at(
+            FENCE_ID, datetime(2026, 8, 28, 12, 4, tzinfo=timezone.utc),
+        )
+
+    assert asyncio.run(run()) is None
+
+
+def test_snapshot_queries_st_intersects(monkeypatch):
+    geofence._cache[:] = [{
+        "id": FENCE_ID, "name": "NC", "geojson": NC_BOX, "active": True,
+    }]
+    sqls: list[str] = []
+
+    class FakeResult:
+        def mappings(self):
+            return self
+
+        def all(self):
+            return []
+
+    class FakeSession:
+        async def execute(self, stmt, params=None):
+            sqls.append(str(stmt))
+            return FakeResult()
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *a):
+            return False
+
+    monkeypatch.setattr(geofence, "async_session", FakeSession)
+
+    async def run():
+        from datetime import datetime, timezone
+        return await geofence.snapshot_at(
+            FENCE_ID, datetime(2026, 8, 28, 12, 4, 30, tzinfo=timezone.utc),
+        )
+
+    body = asyncio.run(run())
+    assert body["aircraft"] == []
+    assert body["vessels"] == []
+    assert body["fires"] == []
+    blob = "\n".join(sqls).lower()
+    assert "st_intersects" in blob
+    assert "aircraft_tracks_1min" in blob
+    assert "vessel_tracks_1min" in blob
+    assert "from fires" in blob

tests/test_geofence_frontend.py

@@ -0,0 +1,56 @@
+"""Geofence layer panel: draw, watch, inbox, delete (HTML contract)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+HTML = (ROOT / "app/static/index.html").read_text()
+
+
+def test_geofence_panel_has_list_and_delete_hook():
+    assert 'id="gf-draw"' in HTML
+    assert 'id="gf-list"' in HTML
+    assert "function deleteGeofence" in HTML
+    assert "method: 'DELETE'" in HTML or 'method: "DELETE"' in HTML
+    assert "/api/geofences/" in HTML
+
+
+def test_load_geofences_renders_delete_controls():
+    js = HTML.split("async function loadGeofences", 1)[1].split(
+        "async function loadFireAircraftHits", 1
+    )[0]
+    assert "gf-list" in js
+    assert "deleteGeofence" in js
+    assert "onEachFeature" in js
+    assert "bindPopup" in js
+
+
+def test_finish_cancel_draw_controls():
+    assert 'id="gf-finish"' in HTML
+    assert 'id="gf-cancel"' in HTML
+    assert "function cancelGeofenceDraw" in HTML
+    assert "function onGfClose" in HTML
+
+
+def test_watch_geofences_ws_payload():
+    assert "watch_geofences" in HTML
+    assert "function sendWatchGeofences" in HTML
+
+
+def test_geofence_alert_inbox():
+    assert 'id="gf-inbox"' in HTML
+    assert "/api/geofence-alerts" in HTML
+    assert "function loadGfInbox" in HTML
+    assert "function pushGfInbox" in HTML
+
+
+def test_delete_geofence_still_present():
+    assert "function deleteGeofence" in HTML
+    assert "method: 'DELETE'" in HTML or 'method: "DELETE"' in HTML
+
+
+def test_fence_dvr_at_endpoint():
+    assert "/at?timestamp=" in HTML or "/at?timestamp=${" in HTML
+    assert "function dvrScrubFence" in HTML
+    assert "gfSelectedId" in HTML

tests/test_gpsjam.py

@@ -0,0 +1,150 @@
+"""GPSJAM GPS-interference overlay: level mapping, CSV→GeoJSON, API contract."""
+
+import asyncio
+
+import httpx
+
+from live_layers import gpsjam_csv_to_geojson, gpsjam_level, overlay_catalog, _cache
+from main import app
+
+BASE = "http://test"
+
+# A valid H3 resolution-4 cell id (the payload hex column carries these).
+HEX_A = "8400c57ffffffff"
+
+CSV = (
+    "hex,count_good_aircraft,count_bad_aircraft\n"
+    f"{HEX_A},0,20\n"        # 100*(20-1)/20 = 95  -> high
+    f"{HEX_A},8,2\n"         # 100*(2-1)/10  = 10  -> medium
+    f"{HEX_A},98,2\n"        # 100*(2-1)/100 = 1   -> low
+    f"{HEX_A},100,0\n"       # bad == 0            -> dropped
+)
+
+
+async def _get(path: str) -> httpx.Response:
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
+        return await client.get(path)
+
+
+def test_gpsjam_level_thresholds():
+    assert gpsjam_level(0.0) == "low"
+    assert gpsjam_level(2.0) == "low"
+    assert gpsjam_level(2.1) == "medium"
+    assert gpsjam_level(10.0) == "medium"
+    assert gpsjam_level(10.1) == "high"
+    assert gpsjam_level(95.0) == "high"
+
+
+def test_gpsjam_csv_to_geojson_levels_and_drop_zero_bad():
+    fc = gpsjam_csv_to_geojson(CSV)
+    assert fc["type"] == "FeatureCollection"
+    assert len(fc["features"]) == 3  # bad==0 row dropped
+    levels = [f["properties"]["level"] for f in fc["features"]]
+    assert levels == ["high", "medium", "low"]
+    for f in fc["features"]:
+        geom = f["geometry"]
+        assert geom["type"] == "Polygon"
+        ring = geom["coordinates"][0]
+        assert len(ring) == 7  # 6 verts + closing point
+        assert ring[0] == ring[-1]
+        assert f["properties"]["hex"] == HEX_A
+        assert set(f["properties"]).issuperset({"level", "percent_bad", "good", "bad", "hex"})
+
+
+def test_gpsjam_csv_skips_malformed_rows():
+    bad_csv = "hex,count_good_aircraft,count_bad_aircraft\n" \
+              ",1,5\n" \
+              f"{HEX_A},x,5\n" \
+              f"{HEX_A},1,notanint\n" \
+              "not_a_cell,1,5\n"
+    fc = gpsjam_csv_to_geojson(bad_csv)
+    assert fc["features"] == []
+
+
+def test_overlay_catalog_has_gpsjam_stub():
+    entry = overlay_catalog()["gpsjam"]
+    assert entry["kind"] == "geojson"
+    assert entry["endpoint"] == "/api/map/gpsjam"
+    assert "GPSJAM" in entry["attribution"]
+
+
+def test_map_gpsjam_returns_featurecollection(monkeypatch):
+    async def fake_fetch(date):
+        return {"type": "FeatureCollection", "features": [{"type": "Feature"}]}
+
+    monkeypatch.setattr("main.fetch_gpsjam", fake_fetch)
+    resp = asyncio.run(_get("/api/map/gpsjam?date=2026-08-28"))
+    assert resp.status_code == 200
+    assert resp.json()["type"] == "FeatureCollection"
+    assert "max-age" in (resp.headers.get("cache-control") or "").lower()
+
+
+def test_map_gpsjam_rejects_bad_date():
+    resp = asyncio.run(_get("/api/map/gpsjam?date=08-28-2026"))
+    assert resp.status_code == 422
+
+
+def test_map_gpsjam_unavailable_on_404(monkeypatch):
+    import httpx as _httpx
+
+    async def fake_fetch(date):
+        exc = _httpx.HTTPStatusError(
+            "404", request=_httpx.Request("GET", "http://x"), response=_httpx.Response(404)
+        )
+        raise exc
+
+    monkeypatch.setattr("main.fetch_gpsjam", fake_fetch)
+    resp = asyncio.run(_get("/api/map/gpsjam?date=2026-08-28"))
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["error"] == "unavailable"
+    assert body["href"] == "https://gpsjam.org/"
+
+
+def test_map_gpsjam_unavailable_on_empty_features(monkeypatch):
+    async def fake_fetch(date):
+        return {"type": "FeatureCollection", "features": []}
+
+    monkeypatch.setattr("main.fetch_gpsjam", fake_fetch)
+    resp = asyncio.run(_get("/api/map/gpsjam?date=2026-08-28"))
+    assert resp.status_code == 200
+    assert resp.json()["error"] == "unavailable"
+
+
+def test_fetch_gpsjam_hits_http_once_within_ttl(monkeypatch):
+    _cache.clear()
+    hits = {"n": 0}
+
+    class FakeResp:
+        text = CSV
+
+        def raise_for_status(self):
+            pass
+
+    class FakeClient:
+        def __init__(self, **kw):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *exc):
+            return False
+
+        async def get(self, url):
+            hits["n"] += 1
+            assert url == "https://gpsjam.org/data/2026-08-28-h3_4.csv"
+            return FakeResp()
+
+    monkeypatch.setattr("live_layers.httpx.AsyncClient", FakeClient)
+    monkeypatch.setattr("live_layers._http", None)
+
+    from live_layers import fetch_gpsjam
+
+    fc1 = asyncio.run(fetch_gpsjam("2026-08-28"))
+    fc2 = asyncio.run(fetch_gpsjam("2026-08-28"))
+    assert len(fc1["features"]) == 3
+    assert fc2 == fc1
+    assert hits["n"] == 1
+    _cache.clear()

tests/test_health_ready.py

@@ -0,0 +1,33 @@
+"""Liveness stays up; readiness/freshness is explicit."""
+
+from __future__ import annotations
+
+import asyncio
+
+import httpx
+
+from main import app
+
+BASE = "http://test"
+
+
+async def _get(path: str) -> httpx.Response:
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
+        return await client.get(path)
+
+
+def test_health_includes_checks_even_when_db_ok(monkeypatch):
+    """HUD must be able to show degraded without docker killing the container."""
+    body = asyncio.run(_get("/api/health")).json()
+    assert "status" in body
+    assert "checks" in body
+    assert "db" in body["checks"]
+
+
+def test_ready_endpoint_exists():
+    resp = asyncio.run(_get("/api/ready"))
+    assert resp.status_code in (200, 503)
+    body = resp.json()
+    assert "checks" in body
+    assert "status" in body

tests/test_hud_chrome.py

@@ -0,0 +1,113 @@
+"""Quiet HUD chrome: VIIRS default, collapsed rail, no Orbitron/MKT dashes."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+HTML = (ROOT / "app/static/index.html").read_text()
+
+
+def _attr(html: str, elem_id: str) -> str:
+    chunk = html.split(f'id="{elem_id}"', 1)[1].split(">", 1)[0]
+    return chunk
+
+
+def test_initmap_prefers_viirs_true_color():
+    init = HTML.split("async function initMap", 1)[1].split("function readMapPrefs", 1)[0]
+    assert "VIIRS_SNPP_CorrectedReflectance_TrueColor" in init
+    assert init.index("VIIRS_SNPP_CorrectedReflectance_TrueColor") < init.index(
+        "MODIS_Terra_CorrectedReflectance_TrueColor"
+    )
+    assert init.index("MODIS_Terra_CorrectedReflectance_TrueColor") < init.index(
+        "BlueMarble_ShadedRelief_Bathymetry"
+    )
+
+
+def test_orbitron_gone():
+    assert "Orbitron" not in HTML
+    assert "IBM Plex Sans" in HTML
+    assert "IBM Plex Mono" in HTML
+
+
+def test_lp_note_stripped_from_layer_list():
+    assert 'class="lp-note"' not in HTML
+    body = HTML.split('class="lp-body"', 1)[1].split("lp-legend", 1)[0]
+    assert "lp-note" not in body
+
+
+def test_default_overlays_basemap_and_firms_only():
+    fires = _attr(HTML, "lp-fires-on")
+    assert "checked" in fires
+    for eid in (
+        "lp-cams-on",
+        "lp-blips-on",
+        "lp-news-on",
+        "lp-radar-on",
+        "lp-alerts-on",
+        "lp-perim-on",
+        "lp-ac-on",
+        "lp-trains-on",
+        "lp-storms-on",
+    ):
+        assert "checked" not in _attr(HTML, eid), eid
+
+
+def test_geofence_markup_before_cameras():
+    assert 'id="gf-draw"' in HTML
+    assert HTML.index('id="gf-draw"') < HTML.index('id="lp-cams-on"')
+    assert HTML.index('id="lp-base-on"') < HTML.index('id="gf-draw"')
+
+
+def test_parent_geofence_hud_survives():
+    assert "watch_geofences" in HTML
+    assert "function deleteGeofence" in HTML
+    assert 'id="gf-finish"' in HTML
+    assert 'id="gf-cancel"' in HTML
+    assert 'id="gf-inbox"' in HTML
+
+
+def test_layer_rail_collapsed_on_load():
+    head = HTML.split('class="lp-head"', 1)[1].split("</div>", 1)[0]
+    assert 'aria-expanded="false"' in head
+    assert 'id="layer-panel" class="collapsed"' in HTML
+
+
+def test_market_ticker_hidden_no_poll():
+    mkt = HTML.split('class="ticker market"', 1)[1].split(">", 1)[0]
+    assert "hidden" in mkt
+    assert "setInterval(probeMarket" not in HTML
+    assert "setInterval(loadMarket" not in HTML
+    init = HTML.split("function initMarketTicker", 1)[1].split("function ", 1)[0]
+    assert "/api/market" in init or "404-poll" in init
+    assert "setInterval" not in init
+
+
+def test_news_ticker_fills_news_only_dock():
+    css = HTML.split("</style>", 1)[0]
+    compact = css.replace(" ", "").replace("\n", "")
+    assert ".dock.news-only{height:32px;}" in compact
+    assert ".dock.news-only.ticker{height:100%;}" in compact
+    assert ".ticker{display:flex;align-items:stretch;height:50%;" in compact
+
+
+def test_news_pins_are_circle_markers():
+    js = HTML.split("async function loadNewsPins", 1)[1].split("function refreshLiveOverlays", 1)[0]
+    assert "L.circleMarker" in js
+    assert "fillOpacity: 0.7" in js or "fillOpacity:0.7" in js
+    assert "rotate(45deg)" not in js
+    assert "L.divIcon" not in js
+
+
+def test_chokepoint_buttons_not_in_toolbar_flow():
+    assert 'id="chokepoint-select"' in HTML
+    css = HTML.split("</style>", 1)[0]
+    assert ".chokepoint-btns { display: none; }" in css or ".chokepoint-btns{display:none" in css.replace(
+        " ", ""
+    )
+
+
+def test_brand_is_osint_slash():
+    assert "GLOBAL SITUATIONAL AWARENESS TERMINAL" not in HTML
+    assert "OSINT" in HTML
+    assert 'class="accent">//</span>' in HTML

tests/test_hud_osiris.py

@@ -0,0 +1,90 @@
+"""HUD: layer-rail stats, shortcuts, terminator, zoom-gated cams, SWPC chip."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+HTML = (ROOT / "app/static/index.html").read_text()
+
+
+def _fn(name: str, nxt: str | None = None) -> str:
+    start = HTML.index(f"function {name}")
+    if nxt:
+        return HTML[start : HTML.index(f"function {nxt}", start + 1)]
+    return HTML[start : start + 4000]
+
+
+def test_stats_poll_uses_api_then_falls_back():
+    assert "/api/stats" in HTML
+    assert "30000" in HTML.split("pollLayerStats")[1][:2500] or "STATS_POLL_MS" in HTML
+    poll = HTML.split("async function pollLayerStats")[1].split("async function ")[0]
+    assert "404" in poll
+    assert "catch" in poll
+    ids = HTML.split("STATS_COUNT_IDS")[1].split("};")[0]
+    assert "aircraft" in ids and "cameras" in ids and "fires" in ids and "vessels" in ids
+    # Overlay loaders still write array lengths when stats is down.
+    assert "setLayerCount('lp-fires-count'" in HTML or 'setLayerCount("lp-fires-count"' in HTML
+    assert "setLayerCount('lp-cams-count'" in HTML or 'setLayerCount("lp-cams-count"' in HTML
+    assert "setLayerCount('lp-ac-count'" in HTML or 'setLayerCount("lp-ac-count"' in HTML
+    assert "setLayerCount('lp-vessels-count'" in HTML or 'setLayerCount("lp-vessels-count"' in HTML
+
+
+def test_keyboard_shortcuts_do_not_steal_osiris_fs():
+    keys = HTML.split("function initHudKeys")[1].split("function ")[0]
+    assert "Escape" in keys
+    assert "cheat-sheet" in keys or "toggleCheatSheet" in keys
+    assert "mapResetView" in keys
+    assert "toggleLayerPanel" in keys or "closeLayerPanel" in keys
+    # Do not bind Osiris's conflicting F/S (flights vs fullscreen / search).
+    assert "e.key === 'f'" not in keys.lower()
+    assert "e.key === 's'" not in keys.lower()
+    assert "case 'f'" not in keys.lower()
+    assert "case 's'" not in keys.lower()
+    assert 'id="cheat-sheet"' in HTML
+    assert "?" in keys or "Shift" in keys
+
+
+def test_terminator_toggle_defaults_off():
+    assert 'id="lp-terminator-on"' in HTML
+    row = HTML.split('id="lp-terminator-on"')[0][-120:] + HTML.split('id="lp-terminator-on"')[1][:80]
+    assert "checked" not in row.split(">")[0]
+    assert "function toggleTerminator" in HTML
+    assert "subsolarPoint" in HTML or "terminator" in HTML.lower()
+
+
+def test_camera_thumbs_gated_at_zoom_12():
+    assert "CAM_THUMB_MIN_ZOOM" in HTML
+    assert "CAM_THUMB_MIN_ZOOM = 12" in HTML
+    thumb = _fn("camThumb", "camPopupHtml")
+    assert "camThumbsAllowed" in thumb or "CAM_THUMB_MIN_ZOOM" in thumb
+    assert "zoom in for preview" in HTML or "zoom for preview" in HTML
+    assert "preview unavailable" in HTML
+    # RTSP still proxy through snapshot; never emit rtsp hrefs.
+    src = _fn("camSourceLink", "youtubeId")
+    assert "rtsp://" in src
+    assert "href=" not in src.split("rtsp://")[1].split("return")[0] or "Never emit" in src
+    assert 'href="${esc(url)}"' in src or "href=\"${esc(url)}\"" in src
+    assert src.index("rtsp://") < src.index("href=")
+
+
+def test_swpc_chip_browser_direct_correct_urls():
+    assert 'id="swpc-chip"' in HTML
+    assert "services.swpc.noaa.gov/json/planetary_k_index_1m.json" in HTML
+    assert "services.swpc.noaa.gov/json/goes/primary/xray-flares-latest.json" in HTML
+    assert "services.swpc.noaa.gov/products/alerts.json" in HTML
+    assert "services.swpc.noaa.gov/json/alerts.json" not in HTML
+    sw = HTML.split("async function pollSwpc")[1].split("async function ")[0]
+    assert "hidden" in sw
+    assert "kp_index" in sw
+    assert "90000" in HTML or "SWPC_POLL_MS" in HTML
+
+
+def test_new_chrome_does_not_cover_mobile_layers_zoom():
+    mobile = HTML.split("@media (max-width: 820px)")[1].split("@media (prefers-reduced-motion")[0]
+    assert "#layer-panel" in mobile
+    assert ".leaflet-top.leaflet-right .leaflet-control-zoom" in mobile
+    assert 'id="cheat-sheet"' in HTML
+    cheat = HTML.split(".cheat-sheet")[1][:500]
+    assert "z-index" in cheat
+    assert "calc(100% - 96px)" in cheat or "96px" in cheat

tests/test_infrastructure.py

@@ -0,0 +1,145 @@
+"""GET /api/infrastructure — Overpass nuclear markers."""
+
+import asyncio
+
+import httpx
+
+from live_layers import (
+    normalize_infra_element,
+    overlay_catalog,
+    overpass_nuclear_to_markers,
+    _cache,
+)
+from main import app
+
+BASE = "http://test"
+
+OVERPASS = {
+    "version": 0.6,
+    "generator": "Overpass API",
+    "elements": [
+        {
+            "type": "node",
+            "id": 12345,
+            "lat": 44.0,
+            "lon": -1.5,
+            "tags": {"name": "Test NPP", "operator": "EDF", "plant:source": "nuclear"},
+        },
+        {
+            "type": "way",
+            "id": 67890,
+            "center": {"lat": 43.5, "lon": -1.25},
+            "tags": {"name": "Test Plant Way", "plant:source": "nuclear"},
+        },
+        {
+            "type": "relation",
+            "id": 999,
+            "center": {"lat": 43.0, "lon": -1.0},
+            "tags": {},
+        },
+    ],
+}
+
+
+async def _get(path: str) -> httpx.Response:
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
+        return await client.get(path)
+
+
+def test_normalize_node_to_marker():
+    m = normalize_infra_element(OVERPASS["elements"][0], "nuclear")
+    assert m["id"] == "node/12345"
+    assert m["name"] == "Test NPP"
+    assert m["lat"] == 44.0
+    assert m["lon"] == -1.5
+    assert m["type"] == "nuclear"
+    assert m["extra"]["operator"] == "EDF"
+    assert "name" not in m["extra"]
+
+
+def test_way_center_and_unnamed_fallback():
+    way = normalize_infra_element(OVERPASS["elements"][1], "nuclear")
+    assert way["lat"] == 43.5
+    assert way["lon"] == -1.25
+    rel = normalize_infra_element(OVERPASS["elements"][2], "nuclear")
+    assert rel["name"] == "relation/999"
+
+
+def test_overpass_json_to_markers():
+    markers = overpass_nuclear_to_markers(OVERPASS)
+    assert len(markers) == 3
+    assert markers[0]["id"] == "node/12345"
+
+
+def test_missing_bbox_400():
+    resp = asyncio.run(_get("/api/infrastructure?types=nuclear"))
+    assert resp.status_code == 400
+
+
+def test_unknown_type_422():
+    resp = asyncio.run(_get("/api/infrastructure?types=military&bbox=-2,43,-1,44"))
+    assert resp.status_code == 422
+
+
+def test_map_infrastructure_returns_markers(monkeypatch):
+    async def fake_fetch(types, bbox):
+        return [
+            {"id": "node/1", "name": "X", "lat": 1.0, "lon": 2.0,
+             "type": "nuclear", "extra": {}}
+        ]
+
+    monkeypatch.setattr("main.fetch_infrastructure", fake_fetch)
+    resp = asyncio.run(_get("/api/infrastructure?types=nuclear&bbox=-2,43,-1,44"))
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body[0]["name"] == "X"
+    assert body[0]["type"] == "nuclear"
+    assert "max-age" in (resp.headers.get("cache-control") or "").lower()
+
+
+def test_overlay_catalog_has_infra_nuclear():
+    entry = overlay_catalog()["infra_nuclear"]
+    assert entry["kind"] == "points"
+    assert "nuclear" in entry["endpoint"]
+
+
+def test_fetch_infrastructure_cache_hit_no_refetch(monkeypatch):
+    _cache.clear()
+    hits = {"n": 0}
+
+    class FakeResp:
+        def raise_for_status(self):
+            pass
+
+        def json(self):
+            return OVERPASS
+
+    class FakeClient:
+        def __init__(self, **kw):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *exc):
+            return False
+
+        async def post(self, url, data=None, timeout=None):
+            hits["n"] += 1
+            assert "overpass-api.de" in url
+            assert "plant:source" in data["data"]
+            assert "nuclear" in data["data"]
+            return FakeResp()
+
+    monkeypatch.setattr("live_layers.httpx.AsyncClient", FakeClient)
+    monkeypatch.setattr("live_layers._http", None)
+
+    from live_layers import fetch_infrastructure
+
+    m1 = asyncio.run(fetch_infrastructure("nuclear", "-2,43,-1,44"))
+    m2 = asyncio.run(fetch_infrastructure("nuclear", "-2,43,-1,44"))
+    assert len(m1) == 3
+    assert m2 == m1
+    assert hits["n"] == 1
+    _cache.clear()

tests/test_ingest_ssrf_and_sources.py

@@ -0,0 +1,67 @@
+"""SSRF guard on ingest triggers + PATCH /api/sources allowlist."""
+
+from __future__ import annotations
+
+import asyncio
+
+import httpx
+import pytest
+from pydantic import ValidationError
+
+from main import app
+
+BASE = "http://test"
+LINK_LOCAL_META = "http://169.254.169.254/latest/meta-data/"
+LOOPBACK = "http://127.0.0.1/secret"
+
+
+async def _req(method: str, path: str, **kw) -> httpx.Response:
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
+        return await client.request(method, path, **kw)
+
+
+def test_rss_ingest_rejects_link_local_metadata_url(monkeypatch):
+    called = {"n": 0}
+
+    async def _boom(*_a, **_k):
+        called["n"] += 1
+        raise AssertionError("ingest_rss_feed must not run for a private URL")
+
+    monkeypatch.setattr("main.ingest_rss_feed", _boom)
+    resp = asyncio.run(_req("POST", "/api/ingest/rss", params={"feed_url": LINK_LOCAL_META}))
+    assert resp.status_code == 400
+    assert called["n"] == 0
+
+
+def test_gdelt_ingest_rejects_private_query_url(monkeypatch):
+    called = {"n": 0}
+
+    async def _boom(*_a, **_k):
+        called["n"] += 1
+        raise AssertionError("ingest_gdelt must not run for a private URL query")
+
+    monkeypatch.setattr("main.ingest_gdelt", _boom)
+    resp = asyncio.run(_req("POST", "/api/ingest/gdelt", params={"query": LOOPBACK}))
+    assert resp.status_code == 400
+    assert called["n"] == 0
+
+
+def test_update_source_rejects_unknown_fields():
+    sid = "00000000-0000-0000-0000-000000000001"
+    resp = asyncio.run(_req("PATCH", f"/api/sources/{sid}", json={"enabled": True, "source_type": "rss"}))
+    assert resp.status_code == 422
+
+
+def test_feed_source_update_allowlist_only():
+    from schemas import FeedSourceUpdate
+
+    payload = FeedSourceUpdate(name="n", url="https://example.com/rss", config={"k": 1}, enabled=False)
+    assert payload.model_dump(exclude_unset=True) == {
+        "name": "n",
+        "url": "https://example.com/rss",
+        "config": {"k": 1},
+        "enabled": False,
+    }
+    with pytest.raises(ValidationError):
+        FeedSourceUpdate.model_validate({"enabled": True, "id": "00000000-0000-0000-0000-000000000001"})

tests/test_live_layers.py

@@ -1,5 +1,7 @@
 """Unit tests for live map-layer mappers (aircraft, trains, AIS, WFIGS, Caltrans)."""
 
+import json
+
 from live_layers import (
     MARKER_FIELDS,
     bbox_center_radius_nm,
@@ -7,7 +9,10 @@ from live_layers import (
     filter_points_bbox,
     parse_bbox,
     quantize_bbox,
+    pick_sentinel_feature,
     rainviewer_tile_url,
+    sign_cog_url,
+    sentinel1_tile_url,
     slim_alert_properties,
     to_marker,
     transform_adsb_lol,
@@ -15,12 +20,14 @@ from live_layers import (
     transform_amtraker,
     transform_nhc_storms,
     transform_wfigs_incidents,
+    SENTINEL1_ATTRIBUTION,
+    TITILER_COG_TILES,
     _cache,
     _ttl_get,
     _wfigs_params,
 )
 
-from camera_scraper import parse_caltrans_json
+from camera_scraper import parse_caltrans_json, parse_udot_ibi_page, parse_odot_json, parse_mdot_json
 
 
 def test_parse_bbox_and_radius_clamps_to_150_nm():
@@ -252,6 +259,186 @@ def test_parse_caltrans_skips_oos_and_maps_jpeg_hls():
     assert "rtsp://" not in cam["snapshot_url"].lower()
 
 
+# ── UDOT IBI 511 parser ──────────────────────────────────────────────────
+
+def _udot_row(cam_id, lng, lat, **img_overrides):
+    img = {
+        "id": cam_id, "cameraSiteId": cam_id,
+        "imageUrl": f"/map/Cctv/{cam_id}", "disabled": False, "blocked": False,
+    }
+    img.update(img_overrides)
+    return {
+        "id": cam_id, "sourceId": "102771", "source": "ADX",
+        "roadway": "Unknown", "direction": "Unknown",
+        "location": "Freedom Blvd / 200 W @ 1100 N, PVO",
+        "latLng": {"geography": {
+            "coordinateSystemId": 4326,
+            "wellKnownText": f"POINT ({lng} {lat})"}},
+        "images": [img],
+    }
+
+
+def _udot_page(rows):
+    import json
+    return json.dumps({"draw": 0, "recordsTotal": len(rows),
+                       "recordsFiltered": len(rows), "data": rows})
+
+
+def test_parse_udot_wkt_maps_lng_lat():
+    cams = parse_udot_ibi_page(_udot_page([_udot_row(112731, -111.66204, 40.24863)]))
+    assert len(cams) == 1
+    cam = cams[0]
+    # WKT is `POINT (lng lat)` — order must not be swapped.
+    assert cam["location_lat"] == 40.24863
+    assert cam["location_lon"] == -111.66204
+    assert cam["discovery_source"] == "udot"
+    assert cam["vendor"] == "UDOT"
+    assert cam["source_url"] == "https://prod-ut.ibi511.com/map/Cctv/112731"
+    assert cam["snapshot_url"] == cam["source_url"]
+    assert "rtsp://" not in cam["source_url"].lower()
+    assert cam["raw"]["udot_id"] == 112731
+
+
+def test_parse_udot_skips_blocked_and_disabled():
+    rows = [
+        _udot_row(1, -111.0, 40.0),
+        _udot_row(2, -111.1, 40.1, blocked=True),
+        _udot_row(3, -111.2, 40.2, disabled=True),
+    ]
+    rows.append(_udot_row(4, -111.3, 40.3))
+    rows[3]["images"] = []  # no images → drop
+    cams = parse_udot_ibi_page(_udot_page(rows))
+    assert [c["raw"]["udot_id"] for c in cams] == [1]
+
+
+def test_parse_udot_drops_out_of_bbox():
+    rows = [
+        _udot_row(1, -111.0, 40.0),   # inside Utah
+        _udot_row(2, -100.0, 40.0),   # east of -108.9
+        _udot_row(3, -120.0, 40.0),   # west of -114.2
+        _udot_row(4, -111.0, 44.0),   # north of 42.1
+        _udot_row(5, -111.0, 30.0),   # south of 36.9
+    ]
+    cams = parse_udot_ibi_page(_udot_page(rows))
+    assert [c["raw"]["udot_id"] for c in cams] == [1]
+
+
+def test_parse_udot_bad_payload_returns_empty():
+    import json
+    assert parse_udot_ibi_page("not json") == []
+    assert parse_udot_ibi_page(json.dumps({"data": None})) == []
+    assert parse_udot_ibi_page(json.dumps({"data": "nope"})) == []
+
+
+def test_parse_udot_missing_wkt_skipped():
+    row = _udot_row(1, -111.0, 40.0)
+    row["latLng"] = {}
+    assert parse_udot_ibi_page(_udot_page([row])) == []
+
+
+def test_parse_odot_tripcheck_keeps_valid_skips_missing_and_oob():
+    payload = """
+    {"features":[
+      {"attributes":{
+        "cameraId":277,"filename":"AstoriaUS101_pid392.jpg",
+        "latitude":46.18785,"longitude":-123.85347,
+        "route":"US101 ","title":"US101 at Astoria"
+      }},
+      {"attributes":{
+        "cameraId":200,"filename":"","latitude":45.0,"longitude":-122.0,
+        "route":"I-5","title":"missing filename"
+      }},
+      {"attributes":{
+        "cameraId":300,"filename":"nocal_pid1.jpg",
+        "latitude":40.0,"longitude":-122.0,
+        "route":"US97","title":"out of bbox"
+      }},
+      {"attributes":{
+        "cameraId":400,"filename":"badcoord_pid2.jpg",
+        "latitude":null,"longitude":-122.0,
+        "route":"OR22","title":"null coord"
+      }}
+    ]}
+    """
+    cams = parse_odot_json(payload, "www.tripcheck.com")
+    assert len(cams) == 1
+    cam = cams[0]
+    assert cam["discovery_source"] == "odot"
+    assert cam["snapshot_url"] == (
+        "https://tripcheck.com/RoadCams/cams/AstoriaUS101_pid392.jpg")
+    assert cam["source_url"] == cam["snapshot_url"]
+    assert cam["location_lat"] == 46.18785
+    assert cam["location_lon"] == -123.85347
+    assert "US101 at Astoria" in cam["location_name"]
+    assert cam["vendor"] == "ODOT"
+    assert cam["device_type"] == "http"
+    assert "rtsp://" not in cam["snapshot_url"].lower()
+
+
+def test_parse_odot_tripcheck_handles_malformed():
+    assert parse_odot_json("not json", "www.tripcheck.com") == []
+    assert parse_odot_json('{"features":null}', "www.tripcheck.com") == []
+
+
+def test_parse_mdot_extracts_html_fields_and_bbox_filters():
+    rows = [
+        # In-bbox, full fields.
+        {
+            "route": "11 Mile",
+            "county": 'Wayne County <a href="/MiDrive/map?cameras=true&lat=42.491304&lon=-83.04479&zoom=15&id=1129"target="_blank">Go to</a>',
+            "location": " @ Mound NB",
+            "direction": "Traffic closest to camera is traveling north.",
+            "image": '<img alt="x" class="cameraImageForActivePane" id="1129Img" src="https://micamerasimages.net/thumbs/semtoc_cam_253.flv.jpg?item=1" height="170" width="250" onerror="cameraImageBroken(this)">',
+        },
+        # Out of bbox (lat 50) → drop.
+        {
+            "route": "Far",
+            "county": 'Nowhere <a href="/MiDrive/map?lat=50.0&lon=-83.0&zoom=15&id=9999">Go to</a>',
+            "location": "",
+            "image": '<img src="https://micamerasimages.net/thumbs/x.jpg">',
+        },
+        # Missing coordinates → drop.
+        {
+            "route": "NoCoords",
+            "county": 'Somewhere <a href="/MiDrive/map?zoom=15&id=8888">Go to</a>',
+            "location": "",
+            "image": '<img src="https://micamerasimages.net/thumbs/y.jpg">',
+        },
+        # Missing image → drop.
+        {
+            "route": "NoImage",
+            "county": 'Kent <a href="/MiDrive/map?lat=42.8841&lon=-85.6646&zoom=15&id=2113">Go to</a>',
+            "location": " @ Division",
+            "image": "",
+        },
+        # RTSP image src → drop.
+        {
+            "route": "Rtsp",
+            "county": 'Wayne <a href="/MiDrive/map?lat=42.4&lon=-83.1&zoom=15&id=1234">Go to</a>',
+            "location": "",
+            "image": '<img src="rtsp://10.0.0.1/stream">',
+        },
+    ]
+    cams = parse_mdot_json(json.dumps(rows), "mdotjboss.state.mi.us")
+    assert len(cams) == 1
+    cam = cams[0]
+    assert cam["discovery_source"] == "mdot"
+    assert cam["location_lat"] == 42.491304
+    assert cam["location_lon"] == -83.04479
+    assert cam["snapshot_url"] == "https://micamerasimages.net/thumbs/semtoc_cam_253.flv.jpg?item=1"
+    assert cam["source_url"] == "https://mdotjboss.state.mi.us/MiDrive/camera/1129"
+    assert cam["device_type"] == "http"
+    assert cam["vendor"] == "MDOT"
+    assert "11 Mile @ Mound NB" in cam["location_name"]
+    assert "Wayne County" in cam["location_name"]
+
+
+def test_parse_mdot_handles_malformed_payload():
+    assert parse_mdot_json("not json", "mdot") == []
+    assert parse_mdot_json('{"not": "a list"}', "mdot") == []
+    assert parse_mdot_json("[]", "mdot") == []
+
+
 def test_quantize_bbox_stable_under_jitter():
     a = quantize_bbox(*parse_bbox("-78.7912,35.7711,-78.6101,35.9102"))
     b = quantize_bbox(*parse_bbox("-78.7900,35.7700,-78.6110,35.9090"))
@@ -362,3 +549,440 @@ def test_wfigs_params_requests_simplified_geometry():
     # Envelope is the quantized cell, not the raw pan box.
     geom = params["geometry"]
     assert geom != "-84.5,33.8,-75.4,36.6"
+
+
+def test_transform_adsb_lol_flags_military_from_dbflags():
+    payload = {
+        "ac": [
+            {
+                "hex": "ae01ab",
+                "flight": "RCH123  ",
+                "r": "04-1234",
+                "t": "C17",
+                "lat": 35.1,
+                "lon": -77.9,
+                "alt_baro": 24000,
+                "gs": 410,
+                "track": 90,
+                "squawk": "5101",
+                "emergency": "none",
+                "category": "A5",
+                "dbFlags": 1,
+                "baro_rate": 64,
+                "alt_geom": 24500,
+                "desc": "Boeing C-17A Globemaster III",
+                "ownOp": "USAF",
+            },
+            {
+                "hex": "a1b2c3",
+                "flight": "AAL123",
+                "r": "N123AA",
+                "t": "B738",
+                "lat": 35.88,
+                "lon": -78.79,
+                "alt_baro": 32000,
+                "gs": 430,
+                "track": 87,
+                "squawk": "1200",
+                "emergency": "none",
+                "category": "A3",
+            },
+        ]
+    }
+    rows = {r["id"]: r for r in transform_adsb_lol(payload)}
+    mil = rows["ae01ab"]["extra"]
+    civ = rows["a1b2c3"]["extra"]
+    assert mil["role"] == "military"
+    assert mil["role_src"] == "dbFlags"
+    assert mil["emitter"] == "heavy"
+    assert mil["desc"] == "Boeing C-17A Globemaster III"
+    assert mil["ownOp"] == "USAF"
+    assert mil["vs"] == 64
+    assert mil["alt_geom"] == 24500
+    assert civ["role"] == "civilian"
+    assert civ["emitter"] == "large"
+
+
+def test_transform_adsb_lol_military_from_icao_type_and_hex():
+    payload = {
+        "ac": [
+            {"hex": "3b76aa", "flight": "FAF123", "t": "F16", "lat": 1, "lon": 2, "category": "A1"},
+            {"hex": "ae1234", "flight": "BOXER1", "t": "C172", "lat": 1, "lon": 2, "category": "A1"},
+        ]
+    }
+    rows = {r["id"]: r for r in transform_adsb_lol(payload)}
+    assert rows["3b76aa"]["extra"]["role"] == "military"
+    assert rows["3b76aa"]["extra"]["role_src"] == "type"
+    assert rows["ae1234"]["extra"]["role"] == "military"
+    assert rows["ae1234"]["extra"]["role_src"] == "hex"
+
+
+def test_transform_ais_static_classifies_military_and_cargo():
+    mil = transform_ais_frame({
+        "MessageType": "ShipStaticData",
+        "MetaData": {"MMSI": 338123456, "ShipName": "USNS BOB", "Latitude": 32.7, "Longitude": -117.2},
+        "Message": {"ShipStaticData": {
+            "Type": 35, "CallSign": "NBXX", "ImoNumber": 0,
+            "Destination": "SAN DIEGO", "MaximumStaticDraught": 8.2,
+            "Dimension": {"A": 80, "B": 20, "C": 8, "D": 8},
+            "Eta": {"Month": 8, "Day": 29, "Hour": 14, "Minute": 0},
+        }},
+    })
+    cargo = transform_ais_frame({
+        "MessageType": "ShipStaticData",
+        "MetaData": {"MMSI": 477123456, "ShipName": "EVER GIVEN", "Latitude": 36.9, "Longitude": -76.3},
+        "Message": {"ShipStaticData": {
+            "Type": 70, "CallSign": "VRXX", "ImoNumber": 9811000,
+            "Destination": "NORFOLK", "MaximumStaticDraught": 14.5,
+            "Dimension": {"A": 200, "B": 150, "C": 20, "D": 20},
+        }},
+    })
+    assert mil is not None and cargo is not None
+    assert mil["extra"]["role"] == "military"
+    assert mil["extra"]["kind"] == "military"
+    assert mil["extra"]["callsign"] == "NBXX"
+    assert mil["extra"]["length"] == 100
+    assert mil["extra"]["beam"] == 16
+    assert mil["extra"]["dest"] == "SAN DIEGO"
+    assert mil["extra"]["country"] == "United States"
+    assert cargo["extra"]["role"] == "civilian"
+    assert cargo["extra"]["kind"] == "cargo"
+    assert cargo["extra"]["imo"] == 9811000
+
+
+def test_transform_ais_position_decodes_navstat():
+    row = transform_ais_frame({
+        "MessageType": "PositionReport",
+        "MetaData": {"MMSI": 366912810, "ShipName": "EVER GIVEN", "latitude": 36.9, "longitude": -76.3},
+        "Message": {"PositionReport": {"Sog": 0.1, "Cog": 88.0, "TrueHeading": 90, "NavigationalStatus": 5}},
+    })
+    assert row is not None
+    assert row["extra"]["nav"] == "moored"
+    assert row["extra"]["navstat"] == 5
+
+
+def test_nws_alerts_does_not_send_bbox_param(monkeypatch):
+    """api.weather.gov/alerts/active 400s on bbox — clip locally instead."""
+    import asyncio
+
+    from live_layers import fetch_weather_alerts, _cache
+
+    seen = []
+
+    async def fake_get(url, params=None):
+        seen.append((url, dict(params or {})))
+        if "weather.gov" in url:
+            return {
+                "type": "FeatureCollection",
+                "features": [{
+                    "type": "Feature",
+                    "properties": {"event": "Tornado Warning", "severity": "Extreme"},
+                    "geometry": {"type": "Point", "coordinates": [-78.7, 35.8]},
+                }],
+            }
+        return {"type": "FeatureCollection", "features": []}
+
+    monkeypatch.setattr("live_layers._get_json", fake_get)
+    _cache.clear()
+    fc = asyncio.run(fetch_weather_alerts(None, "-79.0,35.5,-78.0,36.0"))
+    nws_calls = [p for u, p in seen if "weather.gov" in u]
+    assert nws_calls, "NWS should still be fetched"
+    assert "bbox" not in nws_calls[0]
+    assert fc.get("nws_ok") is True
+    assert len(fc["features"]) == 1
+
+
+def test_nws_alerts_failure_is_flagged(monkeypatch):
+    import asyncio
+
+    from live_layers import fetch_weather_alerts, _cache
+
+    async def fake_get(url, params=None):
+        if "weather.gov" in url:
+            raise RuntimeError("400 Bad Request")
+        return {"type": "FeatureCollection", "features": []}
+
+    monkeypatch.setattr("live_layers._get_json", fake_get)
+    _cache.clear()
+    fc = asyncio.run(fetch_weather_alerts(None, None))
+    assert fc.get("nws_ok") is False
+
+
+def test_fetch_aircraft_get_path_does_not_persist(monkeypatch):
+    """GET /api/aircraft must serve last-known without track/geofence writes."""
+    import asyncio
+
+    from live_layers import (
+        aircraft_last_known, fetch_aircraft, persist_aircraft_snapshot, _cache,
+    )
+
+    aircraft_last_known.clear()
+    aircraft_last_known["abc"] = {
+        "id": "abc", "lat": 35.8, "lon": -78.7, "heading": 90, "speed": 400,
+        "label": "ABC", "extra": {},
+    }
+    writes = {"n": 0}
+
+    async def boom(*a, **k):
+        writes["n"] += 1
+        raise AssertionError("GET path must not persist")
+
+    monkeypatch.setattr("tracks.record_position", boom)
+    monkeypatch.setattr("geofence.record_and_notify", boom)
+    _cache.clear()
+    rows = asyncio.run(fetch_aircraft("-79,35,-78,36", persist=False))
+    assert writes["n"] == 0
+    assert any(r["id"] == "abc" for r in rows)
+
+
+def test_persist_aircraft_snapshot_writes_tracks(monkeypatch):
+    import asyncio
+
+    from live_layers import persist_aircraft_snapshot
+
+    recorded = []
+
+    async def fake_record(kind, marker):
+        recorded.append((kind, marker["id"]))
+        return True
+
+    async def fake_gf(**kw):
+        return 0
+
+    monkeypatch.setattr("tracks.record_position", fake_record)
+    monkeypatch.setattr("geofence.record_and_notify", fake_gf)
+    monkeypatch.setattr("ws_manager.manager.has_clients", lambda: False)
+
+    rows = [{
+        "id": "abc", "lat": 35.8, "lon": -78.7, "heading": 90, "speed": 400,
+        "label": "ABC", "extra": {},
+    }]
+    asyncio.run(persist_aircraft_snapshot(rows))
+    assert recorded == [("aircraft", "abc")]
+
+
+# ── Planespotters.net photo lookup ────────────────────────────────────────
+
+def test_normalize_planespotter_photo_prefers_large_thumbnail():
+    from live_layers import _normalize_planespotter_photo
+
+    out = _normalize_planespotter_photo({
+        "id": "1053982",
+        "thumbnail": {"src": "https://t.plnspttrs.net/x_t.jpg", "size": {"width": 200, "height": 141}},
+        "thumbnail_large": {"src": "https://t.plnspttrs.net/x_280.jpg", "size": {"width": 395, "height": 280}},
+        "link": "https://www.planespotters.net/photo/1053982/foo",
+        "photographer": "Günther Feniuk",
+    })
+    assert out["id"] == "1053982"
+    assert out["src"] == "https://t.plnspttrs.net/x_280.jpg"
+    assert out["width"] == 395
+    assert out["height"] == 280
+    assert out["photographer"] == "Günther Feniuk"
+    assert "planespotters.net" in out["link"]
+
+
+def test_normalize_planespotter_photo_empty_or_malformed_returns_none():
+    from live_layers import _normalize_planespotter_photo
+
+    assert _normalize_planespotter_photo({}) is None
+    assert _normalize_planespotter_photo({"thumbnail": {}}) is None
+    assert _normalize_planespotter_photo(None) is None
+    assert _normalize_planespotter_photo("not-a-dict") is None
+
+
+def test_fetch_planespotters_photo_hex_builds_url_and_normalizes(monkeypatch):
+    import asyncio
+
+    from live_layers import fetch_planespotters_photo, _cache
+
+    seen = []
+
+    async def fake_get(url, params=None, headers=None):
+        seen.append((url, (headers or {}).get("User-Agent", "")))
+        return {"photos": [{
+            "id": "1", "thumbnail": {"src": "https://t.plnspttrs.net/a_t.jpg"},
+            "thumbnail_large": {"src": "https://t.plnspttrs.net/a_280.jpg"},
+            "link": "https://www.planespotters.net/photo/1/x", "photographer": "A",
+        }]}
+
+    monkeypatch.setattr("live_layers._get_json", fake_get)
+    _cache.clear()
+    out = asyncio.run(fetch_planespotters_photo(hex_code="e8027e"))
+    assert out["src"] == "https://t.plnspttrs.net/a_280.jpg"
+    assert seen[0][0] == "https://api.planespotters.net/pub/photos/hex/e8027e"
+    assert "@" in seen[0][1] or "http" in seen[0][1]
+
+
+def test_fetch_planespotters_photo_reg_fallback_and_no_result(monkeypatch):
+    import asyncio
+
+    from live_layers import fetch_planespotters_photo, _cache
+
+    seen = []
+
+    async def fake_get(url, params=None, headers=None):
+        seen.append(url)
+        return {"photos": []}
+
+    monkeypatch.setattr("live_layers._get_json", fake_get)
+    _cache.clear()
+    assert asyncio.run(fetch_planespotters_photo(reg="D-ABCD")) is None
+    assert seen == ["https://api.planespotters.net/pub/photos/reg/D-ABCD"]
+    # no hex and no reg → no upstream call at all
+    assert asyncio.run(fetch_planespotters_photo()) is None
+
+
+def test_planespotters_headers_add_contact_when_ua_is_generic(monkeypatch):
+    import live_layers
+
+    monkeypatch.setattr(live_layers, "OSINT_USER_AGENT", "osint-dashboard/1.0 (self-hosted)")
+    ua = live_layers._planespotters_headers()["User-Agent"]
+    assert "osint-dashboard" in ua
+    assert "@" in ua
+
+
+# ── Sentinel-1 SAR (Planetary Computer STAC → signed COG template) ────────
+
+def test_sign_cog_url_appends_token():
+    # PC returns the token pre-encoded as a query string; append verbatim.
+    assert sign_cog_url("https://blob.example/x.tif", "st=s&se=e&sig=x%3D") == \
+        "https://blob.example/x.tif?st=s&se=e&sig=x%3D"
+    # Existing query string → append with &
+    assert sign_cog_url("https://blob.example/x.tif?foo=1", "st=s&sig=x") == \
+        "https://blob.example/x.tif?foo=1&st=s&sig=x"
+
+
+def test_sentinel1_tile_url_contains_titiler_rescale_and_cfastie():
+    signed = "https://blob.example/x.tif?token=secret"
+    url = sentinel1_tile_url(signed)
+    assert url.startswith(TITILER_COG_TILES + "?")
+    assert "WebMercatorQuad/{z}/{x}/{y}?" in url
+    assert "url=https%3A%2F%2Fblob.example%2Fx.tif%3Ftoken%3Dsecret" in url
+    assert "rescale=0%2C500" in url
+    assert "colormap_name=cfastie" in url
+
+
+def test_sentinel1_tile_url_is_same_origin_relative():
+    # Self-hosted TiTiler: the browser must hit the Pi's nginx vhost, not
+    # titiler.xyz or a raw host:port. The template is a root-relative path.
+    url = sentinel1_tile_url("https://blob.example/x.tif")
+    assert url.startswith("/titiler/cog/tiles/WebMercatorQuad/")
+    assert "://" not in url
+    assert "titiler.xyz" not in url
+
+
+def _stac_feature(assets: dict) -> dict:
+    return {
+        "type": "Feature",
+        "id": "S1A_IW_GRDH_1SDV_20240820T000000",
+        "properties": {"datetime": "2024-08-20T00:00:00Z"},
+        "assets": assets,
+    }
+
+
+def test_fetch_sentinel1_vv_signed_tile_url(monkeypatch):
+    import asyncio
+    from live_layers import fetch_sentinel1, _cache
+
+    calls = []
+
+    async def fake_post(url, json=None, headers=None):
+        calls.append(("post", url, json))
+        return {"features": [_stac_feature({
+            "vv": {"href": "https://blob.example/grd-vv.tif"},
+        })]}
+
+    async def fake_get(url, params=None, headers=None):
+        calls.append(("get", url))
+        return {"token": "sig=abc123"}
+
+    monkeypatch.setattr("live_layers._post_json", fake_post)
+    monkeypatch.setattr("live_layers._get_json", fake_get)
+    _cache.clear()
+
+    out = asyncio.run(fetch_sentinel1("-80,35,-79,36"))
+    assert out["id"] == "sentinel-1-sar"
+    assert out["kind"] == "raster"
+    assert out["polarization"] == "vv"
+    assert out["opacity"] == 0.8
+    assert out["itemId"].startswith("S1A")
+    assert out["attribution"] == SENTINEL1_ATTRIBUTION
+    assert "WebMercatorQuad/{z}/{x}/{y}?" in out["tileUrl"]
+    assert "rescale=0%2C500" in out["tileUrl"]
+    assert "colormap_name=cfastie" in out["tileUrl"]
+    # SAS token "sig=abc123" is appended top-level, then the whole COG URL is
+    # percent-encoded again as a query param (=> sig%3Dabc123).
+    assert "sig%3Dabc123" in out["tileUrl"]
+    # STAC search payload shape
+    post_url, post_json = calls[0][1], calls[0][2]
+    assert post_url.endswith("/api/stac/v1/search")
+    assert post_json["collections"] == ["sentinel-1-grd"]
+    assert post_json["limit"] >= 1
+    assert post_json["sortby"][0]["direction"] == "desc"
+    assert "bbox" in out
+
+
+def test_fetch_sentinel1_uses_hh_when_vv_missing(monkeypatch):
+    import asyncio
+    from live_layers import fetch_sentinel1, _cache
+
+    async def fake_post(url, json=None, headers=None):
+        return {"features": [_stac_feature({
+            "hh": {"href": "https://blob.example/grd-hh.tif"},
+        })]}
+
+    async def fake_get(url, params=None, headers=None):
+        return {"token": "tok"}
+
+    monkeypatch.setattr("live_layers._post_json", fake_post)
+    monkeypatch.setattr("live_layers._get_json", fake_get)
+    _cache.clear()
+
+    out = asyncio.run(fetch_sentinel1("-80,35,-79,36"))
+    assert out["polarization"] == "hh"
+    assert "url=https%3A%2F%2Fblob.example%2Fgrd-hh.tif" in out["tileUrl"]
+
+
+def test_fetch_sentinel1_none_on_empty_features(monkeypatch):
+    import asyncio
+    from live_layers import fetch_sentinel1, _cache
+
+    async def fake_post(url, json=None, headers=None):
+        return {"features": []}
+
+    monkeypatch.setattr("live_layers._post_json", fake_post)
+    _cache.clear()
+
+    assert asyncio.run(fetch_sentinel1("-80,35,-79,36")) is None
+
+
+def test_fetch_sentinel1_none_when_no_vv_or_hh(monkeypatch):
+    import asyncio
+    from live_layers import fetch_sentinel1, _cache
+
+    async def fake_post(url, json=None, headers=None):
+        return {"features": [_stac_feature({"thumbnail": {"href": "https://x"}})]}
+
+    monkeypatch.setattr("live_layers._post_json", fake_post)
+    _cache.clear()
+
+    assert asyncio.run(fetch_sentinel1("-80,35,-79,36")) is None
+
+
+def test_pick_sentinel_feature_prefers_scene_covering_center():
+    features = [
+        {"id": "far", "bbox": [10.0, 10.0, 12.0, 12.0]},
+        {"id": "cover", "bbox": [-80.5, 34.5, -78.5, 36.5]},
+        {"id": "also-far", "bbox": [-10.0, 0.0, -8.0, 2.0]},
+    ]
+    picked = pick_sentinel_feature(features, -79.5, 35.5)
+    assert picked["id"] == "cover"
+
+
+def test_pick_sentinel_feature_falls_back_to_first_when_none_cover():
+    features = [
+        {"id": "a", "bbox": [10.0, 10.0, 12.0, 12.0]},
+        {"id": "b", "bbox": [20.0, 20.0, 22.0, 22.0]},
+    ]
+    assert pick_sentinel_feature(features, -79.5, 35.5)["id"] == "a"
+    assert pick_sentinel_feature([], -79.5, 35.5) is None

tests/test_new_sources.py

@@ -0,0 +1,128 @@
+"""NASA EONET + CISA KEV parsers (no network)."""
+
+from __future__ import annotations
+
+
+def test_parse_eonet_keeps_stable_ids_and_points():
+    from sources import parse_eonet_events
+
+    payload = {
+        "events": [
+            {
+                "id": "EONET_6363",
+                "title": "Etna Volcano",
+                "categories": [{"id": "volcanoes", "title": "Volcanoes"}],
+                "geometry": [
+                    {"date": "2024-01-01T00:00:00Z", "type": "Point", "coordinates": [15.0, 37.7]},
+                ],
+                "link": "https://eonet.gsfc.nasa.gov/api/v3/events/EONET_6363",
+            },
+            {
+                "id": "EONET_skip",
+                "title": "No geometry",
+                "categories": [],
+                "geometry": [],
+                "link": "https://eonet.gsfc.nasa.gov/api/v3/events/EONET_skip",
+            },
+        ]
+    }
+    events = parse_eonet_events(payload)
+    assert len(events) == 1
+    ev = events[0]
+    assert ev["url"] == "https://eonet.gsfc.nasa.gov/api/v3/events/EONET_6363"
+    assert ev["source_type"] == "disaster"
+    assert ev["location_lat"] == 37.7
+    assert ev["location_lon"] == 15.0
+    assert ev["raw"]["eonet_id"] == "EONET_6363"
+    assert "volcanoes" in ev["tags"]
+
+
+def test_parse_cisa_kev_emits_cve_url_no_coords():
+    from sources import parse_cisa_kev
+
+    payload = {
+        "vulnerabilities": [
+            {
+                "cveID": "CVE-2024-1234",
+                "vendorProject": "Acme",
+                "product": "Widget",
+                "vulnerabilityName": "RCE",
+                "dateAdded": "2024-06-01",
+                "shortDescription": "Remote code execution",
+                "requiredAction": "Apply updates",
+                "dueDate": "2024-06-22",
+                "knownRansomwareCampaignUse": "Known",
+            }
+        ]
+    }
+    events = parse_cisa_kev(payload)
+    assert len(events) == 1
+    ev = events[0]
+    assert ev["url"] == "https://nvd.nist.gov/vuln/detail/CVE-2024-1234"
+    assert ev["location_lat"] is None
+    assert ev["location_lon"] is None
+    assert "cisa-kev" in ev["tags"]
+    assert "CVE-2024-1234" in ev["tags"]
+    assert ev["raw"]["cveID"] == "CVE-2024-1234"
+
+
+def test_ingest_cisa_kev_does_not_republish_known_nist_urls(monkeypatch):
+    """Producer must not push the whole KEV catalog to NATS every cycle."""
+    import asyncio
+
+    from sources import ingest_cisa_kev
+
+    payload = {
+        "vulnerabilities": [
+            {
+                "cveID": "CVE-2024-1111",
+                "vulnerabilityName": "old",
+                "dateAdded": "2024-01-01",
+                "shortDescription": "already in db",
+            },
+            {
+                "cveID": "CVE-2024-2222",
+                "vulnerabilityName": "new",
+                "dateAdded": "2024-06-01",
+                "shortDescription": "not in db yet",
+            },
+        ]
+    }
+
+    class FakeResp:
+        def raise_for_status(self):
+            pass
+
+        def json(self):
+            return payload
+
+    class FakeClient:
+        def __init__(self, **kw):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *exc):
+            return False
+
+        async def get(self, url):
+            return FakeResp()
+
+    published: list[str] = []
+
+    async def fake_publish(subject, event):
+        published.append(event["url"])
+
+    known = {"https://nvd.nist.gov/vuln/detail/CVE-2024-1111"}
+
+    async def fake_existing(urls):
+        return {u for u in urls if u in known}
+
+    monkeypatch.setattr("sources.httpx.AsyncClient", FakeClient)
+    monkeypatch.setattr("sources.publish_event", fake_publish)
+    monkeypatch.setattr("sources.existing_event_urls", fake_existing, raising=False)
+
+    n = asyncio.run(ingest_cisa_kev())
+    assert n == 1
+    assert published == ["https://nvd.nist.gov/vuln/detail/CVE-2024-2222"]

tests/test_news_scraper.py

@@ -0,0 +1,42 @@
+"""News spider/pipeline: skip audio, use pubDate, don't log dupes as errors."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+_SCRAPER = ROOT / "news/scraper"
+if str(_SCRAPER) not in sys.path:
+    sys.path.insert(0, str(_SCRAPER))
+
+
+def test_is_audio_enclosure():
+    from newsScraper.feed_util import is_audio_url
+
+    assert is_audio_url("https://cdn.example/podcast.mp3") is True
+    assert is_audio_url("https://cdn.example/show.m4a?x=1") is True
+    assert is_audio_url("https://www.example.com/world/story") is False
+
+
+def test_article_timestamp_prefers_pubdate():
+    from newsScraper.feed_util import article_timestamp
+
+    ts = article_timestamp("Tue, 01 Apr 2025 12:00:00 GMT")
+    assert ts.tzinfo is not None
+    assert ts.year == 2025
+    assert ts.month == 4
+    assert ts.day == 1
+
+
+def test_pipeline_does_not_wrap_dropitem_as_error():
+    src = (ROOT / "news/scraper/newsScraper/pipelines.py").read_text()
+    assert "except DropItem" in src
+    assert "seen_urls.add" in src or "self.seen_urls.add" in src
+
+
+def test_spider_skips_audio_before_request():
+    src = (ROOT / "news/scraper/newsScraper/spiders/news_spider.py").read_text()
+    assert "is_audio_url" in src
+    assert "article_timestamp" in src
+    assert "datetime.datetime.now()" not in src

tests/test_phase1_guardrails.py

@@ -31,3 +31,35 @@ def test_no_redis_kafka_celery():
         assert "kafka" not in blob
         assert "celery" not in blob
     assert "cachetools" in req
+
+
+def test_titiler_image_pinned_by_digest():
+    text = (ROOT / "docker-compose.yml").read_text()
+    assert (
+        "ghcr.io/developmentseed/titiler:latest@sha256:"
+        "1809958d063543e3ec858259536002b2de78e9f8f09a22a8d9591bdc2b550b14"
+        in text
+    )
+    # Unpinned :latest would drift on every pull.
+    for line in text.splitlines():
+        if "titiler" in line.lower() and "image:" in line:
+            assert "@sha256:" in line
+
+
+def test_uvicorn_single_worker_guard():
+    text = (ROOT / "app" / "main.py").read_text()
+    main_block = text.split('if __name__ == "__main__":', 1)[1]
+    assert "workers=1" in main_block
+
+
+def test_bbox_gist_migration_keeps_btree_and_adds_gist():
+    text = (ROOT / "alembic" / "versions" / "010_bbox_gist.py").read_text()
+    assert "down_revision" in text and "009_vessels" in text
+    assert "ix_events_geom_gist" in text
+    assert "ix_fires_geom_gist" in text
+    assert "ST_MakePoint(location_lon, location_lat)" in text
+    assert "ST_MakePoint(longitude, latitude)" in text
+    assert "USING gist" in text
+    models = (ROOT / "app" / "models.py").read_text()
+    assert 'Index("ix_events_location"' in models
+    assert 'Index("ix_fires_bbox"' in models

tests/test_phase2_api.py

@@ -17,6 +17,9 @@ async def _req(method: str, path: str, **kw) -> httpx.Response:
         return await client.request(method, path, **kw)
 
 
+FENCE_ID = "11111111-1111-1111-1111-111111111111"
+
+
 def test_geofence_post_rejects_point():
     resp = asyncio.run(_req(
         "POST", "/api/geofences",
@@ -25,6 +28,111 @@ def test_geofence_post_rejects_point():
     assert resp.status_code == 422
 
 
+def test_delete_geofence_404_when_missing(monkeypatch):
+    async def missing(_gid: str) -> bool:
+        return False
+
+    monkeypatch.setattr("geofence.delete_geofence", missing)
+    resp = asyncio.run(_req("DELETE", f"/api/geofences/{FENCE_ID}"))
+    assert resp.status_code == 404
+
+
+def test_geofence_alerts_passes_filters(monkeypatch):
+    seen = {}
+
+    async def fake_list(**kwargs):
+        seen.update(kwargs)
+        return [{"id": "a", "geofence_id": FENCE_ID, "source_kind": "ais"}]
+
+    monkeypatch.setattr("geofence.list_alerts", fake_list)
+    resp = asyncio.run(_req(
+        "GET", "/api/geofence-alerts",
+        params={
+            "geofence_id": FENCE_ID,
+            "since": "2026-08-28T00:00:00Z",
+            "until": "2026-08-29T00:00:00Z",
+            "source_kind": "ais",
+            "limit": 10,
+        },
+    ))
+    assert resp.status_code == 200
+    assert resp.json()[0]["source_kind"] == "ais"
+    assert seen["geofence_id"] == FENCE_ID
+    assert seen["source_kind"] == "ais"
+    assert seen["limit"] == 10
+    assert seen["since"] is not None
+    assert seen["until"] is not None
+
+
+def test_geofence_alerts_rejects_bad_source_kind():
+    resp = asyncio.run(_req(
+        "GET", "/api/geofence-alerts", params={"source_kind": "camera"},
+    ))
+    assert resp.status_code == 422
+
+
+def test_geofence_at_404_when_missing(monkeypatch):
+    async def no_snap(gid: str, ts):
+        return None
+
+    monkeypatch.setattr("geofence.snapshot_at", no_snap)
+    resp = asyncio.run(_req(
+        "GET", f"/api/geofences/{FENCE_ID}/at",
+        params={"timestamp": "2026-08-28T12:04:00Z"},
+    ))
+    assert resp.status_code == 404
+
+
+def test_geofence_at_empty_lists_when_db_down(monkeypatch):
+    async def empty_snap(gid: str, ts):
+        return {
+            "geofence_id": gid,
+            "timestamp": ts.isoformat(),
+            "aircraft": [],
+            "vessels": [],
+            "fires": [],
+        }
+
+    monkeypatch.setattr("geofence.snapshot_at", empty_snap)
+    resp = asyncio.run(_req(
+        "GET", f"/api/geofences/{FENCE_ID}/at",
+        params={"timestamp": "2026-08-28T12:04:00Z"},
+    ))
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["geofence_id"] == FENCE_ID
+    assert body["aircraft"] == []
+    assert body["vessels"] == []
+    assert body["fires"] == []
+    assert "timestamp" in body
+
+
+def test_geofence_at_does_not_notify(monkeypatch):
+    called = {"notify": 0}
+
+    async def empty_snap(gid: str, ts):
+        return {
+            "geofence_id": gid,
+            "timestamp": ts.isoformat(),
+            "aircraft": [],
+            "vessels": [],
+            "fires": [],
+        }
+
+    async def boom(**_kw):
+        called["notify"] += 1
+        raise AssertionError("GET /at must not record_and_notify")
+
+    monkeypatch.setattr("geofence.snapshot_at", empty_snap)
+    monkeypatch.setattr("geofence.record_and_notify", boom)
+    resp = asyncio.run(_req(
+        "GET", f"/api/geofences/{FENCE_ID}/at",
+        params={"timestamp": "2026-08-28T12:04:00Z"},
+    ))
+    assert resp.status_code == 200
+    assert called["notify"] == 0
+
+
 def test_geofences_list_does_not_collide_with_alerts():
     resp = asyncio.run(_req("GET", "/api/geofences"))
     assert resp.status_code == 200

tests/test_place_dossier_frontend.py

@@ -0,0 +1,59 @@
+"""Right-click place dossier HUD contract."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+HTML = (ROOT / "app/static/index.html").read_text()
+
+
+def test_place_dossier_panel_markup():
+    assert 'id="place-dossier"' in HTML
+    assert "What’s here?" in HTML or "What's here?" in HTML
+    assert 'id="pd-nearby"' in HTML
+    assert 'id="pd-close"' in HTML
+    assert 'role="dialog"' in HTML
+
+
+def test_place_dossier_uses_backend_nominatim_proxy():
+    js = HTML.split("async function openPlaceDossier", 1)[1].split(
+        "/* ═══════════════ INITIAL LOAD", 1
+    )[0]
+    assert "/api/place?lat=" in js
+    assert "nominatim.openstreetmap.org" not in js
+    assert "/api/aircraft" not in js
+    assert "/api/vessels" not in js
+    assert "/api/cameras" not in js
+    assert "/api/fires" not in js
+    assert "/api/weather-alerts" not in js
+    assert "/api/infrastructure" not in js
+
+
+def test_place_dossier_scans_loaded_overlays_5km():
+    assert "const PLACE_PAD_KM = 5" in HTML
+    assert "function collectNearby" in HTML
+    assert "lastCams" in HTML
+    assert "lastAircraft" in HTML
+    assert "lastVessels" in HTML
+    assert "lastFires" in HTML
+    assert "lastAlerts" in HTML
+    assert "function haversineKm" in HTML
+
+
+def test_place_dossier_right_click_and_long_press():
+    assert "map.on('contextmenu'" in HTML
+    assert "function bindPlaceLongPress" in HTML
+    assert "function closePlaceDossier" in HTML
+    assert "Escape" in HTML.split("function initMap", 1)[1][:8000] or "Escape" in HTML.split(
+        "bindPlaceLongPress(map)", 1
+    )[0][-500:]
+
+
+def test_place_dossier_mobile_is_bottom_sheet():
+    mobile = HTML.split("@media (max-width: 820px)")[1].split(
+        "@media (prefers-reduced-motion"
+    )[0]
+    assert "#place-dossier" in mobile
+    assert "bottom: 56px" in mobile
+    assert "max-height: 36vh" in mobile

tests/test_satellites.py

@@ -0,0 +1,200 @@
+"""CelesTrak satellites overlay: GP JSON parser, 2h cache, groups, bbox."""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timezone
+
+import httpx
+
+import satellites
+from live_layers import _cache, overlay_catalog
+from main import app
+
+BASE = "http://test"
+
+# Two real CelesTrak GP JSON records (trimmed to the OMM fields sgp4 needs).
+ISS = {
+    "OBJECT_NAME": "ISS (ZARYA)", "OBJECT_ID": "1998-067A",
+    "EPOCH": "2026-08-31T11:11:23.184384", "MEAN_MOTION": 15.4894954,
+    "ECCENTRICITY": 0.00050456, "INCLINATION": 51.6314,
+    "RA_OF_ASC_NODE": 287.5025, "ARG_OF_PERICENTER": 92.8598,
+    "MEAN_ANOMALY": 267.2968, "EPHEMERIS_TYPE": 0,
+    "CLASSIFICATION_TYPE": "U", "NORAD_CAT_ID": 25544,
+    "ELEMENT_SET_NO": 999, "REV_AT_EPOCH": 58342,
+    "BSTAR": 9.9862358e-5, "MEAN_MOTION_DOT": 5.046e-5,
+    "MEAN_MOTION_DDOT": 0,
+}
+HST = {
+    "OBJECT_NAME": "HST", "OBJECT_ID": "1990-037B",
+    "EPOCH": "2026-08-31T11:11:23.184384", "MEAN_MOTION": 15.0865888,
+    "ECCENTRICITY": 0.0002426, "INCLINATION": 28.4697,
+    "RA_OF_ASC_NODE": 102.1854, "ARG_OF_PERICENTER": 152.8462,
+    "MEAN_ANOMALY": 207.2795, "EPHEMERIS_TYPE": 0,
+    "CLASSIFICATION_TYPE": "U", "NORAD_CAT_ID": 20580,
+    "ELEMENT_SET_NO": 999, "REV_AT_EPOCH": 12345,
+    "BSTAR": 2.9e-5, "MEAN_MOTION_DOT": 0.0,
+    "MEAN_MOTION_DDOT": 0,
+}
+FIXTURE = [ISS, HST]
+
+NOW = datetime(2026, 8, 31, 12, 0, 0, tzinfo=timezone.utc)
+
+
+async def _get(path: str) -> httpx.Response:
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
+        return await client.get(path)
+
+
+def test_propagate_gp_shape():
+    rows = satellites.propagate_gp(FIXTURE, "stations", NOW)
+    assert len(rows) == 2
+    by_id = {r["id"]: r for r in rows}
+    assert set(by_id) == {"25544", "20580"}
+    iss = by_id["25544"]
+    assert iss["name"] == "ISS (ZARYA)"
+    assert iss["group"] == "stations"
+    # ISS is in LEO: ~400 km, |lat| <= inclination 51.63, lon in range.
+    assert 300 < iss["alt_km"] < 500
+    assert -51.7 <= iss["lat"] <= 51.7
+    assert -180 <= iss["lon"] <= 180
+    for key in ("id", "name", "lat", "lon", "alt_km", "group"):
+        assert key in iss
+
+
+def test_propagate_gp_skips_malformed():
+    bad = [{"OBJECT_NAME": "x"}, None, 42, {"NORAD_CAT_ID": 1}]
+    assert satellites.propagate_gp(bad, "stations", NOW) == []
+
+
+def test_parse_groups_defaults_and_validation():
+    assert satellites.parse_groups("stations,weather") == ["stations", "weather"]
+    assert satellites.parse_groups("weather,gps-ops") == ["weather", "gps-ops"]
+    # starlink is allowed only when explicitly requested
+    assert satellites.parse_groups("starlink") == ["starlink"]
+    assert satellites.parse_groups("stations,stations") == ["stations"]
+    for bad in ("", None, "debris", "stations,active", "stations, weather, active"):
+        try:
+            satellites.parse_groups(bad)
+        except ValueError:
+            pass
+        else:
+            raise AssertionError(f"expected ValueError for {bad!r}")
+
+
+def test_overlay_catalog_has_satellites_stub():
+    entry = overlay_catalog()["satellites"]
+    assert entry["kind"] == "points"
+    assert entry["endpoint"] == "/api/satellites"
+    assert "CelesTrak" in entry["attribution"]
+
+
+def test_unknown_group_400():
+    resp = asyncio.run(_get("/api/satellites?groups=debris"))
+    assert resp.status_code == 400
+
+
+def test_default_groups_ok_with_fake_fetch(monkeypatch):
+    async def fake(groups, bbox=None, limit=2000):
+        return {"satellites": [], "source": "celestrak",
+                "tle_epoch": None, "timestamp": "t"}
+
+    monkeypatch.setattr("main.fetch_satellites", fake)
+    resp = asyncio.run(_get("/api/satellites"))
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body["source"] == "celestrak"
+    assert "max-age" in (resp.headers.get("cache-control") or "").lower()
+
+
+def test_2h_cache_does_not_refetch(monkeypatch):
+    satellites._last_good.clear()
+    _cache.clear()
+    hits = {"n": 0}
+
+    class FakeResp:
+        def __init__(self, data):
+            self._data = data
+
+        def raise_for_status(self):
+            pass
+
+        def json(self):
+            return self._data
+
+    class FakeClient:
+        def __init__(self):
+            pass
+
+        async def get(self, url, params=None, headers=None):
+            hits["n"] += 1
+            assert "celestrak.org/NORAD/elements/gp.php" in url
+            return FakeResp(FIXTURE)
+
+    monkeypatch.setattr("live_layers._http", FakeClient())
+
+    async def run():
+        p1 = await satellites.fetch_satellites(["stations"])
+        p2 = await satellites.fetch_satellites(["stations"])
+        return p1, p2
+
+    p1, p2 = asyncio.run(run())
+    assert len(p1["satellites"]) == 2
+    assert p1["tle_epoch"] == "2026-08-31T11:11:23.184384"
+    # Same element blob served from cache (no refetch), same ids/epochs.
+    assert [s["id"] for s in p2["satellites"]] == [s["id"] for s in p1["satellites"]]
+    assert p2["tle_epoch"] == p1["tle_epoch"]
+    assert hits["n"] == 1
+    _cache.clear()
+    satellites._last_good.clear()
+
+
+def test_bbox_culls_satellites(monkeypatch):
+    """bbox filtering in fetch_satellites, deterministic via fake propagation."""
+
+    async def fake_elements(group):
+        return [{"x": 1}], "2026-08-31T11:11:23.184384"
+
+    monkeypatch.setattr("satellites._group_elements", fake_elements)
+
+    def fake_propagate(elements, group, now):
+        return [
+            {"id": "a", "name": "A", "lat": 10.0, "lon": 20.0, "alt_km": 400.0, "group": group},
+            {"id": "b", "name": "B", "lat": 45.0, "lon": -70.0, "alt_km": 400.0, "group": group},
+            {"id": "c", "name": "C", "lat": -10.0, "lon": 30.0, "alt_km": 400.0, "group": group},
+        ]
+
+    monkeypatch.setattr("satellites.propagate_gp", fake_propagate)
+
+    payload = asyncio.run(
+        satellites.fetch_satellites(["stations"], bbox="-80,0,-60,50")
+    )
+    ids = [s["id"] for s in payload["satellites"]]
+    assert ids == ["b"]  # only (45, -70) falls inside the box
+
+
+def test_bbox_culls_nothing_when_empty():
+    from satellites import fetch_satellites
+
+    # No bbox: all rows returned up to limit.
+    # (skip network; just sanity-check the arg is accepted by signature)
+    assert callable(fetch_satellites)
+
+
+def test_satnogs_fallback_parser():
+    payload = [{
+        "tle0": "0 ISS (ZARYA)",
+        "tle1": "1 25544U 98067A   26243.85334329  .00004554  00000-0  90917-4 0  9992",
+        "tle2": "2 25544  51.6312 285.5873 0005057  94.2999 265.8567 15.48953200583481",
+        "norad_cat_id": 25544,
+        "updated": "2026-09-01T01:19:54.327653Z",
+    }]
+    rows, epoch = satellites.propagate_satnogs_tle(payload, "stations", NOW)
+    assert len(rows) == 1
+    row = rows[0]
+    assert row["id"] == "25544"
+    assert row["name"] == "ISS (ZARYA)"
+    assert row["group"] == "stations"
+    assert epoch == "2026-09-01T01:19:54.327653Z"
+    assert 300 < row["alt_km"] < 500

tests/test_sentinel1_frontend.py

@@ -0,0 +1,49 @@
+"""Sentinel-1 SAR layer: Leaflet toggle contract (no browser STAC/TiTiler)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+HTML = (ROOT / "app/static/index.html").read_text()
+
+
+def test_sentinel1_toggle_default_off():
+    assert 'id="lp-sentinel-on"' in HTML
+    assert 'id="sentinel-1-sar"' in HTML
+    assert "Sentinel-1 SAR (Cloud-Penetrating)" in HTML
+    on = HTML.split('id="lp-sentinel-on"', 1)[1].split(">", 1)[0]
+    assert "checked" not in on
+
+
+def test_sentinel1_fetches_backend_not_planetary_computer():
+    assert "/api/map/sentinel1?bbox=" in HTML
+    assert "titiler.xyz" not in HTML
+    js = HTML.split("async function loadSentinel1", 1)[1].split("function loadThermal", 1)[0]
+    assert "planetarycomputer" not in js.lower()
+    assert "stac" not in js.lower()
+
+
+def test_sentinel1_hud_errors_not_alert():
+    js = HTML.split("async function loadSentinel1", 1)[1].split("function loadThermal", 1)[0]
+    assert "alert(" not in js
+    assert "No Sentinel-1 imagery for this view in the last 7 days." in js
+    assert "retry later" in js
+    assert "opacity: sentinelOpacity" in js
+    assert "maxZoom: 18" in js
+    assert "L.tileLayer(body.tileUrl" in js
+
+
+def test_sentinel1_not_fetched_on_init_unless_on():
+    init = HTML.split("function initMap", 1)[1].split("function readMapPrefs", 1)[0]
+    assert "loadSentinel1()" not in init
+    refresh = HTML.split("function refreshLiveOverlays", 1)[1].split("function addExtraAttrib", 1)[0]
+    assert "if (sentinelOn) loadSentinel1();" in refresh
+
+
+def test_sentinel1_reuses_covering_scene_and_clips_tiles():
+    js = HTML.split("async function loadSentinel1", 1)[1].split("function loadThermal", 1)[0]
+    assert "sentinelStillCovers" in HTML
+    assert "itemId" in js
+    assert "L.latLngBounds" in js
+    assert "sentinelBounds" in HTML

tests/test_upstream_cache.py

@@ -22,7 +22,11 @@ def test_firms_and_rss_caches_are_ttlcache():
 
 
 def test_ingest_fires_hits_http_once_within_ttl(monkeypatch):
+    from fire_sources import _csv_digest, _seen_ids
+
     firms_cache.clear()
+    _csv_digest.clear()
+    _seen_ids.clear()
     monkeypatch.setenv("FIRMS_MAP_KEY", "k" * 32)
     monkeypatch.setenv("FIRMS_DATASETS", "VIIRS_NOAA20_NRT")
     # fire_sources already imported FIRMS_DATASETS — patch the module attr
@@ -55,10 +59,10 @@ def test_ingest_fires_hits_http_once_within_ttl(monkeypatch):
     async def fake_publish(points):
         return len(points)
 
-    monkeypatch.setattr("fire_sources.publish_fire_batch", fake_publish)
+    monkeypatch.setattr("fire_sources.persist_hotspots", fake_publish)
 
     assert asyncio.run(ingest_fires()) == 5
-    assert asyncio.run(ingest_fires()) == 5
+    assert asyncio.run(ingest_fires()) == 0
     assert hits["n"] == 1
 
 

tests/test_vesselapi.py

@@ -0,0 +1,387 @@
+"""Unit tests for the VesselAPI poller (no network, no DB).
+
+Covers box span validation, position → marker mapping, glitch skipping,
+and the daily-quota gate (6th 2xx attempt skipped). ``poll_once`` is driven
+with an in-memory fake quota store + fake HTTP client; ``upsert_vessel``'s
+DB/WS side effects are monkeypatched to no-ops so markers can be asserted in
+``vessel_last_known``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timezone
+
+import pytest
+
+import vesselapi
+from vesselapi import (
+    BboxError,
+    budget_allows,
+    choose_box,
+    days_left_in_month,
+    parse_boxes,
+    parse_boxes_validated,
+    poll_once,
+    transform_vesselapi_payload,
+    transform_vesselapi_position,
+    validate_bbox_span,
+)
+from live_layers import vessel_last_known
+
+
+# ── Fake quota store (in-memory, injected) ────────────────────────────────
+
+class FakeStore:
+    def __init__(self, calls: int = 0, remaining: int | None = None):
+        self.calls = calls
+        self.remaining = remaining
+        self.bumps = 0
+
+    async def calls_today(self, day):
+        return self.calls
+
+    async def remaining_today(self, day):
+        return self.remaining
+
+    async def bump(self, day, remaining):
+        self.calls += 1
+        self.remaining = remaining
+        self.bumps += 1
+        return self.calls
+
+
+# ── Fake HTTP client ───────────────────────────────────────────────────────
+
+class FakeResp:
+    def __init__(self, status: int = 200, headers: dict | None = None, data: dict | None = None):
+        self.status_code = status
+        self.headers = headers or {}
+        self._data = data or {}
+
+    def json(self):
+        return self._data
+
+
+class FakeClient:
+    def __init__(self, *responses: FakeResp):
+        self.responses = list(responses)
+        self.calls: list[tuple] = []
+
+    async def get(self, url, params=None, headers=None):
+        self.calls.append((url, params, headers))
+        return self.responses.pop(0)
+
+    async def aclose(self):
+        pass
+
+
+def _sample_payload() -> dict:
+    return {
+        "vessels": [
+            {
+                "mmsi": 422050100,
+                "imo": 9321483,
+                "vessel_name": "HORMUZ STAR",
+                "latitude": 26.5,
+                "longitude": 56.3,
+                "cog": 88.0,
+                "sog": 12.4,
+                "heading": 90,
+                "nav_status": 0,
+                "timestamp": "2026-08-29T12:00:00Z",
+                "suspected_glitch": False,
+            },
+            {
+                "mmsi": 422050101,
+                "vessel_name": "GLITCHY",
+                "latitude": 26.6,
+                "longitude": 56.4,
+                "cog": 45.0,
+                "sog": 5.0,
+                "heading": 45,
+                "nav_status": 5,
+                "suspected_glitch": True,
+            },
+            {"mmsi": 422050102, "vessel_name": "NOFIX"},  # no coords → drop
+        ],
+        "nextToken": "deadbeef",
+    }
+
+
+# ── Box span / parsing ─────────────────────────────────────────────────────
+
+def test_validate_bbox_span_accepts_hormuz():
+    validate_bbox_span(25.5, 55.4, 27.3, 57.2)  # span 3.6 — no raise
+
+
+def test_validate_bbox_span_rejects_conus():
+    with pytest.raises(BboxError):
+        validate_bbox_span(24.0, -125.0, 50.0, -66.0)  # span 85
+
+
+def test_validate_bbox_span_rejects_marine_regions_gazetteer_box():
+    # 25.2732–27.3713 N, 55.1647–57.3419 E → span 4.28 > 4.0.
+    with pytest.raises(BboxError):
+        validate_bbox_span(25.2732, 55.1647, 27.3713, 57.3419)
+
+
+def test_validate_bbox_span_rejects_inverted_axes():
+    with pytest.raises(BboxError):
+        validate_bbox_span(27.0, 55.0, 25.0, 57.0)
+
+
+def test_parse_boxes_semicolon_and_skip_malformed():
+    boxes = parse_boxes("25.5,55.4,27.3,57.2; 10,20,11,21; garbage")
+    assert boxes == [(25.5, 55.4, 27.3, 57.2), (10.0, 20.0, 11.0, 21.0)]
+
+
+def test_parse_boxes_validated_skips_over_span():
+    # Second box is CONUS-sized → dropped, first kept.
+    valid = parse_boxes_validated("25.5,55.4,27.3,57.2;24,-125,50,-66")
+    assert valid == [(25.5, 55.4, 27.3, 57.2)]
+
+
+# ── Position → marker mapping ─────────────────────────────────────────────
+
+def test_transform_position_maps_shared_marker_contract():
+    m = transform_vesselapi_position({
+        "mmsi": 422050100, "imo": 9321483, "vessel_name": "HORMUZ STAR",
+        "latitude": 26.5, "longitude": 56.3, "heading": 90, "cog": 88.0,
+        "sog": 12.4, "nav_status": 0, "timestamp": "2026-08-29T12:00:00Z",
+        "suspected_glitch": False,
+    })
+    assert m is not None
+    assert m["id"] == "422050100"
+    assert m["lat"] == 26.5
+    assert m["lon"] == 56.3
+    assert m["label"] == "HORMUZ STAR"
+    assert m["heading"] == 90
+    assert m["speed"] == 12.4
+    assert m["extra"]["src"] == "vesselapi"
+    assert m["extra"]["mmsi"] == "422050100"
+    assert m["extra"]["imo"] == 9321483
+    assert m["extra"]["navstat"] == 0
+    assert m["extra"]["cog"] == 88.0
+    assert m["extra"]["sog"] == 12.4
+    assert m["extra"]["timestamp"] == "2026-08-29T12:00:00Z"
+
+
+def test_transform_position_heading_falls_back_to_cog():
+    m = transform_vesselapi_position({
+        "mmsi": 123456789, "vessel_name": "X", "latitude": 1.0, "longitude": 2.0,
+        "heading": None, "cog": 123.4, "sog": 5.0,
+    })
+    assert m["heading"] == 123.4
+
+
+def test_transform_position_skips_glitch():
+    assert transform_vesselapi_position({
+        "mmsi": 123456789, "latitude": 1.0, "longitude": 2.0,
+        "suspected_glitch": True,
+    }) is None
+
+
+def test_transform_position_skips_missing_coords():
+    assert transform_vesselapi_position({"mmsi": 123456789, "vessel_name": "NOFIX"}) is None
+
+
+def test_transform_payload_skips_glitch_and_nofix_rows():
+    rows = transform_vesselapi_payload(_sample_payload())
+    assert [r["id"] for r in rows] == ["422050100"]
+
+
+# ── Quota budget / scheduling ─────────────────────────────────────────────
+
+def test_days_left_in_month():
+    assert days_left_in_month(datetime(2026, 8, 29, tzinfo=timezone.utc)) == 3
+
+
+def test_budget_allows_local_daily_cap():
+    # 5 calls already made → 6th is blocked regardless of remaining.
+    assert budget_allows(5, remaining=1000, days_left=3, max_per_day=5) is False
+
+
+def test_budget_allows_monthly_floor():
+    # remaining 14 ≤ 5*3=15 → skip; 16 > 15 → allow.
+    assert budget_allows(2, remaining=14, days_left=3, max_per_day=5) is False
+    assert budget_allows(2, remaining=16, days_left=3, max_per_day=5) is True
+
+
+def test_budget_allows_unknown_remaining():
+    assert budget_allows(2, remaining=None, days_left=3, max_per_day=5) is True
+
+
+def test_choose_box_prefers_primary_when_budget_tight():
+    boxes = [(1, 1, 2, 2), (3, 3, 4, 4), (5, 5, 6, 6)]
+    # 4 calls made, 1 left → always box 0.
+    assert choose_box(boxes, 4, max_per_day=5) == 0
+
+
+def test_choose_box_round_robins_when_budget_covers_all():
+    boxes = [(1, 1, 2, 2), (3, 3, 4, 4), (5, 5, 6, 6)]
+    # 0 calls made, 5 left ≥ 3 boxes → round-robin.
+    assert choose_box(boxes, 0, max_per_day=5) == 0
+    assert choose_box(boxes, 1, max_per_day=5) == 1
+    assert choose_box(boxes, 2, max_per_day=5) == 2
+
+
+# ── poll_once integration (fake store + fake client) ─────────────────────
+
+def _patch_side_effects(monkeypatch):
+    async def _noop(*a, **k):
+        return None
+
+    monkeypatch.setattr("tracks.record_position", _noop)
+    monkeypatch.setattr("geofence.record_and_notify", _noop)
+    monkeypatch.setattr(vesselapi, "persist_vessel_snapshot", _noop)
+    monkeypatch.setattr(vesselapi, "purge_old_vessels", _noop)
+
+
+def test_poll_once_lands_markers_in_vessel_last_known(monkeypatch):
+    _patch_side_effects(monkeypatch)
+    vessel_last_known.clear()
+    client = FakeClient(
+        FakeResp(200, {"X-RateLimit-Remaining": "140"}, _sample_payload()),
+    )
+    monkeypatch.setattr(vesselapi, "_get_client", _make_get_client(client))
+    store = FakeStore()
+
+    ok = asyncio.run(poll_once(store, [(25.5, 55.4, 27.3, 57.2)], "test-key"))
+
+    assert ok is True
+    assert store.calls == 1
+    assert "422050100" in vessel_last_known
+    assert vessel_last_known["422050100"]["extra"]["src"] == "vesselapi"
+    assert "422050101" not in vessel_last_known  # glitch skipped
+    # One HTTP call, bounding-box params, no sat / no nextToken follow.
+    assert len(client.calls) == 1
+    _url, params, headers = client.calls[0]
+    assert params["filter.latBottom"] == "25.5"
+    assert params["filter.latTop"] == "27.3"
+    assert params["filter.lonLeft"] == "55.4"
+    assert params["filter.lonRight"] == "57.2"
+    assert params["pagination.limit"] == "50"
+    assert "sat" not in params
+    assert headers["Authorization"] == "Bearer test-key"
+
+
+def test_poll_once_sixth_2xx_is_skipped_with_zero_http(monkeypatch):
+    client = FakeClient()
+    monkeypatch.setattr(vesselapi, "_get_client", _make_get_client(client))
+    # 5 successful calls already today → 6th poll makes no HTTP request.
+    store = FakeStore(calls=5, remaining=1000)
+
+    ok = asyncio.run(poll_once(store, [(25.5, 55.4, 27.3, 57.2)], "test-key"))
+
+    assert ok is False
+    assert store.bumps == 0
+    assert client.calls == []
+
+
+def test_poll_once_4xx_not_counted_and_no_upsert(monkeypatch):
+    _patch_side_effects(monkeypatch)
+    vessel_last_known.clear()
+    client = FakeClient(FakeResp(400, {}, {"error": {}}))
+    monkeypatch.setattr(vesselapi, "_get_client", _make_get_client(client))
+    store = FakeStore()
+
+    ok = asyncio.run(poll_once(store, [(25.5, 55.4, 27.3, 57.2)], "test-key"))
+
+    assert ok is False
+    assert store.bumps == 0  # 4xx does not count against quota
+    assert vessel_last_known == {}
+
+
+def _make_get_client(client):
+    async def _get_client():
+        return client
+
+    return _get_client
+
+
+def test_eviction_keeps_vesselapi_rows(monkeypatch):
+    """AISStream crowding past _MAX_VESSELS must not reap Hormuz VesselAPI rows."""
+    from ws_manager import manager
+    from live_layers import _MAX_VESSELS, upsert_vessel
+
+    _patch_side_effects(monkeypatch)
+    monkeypatch.setattr("live_layers._MAX_VESSELS", 10)
+    vessel_last_known.clear()
+    manager._queues.clear()
+    manager._viewports.clear()
+
+    # One Hormuz VesselAPI row with the oldest seen_at — the first thing the
+    # old "evict oldest" logic would reap — plus enough AISStream rows to
+    # exceed the cap.
+    vessel_last_known["422050100"] = {
+        "id": "422050100", "lat": 26.5, "lon": 56.3, "label": "HORMUZ STAR",
+        "extra": {"src": "vesselapi"}, "seen_at": "2026-08-29T00:00:00+00:00",
+    }
+    for i in range(10):
+        vid = f"3{i:08d}"
+        vessel_last_known[vid] = {
+            "id": vid, "lat": 35.0 + i * 0.01, "lon": -79.0, "label": vid,
+            "extra": {"src": "aisstream"},
+            "seen_at": f"2026-08-29T0{i}:00:00+00:00",
+        }
+
+    # One more AISStream marker pushes past the cap and triggers eviction.
+    asyncio.run(upsert_vessel({"id": "399999999", "lat": 36.0, "lon": -78.0, "label": "NEW"}))
+
+    assert "422050100" in vessel_last_known  # VesselAPI row survives
+    assert vessel_last_known["422050100"]["extra"]["src"] == "vesselapi"
+    assert len(vessel_last_known) <= 10
+
+
+# ── Independent-provider idle behaviour ───────────────────────────────────
+# VesselAPI must never be gated on AISStream (or vice versa): a missing key on
+# one provider leaves the other running. ``_StopLoop`` is a BaseException so
+# the worker's ``except Exception`` handler can't swallow it — the first
+# ``asyncio.sleep`` aborts the loop after exactly one decision.
+
+class _StopLoop(BaseException):
+    pass
+
+
+async def _stop_sleep(*_a, **_k):
+    raise _StopLoop()
+
+
+async def _resolve(value: str):
+    return value
+
+
+def test_worker_idles_without_vesselapi_key_even_if_aisstream_set(monkeypatch):
+    # AISStream key present, VesselAPI key absent -> no poll, still idles.
+    monkeypatch.setenv("AISSTREAM_API_KEY", "unused-aisstream-key")
+    monkeypatch.setenv("VESSELAPI_API_KEY", "")
+    monkeypatch.setattr(vesselapi, "_resolve_key", lambda: _resolve(""))
+    monkeypatch.setattr(vesselapi.asyncio, "sleep", _stop_sleep)
+    poll_calls: list = []
+    async def _spy_poll(store, boxes, key):
+        poll_calls.append(key)
+        return True
+    monkeypatch.setattr(vesselapi, "poll_once", _spy_poll)
+
+    with pytest.raises(_StopLoop):
+        asyncio.run(vesselapi.run_vesselapi_worker(FakeStore()))
+
+    assert poll_calls == []
+
+
+def test_worker_polls_with_vesselapi_key_even_if_aisstream_unset(monkeypatch):
+    # AISStream key absent, VesselAPI key present -> still polls exactly once.
+    monkeypatch.setenv("AISSTREAM_API_KEY", "")
+    monkeypatch.setattr(vesselapi, "_resolve_key", lambda: _resolve("vesselapi-key"))
+    monkeypatch.setattr(vesselapi.asyncio, "sleep", _stop_sleep)
+    poll_calls: list = []
+    async def _spy_poll(store, boxes, key):
+        poll_calls.append(key)
+        return True
+    monkeypatch.setattr(vesselapi, "poll_once", _spy_poll)
+
+    with pytest.raises(_StopLoop):
+        asyncio.run(vesselapi.run_vesselapi_worker(FakeStore()))
+
+    assert poll_calls == ["vesselapi-key"]

tests/test_vessels_snapshot.py

@@ -0,0 +1,36 @@
+"""VesselAPI daily snapshot store — as-of DVR + UTC-day purge (no DB)."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from vesselapi import pick_poll_at, snapshot_as_of, utc_day_start
+
+
+def test_utc_day_start_floors_to_midnight_utc():
+    now = datetime(2026, 8, 29, 15, 30, 12, tzinfo=timezone.utc)
+    assert utc_day_start(now) == datetime(2026, 8, 29, 0, 0, tzinfo=timezone.utc)
+
+
+def test_pick_poll_at_returns_latest_snapshot_at_or_before_as_of():
+    t1 = datetime(2026, 8, 29, 0, 0, tzinfo=timezone.utc)
+    t2 = datetime(2026, 8, 29, 4, 48, tzinfo=timezone.utc)
+    t3 = datetime(2026, 8, 29, 9, 36, tzinfo=timezone.utc)
+    as_of = datetime(2026, 8, 29, 6, 0, tzinfo=timezone.utc)
+    assert pick_poll_at([t1, t2, t3], as_of) == t2
+    assert pick_poll_at([t1, t2, t3], t1) == t1
+    assert pick_poll_at([t1, t2, t3], datetime(2026, 8, 28, 23, tzinfo=timezone.utc)) is None
+
+
+def test_snapshot_as_of_returns_the_matching_poll_only():
+    t1 = datetime(2026, 8, 29, 0, 0, tzinfo=timezone.utc)
+    t2 = datetime(2026, 8, 29, 4, 48, tzinfo=timezone.utc)
+    rows = [
+        {"id": "1", "poll_at": t1, "lat": 26.5, "lon": 56.0},
+        {"id": "2", "poll_at": t1, "lat": 26.6, "lon": 56.1},
+        {"id": "1", "poll_at": t2, "lat": 26.7, "lon": 56.2},
+    ]
+    out = snapshot_as_of(rows, datetime(2026, 8, 29, 6, 0, tzinfo=timezone.utc))
+    assert {r["id"] for r in out} == {"1"}
+    assert out[0]["lat"] == 26.7
+    assert all(r["poll_at"] == t2 for r in out)