Merge origin/master into HUD polish

Keep terminator .lp-dot.night (this PR) and .lp-dot.conflicts from #39.

app/camera_config.py

@@ -27,6 +27,8 @@ _DEFAULT_SOURCE_URL = ",".join((
     "https://raw.githubusercontent.com/willytop8/Live-Environment-Streams/main/streams.geojson",
     # Official Caltrans CWWP2 JPEG + HLS CCTV (districts 1–12).
     *CALTRANS_CCTV_URLS,
+    # Oregon DOT TripCheck public CCTV JPEG inventory (Esri JSON).
+    "https://www.tripcheck.com/Scripts/map/data/cctvinventory.js",
     # Official MDOT MiDrive CCTV (JPEG stills, Michigan).
     MDOT_CAMERA_URL,
 ))

app/camera_scraper.py

@@ -25,6 +25,7 @@ import hashlib
 import ipaddress
 import json
 import logging
+import math
 import re
 import time
 from datetime import datetime, timezone
@@ -379,6 +380,51 @@ def parse_caltrans_json(text: str, source_name: str) -> list[dict]:
     return out
 
 
+# Oregon DOT TripCheck inventory bounding box (approx state extent).
+ODOT_BBOX = (41.9, 46.3, -124.6, -116.4)  # lat_min, lat_max, lon_min, lon_max
+
+
+def parse_odot_json(text: str, source_name: str) -> list[dict]:
+    """Parse Oregon DOT TripCheck cctvinventory Esri-style JSON.
+
+    Store the JPEG still as snapshot_url (map thumbs); never RTSP. Keep only
+    rows with finite coordinates inside Oregon and a usable filename.
+    """
+    try:
+        payload = json.loads(text)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    lat_min, lat_max, lon_min, lon_max = ODOT_BBOX
+    out: list[dict] = []
+    for feat in payload.get("features") or []:
+        attrs = (feat or {}).get("attributes") or {}
+        filename = (attrs.get("filename") or "").strip()
+        if not filename:
+            continue
+        try:
+            lat = float(attrs.get("latitude"))
+            lon = float(attrs.get("longitude"))
+        except (TypeError, ValueError):
+            continue
+        if not (math.isfinite(lat) and math.isfinite(lon)):
+            continue
+        if not (lat_min <= lat <= lat_max and lon_min <= lon <= lon_max):
+            continue
+        jpeg = f"https://tripcheck.com/RoadCams/cams/{filename}"
+        title = (attrs.get("title") or "").strip()
+        out.append({
+            "source_url": jpeg,
+            "snapshot_url": jpeg,
+            "discovery_source": "odot",
+            "location_lat": lat,
+            "location_lon": lon,
+            "location_name": title or None,
+            "vendor": "ODOT",
+            "device_type": "http",
+        })
+    return out
+
+
 # MDOT MiDrive field extractors (fields carry rendered HTML).
 _MDOT_LAT_RE = re.compile(r"lat=(-?\d+(?:\.\d+)?)", re.I)
 _MDOT_LON_RE = re.compile(r"lon=(-?\d+(?:\.\d+)?)", re.I)
@@ -563,6 +609,8 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder,
     body = resp.text
     if "cwwp2.dot.ca.gov" in src_url or "cctvStatus" in src_url:
         cams = parse_caltrans_json(body, name)
+    elif "cctvinventory" in src_url or "tripcheck.com" in src_url:
+        cams = parse_odot_json(body, name)
     elif "mdotjboss.state.mi.us" in src_url or "/MiDrive/camera/list" in src_url:
         cams = parse_mdot_json(body, name)
     elif ("getCameraDataByLoc" in src_url

app/live_layers.py

@@ -97,6 +97,11 @@ _MAX_VESSELS = 6000
 # Last ADS-B snapshot + WFIGS points for fire↔tanker correlation.
 aircraft_last_known: dict[str, dict] = {}
 fire_last_known: list[dict] = []
+# Last-known counts for the cheap GET /api/stats HUD counter. Updated by the
+# upstream fetchers so the stats endpoint never does its own network/SQL fan-out
+# for these layers; reads are O(1) in-process.
+train_count: int = 0
+nws_alert_count: int = 0
 
 
 def overlay_catalog() -> dict:
@@ -972,6 +977,8 @@ async def fetch_trains(bbox: str | None, limit: int = DEFAULT_LIMIT) -> list[dic
         return transform_amtraker(await _get_json(AMTRAKER_TRAINS))
 
     rows = await _ttl_get("amtraker:trains", 20.0, _load)
+    global train_count
+    train_count = len(rows)
     if bbox:
         minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
         return filter_points_bbox(rows, minlon, minlat, maxlon, maxlat, limit)
@@ -1125,6 +1132,8 @@ async def fetch_weather_alerts(area: str | None, bbox: str | None) -> dict:
             logger.warning("NWS alerts fetch failed: %s", exc)
             nws_ok = False
             nws_fc = {"features": []}
+        global nws_alert_count
+        nws_alert_count = len(nws_fc.get("features") or [])
         sbw_fc = await _ttl_get("iem:sbw", 45.0, _load_iem)
         features = []
         for feat in nws_fc.get("features") or []:

app/main.py

@@ -15,6 +15,7 @@ import asyncio
 import json
 import logging
 import re
+import time
 from contextlib import asynccontextmanager
 from datetime import datetime, timedelta, timezone
 from decimal import Decimal
@@ -273,6 +274,67 @@ def overlay_json(data, max_age: int) -> JSONResponse:
     return resp
 
 
+# ── HUD counters ─────────────────────────────────────────────────────────
+
+# Cheap ~100 B–2 KB counts for the layer rail. Cached in-process so the HUD
+# can poll every second without re-hitting SQL or upstream feeds.
+_STATS_TTL = 20.0
+_stats_cache: dict[str, tuple[float, dict]] = {}
+
+
+async def _stats_counts() -> dict:
+    """Fan out to in-memory last-known / cheap SQL counts. Never raises."""
+    from live_layers import (
+        aircraft_last_known, vessel_last_known, train_count, nws_alert_count,
+    )
+
+    counts: dict[str, int | str] = {
+        "aircraft": len(aircraft_last_known),
+        "vessels": len(vessel_last_known),
+        "trains": train_count,
+        "cameras": 0,
+        "fires": 0,
+        "quakes": 0,
+        "alerts": nws_alert_count,
+    }
+
+    # SQL counts are best-effort: a down DB or missing table must not 500 the
+    # rail — the frontend still renders with zeros.
+    try:
+        from camera_models import cameras as cam_table
+        async with async_session() as session:
+            counts["cameras"] = int(
+                (await session.execute(select(func.count()).select_from(cam_table))).scalar() or 0
+            )
+            counts["fires"] = int(
+                (await session.execute(select(func.count()).select_from(fires))).scalar() or 0
+            )
+            counts["quakes"] = int(
+                (await session.execute(
+                    select(func.count()).select_from(events).where(
+                        events.c.source_type == "earthquake"
+                    )
+                )).scalar() or 0
+            )
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("stats_db_failed", error=str(exc))
+
+    counts["timestamp"] = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+    return counts
+
+
+@app.get("/api/stats")
+async def api_stats():
+    """Cheap HUD counters (counts only — no GeoJSON). Cached ~20 s."""
+    now = time.monotonic()
+    cached = _stats_cache.get("stats")
+    if cached and now - cached[0] < _STATS_TTL:
+        return cached[1]
+    payload = await _stats_counts()
+    _stats_cache["stats"] = (now, payload)
+    return overlay_json(payload, 15)
+
+
 # ── Feed Sources ──────────────────────────────────────────────────────────
 
 @app.get("/api/sources", response_model=list[FeedSourceOut])

app/static/index.html

@@ -356,6 +356,7 @@
             background: linear-gradient(90deg, #0b1c33 50%, #ffb454 50%);
             box-shadow: 0 0 7px #64748b;
         }
+        .lp-dot.conflicts { background: #ff2a6d; box-shadow: 0 0 7px #ff2a6d; }
         .lp-count { font-family: 'Share Tech Mono', monospace; font-size: 0.7rem; color: var(--cyan); }
         .lp-sub { display: flex; justify-content: space-between; align-items: center; gap: 0.5rem; }
         .lp-opacity { display: flex; align-items: center; gap: 0.4rem; font-size: 0.62rem; color: var(--muted); text-transform: uppercase; letter-spacing: 0.05em; }
@@ -444,6 +445,7 @@
         .blip-pop { min-width: 200px; max-width: 260px; }
         .blip-pop .blip-src { font-family: 'Share Tech Mono', monospace; font-size: 0.62rem; color: var(--cyan); text-transform: uppercase; letter-spacing: 0.08em; }
         .blip-pop .blip-time { font-size: 0.7rem; color: var(--muted); font-family: 'Share Tech Mono', monospace; margin: 0.2rem 0 0.3rem; }
+        .blip-pop .blip-desc { font-size: 0.72rem; color: var(--text); line-height: 1.35; margin-top: 0.15rem; }
 
         /* ── Sub-views (News / Events / Alerts / ... ) ── */
         .subview {
@@ -983,6 +985,13 @@
                                 <span class="lp-count" id="lp-storms-count">0</span>
                             </div>
                         </div>
+                        <div class="lp-layer" id="conflicts-layer">
+                            <div class="lp-row">
+                                <label class="lp-name"><input type="checkbox" id="lp-conflicts-on" onchange="toggleConflicts()"> <span class="lp-dot conflicts"></span> Conflicts</label>
+                                <span class="lp-count" id="lp-conflicts-count">0</span>
+                            </div>
+                            <div class="lp-note">Curated theatres · color by severity · news counts, not LiveUAMap.</div>
+                        </div>
 
                         <div class="lp-legend">
                             <h4>Fire heat intensity</h4>
@@ -2140,7 +2149,8 @@ let acGroup = null, acOn = true, acMilOn = false, acMilSupported = false;
 let trainsGroup = null, trainsOn = true;
 let vesselsGroup = null, vesselsOn = false;
 let stormsGroup = null, stormsOn = true;
-let overlayReq = {ac:0, trains:0, vessels:0, alerts:0, perim:0, incidents:0, storms:0, sar:0};
+let conflictsGroup = null, conflictsOn = false, conflictsCache = null;
+let overlayReq = {ac:0, trains:0, vessels:0, alerts:0, perim:0, incidents:0, storms:0, sar:0, conflicts:0};
 let moveDebounce = null;
 let overlayAbort = null;
 let lastCell = '';
@@ -2408,8 +2418,10 @@ async function initMap() {
         trainsOn = document.getElementById('lp-trains-on').checked;
         vesselsOn = document.getElementById('lp-vessels-on').checked;
         stormsOn = document.getElementById('lp-storms-on').checked;
+        conflictsOn = document.getElementById('lp-conflicts-on').checked;
         connectLiveWs();
         loadChokepoints();
+        probeConflicts();
         requestAnimationFrame(() => {
             if (firesOn) loadFires();
             setTimeout(() => {
@@ -3737,6 +3749,96 @@ async function loadStorms() {
     }
 }
 
+function conflictSeverityColor(sev) {
+    const s = String(sev || '').toLowerCase();
+    if (s === 'war') return '#ff2a6d';
+    if (s === 'high') return '#fb923c';
+    if (s === 'elevated') return '#facc15';
+    return '#35e0ff';
+}
+function hideConflictsToggle() {
+    const row = document.getElementById('conflicts-layer');
+    if (row) row.hidden = true;
+    const cb = document.getElementById('lp-conflicts-on');
+    if (cb) cb.checked = false;
+    conflictsOn = false;
+    conflictsCache = null;
+    conflictsGroup = dropLayer(conflictsGroup);
+}
+function paintConflicts() {
+    if (!map || !conflictsOn) return;
+    const zones = (conflictsCache && Array.isArray(conflictsCache.zones)) ? conflictsCache.zones : [];
+    conflictsGroup = dropLayer(conflictsGroup);
+    const markers = [];
+    for (const z of zones) {
+        if (z.lat == null || z.lon == null) continue;
+        const lat = Number(z.lat), lon = Number(z.lon);
+        if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
+        const col = conflictSeverityColor(z.severity);
+        const sev = String(z.severity || '').toLowerCase();
+        const radius = sev === 'war' ? 10 : sev === 'high' ? 8 : 7;
+        const m = L.circleMarker([lat, lon], {
+            radius,
+            color: col,
+            fillColor: col,
+            fillOpacity: 0.28,
+            weight: 2,
+            className: 'conflict-zone',
+        });
+        const n = Number(z.eventCount);
+        const count = Number.isFinite(n) ? n : 0;
+        m.bindPopup(
+            `<div class="blip-pop">` +
+            `<div class="blip-src">${esc(z.severity || '')} · ${esc(count)} events</div>` +
+            `<b>${esc(z.label || '')}</b>` +
+            `<div class="blip-desc">${esc(z.description || '')}</div>` +
+            `</div>`
+        );
+        markers.push(m);
+    }
+    conflictsGroup = L.layerGroup(markers).addTo(map);
+    const countEl = document.getElementById('lp-conflicts-count');
+    if (countEl) countEl.textContent = markers.length.toLocaleString();
+    addExtraAttrib('Curated OSINT conflict catalog');
+}
+async function probeConflicts() {
+    await loadConflicts(false);
+}
+async function toggleConflicts() {
+    const cb = document.getElementById('lp-conflicts-on');
+    conflictsOn = !!(cb && cb.checked);
+    if (conflictsOn) await loadConflicts(true);
+    else conflictsGroup = dropLayer(conflictsGroup);
+}
+async function loadConflicts(paint) {
+    const shouldPaint = paint === true || conflictsOn;
+    const req = ++overlayReq.conflicts;
+    const countEl = document.getElementById('lp-conflicts-count');
+    try {
+        if (!conflictsCache) {
+            // Own fetch — catalog is viewport-independent; overlayAbort on
+            // moveend must not cancel this (and we never refetch on pan).
+            const r = await fetch(`${API}/api/conflicts`);
+            if (req !== overlayReq.conflicts) return;
+            if (r.status === 404) {
+                hideConflictsToggle();
+                return;
+            }
+            if (!r.ok) throw new Error('conflicts ' + r.status);
+            const body = await r.json();
+            if (req !== overlayReq.conflicts) return;
+            conflictsCache = body && typeof body === 'object' ? body : { zones: [] };
+        }
+        const zones = Array.isArray(conflictsCache.zones) ? conflictsCache.zones : [];
+        if (countEl) countEl.textContent = zones.length.toLocaleString();
+        if (shouldPaint) paintConflicts();
+    } catch (e) {
+        if (req !== overlayReq.conflicts) return;
+        console.error('Conflicts load failed', e);
+        if (countEl) countEl.textContent = 'err';
+    }
+}
+
 /* ═══════════════ INITIAL LOAD ═══════════════ */
 initNav();
 initHudKeys();

tests/test_api_stats.py

@@ -0,0 +1,87 @@
+"""GET /api/stats HUD counter contract (counts only, small, never 500)."""
+
+from __future__ import annotations
+
+import asyncio
+import re
+from datetime import timezone
+
+import httpx
+
+from main import app, _stats_counts
+
+BASE = "http://test"
+
+EXPECTED_KEYS = ("aircraft", "vessels", "trains", "cameras",
+                 "fires", "quakes", "alerts", "timestamp")
+
+
+async def _get(path: str) -> httpx.Response:
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
+        return await client.get(path)
+
+
+def test_stats_200_all_keys_present():
+    resp = asyncio.run(_get("/api/stats"))
+    assert resp.status_code == 200
+    body = resp.json()
+    for key in EXPECTED_KEYS:
+        assert key in body, f"missing key {key}"
+    assert "max-age" in (resp.headers.get("cache-control") or "").lower()
+
+
+def test_stats_counters_are_ints():
+    body = asyncio.run(_get("/api/stats")).json()
+    for key in EXPECTED_KEYS:
+        if key == "timestamp":
+            continue
+        assert isinstance(body[key], int), f"{key} is not an int: {body[key]!r}"
+
+
+def test_stats_timestamp_is_iso8601_z():
+    body = asyncio.run(_get("/api/stats")).json()
+    ts = body["timestamp"]
+    # ISO8601 with a trailing Z (we normalize +00:00 -> Z).
+    assert isinstance(ts, str) and ts.endswith("Z")
+    assert re.match(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", ts)
+
+
+def test_stats_payload_is_tiny():
+    resp = asyncio.run(_get("/api/stats"))
+    assert len(resp.content) < 2048, "stats payload must be counts-only, not GeoJSON"
+
+
+def test_stats_counts_reflect_last_known(monkeypatch):
+    """aircraft/vessels/trains/alerts come from in-memory last-known state."""
+    import live_layers
+
+    monkeypatch.setattr(live_layers, "aircraft_last_known", {str(i): {} for i in range(7)})
+    monkeypatch.setattr(live_layers, "vessel_last_known", {str(i): {} for i in range(3)})
+    monkeypatch.setattr(live_layers, "train_count", 11)
+    monkeypatch.setattr(live_layers, "nws_alert_count", 5)
+
+    # _stats_counts imports the dicts/counters inside the function from live_layers,
+    # so monkeypatching the module attributes is what it observes.
+    from main import _stats_counts as fn
+
+    body = asyncio.run(fn())
+    assert body["aircraft"] == 7
+    assert body["vessels"] == 3
+    assert body["trains"] == 11
+    assert body["alerts"] == 5
+
+
+def test_stats_db_failure_degrades_to_zero(monkeypatch):
+    """A down DB yields zeros for the SQL-backed counters, never a 500."""
+    # Make the session factory raise synchronously so the try/except in
+    # _stats_counts degrades the SQL counters to zero (no dangling coroutine).
+    def _raise(*args, **kwargs):
+        raise RuntimeError("db down")
+
+    monkeypatch.setattr("main.async_session", _raise)
+    body = asyncio.run(_stats_counts())
+    assert body["cameras"] == 0
+    assert body["fires"] == 0
+    assert body["quakes"] == 0
+    assert isinstance(body["timestamp"], str)

tests/test_conflicts_frontend.py

@@ -0,0 +1,66 @@
+"""Conflicts Leaflet overlay: default-off toggle, catalog fetch, no jitter."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+HTML = (ROOT / "app/static/index.html").read_text()
+
+
+def _fn(name: str, until: str | None = None) -> str:
+    chunk = HTML.split(f"function {name}", 1)[1]
+    if until:
+        chunk = chunk.split(until, 1)[0]
+    return chunk
+
+
+def test_conflicts_toggle_default_off():
+    assert 'id="lp-conflicts-on"' in HTML
+    assert 'id="conflicts-layer"' in HTML
+    assert "> Conflicts<" in HTML or "> Conflicts</" in HTML
+    on = HTML.split('id="lp-conflicts-on"', 1)[1].split(">", 1)[0]
+    assert "checked" not in on
+
+
+def test_conflicts_fetches_catalog_not_liveuamap():
+    js = _fn("loadConflicts", "/* ═══════════════ INITIAL LOAD")
+    assert "/api/conflicts" in js
+    assert "liveuamap.com" not in HTML.lower()
+    assert "Math.random" not in js
+    assert "jitter" not in js.lower()
+
+
+def test_conflicts_not_refetched_on_moveend():
+    refresh = HTML.split("function refreshLiveOverlays", 1)[1].split(
+        "function addExtraAttrib", 1
+    )[0]
+    assert "loadConflicts" not in refresh
+    assert "probeConflicts" not in refresh
+    init = HTML.split("function initMap", 1)[1].split("function readMapPrefs", 1)[0]
+    assert "probeConflicts()" in init
+    assert "loadConflicts(true)" not in init
+    assert "paintConflicts()" not in init
+
+
+def test_conflicts_hides_toggle_on_404():
+    js = _fn("loadConflicts", "/* ═══════════════ INITIAL LOAD")
+    assert "r.status === 404" in js
+    assert "hideConflictsToggle()" in js
+    hide = _fn("hideConflictsToggle", "function paintConflicts")
+    assert "row.hidden = true" in hide
+    assert "lp-conflicts-on" in hide
+
+
+def test_conflicts_popup_and_severity_colors():
+    paint = _fn("paintConflicts", "async function probeConflicts")
+    assert "z.label" in paint
+    assert "z.description" in paint
+    assert "eventCount" in paint
+    assert "L.circleMarker" in paint
+    assert "z.lat == null || z.lon == null" in paint
+    assert "Number.isFinite(lat)" in paint
+    color = _fn("conflictSeverityColor", "function hideConflictsToggle")
+    assert "war" in color and "#ff2a6d" in color
+    assert "high" in color and "#fb923c" in color
+    assert "elevated" in color and "#facc15" in color

tests/test_live_layers.py

@@ -27,7 +27,7 @@ from live_layers import (
     _wfigs_params,
 )
 
-from camera_scraper import parse_caltrans_json, parse_mdot_json
+from camera_scraper import parse_caltrans_json, parse_odot_json, parse_mdot_json
 
 
 def test_parse_bbox_and_radius_clamps_to_150_nm():
@@ -259,6 +259,50 @@ def test_parse_caltrans_skips_oos_and_maps_jpeg_hls():
     assert "rtsp://" not in cam["snapshot_url"].lower()
 
 
+def test_parse_odot_tripcheck_keeps_valid_skips_missing_and_oob():
+    payload = """
+    {"features":[
+      {"attributes":{
+        "cameraId":277,"filename":"AstoriaUS101_pid392.jpg",
+        "latitude":46.18785,"longitude":-123.85347,
+        "route":"US101 ","title":"US101 at Astoria"
+      }},
+      {"attributes":{
+        "cameraId":200,"filename":"","latitude":45.0,"longitude":-122.0,
+        "route":"I-5","title":"missing filename"
+      }},
+      {"attributes":{
+        "cameraId":300,"filename":"nocal_pid1.jpg",
+        "latitude":40.0,"longitude":-122.0,
+        "route":"US97","title":"out of bbox"
+      }},
+      {"attributes":{
+        "cameraId":400,"filename":"badcoord_pid2.jpg",
+        "latitude":null,"longitude":-122.0,
+        "route":"OR22","title":"null coord"
+      }}
+    ]}
+    """
+    cams = parse_odot_json(payload, "www.tripcheck.com")
+    assert len(cams) == 1
+    cam = cams[0]
+    assert cam["discovery_source"] == "odot"
+    assert cam["snapshot_url"] == (
+        "https://tripcheck.com/RoadCams/cams/AstoriaUS101_pid392.jpg")
+    assert cam["source_url"] == cam["snapshot_url"]
+    assert cam["location_lat"] == 46.18785
+    assert cam["location_lon"] == -123.85347
+    assert "US101 at Astoria" in cam["location_name"]
+    assert cam["vendor"] == "ODOT"
+    assert cam["device_type"] == "http"
+    assert "rtsp://" not in cam["snapshot_url"].lower()
+
+
+def test_parse_odot_tripcheck_handles_malformed():
+    assert parse_odot_json("not json", "www.tripcheck.com") == []
+    assert parse_odot_json('{"features":null}', "www.tripcheck.com") == []
+
+
 def test_parse_mdot_extracts_html_fields_and_bbox_filters():
     rows = [
         # In-bbox, full fields.