diff --git a/app/camera_config.py b/app/camera_config.py index c18f4f0..2ccff69 100644 --- a/app/camera_config.py +++ b/app/camera_config.py @@ -27,6 +27,8 @@ _DEFAULT_SOURCE_URL = ",".join(( "https://raw.githubusercontent.com/willytop8/Live-Environment-Streams/main/streams.geojson", # Official Caltrans CWWP2 JPEG + HLS CCTV (districts 1–12). *CALTRANS_CCTV_URLS, + # Oregon DOT TripCheck public CCTV JPEG inventory (Esri JSON). + "https://www.tripcheck.com/Scripts/map/data/cctvinventory.js", # Official MDOT MiDrive CCTV (JPEG stills, Michigan). MDOT_CAMERA_URL, )) diff --git a/app/camera_scraper.py b/app/camera_scraper.py index f7fdc60..10dde21 100644 --- a/app/camera_scraper.py +++ b/app/camera_scraper.py @@ -25,6 +25,7 @@ import hashlib import ipaddress import json import logging +import math import re import time from datetime import datetime, timezone @@ -466,6 +467,51 @@ def parse_udot_ibi_page(text: str, source_name: str = "udot") -> list[dict]: return out +# Oregon DOT TripCheck inventory bounding box (approx state extent). +ODOT_BBOX = (41.9, 46.3, -124.6, -116.4) # lat_min, lat_max, lon_min, lon_max + + +def parse_odot_json(text: str, source_name: str) -> list[dict]: + """Parse Oregon DOT TripCheck cctvinventory Esri-style JSON. + + Store the JPEG still as snapshot_url (map thumbs); never RTSP. Keep only + rows with finite coordinates inside Oregon and a usable filename. + """ + try: + payload = json.loads(text) + except (json.JSONDecodeError, ValueError): + return [] + lat_min, lat_max, lon_min, lon_max = ODOT_BBOX + out: list[dict] = [] + for feat in payload.get("features") or []: + attrs = (feat or {}).get("attributes") or {} + filename = (attrs.get("filename") or "").strip() + if not filename: + continue + try: + lat = float(attrs.get("latitude")) + lon = float(attrs.get("longitude")) + except (TypeError, ValueError): + continue + if not (math.isfinite(lat) and math.isfinite(lon)): + continue + if not (lat_min <= lat <= lat_max and lon_min <= lon <= lon_max): + continue + jpeg = f"https://tripcheck.com/RoadCams/cams/{filename}" + title = (attrs.get("title") or "").strip() + out.append({ + "source_url": jpeg, + "snapshot_url": jpeg, + "discovery_source": "odot", + "location_lat": lat, + "location_lon": lon, + "location_name": title or None, + "vendor": "ODOT", + "device_type": "http", + }) + return out + + # MDOT MiDrive field extractors (fields carry rendered HTML). _MDOT_LAT_RE = re.compile(r"lat=(-?\d+(?:\.\d+)?)", re.I) _MDOT_LON_RE = re.compile(r"lon=(-?\d+(?:\.\d+)?)", re.I) @@ -650,6 +696,8 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder, body = resp.text if "cwwp2.dot.ca.gov" in src_url or "cctvStatus" in src_url: cams = parse_caltrans_json(body, name) + elif "cctvinventory" in src_url or "tripcheck.com" in src_url: + cams = parse_odot_json(body, name) elif "mdotjboss.state.mi.us" in src_url or "/MiDrive/camera/list" in src_url: cams = parse_mdot_json(body, name) elif ("getCameraDataByLoc" in src_url diff --git a/app/live_layers.py b/app/live_layers.py index 1d8a0f1..1a0e819 100644 --- a/app/live_layers.py +++ b/app/live_layers.py @@ -97,6 +97,11 @@ _MAX_VESSELS = 6000 # Last ADS-B snapshot + WFIGS points for fire↔tanker correlation. aircraft_last_known: dict[str, dict] = {} fire_last_known: list[dict] = [] +# Last-known counts for the cheap GET /api/stats HUD counter. Updated by the +# upstream fetchers so the stats endpoint never does its own network/SQL fan-out +# for these layers; reads are O(1) in-process. +train_count: int = 0 +nws_alert_count: int = 0 def overlay_catalog() -> dict: @@ -972,6 +977,8 @@ async def fetch_trains(bbox: str | None, limit: int = DEFAULT_LIMIT) -> list[dic return transform_amtraker(await _get_json(AMTRAKER_TRAINS)) rows = await _ttl_get("amtraker:trains", 20.0, _load) + global train_count + train_count = len(rows) if bbox: minlon, minlat, maxlon, maxlat = parse_bbox(bbox) return filter_points_bbox(rows, minlon, minlat, maxlon, maxlat, limit) @@ -1125,6 +1132,8 @@ async def fetch_weather_alerts(area: str | None, bbox: str | None) -> dict: logger.warning("NWS alerts fetch failed: %s", exc) nws_ok = False nws_fc = {"features": []} + global nws_alert_count + nws_alert_count = len(nws_fc.get("features") or []) sbw_fc = await _ttl_get("iem:sbw", 45.0, _load_iem) features = [] for feat in nws_fc.get("features") or []: diff --git a/app/main.py b/app/main.py index f659811..1621a07 100644 --- a/app/main.py +++ b/app/main.py @@ -15,6 +15,7 @@ import asyncio import json import logging import re +import time from contextlib import asynccontextmanager from datetime import datetime, timedelta, timezone from decimal import Decimal @@ -273,6 +274,67 @@ def overlay_json(data, max_age: int) -> JSONResponse: return resp +# ── HUD counters ───────────────────────────────────────────────────────── + +# Cheap ~100 B–2 KB counts for the layer rail. Cached in-process so the HUD +# can poll every second without re-hitting SQL or upstream feeds. +_STATS_TTL = 20.0 +_stats_cache: dict[str, tuple[float, dict]] = {} + + +async def _stats_counts() -> dict: + """Fan out to in-memory last-known / cheap SQL counts. Never raises.""" + from live_layers import ( + aircraft_last_known, vessel_last_known, train_count, nws_alert_count, + ) + + counts: dict[str, int | str] = { + "aircraft": len(aircraft_last_known), + "vessels": len(vessel_last_known), + "trains": train_count, + "cameras": 0, + "fires": 0, + "quakes": 0, + "alerts": nws_alert_count, + } + + # SQL counts are best-effort: a down DB or missing table must not 500 the + # rail — the frontend still renders with zeros. + try: + from camera_models import cameras as cam_table + async with async_session() as session: + counts["cameras"] = int( + (await session.execute(select(func.count()).select_from(cam_table))).scalar() or 0 + ) + counts["fires"] = int( + (await session.execute(select(func.count()).select_from(fires))).scalar() or 0 + ) + counts["quakes"] = int( + (await session.execute( + select(func.count()).select_from(events).where( + events.c.source_type == "earthquake" + ) + )).scalar() or 0 + ) + except Exception as exc: # noqa: BLE001 + logger.warning("stats_db_failed", error=str(exc)) + + counts["timestamp"] = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + return counts + + +@app.get("/api/stats") +async def api_stats(): + """Cheap HUD counters (counts only — no GeoJSON). Cached ~20 s.""" + now = time.monotonic() + cached = _stats_cache.get("stats") + if cached and now - cached[0] < _STATS_TTL: + return cached[1] + payload = await _stats_counts() + _stats_cache["stats"] = (now, payload) + return overlay_json(payload, 15) + + # ── Feed Sources ────────────────────────────────────────────────────────── @app.get("/api/sources", response_model=list[FeedSourceOut]) diff --git a/app/static/index.html b/app/static/index.html index 3900d4f..86cd36e 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -352,6 +352,7 @@ .lp-dot.wfigs { background: #ef4444; box-shadow: 0 0 7px #ef4444; } .lp-dot.trains { background: #c084fc; box-shadow: 0 0 7px #c084fc; } .lp-dot.storms { background: #f472b6; box-shadow: 0 0 7px #f472b6; } + .lp-dot.conflicts { background: #ff2a6d; box-shadow: 0 0 7px #ff2a6d; } .lp-count { font-family: 'Share Tech Mono', monospace; font-size: 0.7rem; color: var(--cyan); } .lp-sub { display: flex; justify-content: space-between; align-items: center; gap: 0.5rem; } .lp-opacity { display: flex; align-items: center; gap: 0.4rem; font-size: 0.62rem; color: var(--muted); text-transform: uppercase; letter-spacing: 0.05em; } @@ -411,6 +412,7 @@ .blip-pop { min-width: 200px; max-width: 260px; } .blip-pop .blip-src { font-family: 'Share Tech Mono', monospace; font-size: 0.62rem; color: var(--cyan); text-transform: uppercase; letter-spacing: 0.08em; } .blip-pop .blip-time { font-size: 0.7rem; color: var(--muted); font-family: 'Share Tech Mono', monospace; margin: 0.2rem 0 0.3rem; } + .blip-pop .blip-desc { font-size: 0.72rem; color: var(--text); line-height: 1.35; margin-top: 0.15rem; } /* ── Sub-views (News / Events / Alerts / ... ) ── */ .subview { @@ -941,6 +943,13 @@ 0 +
+
+ + 0 +
+
Curated theatres · color by severity · news counts, not LiveUAMap.
+

Fire heat intensity

@@ -1942,7 +1951,8 @@ let acGroup = null, acOn = true, acMilOn = false, acMilSupported = false; let trainsGroup = null, trainsOn = true; let vesselsGroup = null, vesselsOn = false; let stormsGroup = null, stormsOn = true; -let overlayReq = {ac:0, trains:0, vessels:0, alerts:0, perim:0, incidents:0, storms:0, sar:0}; +let conflictsGroup = null, conflictsOn = false, conflictsCache = null; +let overlayReq = {ac:0, trains:0, vessels:0, alerts:0, perim:0, incidents:0, storms:0, sar:0, conflicts:0}; let moveDebounce = null; let overlayAbort = null; let lastCell = ''; @@ -2210,8 +2220,10 @@ async function initMap() { trainsOn = document.getElementById('lp-trains-on').checked; vesselsOn = document.getElementById('lp-vessels-on').checked; stormsOn = document.getElementById('lp-storms-on').checked; + conflictsOn = document.getElementById('lp-conflicts-on').checked; connectLiveWs(); loadChokepoints(); + probeConflicts(); requestAnimationFrame(() => { if (firesOn) loadFires(); setTimeout(() => { @@ -3484,6 +3496,96 @@ async function loadStorms() { } } +function conflictSeverityColor(sev) { + const s = String(sev || '').toLowerCase(); + if (s === 'war') return '#ff2a6d'; + if (s === 'high') return '#fb923c'; + if (s === 'elevated') return '#facc15'; + return '#35e0ff'; +} +function hideConflictsToggle() { + const row = document.getElementById('conflicts-layer'); + if (row) row.hidden = true; + const cb = document.getElementById('lp-conflicts-on'); + if (cb) cb.checked = false; + conflictsOn = false; + conflictsCache = null; + conflictsGroup = dropLayer(conflictsGroup); +} +function paintConflicts() { + if (!map || !conflictsOn) return; + const zones = (conflictsCache && Array.isArray(conflictsCache.zones)) ? conflictsCache.zones : []; + conflictsGroup = dropLayer(conflictsGroup); + const markers = []; + for (const z of zones) { + if (z.lat == null || z.lon == null) continue; + const lat = Number(z.lat), lon = Number(z.lon); + if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; + const col = conflictSeverityColor(z.severity); + const sev = String(z.severity || '').toLowerCase(); + const radius = sev === 'war' ? 10 : sev === 'high' ? 8 : 7; + const m = L.circleMarker([lat, lon], { + radius, + color: col, + fillColor: col, + fillOpacity: 0.28, + weight: 2, + className: 'conflict-zone', + }); + const n = Number(z.eventCount); + const count = Number.isFinite(n) ? n : 0; + m.bindPopup( + `
` + + `
${esc(z.severity || '')} · ${esc(count)} events
` + + `${esc(z.label || '')}` + + `
${esc(z.description || '')}
` + + `
` + ); + markers.push(m); + } + conflictsGroup = L.layerGroup(markers).addTo(map); + const countEl = document.getElementById('lp-conflicts-count'); + if (countEl) countEl.textContent = markers.length.toLocaleString(); + addExtraAttrib('Curated OSINT conflict catalog'); +} +async function probeConflicts() { + await loadConflicts(false); +} +async function toggleConflicts() { + const cb = document.getElementById('lp-conflicts-on'); + conflictsOn = !!(cb && cb.checked); + if (conflictsOn) await loadConflicts(true); + else conflictsGroup = dropLayer(conflictsGroup); +} +async function loadConflicts(paint) { + const shouldPaint = paint === true || conflictsOn; + const req = ++overlayReq.conflicts; + const countEl = document.getElementById('lp-conflicts-count'); + try { + if (!conflictsCache) { + // Own fetch — catalog is viewport-independent; overlayAbort on + // moveend must not cancel this (and we never refetch on pan). + const r = await fetch(`${API}/api/conflicts`); + if (req !== overlayReq.conflicts) return; + if (r.status === 404) { + hideConflictsToggle(); + return; + } + if (!r.ok) throw new Error('conflicts ' + r.status); + const body = await r.json(); + if (req !== overlayReq.conflicts) return; + conflictsCache = body && typeof body === 'object' ? body : { zones: [] }; + } + const zones = Array.isArray(conflictsCache.zones) ? conflictsCache.zones : []; + if (countEl) countEl.textContent = zones.length.toLocaleString(); + if (shouldPaint) paintConflicts(); + } catch (e) { + if (req !== overlayReq.conflicts) return; + console.error('Conflicts load failed', e); + if (countEl) countEl.textContent = 'err'; + } +} + /* ═══════════════ INITIAL LOAD ═══════════════ */ initNav(); initSettings(); diff --git a/tests/test_api_stats.py b/tests/test_api_stats.py new file mode 100644 index 0000000..870965d --- /dev/null +++ b/tests/test_api_stats.py @@ -0,0 +1,87 @@ +"""GET /api/stats HUD counter contract (counts only, small, never 500).""" + +from __future__ import annotations + +import asyncio +import re +from datetime import timezone + +import httpx + +from main import app, _stats_counts + +BASE = "http://test" + +EXPECTED_KEYS = ("aircraft", "vessels", "trains", "cameras", + "fires", "quakes", "alerts", "timestamp") + + +async def _get(path: str) -> httpx.Response: + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url=BASE) as client: + return await client.get(path) + + +def test_stats_200_all_keys_present(): + resp = asyncio.run(_get("/api/stats")) + assert resp.status_code == 200 + body = resp.json() + for key in EXPECTED_KEYS: + assert key in body, f"missing key {key}" + assert "max-age" in (resp.headers.get("cache-control") or "").lower() + + +def test_stats_counters_are_ints(): + body = asyncio.run(_get("/api/stats")).json() + for key in EXPECTED_KEYS: + if key == "timestamp": + continue + assert isinstance(body[key], int), f"{key} is not an int: {body[key]!r}" + + +def test_stats_timestamp_is_iso8601_z(): + body = asyncio.run(_get("/api/stats")).json() + ts = body["timestamp"] + # ISO8601 with a trailing Z (we normalize +00:00 -> Z). + assert isinstance(ts, str) and ts.endswith("Z") + assert re.match(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", ts) + + +def test_stats_payload_is_tiny(): + resp = asyncio.run(_get("/api/stats")) + assert len(resp.content) < 2048, "stats payload must be counts-only, not GeoJSON" + + +def test_stats_counts_reflect_last_known(monkeypatch): + """aircraft/vessels/trains/alerts come from in-memory last-known state.""" + import live_layers + + monkeypatch.setattr(live_layers, "aircraft_last_known", {str(i): {} for i in range(7)}) + monkeypatch.setattr(live_layers, "vessel_last_known", {str(i): {} for i in range(3)}) + monkeypatch.setattr(live_layers, "train_count", 11) + monkeypatch.setattr(live_layers, "nws_alert_count", 5) + + # _stats_counts imports the dicts/counters inside the function from live_layers, + # so monkeypatching the module attributes is what it observes. + from main import _stats_counts as fn + + body = asyncio.run(fn()) + assert body["aircraft"] == 7 + assert body["vessels"] == 3 + assert body["trains"] == 11 + assert body["alerts"] == 5 + + +def test_stats_db_failure_degrades_to_zero(monkeypatch): + """A down DB yields zeros for the SQL-backed counters, never a 500.""" + # Make the session factory raise synchronously so the try/except in + # _stats_counts degrades the SQL counters to zero (no dangling coroutine). + def _raise(*args, **kwargs): + raise RuntimeError("db down") + + monkeypatch.setattr("main.async_session", _raise) + body = asyncio.run(_stats_counts()) + assert body["cameras"] == 0 + assert body["fires"] == 0 + assert body["quakes"] == 0 + assert isinstance(body["timestamp"], str) \ No newline at end of file diff --git a/tests/test_conflicts_frontend.py b/tests/test_conflicts_frontend.py new file mode 100644 index 0000000..058fd3b --- /dev/null +++ b/tests/test_conflicts_frontend.py @@ -0,0 +1,66 @@ +"""Conflicts Leaflet overlay: default-off toggle, catalog fetch, no jitter.""" + +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +HTML = (ROOT / "app/static/index.html").read_text() + + +def _fn(name: str, until: str | None = None) -> str: + chunk = HTML.split(f"function {name}", 1)[1] + if until: + chunk = chunk.split(until, 1)[0] + return chunk + + +def test_conflicts_toggle_default_off(): + assert 'id="lp-conflicts-on"' in HTML + assert 'id="conflicts-layer"' in HTML + assert "> Conflicts<" in HTML or "> Conflicts", 1)[0] + assert "checked" not in on + + +def test_conflicts_fetches_catalog_not_liveuamap(): + js = _fn("loadConflicts", "/* ═══════════════ INITIAL LOAD") + assert "/api/conflicts" in js + assert "liveuamap.com" not in HTML.lower() + assert "Math.random" not in js + assert "jitter" not in js.lower() + + +def test_conflicts_not_refetched_on_moveend(): + refresh = HTML.split("function refreshLiveOverlays", 1)[1].split( + "function addExtraAttrib", 1 + )[0] + assert "loadConflicts" not in refresh + assert "probeConflicts" not in refresh + init = HTML.split("function initMap", 1)[1].split("function readMapPrefs", 1)[0] + assert "probeConflicts()" in init + assert "loadConflicts(true)" not in init + assert "paintConflicts()" not in init + + +def test_conflicts_hides_toggle_on_404(): + js = _fn("loadConflicts", "/* ═══════════════ INITIAL LOAD") + assert "r.status === 404" in js + assert "hideConflictsToggle()" in js + hide = _fn("hideConflictsToggle", "function paintConflicts") + assert "row.hidden = true" in hide + assert "lp-conflicts-on" in hide + + +def test_conflicts_popup_and_severity_colors(): + paint = _fn("paintConflicts", "async function probeConflicts") + assert "z.label" in paint + assert "z.description" in paint + assert "eventCount" in paint + assert "L.circleMarker" in paint + assert "z.lat == null || z.lon == null" in paint + assert "Number.isFinite(lat)" in paint + color = _fn("conflictSeverityColor", "function hideConflictsToggle") + assert "war" in color and "#ff2a6d" in color + assert "high" in color and "#fb923c" in color + assert "elevated" in color and "#facc15" in color diff --git a/tests/test_live_layers.py b/tests/test_live_layers.py index 283e548..9b31f71 100644 --- a/tests/test_live_layers.py +++ b/tests/test_live_layers.py @@ -27,7 +27,7 @@ from live_layers import ( _wfigs_params, ) -from camera_scraper import parse_caltrans_json, parse_udot_ibi_page, parse_mdot_json +from camera_scraper import parse_caltrans_json, parse_udot_ibi_page, parse_odot_json, parse_mdot_json def test_parse_bbox_and_radius_clamps_to_150_nm(): @@ -336,6 +336,50 @@ def test_parse_udot_missing_wkt_skipped(): assert parse_udot_ibi_page(_udot_page([row])) == [] +def test_parse_odot_tripcheck_keeps_valid_skips_missing_and_oob(): + payload = """ + {"features":[ + {"attributes":{ + "cameraId":277,"filename":"AstoriaUS101_pid392.jpg", + "latitude":46.18785,"longitude":-123.85347, + "route":"US101 ","title":"US101 at Astoria" + }}, + {"attributes":{ + "cameraId":200,"filename":"","latitude":45.0,"longitude":-122.0, + "route":"I-5","title":"missing filename" + }}, + {"attributes":{ + "cameraId":300,"filename":"nocal_pid1.jpg", + "latitude":40.0,"longitude":-122.0, + "route":"US97","title":"out of bbox" + }}, + {"attributes":{ + "cameraId":400,"filename":"badcoord_pid2.jpg", + "latitude":null,"longitude":-122.0, + "route":"OR22","title":"null coord" + }} + ]} + """ + cams = parse_odot_json(payload, "www.tripcheck.com") + assert len(cams) == 1 + cam = cams[0] + assert cam["discovery_source"] == "odot" + assert cam["snapshot_url"] == ( + "https://tripcheck.com/RoadCams/cams/AstoriaUS101_pid392.jpg") + assert cam["source_url"] == cam["snapshot_url"] + assert cam["location_lat"] == 46.18785 + assert cam["location_lon"] == -123.85347 + assert "US101 at Astoria" in cam["location_name"] + assert cam["vendor"] == "ODOT" + assert cam["device_type"] == "http" + assert "rtsp://" not in cam["snapshot_url"].lower() + + +def test_parse_odot_tripcheck_handles_malformed(): + assert parse_odot_json("not json", "www.tripcheck.com") == [] + assert parse_odot_json('{"features":null}', "www.tripcheck.com") == [] + + def test_parse_mdot_extracts_html_fields_and_bbox_filters(): rows = [ # In-bbox, full fields.