perf: faster map overlay load #4

Merged
sirius merged 7 commits from perf/map-layer-load into master 2026-08-27 21:24:24 -04:00
6 changed files with 629 additions and 131 deletions

View file

@ -49,7 +49,8 @@ CONUS = (-125.0, 24.0, -66.0, 50.0)
MAX_RADIUS_NM = 150
DEFAULT_LIMIT = 2000
_HTTP_TIMEOUT = 25.0
_HTTP_TIMEOUT = httpx.Timeout(8.0, connect=3.0)
_http: httpx.AsyncClient | None = None
_COMPASS = {
"N": 0, "NE": 45, "E": 90, "SE": 135,
"S": 180, "SW": 225, "W": 270, "NW": 315,
@ -58,7 +59,9 @@ _COMPASS = {
}
_cache: dict[str, tuple[float, Any]] = {}
_cache_lock = asyncio.Lock()
_key_locks: dict[str, asyncio.Lock] = {}
_key_locks_guard = asyncio.Lock()
_QUANT = 0.25 # degrees — pan jitter inside a cell reuses the TTL entry
# Last-known AIS positions (MMSI -> marker). Filled by ais_stream worker.
vessel_last_known: dict[str, dict] = {}
@ -116,6 +119,33 @@ def parse_bbox(bbox: str) -> tuple[float, float, float, float]:
return minlon, minlat, maxlon, maxlat
def quantize_bbox(
minlon: float, minlat: float, maxlon: float, maxlat: float,
step: float = _QUANT,
) -> tuple[float, float, float, float]:
"""Snap a viewport to a coarse cell so nearby pans share a cache key.
The returned envelope is expanded to cover the original box.
"""
def q_down(v: float, lo: float, hi: float) -> float:
v = max(lo, min(hi, v))
return math.floor(v / step) * step
return (
round(q_down(minlon, -180.0, 180.0), 4),
round(q_down(minlat, -90.0, 90.0), 4),
round(q_down(maxlon, -180.0, 180.0) + step, 4),
round(q_down(maxlat, -90.0, 90.0) + step, 4),
)
def bbox_cell_key(bbox: str | None) -> str:
"""Stable cache-key fragment for a viewport (or 'all')."""
if not bbox:
return "all"
return ",".join(f"{v:.4f}" for v in quantize_bbox(*parse_bbox(bbox)))
def bbox_center_radius_nm(
minlon: float, minlat: float, maxlon: float, maxlat: float,
) -> tuple[float, float, int]:
@ -177,6 +207,78 @@ def filter_points_bbox(
return out
_ALERT_KEEP = ("event", "severity", "headline", "areaDesc", "wfo", "source")
def slim_alert_properties(props: dict | None) -> dict:
"""Keep only the fields the map popup reads."""
src = props or {}
return {k: src.get(k) for k in _ALERT_KEEP}
def _walk_coords(coords: Any, acc: list[float]) -> None:
if not coords:
return
first = coords[0]
if isinstance(first, (int, float)):
lon, lat = float(coords[0]), float(coords[1])
acc[0] = min(acc[0], lon)
acc[1] = min(acc[1], lat)
acc[2] = max(acc[2], lon)
acc[3] = max(acc[3], lat)
return
for child in coords:
_walk_coords(child, acc)
def _geom_envelope(geom: dict | None) -> tuple[float, float, float, float] | None:
if not geom or not isinstance(geom, dict):
return None
if geom.get("type") == "GeometryCollection":
env: list[float] | None = None
for g in geom.get("geometries") or []:
e = _geom_envelope(g)
if e is None:
continue
if env is None:
env = list(e)
else:
env[0] = min(env[0], e[0])
env[1] = min(env[1], e[1])
env[2] = max(env[2], e[2])
env[3] = max(env[3], e[3])
return tuple(env) if env else None # type: ignore[return-value]
coords = geom.get("coordinates")
if coords is None:
return None
acc = [180.0, 90.0, -180.0, -90.0]
try:
_walk_coords(coords, acc)
except (TypeError, ValueError, IndexError):
return None
if acc[0] > acc[2]:
return None
return acc[0], acc[1], acc[2], acc[3]
def clip_fc_to_bbox(
fc: dict | None,
minlon: float, minlat: float, maxlon: float, maxlat: float,
) -> dict:
"""Drop features whose geometry envelope misses the viewport. No shapely."""
box = (minlon, minlat, maxlon, maxlat)
out = []
for feat in (fc or {}).get("features") or []:
if not isinstance(feat, dict):
continue
env = _geom_envelope(feat.get("geometry"))
if env is None:
continue
if env[0] <= box[2] and env[2] >= box[0] and env[1] <= box[3] and env[3] >= box[1]:
out.append(feat)
return {"type": "FeatureCollection", "features": out}
def _f(value: object) -> float | None:
if value is None or value == "":
return None
@ -404,12 +506,22 @@ def _headers() -> dict[str, str]:
return {"User-Agent": OSINT_USER_AGENT, "Accept": "application/json"}
async def _lock_for(key: str) -> asyncio.Lock:
async with _key_locks_guard:
lock = _key_locks.get(key)
if lock is None:
lock = asyncio.Lock()
_key_locks[key] = lock
return lock
async def _ttl_get(key: str, ttl: float, factory: Callable[[], Awaitable[Any]]) -> Any:
now = time.monotonic()
hit = _cache.get(key)
if hit and now - hit[0] < ttl:
return hit[1]
async with _cache_lock:
lock = await _lock_for(key)
async with lock:
hit = _cache.get(key)
if hit and time.monotonic() - hit[0] < ttl:
return hit[1]
@ -418,17 +530,42 @@ async def _ttl_get(key: str, ttl: float, factory: Callable[[], Awaitable[Any]])
return value
async def init_http() -> None:
"""Shared outbound client — one TLS pool for all overlay upstreams."""
global _http
if _http is None:
_http = httpx.AsyncClient(
timeout=_HTTP_TIMEOUT,
follow_redirects=True,
headers=_headers(),
limits=httpx.Limits(max_connections=20, max_keepalive_connections=10),
)
async def close_http() -> None:
global _http
if _http is not None:
await _http.aclose()
_http = None
async def _get_json(url: str, params: dict | None = None) -> Any:
async with httpx.AsyncClient(timeout=_HTTP_TIMEOUT, follow_redirects=True,
headers=_headers()) as client:
resp = await client.get(url, params=params)
resp.raise_for_status()
return resp.json()
if _http is None:
async with httpx.AsyncClient(
timeout=_HTTP_TIMEOUT, follow_redirects=True, headers=_headers(),
) as client:
resp = await client.get(url, params=params)
resp.raise_for_status()
return resp.json()
resp = await _http.get(url, params=params)
resp.raise_for_status()
return resp.json()
async def fetch_aircraft(bbox: str, limit: int = DEFAULT_LIMIT) -> list[dict]:
minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
lat, lon, radius = bbox_center_radius_nm(minlon, minlat, maxlon, maxlat)
qminlon, qminlat, qmaxlon, qmaxlat = quantize_bbox(minlon, minlat, maxlon, maxlat)
lat, lon, radius = bbox_center_radius_nm(qminlon, qminlat, qmaxlon, qmaxlat)
cache_key = f"adsb:{lat:.2f}:{lon:.2f}:{radius}"
async def _load():
@ -485,15 +622,17 @@ async def upsert_vessel(marker: dict) -> None:
}
def _wfigs_params(bbox: str | None) -> dict:
def _wfigs_params(bbox: str | None, *, offset_m: float = 250.0) -> dict:
params = {
"where": "1=1",
"outSR": "4326",
"f": "geojson",
"resultRecordCount": 2000,
"resultRecordCount": 500,
"maxAllowableOffset": offset_m / 111_320.0, # metres → degrees
"geometryPrecision": 5,
}
if bbox:
minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
minlon, minlat, maxlon, maxlat = quantize_bbox(*parse_bbox(bbox))
params["geometry"] = f"{minlon},{minlat},{maxlon},{maxlat}"
params["geometryType"] = "esriGeometryEnvelope"
params["inSR"] = "4326"
@ -511,7 +650,7 @@ async def fetch_fire_incidents(bbox: str | None, limit: int = DEFAULT_LIMIT) ->
async def _load():
return transform_wfigs_incidents(await _get_json(WFIGS_INCIDENTS, params))
rows = await _ttl_get(f"wfigs:inc:{bbox or 'all'}", 600.0, _load)
rows = await _ttl_get(f"wfigs:inc:{bbox_cell_key(bbox)}", 600.0, _load)
return rows[:limit]
@ -525,7 +664,7 @@ async def fetch_fire_perimeters(bbox: str | None) -> dict:
async def _load():
return await _get_json(WFIGS_PERIMETERS, params)
fc = await _ttl_get(f"wfigs:per:{bbox or 'all'}", 600.0, _load)
fc = await _ttl_get(f"wfigs:per:{bbox_cell_key(bbox)}", 600.0, _load)
if not isinstance(fc, dict):
return {"type": "FeatureCollection", "features": []}
return fc
@ -534,44 +673,50 @@ async def fetch_fire_perimeters(bbox: str | None) -> dict:
async def fetch_weather_alerts(area: str | None, bbox: str | None) -> dict:
"""Cached NWS active alerts + IEM storm-based warning polygons."""
async def _load_iem():
try:
data = await _get_json(IEM_SBW)
except Exception:
return {"type": "FeatureCollection", "features": []}
return data if isinstance(data, dict) else {"type": "FeatureCollection", "features": []}
async def _load():
nws_params: dict[str, str] = {"status": "actual"}
clip_box = None
if area:
nws_params["area"] = area.upper()
elif bbox:
minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
nws_params["bbox"] = f"{minlon},{minlat},{maxlon},{maxlat}"
clip_box = quantize_bbox(*parse_bbox(bbox))
nws_params["bbox"] = f"{clip_box[0]},{clip_box[1]},{clip_box[2]},{clip_box[3]}"
nws_fc: dict = {"features": []}
sbw_fc: dict = {"features": []}
try:
nws_fc = await _get_json(NWS_ALERTS, nws_params)
except Exception:
nws_fc = {"features": []}
try:
sbw_fc = await _get_json(IEM_SBW)
except Exception:
sbw_fc = {"features": []}
sbw_fc = await _ttl_get("iem:sbw", 45.0, _load_iem)
features = []
for feat in nws_fc.get("features") or []:
props = feat.get("properties") or {}
if not isinstance(feat, dict):
continue
props = dict(feat.get("properties") or {})
props["source"] = "nws"
feat["properties"] = props
features.append(feat)
for feat in sbw_fc.get("features") or []:
props = feat.get("properties") or {}
features.append({**feat, "properties": slim_alert_properties(props)})
for feat in (sbw_fc or {}).get("features") or []:
if not isinstance(feat, dict):
continue
props = dict(feat.get("properties") or {})
props["source"] = "iem-sbw"
# IEM uses `ps` (phenomenon) / `wfo`; map a display event.
if "event" not in props:
props["event"] = props.get("ps") or "Storm-based warning"
feat["properties"] = props
if bbox:
# Cheap reject: skip if no geometry; keep otherwise (polygons).
if not feat.get("geometry"):
continue
features.append(feat)
return {"type": "FeatureCollection", "features": features}
features.append({**feat, "properties": slim_alert_properties(props)})
merged = {"type": "FeatureCollection", "features": features}
if clip_box:
merged = clip_fc_to_bbox(merged, *clip_box)
elif bbox:
merged = clip_fc_to_bbox(merged, *parse_bbox(bbox))
return merged
key = f"alerts:{area or ''}:{bbox or ''}"
key = f"alerts:{area or ''}:{bbox_cell_key(bbox) if bbox else ''}"
return await _ttl_get(key, 30.0, _load)

View file

@ -14,6 +14,7 @@ from __future__ import annotations
import asyncio
import json
import logging
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
from decimal import Decimal
from pathlib import Path
@ -21,7 +22,8 @@ from uuid import UUID
import structlog
from fastapi import FastAPI, HTTPException, Query
from fastapi.responses import FileResponse, HTMLResponse
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from sqlalchemy import and_, func, or_, select, text
from sqlalchemy.ext.asyncio import AsyncSession
@ -53,11 +55,30 @@ from live_layers import (
logging.basicConfig(level=logging.INFO)
logger = structlog.get_logger("osint.dashboard")
@asynccontextmanager
async def _lifespan(app: FastAPI):
await init_extensions()
from live_layers import close_http, init_http
await init_http()
from config import AISSTREAM_IN_APP
ais_task = None
if AISSTREAM_IN_APP:
from ais_stream import run_ais_worker
ais_task = asyncio.create_task(run_ais_worker())
yield
if ais_task is not None:
ais_task.cancel()
await close_http()
app = FastAPI(
title="OSINT Dashboard",
description="Real-time geospatial OSINT intelligence dashboard",
version="0.1.0",
lifespan=_lifespan,
)
app.add_middleware(GZipMiddleware, minimum_size=1024)
STATIC_DIR = Path(__file__).parent / "static"
@ -69,17 +90,17 @@ def event_to_out(row: dict) -> EventOut:
return EventOut(
id=row["id"],
source_type=row["source_type"],
source_id=row["source_id"],
title=row["title"],
body=row["body"],
url=row["url"],
sentiment_score=row["sentiment_score"],
sentiment_label=row["sentiment_label"],
location_lat=row["location_lat"],
location_lon=row["location_lon"],
location_name=row["location_name"],
entities=row["entities"],
tags=row["tags"],
source_id=row.get("source_id"),
title=row.get("title"),
body=row.get("body"),
url=row.get("url"),
sentiment_score=row.get("sentiment_score"),
sentiment_label=row.get("sentiment_label"),
location_lat=row.get("location_lat"),
location_lon=row.get("location_lon"),
location_name=row.get("location_name"),
entities=row.get("entities"),
tags=row.get("tags"),
ingested_at=row["ingested_at"],
source_timestamp=row["source_timestamp"],
)
@ -131,20 +152,11 @@ async def health():
return {"status": "ok", "db_time": db_time.isoformat() if db_time else None}
# ── Startup ───────────────────────────────────────────────────────────────
@app.on_event("startup")
async def startup():
"""Initialize PostGIS/TimescaleDB extensions on first connection.
Schema migrations are applied by the container entrypoint (alembic upgrade
head) before uvicorn starts, so they don't run nested inside the event loop.
"""
await init_extensions()
from config import AISSTREAM_IN_APP
if AISSTREAM_IN_APP:
from ais_stream import run_ais_worker
asyncio.create_task(run_ais_worker())
def overlay_json(data, max_age: int) -> JSONResponse:
"""JSON overlay payload with a short browser/proxy TTL."""
resp = JSONResponse(content=data)
resp.headers["Cache-Control"] = f"public, max-age={max_age}"
return resp
# ── Feed Sources ──────────────────────────────────────────────────────────
@ -218,7 +230,15 @@ async def list_events(
):
"""List recent ingested events."""
async with async_session() as session:
stmt = select(events).order_by(events.c.ingested_at.desc())
if has_coords:
stmt = select(
events.c.id, events.c.source_type, events.c.source_id,
events.c.title, events.c.url,
events.c.location_lat, events.c.location_lon, events.c.location_name,
events.c.ingested_at, events.c.source_timestamp,
).order_by(events.c.ingested_at.desc())
else:
stmt = select(events).order_by(events.c.ingested_at.desc())
if source_type:
stmt = stmt.where(events.c.source_type == source_type.value)
if since:
@ -272,7 +292,29 @@ async def create_event(payload: EventCreate):
# ── Active Fires / Hotspots (NASA FIRMS) ─────────────────────────────────
@app.get("/api/fires", response_model=list[FireOut])
def fire_heat_row(r) -> dict:
"""Minimal FIRMS point for the heatmap overlay."""
return {
"lat": r["latitude"],
"lon": r["longitude"],
"i": r["brightness"],
"c": r["confidence"],
}
def camera_map_row(r) -> dict:
"""Minimal camera pin — URLs stay off the list payload."""
return {
"id": str(r["id"]),
"lat": r["location_lat"],
"lon": r["location_lon"],
"device_type": r["device_type"],
"discovery_source": r["discovery_source"],
"location_name": r["location_name"],
}
@app.get("/api/fires", response_model=None)
async def list_fires(
bbox: str | None = Query(
None,
@ -286,12 +328,16 @@ async def list_fires(
"(ISO 8601, e.g. '2026-08-24T12:00:00Z').",
),
limit: int = Query(2000, ge=1, le=10000),
format: str = Query("full", description="'full' FireOut rows or 'heat' {lat,lon,i,c}"),
):
"""List stored FIRMS active fire/hotspot detections as JSON.
This is the data contract for the map's fire heatmap overlay: the frontend
calls `GET /api/fires?bbox=...&since=...` and renders the returned points.
calls `GET /api/fires?bbox=...&since=...&format=heat` and renders the points.
"""
fmt = (format or "full").lower()
if fmt not in ("full", "heat"):
raise HTTPException(422, "format must be 'full' or 'heat'")
async with async_session() as session:
stmt = select(fires).order_by(fires.c.acq_time.desc())
if since:
@ -316,6 +362,8 @@ async def list_fires(
)
stmt = stmt.limit(limit)
rows = (await session.execute(stmt)).mappings().all()
if fmt == "heat":
return [fire_heat_row(r) for r in rows]
return [
FireOut(
latitude=r["latitude"], longitude=r["longitude"],
@ -798,7 +846,11 @@ async def list_cameras(
stmt = stmt.where(cam_table.c.discovery_source == source)
rows = (await session.execute(stmt.limit(limit))).mappings().all()
return [{
return [camera_map_row(r) for r in rows]
def camera_detail_row(r) -> dict:
return {
"id": str(r["id"]),
"source_url": r["source_url"],
"snapshot_url": r["snapshot_url"],
@ -808,9 +860,23 @@ async def list_cameras(
"location_name": r["location_name"],
"vendor": r["vendor"],
"device_type": r["device_type"],
"first_seen": r["first_seen"].isoformat(),
"last_seen": r["last_seen"].isoformat(),
} for r in rows]
"first_seen": r["first_seen"].isoformat() if r["first_seen"] else None,
"last_seen": r["last_seen"].isoformat() if r["last_seen"] else None,
}
@app.get("/api/cameras/{camera_id}")
async def get_camera(camera_id: UUID):
"""Full camera row for a map popup. List endpoint stays slim."""
from camera_models import cameras as cam_table
async with async_session() as session:
row = (await session.execute(
select(cam_table).where(cam_table.c.id == camera_id)
)).mappings().one_or_none()
if not row:
raise HTTPException(404, "Camera not found")
return camera_detail_row(row)
@app.get("/api/cameras/{camera_id}/snapshot")
@ -959,6 +1025,10 @@ async def list_news(
),
limit: int = Query(50, ge=1, le=500),
offset: int = Query(0, ge=0),
include_content: bool = Query(
False,
description="Include full article body. Default false — ticker/list only need title/url.",
),
):
"""Most recent scraped news articles (newest first)."""
async with async_session() as session:
@ -974,7 +1044,7 @@ async def list_news(
return [
NewsArticleOut(
id=r["id"], title=r["title"], url=r["url"],
content=r["content"], domain=r["domain"],
content=r["content"] if include_content else None, domain=r["domain"],
timestamp=r["timestamp"],
)
for r in rows
@ -1056,7 +1126,7 @@ def _upstream_or_502(exc: Exception, name: str):
async def map_radar():
"""RainViewer frame list + IEM NEXRAD tile template. Browser fetches tiles."""
try:
return await fetch_radar_meta()
return overlay_json(await fetch_radar_meta(), 60)
except Exception as exc:
_upstream_or_502(exc, "radar")
@ -1069,7 +1139,7 @@ async def list_aircraft(
"""Viewport ADS-B last-known (ADSB.lol). Requires bbox; radius clamped ≤ 150 nm."""
_parse_bbox_query(bbox)
try:
return await fetch_aircraft(bbox, limit)
return overlay_json(await fetch_aircraft(bbox, limit), 5)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
except Exception as exc:
@ -1085,7 +1155,7 @@ async def list_trains(
if bbox:
_parse_bbox_query(bbox)
try:
return await fetch_trains(bbox, limit)
return overlay_json(await fetch_trains(bbox, limit), 20)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
except Exception as exc:
@ -1101,7 +1171,7 @@ async def list_vessels(
if bbox:
_parse_bbox_query(bbox)
try:
return await fetch_vessels(bbox, limit)
return overlay_json(await fetch_vessels(bbox, limit), 5)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
@ -1115,7 +1185,7 @@ async def list_fire_incidents(
if bbox:
_parse_bbox_query(bbox)
try:
return await fetch_fire_incidents(bbox, limit)
return overlay_json(await fetch_fire_incidents(bbox, limit), 30)
except Exception as exc:
_upstream_or_502(exc, "fire-incidents")
@ -1126,7 +1196,7 @@ async def list_fire_perimeters(bbox: str | None = Query(None)):
if bbox:
_parse_bbox_query(bbox)
try:
return await fetch_fire_perimeters(bbox)
return overlay_json(await fetch_fire_perimeters(bbox), 30)
except Exception as exc:
_upstream_or_502(exc, "fire-perimeters")
@ -1144,7 +1214,7 @@ async def list_weather_alerts(
if bbox:
_parse_bbox_query(bbox)
try:
return await fetch_weather_alerts(area, bbox)
return overlay_json(await fetch_weather_alerts(area, bbox), 30)
except Exception as exc:
_upstream_or_502(exc, "weather-alerts")
@ -1153,7 +1223,7 @@ async def list_weather_alerts(
async def list_storms():
"""NHC active tropical cyclones."""
try:
return await fetch_storms()
return overlay_json(await fetch_storms(), 20)
except Exception as exc:
_upstream_or_502(exc, "storms")

View file

@ -1026,10 +1026,22 @@
<script src="/static/vendor/leaflet/leaflet.js"></script>
<script src="/static/vendor/leaflet/leaflet.markercluster.js"></script>
<script src="/static/vendor/leaflet/leaflet.heat.js"></script>
<script src="/static/vendor/hls/hls.min.js"></script>
<script>
const API = '';
function loadHls() {
if (window.Hls) return Promise.resolve();
if (window._hlsLoading) return window._hlsLoading;
window._hlsLoading = new Promise((res, rej) => {
const s = document.createElement('script');
s.src = '/static/vendor/hls/hls.min.js';
s.onload = res;
s.onerror = rej;
document.head.appendChild(s);
});
return window._hlsLoading;
}
/* ═══════════════ BOOT SPLASH ═══════════════ */
(function () {
const boot = document.getElementById('boot');
@ -1136,7 +1148,8 @@ function showView(name) {
}
}
if (name === 'news') loadNews(false);
if (name === 'settings') renderSysInfo();
if (name === 'events') { loadSummary(); loadEvents(); }
if (name === 'settings') { loadSummary(); renderSysInfo(); }
window.scrollTo(0, 0);
}
@ -1637,6 +1650,16 @@ let vesselsGroup = null, vesselsOn = false;
let stormsGroup = null, stormsOn = true;
let overlayReq = {ac:0, trains:0, vessels:0, alerts:0, perim:0, incidents:0, storms:0};
let moveDebounce = null;
let overlayAbort = null;
let lastCell = '';
function bboxCell() {
if (!map) return '';
return currentBBox().split(',').map(n => Number(n).toFixed(2)).join(',') + '@' + map.getZoom();
}
function overlayFetch(url) {
return fetch(url, overlayAbort ? { signal: overlayAbort.signal } : {});
}
function isAbort(e) { return e && e.name === 'AbortError'; }
const pointCanvas = () => L.canvas({ padding: 0.5 });
async function initMap() {
@ -1651,7 +1674,10 @@ async function initMap() {
const sel = document.getElementById('map-layer');
sel.innerHTML = mapLayers.map(l =>
`<option value="${l.id}">${l.title}</option>`).join('');
const preferred = mapLayers.find(l => l.has_time) || mapLayers[0];
const preferred =
mapLayers.find(l => l.id === 'BlueMarble_ShadedRelief_Bathymetry')
|| mapLayers.find(l => !l.has_time)
|| mapLayers[0];
if (preferred) sel.value = preferred.id;
map = L.map('map', {
center: [25, 10], zoom: 2,
@ -1676,20 +1702,24 @@ async function initMap() {
img.alt = 'camera preview';
v.replaceWith(img);
};
if (window.Hls && Hls.isSupported()) {
activeHls = new Hls({ enableWorker: true, lowLatencyMode: true });
activeHls.loadSource(v.dataset.hls);
activeHls.attachMedia(v);
v.play && v.play().catch(() => {});
activeHls.on(Hls.Events.ERROR, (_, data) => {
if (data && data.fatal) fallback();
});
} else if (v.canPlayType && v.canPlayType('application/vnd.apple.mpegurl')) {
v.src = v.dataset.hls;
v.addEventListener('error', fallback, { once: true });
} else {
fallback();
}
const startHls = () => {
if (window.Hls && Hls.isSupported()) {
activeHls = new Hls({ enableWorker: true, lowLatencyMode: true });
activeHls.loadSource(v.dataset.hls);
activeHls.attachMedia(v);
v.play && v.play().catch(() => {});
activeHls.on(Hls.Events.ERROR, (_, data) => {
if (data && data.fatal) fallback();
});
} else if (v.canPlayType && v.canPlayType('application/vnd.apple.mpegurl')) {
v.src = v.dataset.hls;
v.addEventListener('error', fallback, { once: true });
} else {
fallback();
}
};
if (window.Hls) startHls();
else loadHls().then(startHls).catch(fallback);
});
map.on('popupclose', () => {
if (activeHls) { try { activeHls.destroy(); } catch (_) {} activeHls = null; }
@ -1713,13 +1743,19 @@ async function initMap() {
if (camPopupOpen) return; // only the popup's own autopan now
if (moveDebounce) clearTimeout(moveDebounce);
moveDebounce = setTimeout(() => {
const cell = bboxCell();
if (cell === lastCell) return;
lastCell = cell;
if (overlayAbort) overlayAbort.abort();
overlayAbort = new AbortController();
if (firesOn) loadFires();
if (camsOn) loadCams();
if (blipsOn) loadBlips();
refreshLiveOverlays();
}, 300);
});
await mapLayerChanged();
// Static Blue Marble has no time-domain fetch — don't block overlays on GIBS.
mapLayerChanged();
// Overlays default ON (checkboxes in the layer panel + saved settings)
applyMapSettings(readMapPrefs(), false);
firesOn = document.getElementById('lp-fires-on').checked;
@ -1900,10 +1936,12 @@ function setBaseOpacity(v) {
/* ── FIRMS fire heatmap ── */
function firesIntensity(f) {
const confidence = f.confidence || f.c;
const brightness = f.brightness != null ? f.brightness : f.i;
if (firesColor === 'confidence') {
return f.confidence === 'h' ? 1.0 : f.confidence === 'l' ? 0.35 : 0.6;
return confidence === 'h' ? 1.0 : confidence === 'l' ? 0.35 : 0.6;
}
const b = f.brightness || 300;
const b = brightness || 300;
return Math.max(0.05, Math.min(1, (b - 290) / 110));
}
function firesGradient() {
@ -1951,14 +1989,14 @@ async function loadFires() {
if (!map) return;
const req = ++fireReq;
try {
let url = `${API}/api/fires?bbox=${currentBBox()}&limit=2000`;
let url = `${API}/api/fires?bbox=${currentBBox()}&limit=2000&format=heat`;
const since = sinceToISO(firesSince);
if (since) url += `&since=${encodeURIComponent(since)}`;
const r = await fetch(url);
const r = await overlayFetch(url);
const fires = await r.json();
if (req !== fireReq) return; // superseded by a newer pan/zoom
if (firesHeat) map.removeLayer(firesHeat);
const pts = fires.map(f => [f.latitude, f.longitude, firesIntensity(f)]);
const pts = fires.map(f => [f.lat ?? f.latitude, f.lon ?? f.longitude, firesIntensity(f)]);
firesHeat = L.heatLayer(pts, {
radius: 22, blur: 20, maxZoom: 9, max: 1.0, minOpacity: 0.2,
gradient: firesGradient(),
@ -1971,6 +2009,7 @@ async function loadFires() {
`${fires.length.toLocaleString()} fire hotspots in view` +
(since ? ` · since ${since.slice(0,16).replace('T',' ')}Z` : '');
} catch(e) {
if (isAbort(e)) return;
document.getElementById('map-hint').textContent = `Fires load failed: ${e.message || e}`;
console.error('Fires load failed', e);
}
@ -2036,9 +2075,16 @@ function camThumb(c) {
}
async function loadCams() {
if (!map) return;
if (tooZoomedOut()) {
camsGroup = dropLayer(camsGroup);
markZoom('lp-cams-count');
hudCamsCount = null;
syncHud();
return;
}
const req = ++camReq;
try {
const r = await fetch(`${API}/api/cameras?bbox=${currentBBox()}&limit=5000`);
const r = await overlayFetch(`${API}/api/cameras?bbox=${currentBBox()}&limit=2000`);
const cams = await r.json();
if (req !== camReq) return; // superseded by a newer pan/zoom
if (camsGroup) map.removeLayer(camsGroup);
@ -2062,7 +2108,7 @@ async function loadCams() {
iconSize: [12, 12], iconAnchor: [6, 6],
});
camsGroup.addLayer(L.marker([c.lat, c.lon], { icon })
.bindPopup(`<div class="cam-pop">` +
.bindPopup(() => `<div class="cam-pop">` +
`<b>${esc(c.location_name || 'Open camera')}</b>` +
`${c.id ? camThumb(c) : '<div class="thumb placeholder">no snapshot</div>'}` +
`<table>` +
@ -2083,6 +2129,7 @@ async function loadCams() {
document.getElementById('map-hint').textContent =
`${cams.length.toLocaleString()} open cameras in view`;
} catch(e) {
if (isAbort(e)) return;
document.getElementById('map-hint').textContent = `Cameras load failed: ${e.message || e}`;
console.error('Cameras load failed', e);
}
@ -2118,7 +2165,7 @@ async function loadBlips() {
let url = `${API}/api/events?bbox=${currentBBox()}&has_coords=true&limit=500`;
const since = sinceToISO(blipsSince);
if (since) url += `&since=${encodeURIComponent(since)}`;
const r = await fetch(url);
const r = await overlayFetch(url);
const evs = (await r.json()).filter(ev => ev.source_type !== 'camera');
if (req !== blipReq) return; // superseded by a newer pan/zoom
if (blipsGroup) map.removeLayer(blipsGroup);
@ -2148,6 +2195,7 @@ async function loadBlips() {
`${evs.length.toLocaleString()} event blips in view`;
}
} catch(e) {
if (isAbort(e)) return;
document.getElementById('map-hint').textContent = `Blips load failed: ${e.message || e}`;
console.error('Blips load failed', e);
}
@ -2175,6 +2223,14 @@ function dropLayer(ref) {
if (ref && map && map.hasLayer(ref)) map.removeLayer(ref);
return null;
}
/* Polygon/camera layers hitch at world scale. Same idea as aircraft's zoom<=3
skip: do not fetch or render these until the user is zoomed in. */
const HEAVY_MIN_ZOOM = 4;
function tooZoomedOut() { return !map || map.getZoom() <= HEAVY_MIN_ZOOM; }
function markZoom(id) {
const el = document.getElementById(id);
if (el) el.textContent = 'zoom';
}
function intersectsConus() {
if (!map) return false;
const b = map.getBounds();
@ -2251,33 +2307,69 @@ function feedIcon(feed, color, heading) {
}
return ic;
}
function makePointMarker(p, colorFn, feed, renderer) {
if (p.lat == null || p.lon == null) return null;
const col = sanitizeColor(colorFn(p), '#35e0ff');
if (feed) {
const heading = Number(p.heading);
const icon = feedIcon(feed, col, Number.isNaN(heading) ? null : heading);
return L.marker([p.lat, p.lon], { icon }).bindPopup(() => pointPopup(p));
}
return L.circleMarker([p.lat, p.lon], {
radius: 5, color: col, fillColor: col, fillOpacity: 0.9, weight: 1,
renderer,
}).bindPopup(() => pointPopup(p));
}
function renderPoints(existing, points, colorFn, cluster, feed) {
if (existing) map.removeLayer(existing);
const zoom = map.getZoom();
const useCluster = cluster && (zoom < 7 || points.length > 200);
const group = useCluster
? L.markerClusterGroup({ maxClusterRadius: 48, showCoverageOnHover: false, spiderfyOnMaxZoom: true, chunkedLoading: true })
: L.layerGroup();
const canReuse = existing && map.hasLayer(existing)
&& !!existing._osintCluster === !!useCluster
&& existing._osintById;
if (!canReuse) {
if (existing) map.removeLayer(existing);
const group = useCluster
? L.markerClusterGroup({ maxClusterRadius: 48, showCoverageOnHover: false, spiderfyOnMaxZoom: true, chunkedLoading: true })
: L.layerGroup();
group._osintCluster = !!useCluster;
group._osintById = new Map();
const renderer = pointCanvas();
points.forEach(p => {
const m = makePointMarker(p, colorFn, feed, renderer);
if (!m) return;
group.addLayer(m);
if (p.id != null) group._osintById.set(String(p.id), m);
});
group.addTo(map);
return group;
}
const group = existing;
const byId = group._osintById;
const next = new Set();
const renderer = pointCanvas();
points.forEach(p => {
if (p.lat == null || p.lon == null) return;
if (p.lat == null || p.lon == null || p.id == null) return;
const id = String(p.id);
next.add(id);
const col = sanitizeColor(colorFn(p), '#35e0ff');
if (feed) {
const heading = Number(p.heading);
const icon = feedIcon(feed, col, Number.isNaN(heading) ? null : heading);
const m = L.marker([p.lat, p.lon], { icon }).bindPopup(pointPopup(p));
group.addLayer(m);
const m = byId.get(id);
if (m) {
m.setLatLng([p.lat, p.lon]);
if (feed) m.setIcon(feedIcon(feed, col, p.heading));
else if (m.setStyle) m.setStyle({ color: col, fillColor: col });
} else {
const heading = Number(p.heading);
const m = L.circleMarker([p.lat, p.lon], {
radius: 5, color: col, fillColor: col, fillOpacity: 0.9, weight: 1,
renderer,
}).bindPopup(pointPopup(p));
if (!Number.isNaN(heading)) m.setStyle({ className: 'hdg' });
group.addLayer(m);
const nm = makePointMarker(p, colorFn, feed, renderer);
if (!nm) return;
group.addLayer(nm);
byId.set(id, nm);
}
});
byId.forEach((m, id) => {
if (!next.has(id)) {
group.removeLayer(m);
byId.delete(id);
}
});
group.addTo(map);
return group;
}
async function toggleRadar() {
@ -2294,7 +2386,7 @@ async function loadRadar() {
if (!map) return;
try {
if (!radarMeta) {
const r = await fetch(`${API}/api/map/radar`);
const r = await overlayFetch(`${API}/api/map/radar`);
radarMeta = await r.json();
addExtraAttrib('<a href="https://www.rainviewer.com/api.html">Weather data by RainViewer</a>');
addExtraAttrib('Iowa Environmental Mesonet');
@ -2313,6 +2405,7 @@ async function loadRadar() {
radarLayer = L.tileLayer(url, { opacity: radarOpacity, maxZoom: 12, maxNativeZoom: useIem ? 18 : 7, attribution: '' }).addTo(map);
}
} catch (e) {
if (isAbort(e)) return;
console.error('Radar load failed', e);
document.getElementById('lp-radar-count').textContent = 'err';
}
@ -2338,14 +2431,20 @@ async function toggleWxAlerts() {
}
async function loadWxAlerts() {
if (!map) return;
if (tooZoomedOut()) {
wxAlertsGroup = dropLayer(wxAlertsGroup);
markZoom('lp-alerts-count');
return;
}
const req = ++overlayReq.alerts;
try {
const r = await fetch(`${API}/api/weather-alerts?bbox=${currentBBox()}`);
const r = await overlayFetch(`${API}/api/weather-alerts?bbox=${currentBBox()}`);
const fc = await r.json();
if (req !== overlayReq.alerts) return;
wxAlertsGroup = dropLayer(wxAlertsGroup);
const feats = fc.features || [];
wxAlertsGroup = L.geoJSON(fc, {
renderer: L.canvas({ padding: 0.5 }),
style: (f) => ({
color: severityColor((f.properties || {}).severity),
weight: 2, fillOpacity: 0.18,
@ -2358,6 +2457,7 @@ async function loadWxAlerts() {
document.getElementById('lp-alerts-count').textContent = feats.length.toLocaleString();
addExtraAttrib('NWS / IEM storm-based warnings');
} catch (e) {
if (isAbort(e)) return;
console.error('Alerts load failed', e);
document.getElementById('lp-alerts-count').textContent = 'err';
}
@ -2369,14 +2469,20 @@ async function togglePerimeters() {
}
async function loadPerimeters() {
if (!map) return;
if (tooZoomedOut()) {
perimGroup = dropLayer(perimGroup);
markZoom('lp-perim-count');
return;
}
const req = ++overlayReq.perim;
try {
const r = await fetch(`${API}/api/fire-perimeters?bbox=${currentBBox()}`);
const r = await overlayFetch(`${API}/api/fire-perimeters?bbox=${currentBBox()}`);
const fc = await r.json();
if (req !== overlayReq.perim) return;
perimGroup = dropLayer(perimGroup);
const feats = fc.features || [];
perimGroup = L.geoJSON(fc, {
renderer: L.canvas({ padding: 0.5 }),
style: (f) => {
const acres = Number((f.properties || {}).poly_GISAcres || (f.properties || {}).attr_IncidentSize || 0);
return { color: acres > 10000 ? '#ef4444' : '#fb923c', weight: 2, fillOpacity: 0.25, fillColor: '#fb923c' };
@ -2391,6 +2497,7 @@ async function loadPerimeters() {
document.getElementById('lp-perim-count').textContent = feats.length.toLocaleString();
addExtraAttrib('NIFC WFIGS');
} catch (e) {
if (isAbort(e)) return;
console.error('Perimeters load failed', e);
document.getElementById('lp-perim-count').textContent = 'err';
}
@ -2402,15 +2509,21 @@ async function toggleIncidents() {
}
async function loadIncidents() {
if (!map) return;
if (tooZoomedOut()) {
incidentsGroup = dropLayer(incidentsGroup);
markZoom('lp-incidents-count');
return;
}
const req = ++overlayReq.incidents;
try {
const r = await fetch(`${API}/api/fire-incidents?bbox=${currentBBox()}`);
const r = await overlayFetch(`${API}/api/fire-incidents?bbox=${currentBBox()}`);
const pts = await r.json();
if (req !== overlayReq.incidents) return;
incidentsGroup = renderPoints(incidentsGroup, pts, () => '#ef4444', false);
document.getElementById('lp-incidents-count').textContent = pts.length.toLocaleString();
addExtraAttrib('NIFC WFIGS');
} catch (e) {
if (isAbort(e)) return;
console.error('Incidents load failed', e);
document.getElementById('lp-incidents-count').textContent = 'err';
}
@ -2428,13 +2541,14 @@ async function loadAircraft() {
}
const req = ++overlayReq.ac;
try {
const r = await fetch(`${API}/api/aircraft?bbox=${currentBBox()}`);
const r = await overlayFetch(`${API}/api/aircraft?bbox=${currentBBox()}`);
const pts = await r.json();
if (req !== overlayReq.ac) return;
acGroup = renderPoints(acGroup, Array.isArray(pts) ? pts : [], p => altColor((p.extra || {}).alt_baro), true, 'ac');
document.getElementById('lp-ac-count').textContent = (pts.length || 0).toLocaleString();
addExtraAttrib('<a href="https://www.adsb.lol/docs/open-data/api">ADSB.lol</a> ODbL');
} catch (e) {
if (isAbort(e)) return;
console.error('Aircraft load failed', e);
document.getElementById('lp-ac-count').textContent = 'err';
}
@ -2448,13 +2562,14 @@ async function loadTrains() {
if (!map) return;
const req = ++overlayReq.trains;
try {
const r = await fetch(`${API}/api/trains?bbox=${currentBBox()}`);
const r = await overlayFetch(`${API}/api/trains?bbox=${currentBBox()}`);
const pts = await r.json();
if (req !== overlayReq.trains) return;
trainsGroup = renderPoints(trainsGroup, Array.isArray(pts) ? pts : [], p => (p.extra || {}).iconColor || '#c084fc', false, 'train');
document.getElementById('lp-trains-count').textContent = (pts.length || 0).toLocaleString();
addExtraAttrib('<a href="https://amtraker.com/about">Amtraker</a>');
} catch (e) {
if (isAbort(e)) return;
console.error('Trains load failed', e);
document.getElementById('lp-trains-count').textContent = 'err';
}
@ -2472,7 +2587,7 @@ async function loadVessels() {
}
const req = ++overlayReq.vessels;
try {
const r = await fetch(`${API}/api/vessels?bbox=${currentBBox()}`);
const r = await overlayFetch(`${API}/api/vessels?bbox=${currentBBox()}`);
const pts = await r.json();
if (req !== overlayReq.vessels) return;
vesselsGroup = renderPoints(vesselsGroup, Array.isArray(pts) ? pts : [], p => {
@ -2482,6 +2597,7 @@ async function loadVessels() {
document.getElementById('lp-vessels-count').textContent = (pts.length || 0).toLocaleString();
addExtraAttrib('AISStream');
} catch (e) {
if (isAbort(e)) return;
console.error('Vessels load failed', e);
document.getElementById('lp-vessels-count').textContent = 'err';
}
@ -2495,13 +2611,14 @@ async function loadStorms() {
if (!map) return;
const req = ++overlayReq.storms;
try {
const r = await fetch(`${API}/api/storms`);
const r = await overlayFetch(`${API}/api/storms`);
const pts = await r.json();
if (req !== overlayReq.storms) return;
stormsGroup = renderPoints(stormsGroup, Array.isArray(pts) ? pts : [], () => '#f472b6', false);
document.getElementById('lp-storms-count').textContent = (pts.length || 0).toLocaleString();
addExtraAttrib('NHC');
} catch (e) {
if (isAbort(e)) return;
console.error('Storms load failed', e);
document.getElementById('lp-storms-count').textContent = 'err';
}
@ -2511,9 +2628,9 @@ async function loadStorms() {
initNav();
initSettings();
initMarketTicker();
loadSummary(); loadEvents(); loadNews(true); checkHealth();
setInterval(() => { loadSummary(); loadEvents(); checkHealth(); }, 30000);
checkHealth();
initMap();
setInterval(checkHealth, 30000);
</script>
</body>
</html>

View file

@ -36,3 +36,4 @@ def test_vessels_empty_without_ais_key():
resp = asyncio.run(_get("/api/vessels"))
assert resp.status_code == 200
assert resp.json() == []
assert "max-age" in (resp.headers.get("cache-control") or "").lower()

View file

@ -3,15 +3,21 @@
from live_layers import (
MARKER_FIELDS,
bbox_center_radius_nm,
clip_fc_to_bbox,
filter_points_bbox,
parse_bbox,
quantize_bbox,
rainviewer_tile_url,
slim_alert_properties,
to_marker,
transform_adsb_lol,
transform_ais_frame,
transform_amtraker,
transform_nhc_storms,
transform_wfigs_incidents,
_cache,
_ttl_get,
_wfigs_params,
)
from camera_scraper import parse_caltrans_json
@ -244,3 +250,115 @@ def test_parse_caltrans_skips_oos_and_maps_jpeg_hls():
assert "I-80" in cam["location_name"]
assert "rtsp://" not in cam["source_url"].lower()
assert "rtsp://" not in cam["snapshot_url"].lower()
def test_quantize_bbox_stable_under_jitter():
a = quantize_bbox(*parse_bbox("-78.7912,35.7711,-78.6101,35.9102"))
b = quantize_bbox(*parse_bbox("-78.7900,35.7700,-78.6110,35.9090"))
assert a == b
minlon, minlat, maxlon, maxlat = a
assert minlon <= -78.7912
assert minlat <= 35.7700
assert maxlon >= -78.6101
assert maxlat >= 35.9102
def test_ttl_get_does_not_block_other_keys():
import asyncio
_cache.clear()
order = []
async def slow():
order.append("slow-start")
await asyncio.sleep(0.2)
order.append("slow-end")
return "S"
async def fast():
order.append("fast")
return "F"
async def run():
t1 = asyncio.create_task(_ttl_get("slow", 5, slow))
await asyncio.sleep(0.01)
t2 = asyncio.create_task(_ttl_get("fast", 5, fast))
await asyncio.gather(t1, t2)
asyncio.run(run())
assert order.index("fast") < order.index("slow-end")
assert _cache["slow"][1] == "S"
assert _cache["fast"][1] == "F"
_cache.clear()
def test_clip_fc_to_bbox_drops_far_features_and_empty_geometry():
fc = {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {"event": "near"},
"geometry": {"type": "Point", "coordinates": [-78.7, 35.8]},
},
{
"type": "Feature",
"properties": {"event": "far"},
"geometry": {"type": "Point", "coordinates": [-120.0, 45.0]},
},
{
"type": "Feature",
"properties": {"event": "nogeom"},
"geometry": None,
},
{
"type": "Feature",
"properties": {"event": "poly-overlap"},
"geometry": {
"type": "Polygon",
"coordinates": [[
[-79.0, 35.0], [-78.0, 35.0], [-78.0, 36.0],
[-79.0, 36.0], [-79.0, 35.0],
]],
},
},
],
}
clipped = clip_fc_to_bbox(fc, -79.0, 35.5, -78.0, 36.0)
events = [f["properties"]["event"] for f in clipped["features"]]
assert events == ["near", "poly-overlap"]
def test_slim_alert_properties_keeps_popup_fields_only():
fat = {
"event": "Tornado Warning",
"severity": "Extreme",
"headline": "TORNADO WARNING",
"areaDesc": "Wake",
"wfo": "RAH",
"source": "nws",
"parameters": {"WIND": ["70"]},
"description": "A long narrative " * 40,
"instruction": "Take shelter.",
"geocode": {"SAME": ["037183"]},
}
slim = slim_alert_properties(fat)
assert slim == {
"event": "Tornado Warning",
"severity": "Extreme",
"headline": "TORNADO WARNING",
"areaDesc": "Wake",
"wfo": "RAH",
"source": "nws",
}
def test_wfigs_params_requests_simplified_geometry():
params = _wfigs_params("-84.5,33.8,-75.4,36.6")
assert "maxAllowableOffset" in params
assert float(params["maxAllowableOffset"]) > 0
assert params["geometryPrecision"] == 5
assert int(params["resultRecordCount"]) <= 500
# Envelope is the quantized cell, not the raw pan box.
geom = params["geometry"]
assert geom != "-84.5,33.8,-75.4,36.6"

View file

@ -0,0 +1,47 @@
"""Slim map-payload helpers (no DB)."""
from uuid import uuid4
from main import camera_map_row, fire_heat_row
def test_fire_heat_row_is_tiny():
row = {
"latitude": 35.0,
"longitude": -78.0,
"brightness": 340.1,
"confidence": "h",
"satellite": "N21",
"acq_time": "2026-08-27T00:00:00Z",
"instrument": "VIIRS",
"frp": 12.4,
}
out = fire_heat_row(row)
assert out == {"lat": 35.0, "lon": -78.0, "i": 340.1, "c": "h"}
assert "satellite" not in out
assert "frp" not in out
def test_camera_map_row_omits_urls():
cid = uuid4()
row = {
"id": cid,
"location_lat": 37.8,
"location_lon": -122.4,
"device_type": "hls",
"discovery_source": "caltrans",
"location_name": "I-80 WB",
"source_url": "https://example.invalid/playlist.m3u8",
"snapshot_url": "https://example.invalid/cam.jpg",
"vendor": "Caltrans",
"first_seen": None,
"last_seen": None,
}
out = camera_map_row(row)
assert out["id"] == str(cid)
assert out["lat"] == 37.8
assert out["lon"] == -122.4
assert out["device_type"] == "hls"
assert "source_url" not in out
assert "snapshot_url" not in out
assert "vendor" not in out