diff --git a/app/live_layers.py b/app/live_layers.py
index 3e214c6..c2ea170 100644
--- a/app/live_layers.py
+++ b/app/live_layers.py
@@ -49,7 +49,8 @@ CONUS = (-125.0, 24.0, -66.0, 50.0)
MAX_RADIUS_NM = 150
DEFAULT_LIMIT = 2000
-_HTTP_TIMEOUT = 25.0
+_HTTP_TIMEOUT = httpx.Timeout(8.0, connect=3.0)
+_http: httpx.AsyncClient | None = None
_COMPASS = {
"N": 0, "NE": 45, "E": 90, "SE": 135,
"S": 180, "SW": 225, "W": 270, "NW": 315,
@@ -58,7 +59,9 @@ _COMPASS = {
}
_cache: dict[str, tuple[float, Any]] = {}
-_cache_lock = asyncio.Lock()
+_key_locks: dict[str, asyncio.Lock] = {}
+_key_locks_guard = asyncio.Lock()
+_QUANT = 0.25 # degrees — pan jitter inside a cell reuses the TTL entry
# Last-known AIS positions (MMSI -> marker). Filled by ais_stream worker.
vessel_last_known: dict[str, dict] = {}
@@ -116,6 +119,33 @@ def parse_bbox(bbox: str) -> tuple[float, float, float, float]:
return minlon, minlat, maxlon, maxlat
+def quantize_bbox(
+ minlon: float, minlat: float, maxlon: float, maxlat: float,
+ step: float = _QUANT,
+) -> tuple[float, float, float, float]:
+ """Snap a viewport to a coarse cell so nearby pans share a cache key.
+
+ The returned envelope is expanded to cover the original box.
+ """
+ def q_down(v: float, lo: float, hi: float) -> float:
+ v = max(lo, min(hi, v))
+ return math.floor(v / step) * step
+
+ return (
+ round(q_down(minlon, -180.0, 180.0), 4),
+ round(q_down(minlat, -90.0, 90.0), 4),
+ round(q_down(maxlon, -180.0, 180.0) + step, 4),
+ round(q_down(maxlat, -90.0, 90.0) + step, 4),
+ )
+
+
+def bbox_cell_key(bbox: str | None) -> str:
+ """Stable cache-key fragment for a viewport (or 'all')."""
+ if not bbox:
+ return "all"
+ return ",".join(f"{v:.4f}" for v in quantize_bbox(*parse_bbox(bbox)))
+
+
def bbox_center_radius_nm(
minlon: float, minlat: float, maxlon: float, maxlat: float,
) -> tuple[float, float, int]:
@@ -177,6 +207,78 @@ def filter_points_bbox(
return out
+_ALERT_KEEP = ("event", "severity", "headline", "areaDesc", "wfo", "source")
+
+
+def slim_alert_properties(props: dict | None) -> dict:
+ """Keep only the fields the map popup reads."""
+ src = props or {}
+ return {k: src.get(k) for k in _ALERT_KEEP}
+
+
+def _walk_coords(coords: Any, acc: list[float]) -> None:
+ if not coords:
+ return
+ first = coords[0]
+ if isinstance(first, (int, float)):
+ lon, lat = float(coords[0]), float(coords[1])
+ acc[0] = min(acc[0], lon)
+ acc[1] = min(acc[1], lat)
+ acc[2] = max(acc[2], lon)
+ acc[3] = max(acc[3], lat)
+ return
+ for child in coords:
+ _walk_coords(child, acc)
+
+
+def _geom_envelope(geom: dict | None) -> tuple[float, float, float, float] | None:
+ if not geom or not isinstance(geom, dict):
+ return None
+ if geom.get("type") == "GeometryCollection":
+ env: list[float] | None = None
+ for g in geom.get("geometries") or []:
+ e = _geom_envelope(g)
+ if e is None:
+ continue
+ if env is None:
+ env = list(e)
+ else:
+ env[0] = min(env[0], e[0])
+ env[1] = min(env[1], e[1])
+ env[2] = max(env[2], e[2])
+ env[3] = max(env[3], e[3])
+ return tuple(env) if env else None # type: ignore[return-value]
+ coords = geom.get("coordinates")
+ if coords is None:
+ return None
+ acc = [180.0, 90.0, -180.0, -90.0]
+ try:
+ _walk_coords(coords, acc)
+ except (TypeError, ValueError, IndexError):
+ return None
+ if acc[0] > acc[2]:
+ return None
+ return acc[0], acc[1], acc[2], acc[3]
+
+
+def clip_fc_to_bbox(
+ fc: dict | None,
+ minlon: float, minlat: float, maxlon: float, maxlat: float,
+) -> dict:
+ """Drop features whose geometry envelope misses the viewport. No shapely."""
+ box = (minlon, minlat, maxlon, maxlat)
+ out = []
+ for feat in (fc or {}).get("features") or []:
+ if not isinstance(feat, dict):
+ continue
+ env = _geom_envelope(feat.get("geometry"))
+ if env is None:
+ continue
+ if env[0] <= box[2] and env[2] >= box[0] and env[1] <= box[3] and env[3] >= box[1]:
+ out.append(feat)
+ return {"type": "FeatureCollection", "features": out}
+
+
def _f(value: object) -> float | None:
if value is None or value == "":
return None
@@ -404,12 +506,22 @@ def _headers() -> dict[str, str]:
return {"User-Agent": OSINT_USER_AGENT, "Accept": "application/json"}
+async def _lock_for(key: str) -> asyncio.Lock:
+ async with _key_locks_guard:
+ lock = _key_locks.get(key)
+ if lock is None:
+ lock = asyncio.Lock()
+ _key_locks[key] = lock
+ return lock
+
+
async def _ttl_get(key: str, ttl: float, factory: Callable[[], Awaitable[Any]]) -> Any:
now = time.monotonic()
hit = _cache.get(key)
if hit and now - hit[0] < ttl:
return hit[1]
- async with _cache_lock:
+ lock = await _lock_for(key)
+ async with lock:
hit = _cache.get(key)
if hit and time.monotonic() - hit[0] < ttl:
return hit[1]
@@ -418,17 +530,42 @@ async def _ttl_get(key: str, ttl: float, factory: Callable[[], Awaitable[Any]])
return value
+async def init_http() -> None:
+ """Shared outbound client — one TLS pool for all overlay upstreams."""
+ global _http
+ if _http is None:
+ _http = httpx.AsyncClient(
+ timeout=_HTTP_TIMEOUT,
+ follow_redirects=True,
+ headers=_headers(),
+ limits=httpx.Limits(max_connections=20, max_keepalive_connections=10),
+ )
+
+
+async def close_http() -> None:
+ global _http
+ if _http is not None:
+ await _http.aclose()
+ _http = None
+
+
async def _get_json(url: str, params: dict | None = None) -> Any:
- async with httpx.AsyncClient(timeout=_HTTP_TIMEOUT, follow_redirects=True,
- headers=_headers()) as client:
- resp = await client.get(url, params=params)
- resp.raise_for_status()
- return resp.json()
+ if _http is None:
+ async with httpx.AsyncClient(
+ timeout=_HTTP_TIMEOUT, follow_redirects=True, headers=_headers(),
+ ) as client:
+ resp = await client.get(url, params=params)
+ resp.raise_for_status()
+ return resp.json()
+ resp = await _http.get(url, params=params)
+ resp.raise_for_status()
+ return resp.json()
async def fetch_aircraft(bbox: str, limit: int = DEFAULT_LIMIT) -> list[dict]:
minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
- lat, lon, radius = bbox_center_radius_nm(minlon, minlat, maxlon, maxlat)
+ qminlon, qminlat, qmaxlon, qmaxlat = quantize_bbox(minlon, minlat, maxlon, maxlat)
+ lat, lon, radius = bbox_center_radius_nm(qminlon, qminlat, qmaxlon, qmaxlat)
cache_key = f"adsb:{lat:.2f}:{lon:.2f}:{radius}"
async def _load():
@@ -485,15 +622,17 @@ async def upsert_vessel(marker: dict) -> None:
}
-def _wfigs_params(bbox: str | None) -> dict:
+def _wfigs_params(bbox: str | None, *, offset_m: float = 250.0) -> dict:
params = {
"where": "1=1",
"outSR": "4326",
"f": "geojson",
- "resultRecordCount": 2000,
+ "resultRecordCount": 500,
+ "maxAllowableOffset": offset_m / 111_320.0, # metres → degrees
+ "geometryPrecision": 5,
}
if bbox:
- minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
+ minlon, minlat, maxlon, maxlat = quantize_bbox(*parse_bbox(bbox))
params["geometry"] = f"{minlon},{minlat},{maxlon},{maxlat}"
params["geometryType"] = "esriGeometryEnvelope"
params["inSR"] = "4326"
@@ -511,7 +650,7 @@ async def fetch_fire_incidents(bbox: str | None, limit: int = DEFAULT_LIMIT) ->
async def _load():
return transform_wfigs_incidents(await _get_json(WFIGS_INCIDENTS, params))
- rows = await _ttl_get(f"wfigs:inc:{bbox or 'all'}", 600.0, _load)
+ rows = await _ttl_get(f"wfigs:inc:{bbox_cell_key(bbox)}", 600.0, _load)
return rows[:limit]
@@ -525,7 +664,7 @@ async def fetch_fire_perimeters(bbox: str | None) -> dict:
async def _load():
return await _get_json(WFIGS_PERIMETERS, params)
- fc = await _ttl_get(f"wfigs:per:{bbox or 'all'}", 600.0, _load)
+ fc = await _ttl_get(f"wfigs:per:{bbox_cell_key(bbox)}", 600.0, _load)
if not isinstance(fc, dict):
return {"type": "FeatureCollection", "features": []}
return fc
@@ -534,44 +673,50 @@ async def fetch_fire_perimeters(bbox: str | None) -> dict:
async def fetch_weather_alerts(area: str | None, bbox: str | None) -> dict:
"""Cached NWS active alerts + IEM storm-based warning polygons."""
+ async def _load_iem():
+ try:
+ data = await _get_json(IEM_SBW)
+ except Exception:
+ return {"type": "FeatureCollection", "features": []}
+ return data if isinstance(data, dict) else {"type": "FeatureCollection", "features": []}
+
async def _load():
nws_params: dict[str, str] = {"status": "actual"}
+ clip_box = None
if area:
nws_params["area"] = area.upper()
elif bbox:
- minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
- nws_params["bbox"] = f"{minlon},{minlat},{maxlon},{maxlat}"
+ clip_box = quantize_bbox(*parse_bbox(bbox))
+ nws_params["bbox"] = f"{clip_box[0]},{clip_box[1]},{clip_box[2]},{clip_box[3]}"
nws_fc: dict = {"features": []}
- sbw_fc: dict = {"features": []}
try:
nws_fc = await _get_json(NWS_ALERTS, nws_params)
except Exception:
nws_fc = {"features": []}
- try:
- sbw_fc = await _get_json(IEM_SBW)
- except Exception:
- sbw_fc = {"features": []}
+ sbw_fc = await _ttl_get("iem:sbw", 45.0, _load_iem)
features = []
for feat in nws_fc.get("features") or []:
- props = feat.get("properties") or {}
+ if not isinstance(feat, dict):
+ continue
+ props = dict(feat.get("properties") or {})
props["source"] = "nws"
- feat["properties"] = props
- features.append(feat)
- for feat in sbw_fc.get("features") or []:
- props = feat.get("properties") or {}
+ features.append({**feat, "properties": slim_alert_properties(props)})
+ for feat in (sbw_fc or {}).get("features") or []:
+ if not isinstance(feat, dict):
+ continue
+ props = dict(feat.get("properties") or {})
props["source"] = "iem-sbw"
- # IEM uses `ps` (phenomenon) / `wfo`; map a display event.
if "event" not in props:
props["event"] = props.get("ps") or "Storm-based warning"
- feat["properties"] = props
- if bbox:
- # Cheap reject: skip if no geometry; keep otherwise (polygons).
- if not feat.get("geometry"):
- continue
- features.append(feat)
- return {"type": "FeatureCollection", "features": features}
+ features.append({**feat, "properties": slim_alert_properties(props)})
+ merged = {"type": "FeatureCollection", "features": features}
+ if clip_box:
+ merged = clip_fc_to_bbox(merged, *clip_box)
+ elif bbox:
+ merged = clip_fc_to_bbox(merged, *parse_bbox(bbox))
+ return merged
- key = f"alerts:{area or ''}:{bbox or ''}"
+ key = f"alerts:{area or ''}:{bbox_cell_key(bbox) if bbox else ''}"
return await _ttl_get(key, 30.0, _load)
diff --git a/app/main.py b/app/main.py
index 3b4defe..7da2f84 100644
--- a/app/main.py
+++ b/app/main.py
@@ -14,6 +14,7 @@ from __future__ import annotations
import asyncio
import json
import logging
+from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
from decimal import Decimal
from pathlib import Path
@@ -21,7 +22,8 @@ from uuid import UUID
import structlog
from fastapi import FastAPI, HTTPException, Query
-from fastapi.responses import FileResponse, HTMLResponse
+from fastapi.middleware.gzip import GZipMiddleware
+from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from sqlalchemy import and_, func, or_, select, text
from sqlalchemy.ext.asyncio import AsyncSession
@@ -53,11 +55,30 @@ from live_layers import (
logging.basicConfig(level=logging.INFO)
logger = structlog.get_logger("osint.dashboard")
+
+@asynccontextmanager
+async def _lifespan(app: FastAPI):
+ await init_extensions()
+ from live_layers import close_http, init_http
+ await init_http()
+ from config import AISSTREAM_IN_APP
+ ais_task = None
+ if AISSTREAM_IN_APP:
+ from ais_stream import run_ais_worker
+ ais_task = asyncio.create_task(run_ais_worker())
+ yield
+ if ais_task is not None:
+ ais_task.cancel()
+ await close_http()
+
+
app = FastAPI(
title="OSINT Dashboard",
description="Real-time geospatial OSINT intelligence dashboard",
version="0.1.0",
+ lifespan=_lifespan,
)
+app.add_middleware(GZipMiddleware, minimum_size=1024)
STATIC_DIR = Path(__file__).parent / "static"
@@ -69,17 +90,17 @@ def event_to_out(row: dict) -> EventOut:
return EventOut(
id=row["id"],
source_type=row["source_type"],
- source_id=row["source_id"],
- title=row["title"],
- body=row["body"],
- url=row["url"],
- sentiment_score=row["sentiment_score"],
- sentiment_label=row["sentiment_label"],
- location_lat=row["location_lat"],
- location_lon=row["location_lon"],
- location_name=row["location_name"],
- entities=row["entities"],
- tags=row["tags"],
+ source_id=row.get("source_id"),
+ title=row.get("title"),
+ body=row.get("body"),
+ url=row.get("url"),
+ sentiment_score=row.get("sentiment_score"),
+ sentiment_label=row.get("sentiment_label"),
+ location_lat=row.get("location_lat"),
+ location_lon=row.get("location_lon"),
+ location_name=row.get("location_name"),
+ entities=row.get("entities"),
+ tags=row.get("tags"),
ingested_at=row["ingested_at"],
source_timestamp=row["source_timestamp"],
)
@@ -131,20 +152,11 @@ async def health():
return {"status": "ok", "db_time": db_time.isoformat() if db_time else None}
-# ── Startup ───────────────────────────────────────────────────────────────
-
-@app.on_event("startup")
-async def startup():
- """Initialize PostGIS/TimescaleDB extensions on first connection.
-
- Schema migrations are applied by the container entrypoint (alembic upgrade
- head) before uvicorn starts, so they don't run nested inside the event loop.
- """
- await init_extensions()
- from config import AISSTREAM_IN_APP
- if AISSTREAM_IN_APP:
- from ais_stream import run_ais_worker
- asyncio.create_task(run_ais_worker())
+def overlay_json(data, max_age: int) -> JSONResponse:
+ """JSON overlay payload with a short browser/proxy TTL."""
+ resp = JSONResponse(content=data)
+ resp.headers["Cache-Control"] = f"public, max-age={max_age}"
+ return resp
# ── Feed Sources ──────────────────────────────────────────────────────────
@@ -218,7 +230,15 @@ async def list_events(
):
"""List recent ingested events."""
async with async_session() as session:
- stmt = select(events).order_by(events.c.ingested_at.desc())
+ if has_coords:
+ stmt = select(
+ events.c.id, events.c.source_type, events.c.source_id,
+ events.c.title, events.c.url,
+ events.c.location_lat, events.c.location_lon, events.c.location_name,
+ events.c.ingested_at, events.c.source_timestamp,
+ ).order_by(events.c.ingested_at.desc())
+ else:
+ stmt = select(events).order_by(events.c.ingested_at.desc())
if source_type:
stmt = stmt.where(events.c.source_type == source_type.value)
if since:
@@ -272,7 +292,29 @@ async def create_event(payload: EventCreate):
# ── Active Fires / Hotspots (NASA FIRMS) ─────────────────────────────────
-@app.get("/api/fires", response_model=list[FireOut])
+def fire_heat_row(r) -> dict:
+ """Minimal FIRMS point for the heatmap overlay."""
+ return {
+ "lat": r["latitude"],
+ "lon": r["longitude"],
+ "i": r["brightness"],
+ "c": r["confidence"],
+ }
+
+
+def camera_map_row(r) -> dict:
+ """Minimal camera pin — URLs stay off the list payload."""
+ return {
+ "id": str(r["id"]),
+ "lat": r["location_lat"],
+ "lon": r["location_lon"],
+ "device_type": r["device_type"],
+ "discovery_source": r["discovery_source"],
+ "location_name": r["location_name"],
+ }
+
+
+@app.get("/api/fires", response_model=None)
async def list_fires(
bbox: str | None = Query(
None,
@@ -286,12 +328,16 @@ async def list_fires(
"(ISO 8601, e.g. '2026-08-24T12:00:00Z').",
),
limit: int = Query(2000, ge=1, le=10000),
+ format: str = Query("full", description="'full' FireOut rows or 'heat' {lat,lon,i,c}"),
):
"""List stored FIRMS active fire/hotspot detections as JSON.
This is the data contract for the map's fire heatmap overlay: the frontend
- calls `GET /api/fires?bbox=...&since=...` and renders the returned points.
+ calls `GET /api/fires?bbox=...&since=...&format=heat` and renders the points.
"""
+ fmt = (format or "full").lower()
+ if fmt not in ("full", "heat"):
+ raise HTTPException(422, "format must be 'full' or 'heat'")
async with async_session() as session:
stmt = select(fires).order_by(fires.c.acq_time.desc())
if since:
@@ -316,6 +362,8 @@ async def list_fires(
)
stmt = stmt.limit(limit)
rows = (await session.execute(stmt)).mappings().all()
+ if fmt == "heat":
+ return [fire_heat_row(r) for r in rows]
return [
FireOut(
latitude=r["latitude"], longitude=r["longitude"],
@@ -798,7 +846,11 @@ async def list_cameras(
stmt = stmt.where(cam_table.c.discovery_source == source)
rows = (await session.execute(stmt.limit(limit))).mappings().all()
- return [{
+ return [camera_map_row(r) for r in rows]
+
+
+def camera_detail_row(r) -> dict:
+ return {
"id": str(r["id"]),
"source_url": r["source_url"],
"snapshot_url": r["snapshot_url"],
@@ -808,9 +860,23 @@ async def list_cameras(
"location_name": r["location_name"],
"vendor": r["vendor"],
"device_type": r["device_type"],
- "first_seen": r["first_seen"].isoformat(),
- "last_seen": r["last_seen"].isoformat(),
- } for r in rows]
+ "first_seen": r["first_seen"].isoformat() if r["first_seen"] else None,
+ "last_seen": r["last_seen"].isoformat() if r["last_seen"] else None,
+ }
+
+
+@app.get("/api/cameras/{camera_id}")
+async def get_camera(camera_id: UUID):
+ """Full camera row for a map popup. List endpoint stays slim."""
+ from camera_models import cameras as cam_table
+
+ async with async_session() as session:
+ row = (await session.execute(
+ select(cam_table).where(cam_table.c.id == camera_id)
+ )).mappings().one_or_none()
+ if not row:
+ raise HTTPException(404, "Camera not found")
+ return camera_detail_row(row)
@app.get("/api/cameras/{camera_id}/snapshot")
@@ -959,6 +1025,10 @@ async def list_news(
),
limit: int = Query(50, ge=1, le=500),
offset: int = Query(0, ge=0),
+ include_content: bool = Query(
+ False,
+ description="Include full article body. Default false — ticker/list only need title/url.",
+ ),
):
"""Most recent scraped news articles (newest first)."""
async with async_session() as session:
@@ -974,7 +1044,7 @@ async def list_news(
return [
NewsArticleOut(
id=r["id"], title=r["title"], url=r["url"],
- content=r["content"], domain=r["domain"],
+ content=r["content"] if include_content else None, domain=r["domain"],
timestamp=r["timestamp"],
)
for r in rows
@@ -1056,7 +1126,7 @@ def _upstream_or_502(exc: Exception, name: str):
async def map_radar():
"""RainViewer frame list + IEM NEXRAD tile template. Browser fetches tiles."""
try:
- return await fetch_radar_meta()
+ return overlay_json(await fetch_radar_meta(), 60)
except Exception as exc:
_upstream_or_502(exc, "radar")
@@ -1069,7 +1139,7 @@ async def list_aircraft(
"""Viewport ADS-B last-known (ADSB.lol). Requires bbox; radius clamped ≤ 150 nm."""
_parse_bbox_query(bbox)
try:
- return await fetch_aircraft(bbox, limit)
+ return overlay_json(await fetch_aircraft(bbox, limit), 5)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
except Exception as exc:
@@ -1085,7 +1155,7 @@ async def list_trains(
if bbox:
_parse_bbox_query(bbox)
try:
- return await fetch_trains(bbox, limit)
+ return overlay_json(await fetch_trains(bbox, limit), 20)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
except Exception as exc:
@@ -1101,7 +1171,7 @@ async def list_vessels(
if bbox:
_parse_bbox_query(bbox)
try:
- return await fetch_vessels(bbox, limit)
+ return overlay_json(await fetch_vessels(bbox, limit), 5)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
@@ -1115,7 +1185,7 @@ async def list_fire_incidents(
if bbox:
_parse_bbox_query(bbox)
try:
- return await fetch_fire_incidents(bbox, limit)
+ return overlay_json(await fetch_fire_incidents(bbox, limit), 30)
except Exception as exc:
_upstream_or_502(exc, "fire-incidents")
@@ -1126,7 +1196,7 @@ async def list_fire_perimeters(bbox: str | None = Query(None)):
if bbox:
_parse_bbox_query(bbox)
try:
- return await fetch_fire_perimeters(bbox)
+ return overlay_json(await fetch_fire_perimeters(bbox), 30)
except Exception as exc:
_upstream_or_502(exc, "fire-perimeters")
@@ -1144,7 +1214,7 @@ async def list_weather_alerts(
if bbox:
_parse_bbox_query(bbox)
try:
- return await fetch_weather_alerts(area, bbox)
+ return overlay_json(await fetch_weather_alerts(area, bbox), 30)
except Exception as exc:
_upstream_or_502(exc, "weather-alerts")
@@ -1153,7 +1223,7 @@ async def list_weather_alerts(
async def list_storms():
"""NHC active tropical cyclones."""
try:
- return await fetch_storms()
+ return overlay_json(await fetch_storms(), 20)
except Exception as exc:
_upstream_or_502(exc, "storms")
diff --git a/app/static/index.html b/app/static/index.html
index 98c44a0..4012750 100644
--- a/app/static/index.html
+++ b/app/static/index.html
@@ -1026,10 +1026,22 @@
-