perf: faster map overlay load #4

Merged
sirius merged 7 commits from perf/map-layer-load into master 2026-08-27 21:24:24 -04:00
3 changed files with 67 additions and 29 deletions
Showing only changes of commit 435f63e473 - Show all commits

View file

@ -49,7 +49,8 @@ CONUS = (-125.0, 24.0, -66.0, 50.0)
MAX_RADIUS_NM = 150
DEFAULT_LIMIT = 2000
_HTTP_TIMEOUT = 25.0
_HTTP_TIMEOUT = httpx.Timeout(8.0, connect=3.0)
_http: httpx.AsyncClient | None = None
_COMPASS = {
"N": 0, "NE": 45, "E": 90, "SE": 135,
"S": 180, "SW": 225, "W": 270, "NW": 315,
@ -529,12 +530,36 @@ async def _ttl_get(key: str, ttl: float, factory: Callable[[], Awaitable[Any]])
return value
async def init_http() -> None:
"""Shared outbound client — one TLS pool for all overlay upstreams."""
global _http
if _http is None:
_http = httpx.AsyncClient(
timeout=_HTTP_TIMEOUT,
follow_redirects=True,
headers=_headers(),
limits=httpx.Limits(max_connections=20, max_keepalive_connections=10),
)
async def close_http() -> None:
global _http
if _http is not None:
await _http.aclose()
_http = None
async def _get_json(url: str, params: dict | None = None) -> Any:
async with httpx.AsyncClient(timeout=_HTTP_TIMEOUT, follow_redirects=True,
headers=_headers()) as client:
resp = await client.get(url, params=params)
resp.raise_for_status()
return resp.json()
if _http is None:
async with httpx.AsyncClient(
timeout=_HTTP_TIMEOUT, follow_redirects=True, headers=_headers(),
) as client:
resp = await client.get(url, params=params)
resp.raise_for_status()
return resp.json()
resp = await _http.get(url, params=params)
resp.raise_for_status()
return resp.json()
async def fetch_aircraft(bbox: str, limit: int = DEFAULT_LIMIT) -> list[dict]:

View file

@ -14,6 +14,7 @@ from __future__ import annotations
import asyncio
import json
import logging
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
from decimal import Decimal
from pathlib import Path
@ -21,7 +22,8 @@ from uuid import UUID
import structlog
from fastapi import FastAPI, HTTPException, Query
from fastapi.responses import FileResponse, HTMLResponse
from fastapi.middleware.gzip import GZipMiddleware
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
from fastapi.staticfiles import StaticFiles
from sqlalchemy import and_, func, or_, select, text
from sqlalchemy.ext.asyncio import AsyncSession
@ -53,11 +55,30 @@ from live_layers import (
logging.basicConfig(level=logging.INFO)
logger = structlog.get_logger("osint.dashboard")
@asynccontextmanager
async def _lifespan(app: FastAPI):
await init_extensions()
from live_layers import close_http, init_http
await init_http()
from config import AISSTREAM_IN_APP
ais_task = None
if AISSTREAM_IN_APP:
from ais_stream import run_ais_worker
ais_task = asyncio.create_task(run_ais_worker())
yield
if ais_task is not None:
ais_task.cancel()
await close_http()
app = FastAPI(
title="OSINT Dashboard",
description="Real-time geospatial OSINT intelligence dashboard",
version="0.1.0",
lifespan=_lifespan,
)
app.add_middleware(GZipMiddleware, minimum_size=1024)
STATIC_DIR = Path(__file__).parent / "static"
@ -131,20 +152,11 @@ async def health():
return {"status": "ok", "db_time": db_time.isoformat() if db_time else None}
# ── Startup ───────────────────────────────────────────────────────────────
@app.on_event("startup")
async def startup():
"""Initialize PostGIS/TimescaleDB extensions on first connection.
Schema migrations are applied by the container entrypoint (alembic upgrade
head) before uvicorn starts, so they don't run nested inside the event loop.
"""
await init_extensions()
from config import AISSTREAM_IN_APP
if AISSTREAM_IN_APP:
from ais_stream import run_ais_worker
asyncio.create_task(run_ais_worker())
def overlay_json(data, max_age: int) -> JSONResponse:
"""JSON overlay payload with a short browser/proxy TTL."""
resp = JSONResponse(content=data)
resp.headers["Cache-Control"] = f"public, max-age={max_age}"
return resp
# ── Feed Sources ──────────────────────────────────────────────────────────
@ -1114,7 +1126,7 @@ def _upstream_or_502(exc: Exception, name: str):
async def map_radar():
"""RainViewer frame list + IEM NEXRAD tile template. Browser fetches tiles."""
try:
return await fetch_radar_meta()
return overlay_json(await fetch_radar_meta(), 60)
except Exception as exc:
_upstream_or_502(exc, "radar")
@ -1127,7 +1139,7 @@ async def list_aircraft(
"""Viewport ADS-B last-known (ADSB.lol). Requires bbox; radius clamped ≤ 150 nm."""
_parse_bbox_query(bbox)
try:
return await fetch_aircraft(bbox, limit)
return overlay_json(await fetch_aircraft(bbox, limit), 5)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
except Exception as exc:
@ -1143,7 +1155,7 @@ async def list_trains(
if bbox:
_parse_bbox_query(bbox)
try:
return await fetch_trains(bbox, limit)
return overlay_json(await fetch_trains(bbox, limit), 20)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
except Exception as exc:
@ -1159,7 +1171,7 @@ async def list_vessels(
if bbox:
_parse_bbox_query(bbox)
try:
return await fetch_vessels(bbox, limit)
return overlay_json(await fetch_vessels(bbox, limit), 5)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
@ -1173,7 +1185,7 @@ async def list_fire_incidents(
if bbox:
_parse_bbox_query(bbox)
try:
return await fetch_fire_incidents(bbox, limit)
return overlay_json(await fetch_fire_incidents(bbox, limit), 30)
except Exception as exc:
_upstream_or_502(exc, "fire-incidents")
@ -1184,7 +1196,7 @@ async def list_fire_perimeters(bbox: str | None = Query(None)):
if bbox:
_parse_bbox_query(bbox)
try:
return await fetch_fire_perimeters(bbox)
return overlay_json(await fetch_fire_perimeters(bbox), 30)
except Exception as exc:
_upstream_or_502(exc, "fire-perimeters")
@ -1202,7 +1214,7 @@ async def list_weather_alerts(
if bbox:
_parse_bbox_query(bbox)
try:
return await fetch_weather_alerts(area, bbox)
return overlay_json(await fetch_weather_alerts(area, bbox), 30)
except Exception as exc:
_upstream_or_502(exc, "weather-alerts")
@ -1211,7 +1223,7 @@ async def list_weather_alerts(
async def list_storms():
"""NHC active tropical cyclones."""
try:
return await fetch_storms()
return overlay_json(await fetch_storms(), 20)
except Exception as exc:
_upstream_or_502(exc, "storms")

View file

@ -36,3 +36,4 @@ def test_vessels_empty_without_ais_key():
resp = asyncio.run(_get("/api/vessels"))
assert resp.status_code == 200
assert resp.json() == []
assert "max-age" in (resp.headers.get("cache-control") or "").lower()