perf: faster map overlay load #4
3 changed files with 67 additions and 29 deletions
|
|
@ -49,7 +49,8 @@ CONUS = (-125.0, 24.0, -66.0, 50.0)
|
|||
MAX_RADIUS_NM = 150
|
||||
DEFAULT_LIMIT = 2000
|
||||
|
||||
_HTTP_TIMEOUT = 25.0
|
||||
_HTTP_TIMEOUT = httpx.Timeout(8.0, connect=3.0)
|
||||
_http: httpx.AsyncClient | None = None
|
||||
_COMPASS = {
|
||||
"N": 0, "NE": 45, "E": 90, "SE": 135,
|
||||
"S": 180, "SW": 225, "W": 270, "NW": 315,
|
||||
|
|
@ -529,12 +530,36 @@ async def _ttl_get(key: str, ttl: float, factory: Callable[[], Awaitable[Any]])
|
|||
return value
|
||||
|
||||
|
||||
async def init_http() -> None:
|
||||
"""Shared outbound client — one TLS pool for all overlay upstreams."""
|
||||
global _http
|
||||
if _http is None:
|
||||
_http = httpx.AsyncClient(
|
||||
timeout=_HTTP_TIMEOUT,
|
||||
follow_redirects=True,
|
||||
headers=_headers(),
|
||||
limits=httpx.Limits(max_connections=20, max_keepalive_connections=10),
|
||||
)
|
||||
|
||||
|
||||
async def close_http() -> None:
|
||||
global _http
|
||||
if _http is not None:
|
||||
await _http.aclose()
|
||||
_http = None
|
||||
|
||||
|
||||
async def _get_json(url: str, params: dict | None = None) -> Any:
|
||||
async with httpx.AsyncClient(timeout=_HTTP_TIMEOUT, follow_redirects=True,
|
||||
headers=_headers()) as client:
|
||||
resp = await client.get(url, params=params)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
if _http is None:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=_HTTP_TIMEOUT, follow_redirects=True, headers=_headers(),
|
||||
) as client:
|
||||
resp = await client.get(url, params=params)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
resp = await _http.get(url, params=params)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
async def fetch_aircraft(bbox: str, limit: int = DEFAULT_LIMIT) -> list[dict]:
|
||||
|
|
|
|||
58
app/main.py
58
app/main.py
|
|
@ -14,6 +14,7 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
|
@ -21,7 +22,8 @@ from uuid import UUID
|
|||
|
||||
import structlog
|
||||
from fastapi import FastAPI, HTTPException, Query
|
||||
from fastapi.responses import FileResponse, HTMLResponse
|
||||
from fastapi.middleware.gzip import GZipMiddleware
|
||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqlalchemy import and_, func, or_, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
|
@ -53,11 +55,30 @@ from live_layers import (
|
|||
logging.basicConfig(level=logging.INFO)
|
||||
logger = structlog.get_logger("osint.dashboard")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _lifespan(app: FastAPI):
|
||||
await init_extensions()
|
||||
from live_layers import close_http, init_http
|
||||
await init_http()
|
||||
from config import AISSTREAM_IN_APP
|
||||
ais_task = None
|
||||
if AISSTREAM_IN_APP:
|
||||
from ais_stream import run_ais_worker
|
||||
ais_task = asyncio.create_task(run_ais_worker())
|
||||
yield
|
||||
if ais_task is not None:
|
||||
ais_task.cancel()
|
||||
await close_http()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="OSINT Dashboard",
|
||||
description="Real-time geospatial OSINT intelligence dashboard",
|
||||
version="0.1.0",
|
||||
lifespan=_lifespan,
|
||||
)
|
||||
app.add_middleware(GZipMiddleware, minimum_size=1024)
|
||||
|
||||
STATIC_DIR = Path(__file__).parent / "static"
|
||||
|
||||
|
|
@ -131,20 +152,11 @@ async def health():
|
|||
return {"status": "ok", "db_time": db_time.isoformat() if db_time else None}
|
||||
|
||||
|
||||
# ── Startup ───────────────────────────────────────────────────────────────
|
||||
|
||||
@app.on_event("startup")
|
||||
async def startup():
|
||||
"""Initialize PostGIS/TimescaleDB extensions on first connection.
|
||||
|
||||
Schema migrations are applied by the container entrypoint (alembic upgrade
|
||||
head) before uvicorn starts, so they don't run nested inside the event loop.
|
||||
"""
|
||||
await init_extensions()
|
||||
from config import AISSTREAM_IN_APP
|
||||
if AISSTREAM_IN_APP:
|
||||
from ais_stream import run_ais_worker
|
||||
asyncio.create_task(run_ais_worker())
|
||||
def overlay_json(data, max_age: int) -> JSONResponse:
|
||||
"""JSON overlay payload with a short browser/proxy TTL."""
|
||||
resp = JSONResponse(content=data)
|
||||
resp.headers["Cache-Control"] = f"public, max-age={max_age}"
|
||||
return resp
|
||||
|
||||
|
||||
# ── Feed Sources ──────────────────────────────────────────────────────────
|
||||
|
|
@ -1114,7 +1126,7 @@ def _upstream_or_502(exc: Exception, name: str):
|
|||
async def map_radar():
|
||||
"""RainViewer frame list + IEM NEXRAD tile template. Browser fetches tiles."""
|
||||
try:
|
||||
return await fetch_radar_meta()
|
||||
return overlay_json(await fetch_radar_meta(), 60)
|
||||
except Exception as exc:
|
||||
_upstream_or_502(exc, "radar")
|
||||
|
||||
|
|
@ -1127,7 +1139,7 @@ async def list_aircraft(
|
|||
"""Viewport ADS-B last-known (ADSB.lol). Requires bbox; radius clamped ≤ 150 nm."""
|
||||
_parse_bbox_query(bbox)
|
||||
try:
|
||||
return await fetch_aircraft(bbox, limit)
|
||||
return overlay_json(await fetch_aircraft(bbox, limit), 5)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(422, str(exc)) from exc
|
||||
except Exception as exc:
|
||||
|
|
@ -1143,7 +1155,7 @@ async def list_trains(
|
|||
if bbox:
|
||||
_parse_bbox_query(bbox)
|
||||
try:
|
||||
return await fetch_trains(bbox, limit)
|
||||
return overlay_json(await fetch_trains(bbox, limit), 20)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(422, str(exc)) from exc
|
||||
except Exception as exc:
|
||||
|
|
@ -1159,7 +1171,7 @@ async def list_vessels(
|
|||
if bbox:
|
||||
_parse_bbox_query(bbox)
|
||||
try:
|
||||
return await fetch_vessels(bbox, limit)
|
||||
return overlay_json(await fetch_vessels(bbox, limit), 5)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(422, str(exc)) from exc
|
||||
|
||||
|
|
@ -1173,7 +1185,7 @@ async def list_fire_incidents(
|
|||
if bbox:
|
||||
_parse_bbox_query(bbox)
|
||||
try:
|
||||
return await fetch_fire_incidents(bbox, limit)
|
||||
return overlay_json(await fetch_fire_incidents(bbox, limit), 30)
|
||||
except Exception as exc:
|
||||
_upstream_or_502(exc, "fire-incidents")
|
||||
|
||||
|
|
@ -1184,7 +1196,7 @@ async def list_fire_perimeters(bbox: str | None = Query(None)):
|
|||
if bbox:
|
||||
_parse_bbox_query(bbox)
|
||||
try:
|
||||
return await fetch_fire_perimeters(bbox)
|
||||
return overlay_json(await fetch_fire_perimeters(bbox), 30)
|
||||
except Exception as exc:
|
||||
_upstream_or_502(exc, "fire-perimeters")
|
||||
|
||||
|
|
@ -1202,7 +1214,7 @@ async def list_weather_alerts(
|
|||
if bbox:
|
||||
_parse_bbox_query(bbox)
|
||||
try:
|
||||
return await fetch_weather_alerts(area, bbox)
|
||||
return overlay_json(await fetch_weather_alerts(area, bbox), 30)
|
||||
except Exception as exc:
|
||||
_upstream_or_502(exc, "weather-alerts")
|
||||
|
||||
|
|
@ -1211,7 +1223,7 @@ async def list_weather_alerts(
|
|||
async def list_storms():
|
||||
"""NHC active tropical cyclones."""
|
||||
try:
|
||||
return await fetch_storms()
|
||||
return overlay_json(await fetch_storms(), 20)
|
||||
except Exception as exc:
|
||||
_upstream_or_502(exc, "storms")
|
||||
|
||||
|
|
|
|||
|
|
@ -36,3 +36,4 @@ def test_vessels_empty_without_ais_key():
|
|||
resp = asyncio.run(_get("/api/vessels"))
|
||||
assert resp.status_code == 200
|
||||
assert resp.json() == []
|
||||
assert "max-age" in (resp.headers.get("cache-control") or "").lower()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue