From 435f63e473af6510060bcbff4afebd2af2f46b45 Mon Sep 17 00:00:00 2001 From: Sirius DevOps Date: Thu, 27 Aug 2026 21:21:19 -0400 Subject: [PATCH] perf: shared httpx client, gzip, overlay cache-control Reuse one TLS pool for overlay upstreams (8s/3s timeouts). Gzip JSON over 1 KB. Aircraft/vessels Cache-Control max-age=5, alerts/perimeters 30. Lifespan replaces deprecated on_event startup. --- app/live_layers.py | 37 ++++++++++++++++++---- app/main.py | 58 +++++++++++++++++++++-------------- tests/test_api_live_layers.py | 1 + 3 files changed, 67 insertions(+), 29 deletions(-) diff --git a/app/live_layers.py b/app/live_layers.py index aeb9736..c2ea170 100644 --- a/app/live_layers.py +++ b/app/live_layers.py @@ -49,7 +49,8 @@ CONUS = (-125.0, 24.0, -66.0, 50.0) MAX_RADIUS_NM = 150 DEFAULT_LIMIT = 2000 -_HTTP_TIMEOUT = 25.0 +_HTTP_TIMEOUT = httpx.Timeout(8.0, connect=3.0) +_http: httpx.AsyncClient | None = None _COMPASS = { "N": 0, "NE": 45, "E": 90, "SE": 135, "S": 180, "SW": 225, "W": 270, "NW": 315, @@ -529,12 +530,36 @@ async def _ttl_get(key: str, ttl: float, factory: Callable[[], Awaitable[Any]]) return value +async def init_http() -> None: + """Shared outbound client — one TLS pool for all overlay upstreams.""" + global _http + if _http is None: + _http = httpx.AsyncClient( + timeout=_HTTP_TIMEOUT, + follow_redirects=True, + headers=_headers(), + limits=httpx.Limits(max_connections=20, max_keepalive_connections=10), + ) + + +async def close_http() -> None: + global _http + if _http is not None: + await _http.aclose() + _http = None + + async def _get_json(url: str, params: dict | None = None) -> Any: - async with httpx.AsyncClient(timeout=_HTTP_TIMEOUT, follow_redirects=True, - headers=_headers()) as client: - resp = await client.get(url, params=params) - resp.raise_for_status() - return resp.json() + if _http is None: + async with httpx.AsyncClient( + timeout=_HTTP_TIMEOUT, follow_redirects=True, headers=_headers(), + ) as client: + resp = await client.get(url, params=params) + resp.raise_for_status() + return resp.json() + resp = await _http.get(url, params=params) + resp.raise_for_status() + return resp.json() async def fetch_aircraft(bbox: str, limit: int = DEFAULT_LIMIT) -> list[dict]: diff --git a/app/main.py b/app/main.py index 8d3b7ec..7da2f84 100644 --- a/app/main.py +++ b/app/main.py @@ -14,6 +14,7 @@ from __future__ import annotations import asyncio import json import logging +from contextlib import asynccontextmanager from datetime import datetime, timedelta, timezone from decimal import Decimal from pathlib import Path @@ -21,7 +22,8 @@ from uuid import UUID import structlog from fastapi import FastAPI, HTTPException, Query -from fastapi.responses import FileResponse, HTMLResponse +from fastapi.middleware.gzip import GZipMiddleware +from fastapi.responses import FileResponse, HTMLResponse, JSONResponse from fastapi.staticfiles import StaticFiles from sqlalchemy import and_, func, or_, select, text from sqlalchemy.ext.asyncio import AsyncSession @@ -53,11 +55,30 @@ from live_layers import ( logging.basicConfig(level=logging.INFO) logger = structlog.get_logger("osint.dashboard") + +@asynccontextmanager +async def _lifespan(app: FastAPI): + await init_extensions() + from live_layers import close_http, init_http + await init_http() + from config import AISSTREAM_IN_APP + ais_task = None + if AISSTREAM_IN_APP: + from ais_stream import run_ais_worker + ais_task = asyncio.create_task(run_ais_worker()) + yield + if ais_task is not None: + ais_task.cancel() + await close_http() + + app = FastAPI( title="OSINT Dashboard", description="Real-time geospatial OSINT intelligence dashboard", version="0.1.0", + lifespan=_lifespan, ) +app.add_middleware(GZipMiddleware, minimum_size=1024) STATIC_DIR = Path(__file__).parent / "static" @@ -131,20 +152,11 @@ async def health(): return {"status": "ok", "db_time": db_time.isoformat() if db_time else None} -# ── Startup ─────────────────────────────────────────────────────────────── - -@app.on_event("startup") -async def startup(): - """Initialize PostGIS/TimescaleDB extensions on first connection. - - Schema migrations are applied by the container entrypoint (alembic upgrade - head) before uvicorn starts, so they don't run nested inside the event loop. - """ - await init_extensions() - from config import AISSTREAM_IN_APP - if AISSTREAM_IN_APP: - from ais_stream import run_ais_worker - asyncio.create_task(run_ais_worker()) +def overlay_json(data, max_age: int) -> JSONResponse: + """JSON overlay payload with a short browser/proxy TTL.""" + resp = JSONResponse(content=data) + resp.headers["Cache-Control"] = f"public, max-age={max_age}" + return resp # ── Feed Sources ────────────────────────────────────────────────────────── @@ -1114,7 +1126,7 @@ def _upstream_or_502(exc: Exception, name: str): async def map_radar(): """RainViewer frame list + IEM NEXRAD tile template. Browser fetches tiles.""" try: - return await fetch_radar_meta() + return overlay_json(await fetch_radar_meta(), 60) except Exception as exc: _upstream_or_502(exc, "radar") @@ -1127,7 +1139,7 @@ async def list_aircraft( """Viewport ADS-B last-known (ADSB.lol). Requires bbox; radius clamped ≤ 150 nm.""" _parse_bbox_query(bbox) try: - return await fetch_aircraft(bbox, limit) + return overlay_json(await fetch_aircraft(bbox, limit), 5) except ValueError as exc: raise HTTPException(422, str(exc)) from exc except Exception as exc: @@ -1143,7 +1155,7 @@ async def list_trains( if bbox: _parse_bbox_query(bbox) try: - return await fetch_trains(bbox, limit) + return overlay_json(await fetch_trains(bbox, limit), 20) except ValueError as exc: raise HTTPException(422, str(exc)) from exc except Exception as exc: @@ -1159,7 +1171,7 @@ async def list_vessels( if bbox: _parse_bbox_query(bbox) try: - return await fetch_vessels(bbox, limit) + return overlay_json(await fetch_vessels(bbox, limit), 5) except ValueError as exc: raise HTTPException(422, str(exc)) from exc @@ -1173,7 +1185,7 @@ async def list_fire_incidents( if bbox: _parse_bbox_query(bbox) try: - return await fetch_fire_incidents(bbox, limit) + return overlay_json(await fetch_fire_incidents(bbox, limit), 30) except Exception as exc: _upstream_or_502(exc, "fire-incidents") @@ -1184,7 +1196,7 @@ async def list_fire_perimeters(bbox: str | None = Query(None)): if bbox: _parse_bbox_query(bbox) try: - return await fetch_fire_perimeters(bbox) + return overlay_json(await fetch_fire_perimeters(bbox), 30) except Exception as exc: _upstream_or_502(exc, "fire-perimeters") @@ -1202,7 +1214,7 @@ async def list_weather_alerts( if bbox: _parse_bbox_query(bbox) try: - return await fetch_weather_alerts(area, bbox) + return overlay_json(await fetch_weather_alerts(area, bbox), 30) except Exception as exc: _upstream_or_502(exc, "weather-alerts") @@ -1211,7 +1223,7 @@ async def list_weather_alerts( async def list_storms(): """NHC active tropical cyclones.""" try: - return await fetch_storms() + return overlay_json(await fetch_storms(), 20) except Exception as exc: _upstream_or_502(exc, "storms") diff --git a/tests/test_api_live_layers.py b/tests/test_api_live_layers.py index a9e2c1d..ec17da8 100644 --- a/tests/test_api_live_layers.py +++ b/tests/test_api_live_layers.py @@ -36,3 +36,4 @@ def test_vessels_empty_without_ais_key(): resp = asyncio.run(_get("/api/vessels")) assert resp.status_code == 200 assert resp.json() == [] + assert "max-age" in (resp.headers.get("cache-control") or "").lower()