perf: shared httpx client, gzip, overlay cache-control
Reuse one TLS pool for overlay upstreams (8s/3s timeouts). Gzip JSON over 1 KB. Aircraft/vessels Cache-Control max-age=5, alerts/perimeters 30. Lifespan replaces deprecated on_event startup.
This commit is contained in:
parent
8f89994201
commit
435f63e473
3 changed files with 67 additions and 29 deletions
|
|
@ -49,7 +49,8 @@ CONUS = (-125.0, 24.0, -66.0, 50.0)
|
||||||
MAX_RADIUS_NM = 150
|
MAX_RADIUS_NM = 150
|
||||||
DEFAULT_LIMIT = 2000
|
DEFAULT_LIMIT = 2000
|
||||||
|
|
||||||
_HTTP_TIMEOUT = 25.0
|
_HTTP_TIMEOUT = httpx.Timeout(8.0, connect=3.0)
|
||||||
|
_http: httpx.AsyncClient | None = None
|
||||||
_COMPASS = {
|
_COMPASS = {
|
||||||
"N": 0, "NE": 45, "E": 90, "SE": 135,
|
"N": 0, "NE": 45, "E": 90, "SE": 135,
|
||||||
"S": 180, "SW": 225, "W": 270, "NW": 315,
|
"S": 180, "SW": 225, "W": 270, "NW": 315,
|
||||||
|
|
@ -529,12 +530,36 @@ async def _ttl_get(key: str, ttl: float, factory: Callable[[], Awaitable[Any]])
|
||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
async def init_http() -> None:
|
||||||
|
"""Shared outbound client — one TLS pool for all overlay upstreams."""
|
||||||
|
global _http
|
||||||
|
if _http is None:
|
||||||
|
_http = httpx.AsyncClient(
|
||||||
|
timeout=_HTTP_TIMEOUT,
|
||||||
|
follow_redirects=True,
|
||||||
|
headers=_headers(),
|
||||||
|
limits=httpx.Limits(max_connections=20, max_keepalive_connections=10),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def close_http() -> None:
|
||||||
|
global _http
|
||||||
|
if _http is not None:
|
||||||
|
await _http.aclose()
|
||||||
|
_http = None
|
||||||
|
|
||||||
|
|
||||||
async def _get_json(url: str, params: dict | None = None) -> Any:
|
async def _get_json(url: str, params: dict | None = None) -> Any:
|
||||||
async with httpx.AsyncClient(timeout=_HTTP_TIMEOUT, follow_redirects=True,
|
if _http is None:
|
||||||
headers=_headers()) as client:
|
async with httpx.AsyncClient(
|
||||||
resp = await client.get(url, params=params)
|
timeout=_HTTP_TIMEOUT, follow_redirects=True, headers=_headers(),
|
||||||
resp.raise_for_status()
|
) as client:
|
||||||
return resp.json()
|
resp = await client.get(url, params=params)
|
||||||
|
resp.raise_for_status()
|
||||||
|
return resp.json()
|
||||||
|
resp = await _http.get(url, params=params)
|
||||||
|
resp.raise_for_status()
|
||||||
|
return resp.json()
|
||||||
|
|
||||||
|
|
||||||
async def fetch_aircraft(bbox: str, limit: int = DEFAULT_LIMIT) -> list[dict]:
|
async def fetch_aircraft(bbox: str, limit: int = DEFAULT_LIMIT) -> list[dict]:
|
||||||
|
|
|
||||||
58
app/main.py
58
app/main.py
|
|
@ -14,6 +14,7 @@ from __future__ import annotations
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
@ -21,7 +22,8 @@ from uuid import UUID
|
||||||
|
|
||||||
import structlog
|
import structlog
|
||||||
from fastapi import FastAPI, HTTPException, Query
|
from fastapi import FastAPI, HTTPException, Query
|
||||||
from fastapi.responses import FileResponse, HTMLResponse
|
from fastapi.middleware.gzip import GZipMiddleware
|
||||||
|
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
from sqlalchemy import and_, func, or_, select, text
|
from sqlalchemy import and_, func, or_, select, text
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
@ -53,11 +55,30 @@ from live_layers import (
|
||||||
logging.basicConfig(level=logging.INFO)
|
logging.basicConfig(level=logging.INFO)
|
||||||
logger = structlog.get_logger("osint.dashboard")
|
logger = structlog.get_logger("osint.dashboard")
|
||||||
|
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def _lifespan(app: FastAPI):
|
||||||
|
await init_extensions()
|
||||||
|
from live_layers import close_http, init_http
|
||||||
|
await init_http()
|
||||||
|
from config import AISSTREAM_IN_APP
|
||||||
|
ais_task = None
|
||||||
|
if AISSTREAM_IN_APP:
|
||||||
|
from ais_stream import run_ais_worker
|
||||||
|
ais_task = asyncio.create_task(run_ais_worker())
|
||||||
|
yield
|
||||||
|
if ais_task is not None:
|
||||||
|
ais_task.cancel()
|
||||||
|
await close_http()
|
||||||
|
|
||||||
|
|
||||||
app = FastAPI(
|
app = FastAPI(
|
||||||
title="OSINT Dashboard",
|
title="OSINT Dashboard",
|
||||||
description="Real-time geospatial OSINT intelligence dashboard",
|
description="Real-time geospatial OSINT intelligence dashboard",
|
||||||
version="0.1.0",
|
version="0.1.0",
|
||||||
|
lifespan=_lifespan,
|
||||||
)
|
)
|
||||||
|
app.add_middleware(GZipMiddleware, minimum_size=1024)
|
||||||
|
|
||||||
STATIC_DIR = Path(__file__).parent / "static"
|
STATIC_DIR = Path(__file__).parent / "static"
|
||||||
|
|
||||||
|
|
@ -131,20 +152,11 @@ async def health():
|
||||||
return {"status": "ok", "db_time": db_time.isoformat() if db_time else None}
|
return {"status": "ok", "db_time": db_time.isoformat() if db_time else None}
|
||||||
|
|
||||||
|
|
||||||
# ── Startup ───────────────────────────────────────────────────────────────
|
def overlay_json(data, max_age: int) -> JSONResponse:
|
||||||
|
"""JSON overlay payload with a short browser/proxy TTL."""
|
||||||
@app.on_event("startup")
|
resp = JSONResponse(content=data)
|
||||||
async def startup():
|
resp.headers["Cache-Control"] = f"public, max-age={max_age}"
|
||||||
"""Initialize PostGIS/TimescaleDB extensions on first connection.
|
return resp
|
||||||
|
|
||||||
Schema migrations are applied by the container entrypoint (alembic upgrade
|
|
||||||
head) before uvicorn starts, so they don't run nested inside the event loop.
|
|
||||||
"""
|
|
||||||
await init_extensions()
|
|
||||||
from config import AISSTREAM_IN_APP
|
|
||||||
if AISSTREAM_IN_APP:
|
|
||||||
from ais_stream import run_ais_worker
|
|
||||||
asyncio.create_task(run_ais_worker())
|
|
||||||
|
|
||||||
|
|
||||||
# ── Feed Sources ──────────────────────────────────────────────────────────
|
# ── Feed Sources ──────────────────────────────────────────────────────────
|
||||||
|
|
@ -1114,7 +1126,7 @@ def _upstream_or_502(exc: Exception, name: str):
|
||||||
async def map_radar():
|
async def map_radar():
|
||||||
"""RainViewer frame list + IEM NEXRAD tile template. Browser fetches tiles."""
|
"""RainViewer frame list + IEM NEXRAD tile template. Browser fetches tiles."""
|
||||||
try:
|
try:
|
||||||
return await fetch_radar_meta()
|
return overlay_json(await fetch_radar_meta(), 60)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
_upstream_or_502(exc, "radar")
|
_upstream_or_502(exc, "radar")
|
||||||
|
|
||||||
|
|
@ -1127,7 +1139,7 @@ async def list_aircraft(
|
||||||
"""Viewport ADS-B last-known (ADSB.lol). Requires bbox; radius clamped ≤ 150 nm."""
|
"""Viewport ADS-B last-known (ADSB.lol). Requires bbox; radius clamped ≤ 150 nm."""
|
||||||
_parse_bbox_query(bbox)
|
_parse_bbox_query(bbox)
|
||||||
try:
|
try:
|
||||||
return await fetch_aircraft(bbox, limit)
|
return overlay_json(await fetch_aircraft(bbox, limit), 5)
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
raise HTTPException(422, str(exc)) from exc
|
raise HTTPException(422, str(exc)) from exc
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
|
@ -1143,7 +1155,7 @@ async def list_trains(
|
||||||
if bbox:
|
if bbox:
|
||||||
_parse_bbox_query(bbox)
|
_parse_bbox_query(bbox)
|
||||||
try:
|
try:
|
||||||
return await fetch_trains(bbox, limit)
|
return overlay_json(await fetch_trains(bbox, limit), 20)
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
raise HTTPException(422, str(exc)) from exc
|
raise HTTPException(422, str(exc)) from exc
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
|
@ -1159,7 +1171,7 @@ async def list_vessels(
|
||||||
if bbox:
|
if bbox:
|
||||||
_parse_bbox_query(bbox)
|
_parse_bbox_query(bbox)
|
||||||
try:
|
try:
|
||||||
return await fetch_vessels(bbox, limit)
|
return overlay_json(await fetch_vessels(bbox, limit), 5)
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
raise HTTPException(422, str(exc)) from exc
|
raise HTTPException(422, str(exc)) from exc
|
||||||
|
|
||||||
|
|
@ -1173,7 +1185,7 @@ async def list_fire_incidents(
|
||||||
if bbox:
|
if bbox:
|
||||||
_parse_bbox_query(bbox)
|
_parse_bbox_query(bbox)
|
||||||
try:
|
try:
|
||||||
return await fetch_fire_incidents(bbox, limit)
|
return overlay_json(await fetch_fire_incidents(bbox, limit), 30)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
_upstream_or_502(exc, "fire-incidents")
|
_upstream_or_502(exc, "fire-incidents")
|
||||||
|
|
||||||
|
|
@ -1184,7 +1196,7 @@ async def list_fire_perimeters(bbox: str | None = Query(None)):
|
||||||
if bbox:
|
if bbox:
|
||||||
_parse_bbox_query(bbox)
|
_parse_bbox_query(bbox)
|
||||||
try:
|
try:
|
||||||
return await fetch_fire_perimeters(bbox)
|
return overlay_json(await fetch_fire_perimeters(bbox), 30)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
_upstream_or_502(exc, "fire-perimeters")
|
_upstream_or_502(exc, "fire-perimeters")
|
||||||
|
|
||||||
|
|
@ -1202,7 +1214,7 @@ async def list_weather_alerts(
|
||||||
if bbox:
|
if bbox:
|
||||||
_parse_bbox_query(bbox)
|
_parse_bbox_query(bbox)
|
||||||
try:
|
try:
|
||||||
return await fetch_weather_alerts(area, bbox)
|
return overlay_json(await fetch_weather_alerts(area, bbox), 30)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
_upstream_or_502(exc, "weather-alerts")
|
_upstream_or_502(exc, "weather-alerts")
|
||||||
|
|
||||||
|
|
@ -1211,7 +1223,7 @@ async def list_weather_alerts(
|
||||||
async def list_storms():
|
async def list_storms():
|
||||||
"""NHC active tropical cyclones."""
|
"""NHC active tropical cyclones."""
|
||||||
try:
|
try:
|
||||||
return await fetch_storms()
|
return overlay_json(await fetch_storms(), 20)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
_upstream_or_502(exc, "storms")
|
_upstream_or_502(exc, "storms")
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -36,3 +36,4 @@ def test_vessels_empty_without_ais_key():
|
||||||
resp = asyncio.run(_get("/api/vessels"))
|
resp = asyncio.run(_get("/api/vessels"))
|
||||||
assert resp.status_code == 200
|
assert resp.status_code == 200
|
||||||
assert resp.json() == []
|
assert resp.json() == []
|
||||||
|
assert "max-age" in (resp.headers.get("cache-control") or "").lower()
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue