From 42ca6295f045ff298cfdf3edb6600023b0b069ad Mon Sep 17 00:00:00 2001 From: Sirius DevOps Date: Sat, 29 Aug 2026 14:14:08 -0400 Subject: [PATCH] feat(gpsjam): GPS interference hex overlay API (GPSJAM/ADS-B Exchange) GET /api/map/gpsjam?date=YYYY-MM-DD (default yesterday UTC) fetches the daily gpsjam.org H3 resolution-4 CSV, converts hexes to GeoJSON polygons, and tags each with level low|medium|high (0-2% / 2-10% / >10%) via GPSJAM's denoise formula. Whole world once, 1h TTL, graceful unavailable fallback. - overlay_catalog: add kind=geojson gpsjam stub - tests: mocked-HTTP unit + API contract (9 new) --- app/live_layers.py | 89 +++++++++++++++++++++++++ app/main.py | 43 ++++++++++++- app/requirements.txt | 1 + tests/test_gpsjam.py | 150 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 280 insertions(+), 3 deletions(-) create mode 100644 tests/test_gpsjam.py diff --git a/app/live_layers.py b/app/live_layers.py index 7022387..e243fd6 100644 --- a/app/live_layers.py +++ b/app/live_layers.py @@ -54,6 +54,13 @@ TITILER_COG_TILES = f"{TITILER_PUBLIC_BASE}/cog/tiles/WebMercatorQuad/{{z}}/{{x} SENTINEL1_TTL = 20 * 60 # 15–30 min quota-friendly window SENTINEL1_ATTRIBUTION = "Copernicus Sentinel-1 / Microsoft Planetary Computer" +# GPSJAM (John Wiseman / ADS-B Exchange): daily H3 hexes of aircraft nav +# accuracy. Hexes are published as a gzip CSV at a stable per-date URL, soon +# after midnight UTC. Red/yellow != proven jamming. +GPSJAM_BASE = "https://gpsjam.org" +GPSJAM_RES = 4 +GPSJAM_TTL = 3600.0 # 1h — whole-world layer, fetched once per day effectively + IEM_NEXRAD = "https://mesonet.agron.iastate.edu/cache/tile.py/1.0.0/nexrad-n0q/{z}/{x}/{y}.png" GIBS_THERMAL = ( "https://gibs.earthdata.nasa.gov/wmts/epsg3857/best/" @@ -135,6 +142,12 @@ def overlay_catalog() -> dict: "vessels": {"id": "vessels", "kind": "points", "endpoint": "/api/vessels"}, "trains": {"id": "trains", "kind": "points", "endpoint": "/api/trains"}, "storms": {"id": "storms", "kind": "points", "endpoint": "/api/storms"}, + "gpsjam": { + "id": "gpsjam", + "kind": "geojson", + "endpoint": "/api/map/gpsjam", + "attribution": "GPSJAM / John Wiseman / ADS-B Exchange", + }, } @@ -1283,3 +1296,79 @@ async def fetch_sentinel1(bbox: str) -> dict | None: } return await _ttl_get(key, float(SENTINEL1_TTL), _load) +# ── GPSJAM ────────────────────────────────────────────────────────────────── + + +def gpsjam_level(percent_bad: float) -> str: + """Map a denoised bad-aircraft percentage to GPSJAM's three tiers.""" + if percent_bad > 10.0: + return "high" + if percent_bad > 2.0: + return "medium" + return "low" + + +def gpsjam_csv_to_geojson(text: str) -> dict: + """Convert a GPSJAM daily CSV to a world FeatureCollection of hex polygons. + + Rows are ``hex,count_good_aircraft,count_bad_aircraft``. The interference + percentage uses GPSJAM's published denoise formula + ``100 * (bad - 1) / (good + bad)``; hexes with zero bad aircraft are the + "normal" background and are dropped (the base map already shows nothing). + """ + import csv + import io + + import h3 + + features = [] + for row in csv.DictReader(io.StringIO(text)): + hex_id = (row.get("hex") or "").strip() + if not hex_id: + continue + try: + good = int(row.get("count_good_aircraft") or 0) + bad = int(row.get("count_bad_aircraft") or 0) + except (TypeError, ValueError): + continue + if bad < 1: + continue + denom = good + bad + percent = 100.0 * (bad - 1) / denom if denom > 0 else 0.0 + try: + # h3 returns (lat, lng); GeoJSON needs (lng, lat) closed rings. + ring = [[lng, lat] for lat, lng in h3.cell_to_boundary(hex_id)] + except Exception: # malformed/unknown cell id — skip + continue + ring.append(ring[0]) + features.append({ + "type": "Feature", + "geometry": {"type": "Polygon", "coordinates": [ring]}, + "properties": { + "level": gpsjam_level(percent), + "percent_bad": round(percent, 2), + "good": good, + "bad": bad, + "hex": hex_id, + }, + }) + return {"type": "FeatureCollection", "features": features} + + +async def fetch_gpsjam(date: str) -> dict: + """Fetch + convert one GPSJAM daily hex layer (whole world, 1h TTL).""" + url = f"{GPSJAM_BASE}/data/{date}-h3_{GPSJAM_RES}.csv" + + async def _load(): + if _http is None: + async with httpx.AsyncClient( + timeout=_HTTP_TIMEOUT, follow_redirects=True, headers=_headers(), + ) as client: + resp = await client.get(url) + resp.raise_for_status() + return gpsjam_csv_to_geojson(resp.text) + resp = await _http.get(url) + resp.raise_for_status() + return gpsjam_csv_to_geojson(resp.text) + + return await _ttl_get(f"gpsjam:{date}", GPSJAM_TTL, _load) diff --git a/app/main.py b/app/main.py index 6b39d71..e2ce82b 100644 --- a/app/main.py +++ b/app/main.py @@ -14,6 +14,7 @@ from __future__ import annotations import asyncio import json import logging +import re from contextlib import asynccontextmanager from datetime import datetime, timedelta, timezone from decimal import Decimal @@ -21,6 +22,7 @@ from pathlib import Path from typing import NoReturn from uuid import UUID +import httpx import structlog from fastapi import BackgroundTasks, FastAPI, HTTPException, Query, WebSocket, WebSocketDisconnect from fastapi.middleware.gzip import GZipMiddleware @@ -53,9 +55,9 @@ from keystore import KeyFormatError, delete_key, list_keys, set_key from settings_store import SettingsError, get_app_settings, list_models, set_summary_model from live_layers import ( fetch_aircraft, fetch_fire_incidents, fetch_fire_perimeters, - fetch_planespotters_photo, fetch_radar_meta, fetch_sentinel1, fetch_storms, - fetch_trains, fetch_vessels, fetch_weather_alerts, overlay_catalog, - parse_bbox, UpstreamRateLimited, + fetch_gpsjam, fetch_planespotters_photo, fetch_radar_meta, fetch_sentinel1, + fetch_storms, fetch_trains, fetch_vessels, fetch_weather_alerts, + overlay_catalog, parse_bbox, UpstreamRateLimited, ) logging.basicConfig(level=logging.INFO) @@ -1682,6 +1684,41 @@ async def list_storms(): _upstream_or_502(exc, "storms") +_GPSJAM_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +@app.get("/api/map/gpsjam") +async def map_gpsjam( + date: str | None = Query(None, description="YYYY-MM-DD (default: yesterday UTC)"), +): + """GPSJAM daily GPS-interference hex layer (whole world, GeoJSON). + + Red/yellow hexes correlate with suspected jamming but are NOT proof of it. + Fetched once per day from gpsjam.org (ADS-B Exchange data) and cached 1h. + """ + target = date + if target is None: + target = (datetime.now(timezone.utc) - timedelta(days=1)).strftime("%Y-%m-%d") + if not _GPSJAM_DATE.match(target): + raise HTTPException(422, "date must be YYYY-MM-DD") + try: + fc = await fetch_gpsjam(target) + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return JSONResponse( + {"error": "unavailable", "href": "https://gpsjam.org/", + "date": target}, + ) + _upstream_or_502(exc, "gpsjam") + except Exception as exc: + _upstream_or_502(exc, "gpsjam") + if not fc.get("features"): + return JSONResponse( + {"error": "unavailable", "href": "https://gpsjam.org/", "date": target}, + ) + return overlay_json(fc, 3600) + + @app.get("/api/map/times") async def map_layer_times( layer: str = Query(..., description="GIBS layer identifier, e.g. VIIRS_SNPP_CorrectedReflectance_TrueColor"), diff --git a/app/requirements.txt b/app/requirements.txt index 691ab71..2389199 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -12,3 +12,4 @@ python-dateutil>=2.9 structlog>=24.4 websockets>=14 cachetools>=5.5 +h3>=4.0 diff --git a/tests/test_gpsjam.py b/tests/test_gpsjam.py new file mode 100644 index 0000000..4976523 --- /dev/null +++ b/tests/test_gpsjam.py @@ -0,0 +1,150 @@ +"""GPSJAM GPS-interference overlay: level mapping, CSV→GeoJSON, API contract.""" + +import asyncio + +import httpx + +from live_layers import gpsjam_csv_to_geojson, gpsjam_level, overlay_catalog, _cache +from main import app + +BASE = "http://test" + +# A valid H3 resolution-4 cell id (the payload hex column carries these). +HEX_A = "8400c57ffffffff" + +CSV = ( + "hex,count_good_aircraft,count_bad_aircraft\n" + f"{HEX_A},0,20\n" # 100*(20-1)/20 = 95 -> high + f"{HEX_A},8,2\n" # 100*(2-1)/10 = 10 -> medium + f"{HEX_A},98,2\n" # 100*(2-1)/100 = 1 -> low + f"{HEX_A},100,0\n" # bad == 0 -> dropped +) + + +async def _get(path: str) -> httpx.Response: + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url=BASE) as client: + return await client.get(path) + + +def test_gpsjam_level_thresholds(): + assert gpsjam_level(0.0) == "low" + assert gpsjam_level(2.0) == "low" + assert gpsjam_level(2.1) == "medium" + assert gpsjam_level(10.0) == "medium" + assert gpsjam_level(10.1) == "high" + assert gpsjam_level(95.0) == "high" + + +def test_gpsjam_csv_to_geojson_levels_and_drop_zero_bad(): + fc = gpsjam_csv_to_geojson(CSV) + assert fc["type"] == "FeatureCollection" + assert len(fc["features"]) == 3 # bad==0 row dropped + levels = [f["properties"]["level"] for f in fc["features"]] + assert levels == ["high", "medium", "low"] + for f in fc["features"]: + geom = f["geometry"] + assert geom["type"] == "Polygon" + ring = geom["coordinates"][0] + assert len(ring) == 7 # 6 verts + closing point + assert ring[0] == ring[-1] + assert f["properties"]["hex"] == HEX_A + assert set(f["properties"]).issuperset({"level", "percent_bad", "good", "bad", "hex"}) + + +def test_gpsjam_csv_skips_malformed_rows(): + bad_csv = "hex,count_good_aircraft,count_bad_aircraft\n" \ + ",1,5\n" \ + f"{HEX_A},x,5\n" \ + f"{HEX_A},1,notanint\n" \ + "not_a_cell,1,5\n" + fc = gpsjam_csv_to_geojson(bad_csv) + assert fc["features"] == [] + + +def test_overlay_catalog_has_gpsjam_stub(): + entry = overlay_catalog()["gpsjam"] + assert entry["kind"] == "geojson" + assert entry["endpoint"] == "/api/map/gpsjam" + assert "GPSJAM" in entry["attribution"] + + +def test_map_gpsjam_returns_featurecollection(monkeypatch): + async def fake_fetch(date): + return {"type": "FeatureCollection", "features": [{"type": "Feature"}]} + + monkeypatch.setattr("main.fetch_gpsjam", fake_fetch) + resp = asyncio.run(_get("/api/map/gpsjam?date=2026-08-28")) + assert resp.status_code == 200 + assert resp.json()["type"] == "FeatureCollection" + assert "max-age" in (resp.headers.get("cache-control") or "").lower() + + +def test_map_gpsjam_rejects_bad_date(): + resp = asyncio.run(_get("/api/map/gpsjam?date=08-28-2026")) + assert resp.status_code == 422 + + +def test_map_gpsjam_unavailable_on_404(monkeypatch): + import httpx as _httpx + + async def fake_fetch(date): + exc = _httpx.HTTPStatusError( + "404", request=_httpx.Request("GET", "http://x"), response=_httpx.Response(404) + ) + raise exc + + monkeypatch.setattr("main.fetch_gpsjam", fake_fetch) + resp = asyncio.run(_get("/api/map/gpsjam?date=2026-08-28")) + assert resp.status_code == 200 + body = resp.json() + assert body["error"] == "unavailable" + assert body["href"] == "https://gpsjam.org/" + + +def test_map_gpsjam_unavailable_on_empty_features(monkeypatch): + async def fake_fetch(date): + return {"type": "FeatureCollection", "features": []} + + monkeypatch.setattr("main.fetch_gpsjam", fake_fetch) + resp = asyncio.run(_get("/api/map/gpsjam?date=2026-08-28")) + assert resp.status_code == 200 + assert resp.json()["error"] == "unavailable" + + +def test_fetch_gpsjam_hits_http_once_within_ttl(monkeypatch): + _cache.clear() + hits = {"n": 0} + + class FakeResp: + text = CSV + + def raise_for_status(self): + pass + + class FakeClient: + def __init__(self, **kw): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def get(self, url): + hits["n"] += 1 + assert url == "https://gpsjam.org/data/2026-08-28-h3_4.csv" + return FakeResp() + + monkeypatch.setattr("live_layers.httpx.AsyncClient", FakeClient) + monkeypatch.setattr("live_layers._http", None) + + from live_layers import fetch_gpsjam + + fc1 = asyncio.run(fetch_gpsjam("2026-08-28")) + fc2 = asyncio.run(fetch_gpsjam("2026-08-28")) + assert len(fc1["features"]) == 3 + assert fc2 == fc1 + assert hits["n"] == 1 + _cache.clear() -- 2.45.3