diff --git a/app/config.py b/app/config.py index 8b8fd82..d93f492 100644 --- a/app/config.py +++ b/app/config.py @@ -72,6 +72,13 @@ OSINT_USER_AGENT = os.getenv( "OSINT_USER_AGENT", "osint-dashboard/1.0 (self-hosted; lancewalters94@gmail.com)" ) +# Self-hosted TiTiler (warps Sentinel-1 signed COGs into XYZ tiles on the Pi). +# TITILER_PUBLIC_BASE is the same-origin path prefix the browser hits through +# the osint.rpi.local nginx vhost (`location /titiler/` → 127.0.0.1:8001). +# TITILER_INTERNAL_URL is the compose-DNS address, used only for health checks. +TITILER_PUBLIC_BASE = os.getenv("TITILER_PUBLIC_BASE", "/titiler").rstrip("/") +TITILER_INTERNAL_URL = os.getenv("TITILER_INTERNAL_URL", "http://titiler:8000") + # AISStream (server-side WebSocket only). Idle when unset. AISSTREAM_API_KEY = os.getenv("AISSTREAM_API_KEY", "") # Bounding box(es) as minlat,minlon,maxlat,maxlon — note lat/lon order (AISStream). diff --git a/app/live_layers.py b/app/live_layers.py index 4e3b1fd..b6e80d1 100644 --- a/app/live_layers.py +++ b/app/live_layers.py @@ -15,13 +15,13 @@ import asyncio import logging import math import time -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Any, Awaitable, Callable import httpx -from urllib.parse import quote +from urllib.parse import quote, urlencode -from config import OSINT_USER_AGENT +from config import OSINT_USER_AGENT, TITILER_PUBLIC_BASE logger = logging.getLogger("osint.live_layers") @@ -45,6 +45,15 @@ WFIGS_PERIMETERS = ( ) NHC_STORMS = "https://www.nhc.noaa.gov/CurrentStorms.json" +PC_STAC_SEARCH = "https://planetarycomputer.microsoft.com/api/stac/v1/search" +PC_SAS_TOKEN = "https://planetarycomputer.microsoft.com/api/sas/v1/token/sentinel-1-grd" +# Self-hosted TiTiler on the Pi, exposed same-origin through the osint.rpi.local +# nginx vhost. Relative template — Leaflet resolves it against the page origin, +# so the browser never touches a raw loopback port or titiler.xyz. +TITILER_COG_TILES = f"{TITILER_PUBLIC_BASE}/cog/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}@1x" +SENTINEL1_TTL = 20 * 60 # 15–30 min quota-friendly window +SENTINEL1_ATTRIBUTION = "Copernicus Sentinel-1 / Microsoft Planetary Computer" + IEM_NEXRAD = "https://mesonet.agron.iastate.edu/cache/tile.py/1.0.0/nexrad-n0q/{z}/{x}/{y}.png" GIBS_THERMAL = ( "https://gibs.earthdata.nasa.gov/wmts/epsg3857/best/" @@ -110,6 +119,15 @@ def overlay_catalog() -> dict: "maxZoom": 9, "attribution": "NASA GIBS / EOSDIS", }, + "sentinel1_sar": { + "id": "sentinel1_sar", + "title": "Sentinel-1 SAR (Cloud-Penetrating)", + "kind": "raster", + "tileUrl": None, # filled from /api/map/sentinel1 (signed COG) + "endpoint": "/api/map/sentinel1", + "maxZoom": 14, + "attribution": SENTINEL1_ATTRIBUTION, + }, "nws_alerts": {"id": "nws_alerts", "kind": "geojson", "endpoint": "/api/weather-alerts"}, "wfigs_incidents": {"id": "wfigs_incidents", "kind": "points", "endpoint": "/api/fire-incidents"}, "wfigs_perimeters": {"id": "wfigs_perimeters", "kind": "geojson", "endpoint": "/api/fire-perimeters"}, @@ -1136,3 +1154,116 @@ async def fetch_storms() -> list[dict]: return transform_nhc_storms(await _get_json(NHC_STORMS)) return await _ttl_get("nhc:storms", 300.0, _load) + + +# ── Sentinel-1 SAR (Planetary Computer STAC → signed COG tile template) ──── + +class UpstreamRateLimited(Exception): + """Planetary Computer returned 429. Carries Retry-After for the client.""" + + def __init__(self, retry_after: str | None = None): + self.retry_after = retry_after + super().__init__("planetary computer rate limited") + + +async def _post_json( + url: str, json: dict | None = None, headers: dict | None = None, +) -> Any: + if _http is None: + async with httpx.AsyncClient( + timeout=_HTTP_TIMEOUT, follow_redirects=True, headers=_headers(), + ) as client: + resp = await client.post(url, json=json, headers=headers) + resp.raise_for_status() + return resp.json() + resp = await _http.post(url, json=json, headers=headers) + resp.raise_for_status() + return resp.json() + + +async def _pc_call(coro: Awaitable[Any]) -> Any: + """Run a Planetary Computer call, mapping 429 → UpstreamRateLimited.""" + try: + return await coro + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 429: + raise UpstreamRateLimited( + exc.response.headers.get("Retry-After") + ) from exc + raise + + +def sign_cog_url(href: str, token: str) -> str: + """Append a SAS token to a PC blob URL (respect existing query string).""" + sep = "&" if "?" in href else "?" + return f"{href}{sep}token={quote(token, safe='')}" + + +def sentinel1_tile_url(signed_cog: str) -> str: + """TiTiler XYZ template for a signed COG (Leaflet substitutes {z}/{x}/{y}).""" + params = urlencode({ + "url": signed_cog, + "rescale": "0,500", + "colormap_name": "cfastie", + }) + return f"{TITILER_COG_TILES}?{params}" + + +async def fetch_sentinel1(bbox: str) -> dict | None: + """Most recent Sentinel-1 GRD COG for a viewport, signed and TiTiler-ready. + + Returns the overlay tile-template dict, or ``None`` when no GRD imagery + covers the bbox in the last 7 days (caller maps to 404). Queries Planetary + Computer only when called; cached per quantized bbox + UTC day. + """ + minlon, minlat, maxlon, maxlat = parse_bbox(bbox) + day = datetime.now(timezone.utc).date().isoformat() + key = f"sentinel1:{day}:{bbox_cell_key(bbox)}" + + async def _load() -> dict | None: + now = datetime.now(timezone.utc) + week_ago = now - timedelta(days=7) + payload = { + "collections": ["sentinel-1-grd"], + "bbox": [minlon, minlat, maxlon, maxlat], + "datetime": f"{week_ago.isoformat()}/{now.isoformat()}", + "limit": 1, + "sortby": [{"field": "datetime", "direction": "desc"}], + } + data = await _pc_call(_post_json(PC_STAC_SEARCH, json=payload)) + features = data.get("features") or [] + if not features: + return None + + feat = features[0] + assets = feat.get("assets") or {} + chosen_href: str | None = None + polarization: str | None = None + for pol in ("vv", "hh"): + href = (assets.get(pol) or {}).get("href") + if href: + chosen_href = href + polarization = pol + break + if not chosen_href: + return None + + sas = await _pc_call(_get_json(PC_SAS_TOKEN)) + token = (sas or {}).get("token") + if not token: + raise RuntimeError("planetarycomputer SAS token missing") + signed = sign_cog_url(chosen_href, token) + + props = feat.get("properties") or {} + return { + "id": "sentinel-1-sar", + "kind": "raster", + "tileUrl": sentinel1_tile_url(signed), + "opacity": 0.8, + "itemId": feat.get("id"), + "datetime": props.get("datetime") or feat.get("datetime"), + "polarization": polarization, + "attribution": SENTINEL1_ATTRIBUTION, + } + + return await _ttl_get(key, float(SENTINEL1_TTL), _load) diff --git a/app/main.py b/app/main.py index 0b9e47b..6b39d71 100644 --- a/app/main.py +++ b/app/main.py @@ -53,8 +53,9 @@ from keystore import KeyFormatError, delete_key, list_keys, set_key from settings_store import SettingsError, get_app_settings, list_models, set_summary_model from live_layers import ( fetch_aircraft, fetch_fire_incidents, fetch_fire_perimeters, - fetch_planespotters_photo, fetch_radar_meta, fetch_storms, fetch_trains, - fetch_vessels, fetch_weather_alerts, overlay_catalog, parse_bbox, + fetch_planespotters_photo, fetch_radar_meta, fetch_sentinel1, fetch_storms, + fetch_trains, fetch_vessels, fetch_weather_alerts, overlay_catalog, + parse_bbox, UpstreamRateLimited, ) logging.basicConfig(level=logging.INFO) @@ -1413,6 +1414,33 @@ async def map_radar(): _upstream_or_502(exc, "radar") +@app.get("/api/map/sentinel1") +async def map_sentinel1(bbox: str = Query(..., description="minlon,minlat,maxlon,maxlat")): + """Most recent Sentinel-1 GRD as a signed COG tile template (TiTiler). + + Queries Planetary Computer only on demand; no tiles proxied through the Pi. + """ + _parse_bbox_query(bbox) + try: + result = await fetch_sentinel1(bbox) + except UpstreamRateLimited as exc: + headers = {"Retry-After": exc.retry_after} if exc.retry_after else None + raise HTTPException( + 429, "Planetary Computer rate limit", headers=headers, + ) from exc + except Exception as exc: + _upstream_or_502(exc, "sentinel1") + if result is None: + return JSONResponse( + status_code=404, + content={ + "error": "no_imagery", + "message": "No Sentinel-1 GRD in the last 7 days for this bbox", + }, + ) + return overlay_json(result, 300) + + @app.get("/api/aircraft") async def list_aircraft( bbox: str = Query(..., description="minlon,minlat,maxlon,maxlat"), diff --git a/deploy/osint-titiler.nginx.conf b/deploy/osint-titiler.nginx.conf new file mode 100644 index 0000000..829ab66 --- /dev/null +++ b/deploy/osint-titiler.nginx.conf @@ -0,0 +1,21 @@ +# osint.rpi.local — Sentinel-1 SAR tile proxy (/titiler/) +# +# GitOps: this file is the source of truth. On the Pi: +# sudo cp deploy/osint-titiler.nginx.conf /etc/nginx/snippets/osint-titiler.conf +# then `include snippets/osint-titiler.conf;` inside the osint.rpi.local server +# block (before `location /`), `nginx -t && systemctl reload nginx`. +# +# The browser hits /titiler/cog/tiles/... (same-origin). We strip the /titiler +# prefix so self-hosted TiTiler (127.0.0.1:8001) sees /cog/tiles/... and proxy +# its response straight back. Tiles are heavy PNGs — disable buffering so a +# slow client doesn't hold a worker open. + +location /titiler/ { + proxy_pass http://127.0.0.1:8001/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_buffering off; + proxy_read_timeout 300s; +} diff --git a/docker-compose.yml b/docker-compose.yml index f908975..6e38729 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -141,6 +141,9 @@ services: VESSELAPI_INTERVAL: ${VESSELAPI_INTERVAL:-17280} VESSELAPI_MAX_CALLS_PER_DAY: ${VESSELAPI_MAX_CALLS_PER_DAY:-5} VESSELAPI_IN_APP: ${VESSELAPI_IN_APP:-1} + # ── Self-hosted TiTiler (Sentinel-1 SAR tiles) ── + TITILER_PUBLIC_BASE: ${TITILER_PUBLIC_BASE:-/titiler} + TITILER_INTERNAL_URL: ${TITILER_INTERNAL_URL:-http://titiler:8000} ports: - "127.0.0.1:8000:8000" deploy: @@ -153,6 +156,27 @@ services: timeout: 5s retries: 5 + # ── Self-hosted TiTiler (Sentinel-1 SAR COG → XYZ tiles) ──────────────── + # Warps the signed Planetary Computer COG into WebMercator XYZ tiles so the + # browser never loads a multi-GB GeoTIFF. The FastAPI app signs the COG URL + # and returns a /titiler/... template; nginx routes /titiler/ here. + # Listens on 8000 INSIDE the container (the app already owns host 8000); + # published on host loopback 127.0.0.1:8001 only. + titiler: + image: ghcr.io/developmentseed/titiler:latest + container_name: osint-titiler + platform: linux/arm64 + restart: unless-stopped + environment: + - PORT=8000 + - WORKERS_PER_CORE=1 + ports: + - "127.0.0.1:8001:8000" + deploy: + resources: + limits: + memory: 1G + camera-service: build: context: . diff --git a/tests/test_live_layers.py b/tests/test_live_layers.py index ff70c8a..ebd4340 100644 --- a/tests/test_live_layers.py +++ b/tests/test_live_layers.py @@ -8,6 +8,8 @@ from live_layers import ( parse_bbox, quantize_bbox, rainviewer_tile_url, + sign_cog_url, + sentinel1_tile_url, slim_alert_properties, to_marker, transform_adsb_lol, @@ -15,6 +17,8 @@ from live_layers import ( transform_amtraker, transform_nhc_storms, transform_wfigs_incidents, + SENTINEL1_ATTRIBUTION, + TITILER_COG_TILES, _cache, _ttl_get, _wfigs_params, @@ -652,3 +656,129 @@ def test_planespotters_headers_add_contact_when_ua_is_generic(monkeypatch): ua = live_layers._planespotters_headers()["User-Agent"] assert "osint-dashboard" in ua assert "@" in ua + + +# ── Sentinel-1 SAR (Planetary Computer STAC → signed COG template) ──────── + +def test_sign_cog_url_appends_token(): + assert sign_cog_url("https://blob.example/x.tif", "tok=abc") == \ + "https://blob.example/x.tif?token=tok%3Dabc" + # Existing query string → append with & + assert sign_cog_url("https://blob.example/x.tif?st=1", "tok") == \ + "https://blob.example/x.tif?st=1&token=tok" + + +def test_sentinel1_tile_url_contains_titiler_rescale_and_cfastie(): + signed = "https://blob.example/x.tif?token=secret" + url = sentinel1_tile_url(signed) + assert url.startswith(TITILER_COG_TILES + "?") + assert "WebMercatorQuad/{z}/{x}/{y}@1x" in url + assert "url=https%3A%2F%2Fblob.example%2Fx.tif%3Ftoken%3Dsecret" in url + assert "rescale=0%2C500" in url + assert "colormap_name=cfastie" in url + + +def test_sentinel1_tile_url_is_same_origin_relative(): + # Self-hosted TiTiler: the browser must hit the Pi's nginx vhost, not + # titiler.xyz or a raw host:port. The template is a root-relative path. + url = sentinel1_tile_url("https://blob.example/x.tif") + assert url.startswith("/titiler/cog/tiles/WebMercatorQuad/") + assert "://" not in url + assert "titiler.xyz" not in url + + +def _stac_feature(assets: dict) -> dict: + return { + "type": "Feature", + "id": "S1A_IW_GRDH_1SDV_20240820T000000", + "properties": {"datetime": "2024-08-20T00:00:00Z"}, + "assets": assets, + } + + +def test_fetch_sentinel1_vv_signed_tile_url(monkeypatch): + import asyncio + from live_layers import fetch_sentinel1, _cache + + calls = [] + + async def fake_post(url, json=None, headers=None): + calls.append(("post", url, json)) + return {"features": [_stac_feature({ + "vv": {"href": "https://blob.example/grd-vv.tif"}, + })]} + + async def fake_get(url, params=None, headers=None): + calls.append(("get", url)) + return {"token": "sig=abc123"} + + monkeypatch.setattr("live_layers._post_json", fake_post) + monkeypatch.setattr("live_layers._get_json", fake_get) + _cache.clear() + + out = asyncio.run(fetch_sentinel1("-80,35,-79,36")) + assert out["id"] == "sentinel-1-sar" + assert out["kind"] == "raster" + assert out["polarization"] == "vv" + assert out["opacity"] == 0.8 + assert out["itemId"].startswith("S1A") + assert out["attribution"] == SENTINEL1_ATTRIBUTION + assert "WebMercatorQuad/{z}/{x}/{y}@1x" in out["tileUrl"] + assert "rescale=0%2C500" in out["tileUrl"] + assert "colormap_name=cfastie" in out["tileUrl"] + # SAS token "sig=abc123" is signed as ?token=sig%3Dabc123, then the whole + # COG URL is percent-encoded again as a query param (=> sig%253Dabc123). + assert "sig%253Dabc123" in out["tileUrl"] + # STAC search payload shape + post_url, post_json = calls[0][1], calls[0][2] + assert post_url.endswith("/api/stac/v1/search") + assert post_json["collections"] == ["sentinel-1-grd"] + assert post_json["limit"] == 1 + assert post_json["sortby"][0]["direction"] == "desc" + + +def test_fetch_sentinel1_uses_hh_when_vv_missing(monkeypatch): + import asyncio + from live_layers import fetch_sentinel1, _cache + + async def fake_post(url, json=None, headers=None): + return {"features": [_stac_feature({ + "hh": {"href": "https://blob.example/grd-hh.tif"}, + })]} + + async def fake_get(url, params=None, headers=None): + return {"token": "tok"} + + monkeypatch.setattr("live_layers._post_json", fake_post) + monkeypatch.setattr("live_layers._get_json", fake_get) + _cache.clear() + + out = asyncio.run(fetch_sentinel1("-80,35,-79,36")) + assert out["polarization"] == "hh" + assert "url=https%3A%2F%2Fblob.example%2Fgrd-hh.tif" in out["tileUrl"] + + +def test_fetch_sentinel1_none_on_empty_features(monkeypatch): + import asyncio + from live_layers import fetch_sentinel1, _cache + + async def fake_post(url, json=None, headers=None): + return {"features": []} + + monkeypatch.setattr("live_layers._post_json", fake_post) + _cache.clear() + + assert asyncio.run(fetch_sentinel1("-80,35,-79,36")) is None + + +def test_fetch_sentinel1_none_when_no_vv_or_hh(monkeypatch): + import asyncio + from live_layers import fetch_sentinel1, _cache + + async def fake_post(url, json=None, headers=None): + return {"features": [_stac_feature({"thumbnail": {"href": "https://x"}})]} + + monkeypatch.setattr("live_layers._post_json", fake_post) + _cache.clear() + + assert asyncio.run(fetch_sentinel1("-80,35,-79,36")) is None