Merge pull request 'feat(sentinel1): Sentinel-1 SAR STAC -> self-hosted TiTiler tile template' (#20) from feat/sentinel1-sar-titiler into master

Reviewed-on: #20

app/config.py

@@ -72,6 +72,13 @@ OSINT_USER_AGENT = os.getenv(
     "OSINT_USER_AGENT", "osint-dashboard/1.0 (self-hosted; lancewalters94@gmail.com)"
 )
 
+# Self-hosted TiTiler (warps Sentinel-1 signed COGs into XYZ tiles on the Pi).
+# TITILER_PUBLIC_BASE is the same-origin path prefix the browser hits through
+# the osint.rpi.local nginx vhost (`location /titiler/` → 127.0.0.1:8001).
+# TITILER_INTERNAL_URL is the compose-DNS address, used only for health checks.
+TITILER_PUBLIC_BASE = os.getenv("TITILER_PUBLIC_BASE", "/titiler").rstrip("/")
+TITILER_INTERNAL_URL = os.getenv("TITILER_INTERNAL_URL", "http://titiler:8000")
+
 # AISStream (server-side WebSocket only). Idle when unset.
 AISSTREAM_API_KEY = os.getenv("AISSTREAM_API_KEY", "")
 # Bounding box(es) as minlat,minlon,maxlat,maxlon — note lat/lon order (AISStream).

app/live_layers.py

@@ -15,13 +15,13 @@ import asyncio
 import logging
 import math
 import time
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from typing import Any, Awaitable, Callable
 
 import httpx
-from urllib.parse import quote
+from urllib.parse import quote, urlencode
 
-from config import OSINT_USER_AGENT
+from config import OSINT_USER_AGENT, TITILER_PUBLIC_BASE
 
 logger = logging.getLogger("osint.live_layers")
 
@@ -45,6 +45,15 @@ WFIGS_PERIMETERS = (
 )
 NHC_STORMS = "https://www.nhc.noaa.gov/CurrentStorms.json"
 
+PC_STAC_SEARCH = "https://planetarycomputer.microsoft.com/api/stac/v1/search"
+PC_SAS_TOKEN = "https://planetarycomputer.microsoft.com/api/sas/v1/token/sentinel-1-grd"
+# Self-hosted TiTiler on the Pi, exposed same-origin through the osint.rpi.local
+# nginx vhost. Relative template — Leaflet resolves it against the page origin,
+# so the browser never touches a raw loopback port or titiler.xyz.
+TITILER_COG_TILES = f"{TITILER_PUBLIC_BASE}/cog/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}@1x"
+SENTINEL1_TTL = 20 * 60  # 15–30 min quota-friendly window
+SENTINEL1_ATTRIBUTION = "Copernicus Sentinel-1 / Microsoft Planetary Computer"
+
 IEM_NEXRAD = "https://mesonet.agron.iastate.edu/cache/tile.py/1.0.0/nexrad-n0q/{z}/{x}/{y}.png"
 GIBS_THERMAL = (
     "https://gibs.earthdata.nasa.gov/wmts/epsg3857/best/"
@@ -110,6 +119,15 @@ def overlay_catalog() -> dict:
             "maxZoom": 9,
             "attribution": "NASA GIBS / EOSDIS",
         },
+        "sentinel1_sar": {
+            "id": "sentinel1_sar",
+            "title": "Sentinel-1 SAR (Cloud-Penetrating)",
+            "kind": "raster",
+            "tileUrl": None,  # filled from /api/map/sentinel1 (signed COG)
+            "endpoint": "/api/map/sentinel1",
+            "maxZoom": 14,
+            "attribution": SENTINEL1_ATTRIBUTION,
+        },
         "nws_alerts": {"id": "nws_alerts", "kind": "geojson", "endpoint": "/api/weather-alerts"},
         "wfigs_incidents": {"id": "wfigs_incidents", "kind": "points", "endpoint": "/api/fire-incidents"},
         "wfigs_perimeters": {"id": "wfigs_perimeters", "kind": "geojson", "endpoint": "/api/fire-perimeters"},
@@ -1136,3 +1154,116 @@ async def fetch_storms() -> list[dict]:
         return transform_nhc_storms(await _get_json(NHC_STORMS))
 
     return await _ttl_get("nhc:storms", 300.0, _load)
+
+
+# ── Sentinel-1 SAR (Planetary Computer STAC → signed COG tile template) ────
+
+class UpstreamRateLimited(Exception):
+    """Planetary Computer returned 429. Carries Retry-After for the client."""
+
+    def __init__(self, retry_after: str | None = None):
+        self.retry_after = retry_after
+        super().__init__("planetary computer rate limited")
+
+
+async def _post_json(
+    url: str, json: dict | None = None, headers: dict | None = None,
+) -> Any:
+    if _http is None:
+        async with httpx.AsyncClient(
+            timeout=_HTTP_TIMEOUT, follow_redirects=True, headers=_headers(),
+        ) as client:
+            resp = await client.post(url, json=json, headers=headers)
+            resp.raise_for_status()
+            return resp.json()
+    resp = await _http.post(url, json=json, headers=headers)
+    resp.raise_for_status()
+    return resp.json()
+
+
+async def _pc_call(coro: Awaitable[Any]) -> Any:
+    """Run a Planetary Computer call, mapping 429 → UpstreamRateLimited."""
+    try:
+        return await coro
+    except httpx.HTTPStatusError as exc:
+        if exc.response.status_code == 429:
+            raise UpstreamRateLimited(
+                exc.response.headers.get("Retry-After")
+            ) from exc
+        raise
+
+
+def sign_cog_url(href: str, token: str) -> str:
+    """Append a SAS token to a PC blob URL (respect existing query string)."""
+    sep = "&" if "?" in href else "?"
+    return f"{href}{sep}token={quote(token, safe='')}"
+
+
+def sentinel1_tile_url(signed_cog: str) -> str:
+    """TiTiler XYZ template for a signed COG (Leaflet substitutes {z}/{x}/{y})."""
+    params = urlencode({
+        "url": signed_cog,
+        "rescale": "0,500",
+        "colormap_name": "cfastie",
+    })
+    return f"{TITILER_COG_TILES}?{params}"
+
+
+async def fetch_sentinel1(bbox: str) -> dict | None:
+    """Most recent Sentinel-1 GRD COG for a viewport, signed and TiTiler-ready.
+
+    Returns the overlay tile-template dict, or ``None`` when no GRD imagery
+    covers the bbox in the last 7 days (caller maps to 404). Queries Planetary
+    Computer only when called; cached per quantized bbox + UTC day.
+    """
+    minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
+    day = datetime.now(timezone.utc).date().isoformat()
+    key = f"sentinel1:{day}:{bbox_cell_key(bbox)}"
+
+    async def _load() -> dict | None:
+        now = datetime.now(timezone.utc)
+        week_ago = now - timedelta(days=7)
+        payload = {
+            "collections": ["sentinel-1-grd"],
+            "bbox": [minlon, minlat, maxlon, maxlat],
+            "datetime": f"{week_ago.isoformat()}/{now.isoformat()}",
+            "limit": 1,
+            "sortby": [{"field": "datetime", "direction": "desc"}],
+        }
+        data = await _pc_call(_post_json(PC_STAC_SEARCH, json=payload))
+        features = data.get("features") or []
+        if not features:
+            return None
+
+        feat = features[0]
+        assets = feat.get("assets") or {}
+        chosen_href: str | None = None
+        polarization: str | None = None
+        for pol in ("vv", "hh"):
+            href = (assets.get(pol) or {}).get("href")
+            if href:
+                chosen_href = href
+                polarization = pol
+                break
+        if not chosen_href:
+            return None
+
+        sas = await _pc_call(_get_json(PC_SAS_TOKEN))
+        token = (sas or {}).get("token")
+        if not token:
+            raise RuntimeError("planetarycomputer SAS token missing")
+        signed = sign_cog_url(chosen_href, token)
+
+        props = feat.get("properties") or {}
+        return {
+            "id": "sentinel-1-sar",
+            "kind": "raster",
+            "tileUrl": sentinel1_tile_url(signed),
+            "opacity": 0.8,
+            "itemId": feat.get("id"),
+            "datetime": props.get("datetime") or feat.get("datetime"),
+            "polarization": polarization,
+            "attribution": SENTINEL1_ATTRIBUTION,
+        }
+
+    return await _ttl_get(key, float(SENTINEL1_TTL), _load)

app/main.py

@@ -53,8 +53,9 @@ from keystore import KeyFormatError, delete_key, list_keys, set_key
 from settings_store import SettingsError, get_app_settings, list_models, set_summary_model
 from live_layers import (
     fetch_aircraft, fetch_fire_incidents, fetch_fire_perimeters,
-    fetch_planespotters_photo, fetch_radar_meta, fetch_storms, fetch_trains,
+    fetch_planespotters_photo, fetch_radar_meta, fetch_sentinel1, fetch_storms,
-    fetch_vessels, fetch_weather_alerts, overlay_catalog, parse_bbox,
+    fetch_trains, fetch_vessels, fetch_weather_alerts, overlay_catalog,
+    parse_bbox, UpstreamRateLimited,
 )
 
 logging.basicConfig(level=logging.INFO)
@@ -1413,6 +1414,33 @@ async def map_radar():
         _upstream_or_502(exc, "radar")
 
 
+@app.get("/api/map/sentinel1")
+async def map_sentinel1(bbox: str = Query(..., description="minlon,minlat,maxlon,maxlat")):
+    """Most recent Sentinel-1 GRD as a signed COG tile template (TiTiler).
+
+    Queries Planetary Computer only on demand; no tiles proxied through the Pi.
+    """
+    _parse_bbox_query(bbox)
+    try:
+        result = await fetch_sentinel1(bbox)
+    except UpstreamRateLimited as exc:
+        headers = {"Retry-After": exc.retry_after} if exc.retry_after else None
+        raise HTTPException(
+            429, "Planetary Computer rate limit", headers=headers,
+        ) from exc
+    except Exception as exc:
+        _upstream_or_502(exc, "sentinel1")
+    if result is None:
+        return JSONResponse(
+            status_code=404,
+            content={
+                "error": "no_imagery",
+                "message": "No Sentinel-1 GRD in the last 7 days for this bbox",
+            },
+        )
+    return overlay_json(result, 300)
+
+
 @app.get("/api/aircraft")
 async def list_aircraft(
     bbox: str = Query(..., description="minlon,minlat,maxlon,maxlat"),

deploy/osint-titiler.nginx.conf

@@ -0,0 +1,21 @@
+# osint.rpi.local — Sentinel-1 SAR tile proxy (/titiler/)
+#
+# GitOps: this file is the source of truth. On the Pi:
+#   sudo cp deploy/osint-titiler.nginx.conf /etc/nginx/snippets/osint-titiler.conf
+# then `include snippets/osint-titiler.conf;` inside the osint.rpi.local server
+# block (before `location /`), `nginx -t && systemctl reload nginx`.
+#
+# The browser hits /titiler/cog/tiles/... (same-origin). We strip the /titiler
+# prefix so self-hosted TiTiler (127.0.0.1:8001) sees /cog/tiles/... and proxy
+# its response straight back. Tiles are heavy PNGs — disable buffering so a
+# slow client doesn't hold a worker open.
+
+location /titiler/ {
+    proxy_pass http://127.0.0.1:8001/;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_buffering off;
+    proxy_read_timeout 300s;
+}

docker-compose.yml

@@ -141,6 +141,9 @@ services:
       VESSELAPI_INTERVAL: ${VESSELAPI_INTERVAL:-17280}
       VESSELAPI_MAX_CALLS_PER_DAY: ${VESSELAPI_MAX_CALLS_PER_DAY:-5}
       VESSELAPI_IN_APP: ${VESSELAPI_IN_APP:-1}
+      # ── Self-hosted TiTiler (Sentinel-1 SAR tiles) ──
+      TITILER_PUBLIC_BASE: ${TITILER_PUBLIC_BASE:-/titiler}
+      TITILER_INTERNAL_URL: ${TITILER_INTERNAL_URL:-http://titiler:8000}
     ports:
       - "127.0.0.1:8000:8000"
     deploy:
@@ -153,6 +156,27 @@ services:
       timeout: 5s
       retries: 5
 
+  # ── Self-hosted TiTiler (Sentinel-1 SAR COG → XYZ tiles) ────────────────
+  # Warps the signed Planetary Computer COG into WebMercator XYZ tiles so the
+  # browser never loads a multi-GB GeoTIFF. The FastAPI app signs the COG URL
+  # and returns a /titiler/... template; nginx routes /titiler/ here.
+  # Listens on 8000 INSIDE the container (the app already owns host 8000);
+  # published on host loopback 127.0.0.1:8001 only.
+  titiler:
+    image: ghcr.io/developmentseed/titiler:latest
+    container_name: osint-titiler
+    platform: linux/arm64
+    restart: unless-stopped
+    environment:
+      - PORT=8000
+      - WORKERS_PER_CORE=1
+    ports:
+      - "127.0.0.1:8001:8000"
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+
   camera-service:
     build:
       context: .

tests/test_live_layers.py

@@ -8,6 +8,8 @@ from live_layers import (
     parse_bbox,
     quantize_bbox,
     rainviewer_tile_url,
+    sign_cog_url,
+    sentinel1_tile_url,
     slim_alert_properties,
     to_marker,
     transform_adsb_lol,
@@ -15,6 +17,8 @@ from live_layers import (
     transform_amtraker,
     transform_nhc_storms,
     transform_wfigs_incidents,
+    SENTINEL1_ATTRIBUTION,
+    TITILER_COG_TILES,
     _cache,
     _ttl_get,
     _wfigs_params,
@@ -652,3 +656,129 @@ def test_planespotters_headers_add_contact_when_ua_is_generic(monkeypatch):
     ua = live_layers._planespotters_headers()["User-Agent"]
     assert "osint-dashboard" in ua
     assert "@" in ua
+
+
+# ── Sentinel-1 SAR (Planetary Computer STAC → signed COG template) ────────
+
+def test_sign_cog_url_appends_token():
+    assert sign_cog_url("https://blob.example/x.tif", "tok=abc") == \
+        "https://blob.example/x.tif?token=tok%3Dabc"
+    # Existing query string → append with &
+    assert sign_cog_url("https://blob.example/x.tif?st=1", "tok") == \
+        "https://blob.example/x.tif?st=1&token=tok"
+
+
+def test_sentinel1_tile_url_contains_titiler_rescale_and_cfastie():
+    signed = "https://blob.example/x.tif?token=secret"
+    url = sentinel1_tile_url(signed)
+    assert url.startswith(TITILER_COG_TILES + "?")
+    assert "WebMercatorQuad/{z}/{x}/{y}@1x" in url
+    assert "url=https%3A%2F%2Fblob.example%2Fx.tif%3Ftoken%3Dsecret" in url
+    assert "rescale=0%2C500" in url
+    assert "colormap_name=cfastie" in url
+
+
+def test_sentinel1_tile_url_is_same_origin_relative():
+    # Self-hosted TiTiler: the browser must hit the Pi's nginx vhost, not
+    # titiler.xyz or a raw host:port. The template is a root-relative path.
+    url = sentinel1_tile_url("https://blob.example/x.tif")
+    assert url.startswith("/titiler/cog/tiles/WebMercatorQuad/")
+    assert "://" not in url
+    assert "titiler.xyz" not in url
+
+
+def _stac_feature(assets: dict) -> dict:
+    return {
+        "type": "Feature",
+        "id": "S1A_IW_GRDH_1SDV_20240820T000000",
+        "properties": {"datetime": "2024-08-20T00:00:00Z"},
+        "assets": assets,
+    }
+
+
+def test_fetch_sentinel1_vv_signed_tile_url(monkeypatch):
+    import asyncio
+    from live_layers import fetch_sentinel1, _cache
+
+    calls = []
+
+    async def fake_post(url, json=None, headers=None):
+        calls.append(("post", url, json))
+        return {"features": [_stac_feature({
+            "vv": {"href": "https://blob.example/grd-vv.tif"},
+        })]}
+
+    async def fake_get(url, params=None, headers=None):
+        calls.append(("get", url))
+        return {"token": "sig=abc123"}
+
+    monkeypatch.setattr("live_layers._post_json", fake_post)
+    monkeypatch.setattr("live_layers._get_json", fake_get)
+    _cache.clear()
+
+    out = asyncio.run(fetch_sentinel1("-80,35,-79,36"))
+    assert out["id"] == "sentinel-1-sar"
+    assert out["kind"] == "raster"
+    assert out["polarization"] == "vv"
+    assert out["opacity"] == 0.8
+    assert out["itemId"].startswith("S1A")
+    assert out["attribution"] == SENTINEL1_ATTRIBUTION
+    assert "WebMercatorQuad/{z}/{x}/{y}@1x" in out["tileUrl"]
+    assert "rescale=0%2C500" in out["tileUrl"]
+    assert "colormap_name=cfastie" in out["tileUrl"]
+    # SAS token "sig=abc123" is signed as ?token=sig%3Dabc123, then the whole
+    # COG URL is percent-encoded again as a query param (=> sig%253Dabc123).
+    assert "sig%253Dabc123" in out["tileUrl"]
+    # STAC search payload shape
+    post_url, post_json = calls[0][1], calls[0][2]
+    assert post_url.endswith("/api/stac/v1/search")
+    assert post_json["collections"] == ["sentinel-1-grd"]
+    assert post_json["limit"] == 1
+    assert post_json["sortby"][0]["direction"] == "desc"
+
+
+def test_fetch_sentinel1_uses_hh_when_vv_missing(monkeypatch):
+    import asyncio
+    from live_layers import fetch_sentinel1, _cache
+
+    async def fake_post(url, json=None, headers=None):
+        return {"features": [_stac_feature({
+            "hh": {"href": "https://blob.example/grd-hh.tif"},
+        })]}
+
+    async def fake_get(url, params=None, headers=None):
+        return {"token": "tok"}
+
+    monkeypatch.setattr("live_layers._post_json", fake_post)
+    monkeypatch.setattr("live_layers._get_json", fake_get)
+    _cache.clear()
+
+    out = asyncio.run(fetch_sentinel1("-80,35,-79,36"))
+    assert out["polarization"] == "hh"
+    assert "url=https%3A%2F%2Fblob.example%2Fgrd-hh.tif" in out["tileUrl"]
+
+
+def test_fetch_sentinel1_none_on_empty_features(monkeypatch):
+    import asyncio
+    from live_layers import fetch_sentinel1, _cache
+
+    async def fake_post(url, json=None, headers=None):
+        return {"features": []}
+
+    monkeypatch.setattr("live_layers._post_json", fake_post)
+    _cache.clear()
+
+    assert asyncio.run(fetch_sentinel1("-80,35,-79,36")) is None
+
+
+def test_fetch_sentinel1_none_when_no_vv_or_hh(monkeypatch):
+    import asyncio
+    from live_layers import fetch_sentinel1, _cache
+
+    async def fake_post(url, json=None, headers=None):
+        return {"features": [_stac_feature({"thumbnail": {"href": "https://x"}})]}
+
+    monkeypatch.setattr("live_layers._post_json", fake_post)
+    _cache.clear()
+
+    assert asyncio.run(fetch_sentinel1("-80,35,-79,36")) is None