feat(sentinel1): Sentinel-1 SAR STAC -> self-hosted TiTiler tile template
Add GET /api/map/sentinel1?bbox=... which queries Planetary Computer STAC (sentinel-1-grd, last 7d, most recent), signs the vv/hh COG with a SAS token, and returns a same-origin /titiler/... XYZ tile template. Cache keyed on quantized bbox + UTC day (20 min TTL). 429 -> 429 (Retry-After), no imagery -> 404, other upstream errors -> 502. Self-host TiTiler on the Pi (ghcr.io/developmentseed/titiler, arm64, 1G cap, host loopback 8001 -> container 8000) instead of titiler.xyz. nginx /titiler/ proxy snippet routes browser tiles to it; TITILER_PUBLIC_BASE and TITILER_INTERNAL_URL are env-driven (no hardcoded hostnames).
This commit is contained in:
Sirius DevOps
parent
30a9c9e9b1
commit
5212aaa5cd
6 changed files with 346 additions and 5 deletions
|
|
@ -72,6 +72,13 @@ OSINT_USER_AGENT = os.getenv(
|
|||
"OSINT_USER_AGENT", "osint-dashboard/1.0 (self-hosted; lancewalters94@gmail.com)"
|
||||
)
|
||||
|
||||
# Self-hosted TiTiler (warps Sentinel-1 signed COGs into XYZ tiles on the Pi).
|
||||
# TITILER_PUBLIC_BASE is the same-origin path prefix the browser hits through
|
||||
# the osint.rpi.local nginx vhost (`location /titiler/` → 127.0.0.1:8001).
|
||||
# TITILER_INTERNAL_URL is the compose-DNS address, used only for health checks.
|
||||
TITILER_PUBLIC_BASE = os.getenv("TITILER_PUBLIC_BASE", "/titiler").rstrip("/")
|
||||
TITILER_INTERNAL_URL = os.getenv("TITILER_INTERNAL_URL", "http://titiler:8000")
|
||||
|
||||
# AISStream (server-side WebSocket only). Idle when unset.
|
||||
AISSTREAM_API_KEY = os.getenv("AISSTREAM_API_KEY", "")
|
||||
# Bounding box(es) as minlat,minlon,maxlat,maxlon — note lat/lon order (AISStream).
|
||||
|
|
|
|||
|
|
@ -15,13 +15,13 @@ import asyncio
|
|||
import logging
|
||||
import math
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Awaitable, Callable
|
||||
|
||||
import httpx
|
||||
from urllib.parse import quote
|
||||
from urllib.parse import quote, urlencode
|
||||
|
||||
from config import OSINT_USER_AGENT
|
||||
from config import OSINT_USER_AGENT, TITILER_PUBLIC_BASE
|
||||
|
||||
logger = logging.getLogger("osint.live_layers")
|
||||
|
||||
|
|
@ -45,6 +45,15 @@ WFIGS_PERIMETERS = (
|
|||
)
|
||||
NHC_STORMS = "https://www.nhc.noaa.gov/CurrentStorms.json"
|
||||
|
||||
PC_STAC_SEARCH = "https://planetarycomputer.microsoft.com/api/stac/v1/search"
|
||||
PC_SAS_TOKEN = "https://planetarycomputer.microsoft.com/api/sas/v1/token/sentinel-1-grd"
|
||||
# Self-hosted TiTiler on the Pi, exposed same-origin through the osint.rpi.local
|
||||
# nginx vhost. Relative template — Leaflet resolves it against the page origin,
|
||||
# so the browser never touches a raw loopback port or titiler.xyz.
|
||||
TITILER_COG_TILES = f"{TITILER_PUBLIC_BASE}/cog/tiles/WebMercatorQuad/{{z}}/{{x}}/{{y}}@1x"
|
||||
SENTINEL1_TTL = 20 * 60 # 15–30 min quota-friendly window
|
||||
SENTINEL1_ATTRIBUTION = "Copernicus Sentinel-1 / Microsoft Planetary Computer"
|
||||
|
||||
IEM_NEXRAD = "https://mesonet.agron.iastate.edu/cache/tile.py/1.0.0/nexrad-n0q/{z}/{x}/{y}.png"
|
||||
GIBS_THERMAL = (
|
||||
"https://gibs.earthdata.nasa.gov/wmts/epsg3857/best/"
|
||||
|
|
@ -110,6 +119,15 @@ def overlay_catalog() -> dict:
|
|||
"maxZoom": 9,
|
||||
"attribution": "NASA GIBS / EOSDIS",
|
||||
},
|
||||
"sentinel1_sar": {
|
||||
"id": "sentinel1_sar",
|
||||
"title": "Sentinel-1 SAR (Cloud-Penetrating)",
|
||||
"kind": "raster",
|
||||
"tileUrl": None, # filled from /api/map/sentinel1 (signed COG)
|
||||
"endpoint": "/api/map/sentinel1",
|
||||
"maxZoom": 14,
|
||||
"attribution": SENTINEL1_ATTRIBUTION,
|
||||
},
|
||||
"nws_alerts": {"id": "nws_alerts", "kind": "geojson", "endpoint": "/api/weather-alerts"},
|
||||
"wfigs_incidents": {"id": "wfigs_incidents", "kind": "points", "endpoint": "/api/fire-incidents"},
|
||||
"wfigs_perimeters": {"id": "wfigs_perimeters", "kind": "geojson", "endpoint": "/api/fire-perimeters"},
|
||||
|
|
@ -1136,3 +1154,116 @@ async def fetch_storms() -> list[dict]:
|
|||
return transform_nhc_storms(await _get_json(NHC_STORMS))
|
||||
|
||||
return await _ttl_get("nhc:storms", 300.0, _load)
|
||||
|
||||
|
||||
# ── Sentinel-1 SAR (Planetary Computer STAC → signed COG tile template) ────
|
||||
|
||||
class UpstreamRateLimited(Exception):
|
||||
"""Planetary Computer returned 429. Carries Retry-After for the client."""
|
||||
|
||||
def __init__(self, retry_after: str | None = None):
|
||||
self.retry_after = retry_after
|
||||
super().__init__("planetary computer rate limited")
|
||||
|
||||
|
||||
async def _post_json(
|
||||
url: str, json: dict | None = None, headers: dict | None = None,
|
||||
) -> Any:
|
||||
if _http is None:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=_HTTP_TIMEOUT, follow_redirects=True, headers=_headers(),
|
||||
) as client:
|
||||
resp = await client.post(url, json=json, headers=headers)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
resp = await _http.post(url, json=json, headers=headers)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
async def _pc_call(coro: Awaitable[Any]) -> Any:
|
||||
"""Run a Planetary Computer call, mapping 429 → UpstreamRateLimited."""
|
||||
try:
|
||||
return await coro
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if exc.response.status_code == 429:
|
||||
raise UpstreamRateLimited(
|
||||
exc.response.headers.get("Retry-After")
|
||||
) from exc
|
||||
raise
|
||||
|
||||
|
||||
def sign_cog_url(href: str, token: str) -> str:
|
||||
"""Append a SAS token to a PC blob URL (respect existing query string)."""
|
||||
sep = "&" if "?" in href else "?"
|
||||
return f"{href}{sep}token={quote(token, safe='')}"
|
||||
|
||||
|
||||
def sentinel1_tile_url(signed_cog: str) -> str:
|
||||
"""TiTiler XYZ template for a signed COG (Leaflet substitutes {z}/{x}/{y})."""
|
||||
params = urlencode({
|
||||
"url": signed_cog,
|
||||
"rescale": "0,500",
|
||||
"colormap_name": "cfastie",
|
||||
})
|
||||
return f"{TITILER_COG_TILES}?{params}"
|
||||
|
||||
|
||||
async def fetch_sentinel1(bbox: str) -> dict | None:
|
||||
"""Most recent Sentinel-1 GRD COG for a viewport, signed and TiTiler-ready.
|
||||
|
||||
Returns the overlay tile-template dict, or ``None`` when no GRD imagery
|
||||
covers the bbox in the last 7 days (caller maps to 404). Queries Planetary
|
||||
Computer only when called; cached per quantized bbox + UTC day.
|
||||
"""
|
||||
minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
|
||||
day = datetime.now(timezone.utc).date().isoformat()
|
||||
key = f"sentinel1:{day}:{bbox_cell_key(bbox)}"
|
||||
|
||||
async def _load() -> dict | None:
|
||||
now = datetime.now(timezone.utc)
|
||||
week_ago = now - timedelta(days=7)
|
||||
payload = {
|
||||
"collections": ["sentinel-1-grd"],
|
||||
"bbox": [minlon, minlat, maxlon, maxlat],
|
||||
"datetime": f"{week_ago.isoformat()}/{now.isoformat()}",
|
||||
"limit": 1,
|
||||
"sortby": [{"field": "datetime", "direction": "desc"}],
|
||||
}
|
||||
data = await _pc_call(_post_json(PC_STAC_SEARCH, json=payload))
|
||||
features = data.get("features") or []
|
||||
if not features:
|
||||
return None
|
||||
|
||||
feat = features[0]
|
||||
assets = feat.get("assets") or {}
|
||||
chosen_href: str | None = None
|
||||
polarization: str | None = None
|
||||
for pol in ("vv", "hh"):
|
||||
href = (assets.get(pol) or {}).get("href")
|
||||
if href:
|
||||
chosen_href = href
|
||||
polarization = pol
|
||||
break
|
||||
if not chosen_href:
|
||||
return None
|
||||
|
||||
sas = await _pc_call(_get_json(PC_SAS_TOKEN))
|
||||
token = (sas or {}).get("token")
|
||||
if not token:
|
||||
raise RuntimeError("planetarycomputer SAS token missing")
|
||||
signed = sign_cog_url(chosen_href, token)
|
||||
|
||||
props = feat.get("properties") or {}
|
||||
return {
|
||||
"id": "sentinel-1-sar",
|
||||
"kind": "raster",
|
||||
"tileUrl": sentinel1_tile_url(signed),
|
||||
"opacity": 0.8,
|
||||
"itemId": feat.get("id"),
|
||||
"datetime": props.get("datetime") or feat.get("datetime"),
|
||||
"polarization": polarization,
|
||||
"attribution": SENTINEL1_ATTRIBUTION,
|
||||
}
|
||||
|
||||
return await _ttl_get(key, float(SENTINEL1_TTL), _load)
|
||||
|
|
|
|||
32
app/main.py
32
app/main.py
|
|
@ -53,8 +53,9 @@ from keystore import KeyFormatError, delete_key, list_keys, set_key
|
|||
from settings_store import SettingsError, get_app_settings, list_models, set_summary_model
|
||||
from live_layers import (
|
||||
fetch_aircraft, fetch_fire_incidents, fetch_fire_perimeters,
|
||||
fetch_planespotters_photo, fetch_radar_meta, fetch_storms, fetch_trains,
|
||||
fetch_vessels, fetch_weather_alerts, overlay_catalog, parse_bbox,
|
||||
fetch_planespotters_photo, fetch_radar_meta, fetch_sentinel1, fetch_storms,
|
||||
fetch_trains, fetch_vessels, fetch_weather_alerts, overlay_catalog,
|
||||
parse_bbox, UpstreamRateLimited,
|
||||
)
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
|
@ -1405,6 +1406,33 @@ async def map_radar():
|
|||
_upstream_or_502(exc, "radar")
|
||||
|
||||
|
||||
@app.get("/api/map/sentinel1")
|
||||
async def map_sentinel1(bbox: str = Query(..., description="minlon,minlat,maxlon,maxlat")):
|
||||
"""Most recent Sentinel-1 GRD as a signed COG tile template (TiTiler).
|
||||
|
||||
Queries Planetary Computer only on demand; no tiles proxied through the Pi.
|
||||
"""
|
||||
_parse_bbox_query(bbox)
|
||||
try:
|
||||
result = await fetch_sentinel1(bbox)
|
||||
except UpstreamRateLimited as exc:
|
||||
headers = {"Retry-After": exc.retry_after} if exc.retry_after else None
|
||||
raise HTTPException(
|
||||
429, "Planetary Computer rate limit", headers=headers,
|
||||
) from exc
|
||||
except Exception as exc:
|
||||
_upstream_or_502(exc, "sentinel1")
|
||||
if result is None:
|
||||
return JSONResponse(
|
||||
status_code=404,
|
||||
content={
|
||||
"error": "no_imagery",
|
||||
"message": "No Sentinel-1 GRD in the last 7 days for this bbox",
|
||||
},
|
||||
)
|
||||
return overlay_json(result, 300)
|
||||
|
||||
|
||||
@app.get("/api/aircraft")
|
||||
async def list_aircraft(
|
||||
bbox: str = Query(..., description="minlon,minlat,maxlon,maxlat"),
|
||||
|
|
|
|||
21
deploy/osint-titiler.nginx.conf
Normal file
21
deploy/osint-titiler.nginx.conf
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
# osint.rpi.local — Sentinel-1 SAR tile proxy (/titiler/)
|
||||
#
|
||||
# GitOps: this file is the source of truth. On the Pi:
|
||||
# sudo cp deploy/osint-titiler.nginx.conf /etc/nginx/snippets/osint-titiler.conf
|
||||
# then `include snippets/osint-titiler.conf;` inside the osint.rpi.local server
|
||||
# block (before `location /`), `nginx -t && systemctl reload nginx`.
|
||||
#
|
||||
# The browser hits /titiler/cog/tiles/... (same-origin). We strip the /titiler
|
||||
# prefix so self-hosted TiTiler (127.0.0.1:8001) sees /cog/tiles/... and proxy
|
||||
# its response straight back. Tiles are heavy PNGs — disable buffering so a
|
||||
# slow client doesn't hold a worker open.
|
||||
|
||||
location /titiler/ {
|
||||
proxy_pass http://127.0.0.1:8001/;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 300s;
|
||||
}
|
||||
|
|
@ -131,6 +131,9 @@ services:
|
|||
AISSTREAM_API_KEY: ${AISSTREAM_API_KEY:-}
|
||||
AISSTREAM_BBOX: ${AISSTREAM_BBOX:-24,-125,50,-66}
|
||||
AISSTREAM_IN_APP: ${AISSTREAM_IN_APP:-1}
|
||||
# ── Self-hosted TiTiler (Sentinel-1 SAR tiles) ──
|
||||
TITILER_PUBLIC_BASE: ${TITILER_PUBLIC_BASE:-/titiler}
|
||||
TITILER_INTERNAL_URL: ${TITILER_INTERNAL_URL:-http://titiler:8000}
|
||||
ports:
|
||||
- "127.0.0.1:8000:8000"
|
||||
deploy:
|
||||
|
|
@ -143,6 +146,27 @@ services:
|
|||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# ── Self-hosted TiTiler (Sentinel-1 SAR COG → XYZ tiles) ────────────────
|
||||
# Warps the signed Planetary Computer COG into WebMercator XYZ tiles so the
|
||||
# browser never loads a multi-GB GeoTIFF. The FastAPI app signs the COG URL
|
||||
# and returns a /titiler/... template; nginx routes /titiler/ here.
|
||||
# Listens on 8000 INSIDE the container (the app already owns host 8000);
|
||||
# published on host loopback 127.0.0.1:8001 only.
|
||||
titiler:
|
||||
image: ghcr.io/developmentseed/titiler:latest
|
||||
container_name: osint-titiler
|
||||
platform: linux/arm64
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- PORT=8000
|
||||
- WORKERS_PER_CORE=1
|
||||
ports:
|
||||
- "127.0.0.1:8001:8000"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 1G
|
||||
|
||||
camera-service:
|
||||
build:
|
||||
context: .
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ from live_layers import (
|
|||
parse_bbox,
|
||||
quantize_bbox,
|
||||
rainviewer_tile_url,
|
||||
sign_cog_url,
|
||||
sentinel1_tile_url,
|
||||
slim_alert_properties,
|
||||
to_marker,
|
||||
transform_adsb_lol,
|
||||
|
|
@ -15,6 +17,8 @@ from live_layers import (
|
|||
transform_amtraker,
|
||||
transform_nhc_storms,
|
||||
transform_wfigs_incidents,
|
||||
SENTINEL1_ATTRIBUTION,
|
||||
TITILER_COG_TILES,
|
||||
_cache,
|
||||
_ttl_get,
|
||||
_wfigs_params,
|
||||
|
|
@ -652,3 +656,129 @@ def test_planespotters_headers_add_contact_when_ua_is_generic(monkeypatch):
|
|||
ua = live_layers._planespotters_headers()["User-Agent"]
|
||||
assert "osint-dashboard" in ua
|
||||
assert "@" in ua
|
||||
|
||||
|
||||
# ── Sentinel-1 SAR (Planetary Computer STAC → signed COG template) ────────
|
||||
|
||||
def test_sign_cog_url_appends_token():
|
||||
assert sign_cog_url("https://blob.example/x.tif", "tok=abc") == \
|
||||
"https://blob.example/x.tif?token=tok%3Dabc"
|
||||
# Existing query string → append with &
|
||||
assert sign_cog_url("https://blob.example/x.tif?st=1", "tok") == \
|
||||
"https://blob.example/x.tif?st=1&token=tok"
|
||||
|
||||
|
||||
def test_sentinel1_tile_url_contains_titiler_rescale_and_cfastie():
|
||||
signed = "https://blob.example/x.tif?token=secret"
|
||||
url = sentinel1_tile_url(signed)
|
||||
assert url.startswith(TITILER_COG_TILES + "?")
|
||||
assert "WebMercatorQuad/{z}/{x}/{y}@1x" in url
|
||||
assert "url=https%3A%2F%2Fblob.example%2Fx.tif%3Ftoken%3Dsecret" in url
|
||||
assert "rescale=0%2C500" in url
|
||||
assert "colormap_name=cfastie" in url
|
||||
|
||||
|
||||
def test_sentinel1_tile_url_is_same_origin_relative():
|
||||
# Self-hosted TiTiler: the browser must hit the Pi's nginx vhost, not
|
||||
# titiler.xyz or a raw host:port. The template is a root-relative path.
|
||||
url = sentinel1_tile_url("https://blob.example/x.tif")
|
||||
assert url.startswith("/titiler/cog/tiles/WebMercatorQuad/")
|
||||
assert "://" not in url
|
||||
assert "titiler.xyz" not in url
|
||||
|
||||
|
||||
def _stac_feature(assets: dict) -> dict:
|
||||
return {
|
||||
"type": "Feature",
|
||||
"id": "S1A_IW_GRDH_1SDV_20240820T000000",
|
||||
"properties": {"datetime": "2024-08-20T00:00:00Z"},
|
||||
"assets": assets,
|
||||
}
|
||||
|
||||
|
||||
def test_fetch_sentinel1_vv_signed_tile_url(monkeypatch):
|
||||
import asyncio
|
||||
from live_layers import fetch_sentinel1, _cache
|
||||
|
||||
calls = []
|
||||
|
||||
async def fake_post(url, json=None, headers=None):
|
||||
calls.append(("post", url, json))
|
||||
return {"features": [_stac_feature({
|
||||
"vv": {"href": "https://blob.example/grd-vv.tif"},
|
||||
})]}
|
||||
|
||||
async def fake_get(url, params=None, headers=None):
|
||||
calls.append(("get", url))
|
||||
return {"token": "sig=abc123"}
|
||||
|
||||
monkeypatch.setattr("live_layers._post_json", fake_post)
|
||||
monkeypatch.setattr("live_layers._get_json", fake_get)
|
||||
_cache.clear()
|
||||
|
||||
out = asyncio.run(fetch_sentinel1("-80,35,-79,36"))
|
||||
assert out["id"] == "sentinel-1-sar"
|
||||
assert out["kind"] == "raster"
|
||||
assert out["polarization"] == "vv"
|
||||
assert out["opacity"] == 0.8
|
||||
assert out["itemId"].startswith("S1A")
|
||||
assert out["attribution"] == SENTINEL1_ATTRIBUTION
|
||||
assert "WebMercatorQuad/{z}/{x}/{y}@1x" in out["tileUrl"]
|
||||
assert "rescale=0%2C500" in out["tileUrl"]
|
||||
assert "colormap_name=cfastie" in out["tileUrl"]
|
||||
# SAS token "sig=abc123" is signed as ?token=sig%3Dabc123, then the whole
|
||||
# COG URL is percent-encoded again as a query param (=> sig%253Dabc123).
|
||||
assert "sig%253Dabc123" in out["tileUrl"]
|
||||
# STAC search payload shape
|
||||
post_url, post_json = calls[0][1], calls[0][2]
|
||||
assert post_url.endswith("/api/stac/v1/search")
|
||||
assert post_json["collections"] == ["sentinel-1-grd"]
|
||||
assert post_json["limit"] == 1
|
||||
assert post_json["sortby"][0]["direction"] == "desc"
|
||||
|
||||
|
||||
def test_fetch_sentinel1_uses_hh_when_vv_missing(monkeypatch):
|
||||
import asyncio
|
||||
from live_layers import fetch_sentinel1, _cache
|
||||
|
||||
async def fake_post(url, json=None, headers=None):
|
||||
return {"features": [_stac_feature({
|
||||
"hh": {"href": "https://blob.example/grd-hh.tif"},
|
||||
})]}
|
||||
|
||||
async def fake_get(url, params=None, headers=None):
|
||||
return {"token": "tok"}
|
||||
|
||||
monkeypatch.setattr("live_layers._post_json", fake_post)
|
||||
monkeypatch.setattr("live_layers._get_json", fake_get)
|
||||
_cache.clear()
|
||||
|
||||
out = asyncio.run(fetch_sentinel1("-80,35,-79,36"))
|
||||
assert out["polarization"] == "hh"
|
||||
assert "url=https%3A%2F%2Fblob.example%2Fgrd-hh.tif" in out["tileUrl"]
|
||||
|
||||
|
||||
def test_fetch_sentinel1_none_on_empty_features(monkeypatch):
|
||||
import asyncio
|
||||
from live_layers import fetch_sentinel1, _cache
|
||||
|
||||
async def fake_post(url, json=None, headers=None):
|
||||
return {"features": []}
|
||||
|
||||
monkeypatch.setattr("live_layers._post_json", fake_post)
|
||||
_cache.clear()
|
||||
|
||||
assert asyncio.run(fetch_sentinel1("-80,35,-79,36")) is None
|
||||
|
||||
|
||||
def test_fetch_sentinel1_none_when_no_vv_or_hh(monkeypatch):
|
||||
import asyncio
|
||||
from live_layers import fetch_sentinel1, _cache
|
||||
|
||||
async def fake_post(url, json=None, headers=None):
|
||||
return {"features": [_stac_feature({"thumbnail": {"href": "https://x"}})]}
|
||||
|
||||
monkeypatch.setattr("live_layers._post_json", fake_post)
|
||||
_cache.clear()
|
||||
|
||||
assert asyncio.run(fetch_sentinel1("-80,35,-79,36")) is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue