cameras: Live-Environment-Streams + HLS.js player

Ingest ~4.2k direct HLS/YouTube feeds (VDOT, MDSHA, DelDOT, Iowa DOT,
OpenCCTV, etc.) from the public GeoJSON catalog. Popup plays HLS via a
CORS-safe playlist proxy + vendored hls.js, YouTube via embed, with
ffmpeg-MJPEG fallback. ALERTWest rows now use a stable camera id so
hourly scrapes do not duplicate.

.env.example

@@ -21,7 +21,8 @@ MINIO_SECURE=false
 # ── Camera discovery scraper ───────────────────────────────────────────────
 # Comma-separated public directory/list/API URLs (Insecam-style pages,
 # plain-text lists, or the ALERTWest JSON API). Empty = built-in defaults
-# (public-ip-cams README + ALERTCalifornia/ALERTWest official JPEGs).
+# (public-ip-cams README + ALERTCalifornia/ALERTWest official JPEGs +
+# Live-Environment-Streams HLS/YouTube GeoJSON).
 CAMERA_SOURCE_URLS=
 CAMERA_SCRAPE_INTERVAL=3600
 CAMERA_REQUEST_DELAY=2.0

app/camera_config.py

@@ -17,6 +17,8 @@ _DEFAULT_SOURCE_URL = ",".join((
     "https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md",
     # ALERTCalifornia / ALERTWest official public JPEG API (wildfire + DOT + FAA).
     "https://api.cdn.prod.alertwest.com/api/getCameraDataByLoc",
+    # Curated global outdoor streams (HLS / YouTube / JPEG) — VDOT, MDSHA, etc.
+    "https://raw.githubusercontent.com/willytop8/Live-Environment-Streams/main/streams.geojson",
 ))
 CAMERA_SOURCE_URLS = [
     u.strip()

app/camera_hls.py

@@ -0,0 +1,112 @@
+"""HLS playlist rewriter so the in-page player can play CORS-blocked feeds.
+
+The browser talks only to /api/cameras/{id}/hls.m3u8 and /hlsseg. We fetch
+the real playlist, rewrite every URI to our proxy, and remember the hosts
+that appeared so /hlsseg cannot be used as an open proxy.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from urllib.parse import quote, unquote, urljoin, urlparse
+
+import httpx
+from fastapi import HTTPException
+from fastapi.responses import Response
+
+from camera_scraper import is_public_url
+
+_UA = {"User-Agent": "osint-dashboard-hls/1.0"}
+# camera_id -> (expiry_epoch, allowed_hosts)
+_ALLOWED: dict[str, tuple[float, set[str]]] = {}
+_TTL = 600.0
+_URI_ATTR = re.compile(r'URI="([^"]+)"', re.I)
+
+
+def _remember(camera_id: str, url: str) -> None:
+    host = urlparse(url).hostname
+    if not host:
+        return
+    now = time.monotonic()
+    exp, hosts = _ALLOWED.get(camera_id, (now + _TTL, set()))
+    hosts.add(host.lower())
+    _ALLOWED[camera_id] = (now + _TTL, hosts)
+
+
+def _host_ok(camera_id: str, url: str) -> bool:
+    host = (urlparse(url).hostname or "").lower()
+    if not host:
+        return False
+    rec = _ALLOWED.get(camera_id)
+    if not rec or rec[0] < time.monotonic():
+        return False
+    return host in rec[1]
+
+
+def _proxied(camera_id: str, abs_url: str) -> str:
+    _remember(camera_id, abs_url)
+    return f"/api/cameras/{camera_id}/hlsseg?u={quote(abs_url, safe='')}"
+
+
+def rewrite_m3u8(text: str, base: str, camera_id: str) -> str:
+    out: list[str] = []
+    for line in text.splitlines():
+        stripped = line.strip()
+        if not stripped:
+            out.append(line)
+            continue
+        if stripped.startswith("#"):
+            def repl(m: re.Match[str]) -> str:
+                return f'URI="{_proxied(camera_id, urljoin(base, m.group(1)))}"'
+            out.append(_URI_ATTR.sub(repl, line))
+            continue
+        out.append(_proxied(camera_id, urljoin(base, stripped)))
+    return "\n".join(out) + "\n"
+
+
+async def fetch_playlist(camera_id: str, url: str) -> Response:
+    if not is_public_url(url):
+        raise HTTPException(400, "HLS URL is not public")
+    _remember(camera_id, url)
+    try:
+        async with httpx.AsyncClient(timeout=15, follow_redirects=True, headers=_UA) as c:
+            r = await c.get(url)
+    except Exception as exc:  # noqa: BLE001
+        raise HTTPException(502, "HLS playlist unavailable") from exc
+    if r.status_code != 200:
+        raise HTTPException(502, "HLS playlist unavailable")
+    base = str(r.url)
+    body = rewrite_m3u8(r.text, base, camera_id)
+    return Response(
+        content=body,
+        media_type="application/vnd.apple.mpegurl",
+        headers={"Cache-Control": "no-store"},
+    )
+
+
+async def fetch_segment(camera_id: str, raw_url: str) -> Response:
+    url = unquote(raw_url)
+    if not url.lower().startswith(("http://", "https://")):
+        raise HTTPException(400, "invalid segment URL")
+    if not is_public_url(url):
+        raise HTTPException(400, "segment URL is not public")
+    if not _host_ok(camera_id, url):
+        raise HTTPException(400, "segment host not in playlist")
+    try:
+        async with httpx.AsyncClient(timeout=20, follow_redirects=True, headers=_UA) as c:
+            r = await c.get(url)
+    except Exception as exc:  # noqa: BLE001
+        raise HTTPException(502, "HLS segment unavailable") from exc
+    if r.status_code != 200:
+        raise HTTPException(502, "HLS segment unavailable")
+    ctype = (r.headers.get("content-type") or "").lower()
+    if "mpegurl" in ctype or "m3u8" in url.lower().split("?")[0]:
+        body = rewrite_m3u8(r.text, str(r.url), camera_id)
+        return Response(content=body, media_type="application/vnd.apple.mpegurl",
+                        headers={"Cache-Control": "no-store"})
+    return Response(
+        content=r.content,
+        media_type=r.headers.get("content-type") or "application/octet-stream",
+        headers={"Cache-Control": "no-store"},
+    )

app/camera_preview.py

@@ -177,8 +177,10 @@ async def ffmpeg_mjpeg_stream(url: str):
         return
     cmd = [
         _FFMPEG, "-hide_banner", "-loglevel", "error", "-nostdin",
-        "-rtsp_transport", "tcp",
-        "-timeout", "4000000",
+    ]
+    if url.lower().startswith("rtsp://"):
+        cmd += ["-rtsp_transport", "tcp", "-timeout", "4000000"]
+    cmd += [
         "-i", url,
         "-an", "-c:v", "mjpeg", "-q:v", "8",
         "-f", "mpjpeg", "pipe:1",

app/camera_scraper.py

@@ -303,6 +303,8 @@ def parse_alertwest_json(text: str, source_name: str) -> list[dict]:
             f"https://img.cdn.prod.alertwest.com/data/img/"
             f"{cid}/{dt:%Y}/{dt:%m}/{dt:%d}/{img}"
         )
+        # Stable identity (the JPEG filename changes every capture).
+        ident = f"https://img.cdn.prod.alertwest.com/cam/{cid}"
         loc = locs.get(cam.get("lid")) or {}
         try:
             lat = float(loc["lat"]) if loc.get("lat") is not None else None
@@ -312,7 +314,7 @@ def parse_alertwest_json(text: str, source_name: str) -> list[dict]:
         bits = [cam.get("cn"), cam.get("co"), loc.get("st") or cam.get("st")]
         name = ", ".join(str(b) for b in bits if b)
         out.append({
-            "source_url": snap,
+            "source_url": ident,
             "snapshot_url": snap,
             "discovery_source": source_name,
             "location_lat": lat,
@@ -324,6 +326,49 @@ def parse_alertwest_json(text: str, source_name: str) -> list[dict]:
     return out
 
 
+def parse_live_streams_geojson(text: str, source_name: str) -> list[dict]:
+    """Parse willytop8/Live-Environment-Streams GeoJSON.
+
+    Only direct, playable URLs: HLS, YouTube, HTTP stills. Skip html_page
+    feeds that need token extraction or a headless browser.
+    """
+    try:
+        payload = json.loads(text)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    usable = {"hls", "youtube", "http_image"}
+    out: list[dict] = []
+    for feat in payload.get("features") or []:
+        props = feat.get("properties") or {}
+        ut = (props.get("url_type") or "").lower()
+        if ut not in usable:
+            continue
+        if props.get("source_url_requires"):
+            continue
+        url = (props.get("url") or "").strip()
+        if not url:
+            continue
+        coords = (feat.get("geometry") or {}).get("coordinates") or []
+        lon = lat = None
+        if len(coords) >= 2:
+            try:
+                lon, lat = float(coords[0]), float(coords[1])
+            except (TypeError, ValueError):
+                lon = lat = None
+        dtype = "ip-cam" if ut == "http_image" else ut
+        out.append({
+            "source_url": url,
+            "snapshot_url": url,
+            "discovery_source": source_name,
+            "location_lat": lat,
+            "location_lon": lon,
+            "location_name": props.get("display_name") or props.get("name"),
+            "vendor": props.get("source_family"),
+            "device_type": dtype,
+        })
+    return out
+
+
 def parse_directory_html(html: str, base_url: str, source_name: str) -> list[dict]:
     """Generic Insecam-style directory parser."""
     cams = []
@@ -393,6 +438,9 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder,
     if ("getCameraDataByLoc" in src_url
             or ("json" in ctype and '"locs"' in body[:4000] and '"cams"' in body[:8000])):
         cams = parse_alertwest_json(body, name)
+    elif (src_url.endswith(".geojson") or src_url.endswith("/streams.geojson")
+            or '"FeatureCollection"' in body[:400]):
+        cams = parse_live_streams_geojson(body, name)
     elif "html" in ctype:
         cams = parse_directory_html(body, str(resp.url), name)
     else:

app/main.py

@@ -812,10 +812,13 @@ async def camera_stream(camera_id: UUID):
     if not row:
         raise HTTPException(404, "Camera not found")
     url = row["snapshot_url"] or row["source_url"]
-    if str(url or "").lower().startswith("rtsp://"):
+    low = str(url or "").lower()
+    if low.startswith("rtsp://") or ".m3u8" in low or row.get("device_type") == "hls":
         from camera_preview import ffmpeg_mjpeg_stream, _FFMPEG
         if not _FFMPEG:
-            raise HTTPException(502, "RTSP preview requires ffmpeg")
+            raise HTTPException(502, "Live preview requires ffmpeg")
+        if not low.startswith("rtsp://") and not low.startswith(("http://", "https://")):
+            raise HTTPException(404, "Camera or snapshot not found")
         return StreamingResponse(
             ffmpeg_mjpeg_stream(url),
             media_type="multipart/x-mixed-replace; boundary=ffmpeg",
@@ -858,6 +861,39 @@ async def camera_stream(camera_id: UUID):
     return StreamingResponse(gen(), media_type=media)
 
 
+@app.get("/api/cameras/{camera_id}/hls.m3u8")
+async def camera_hls_playlist(camera_id: UUID):
+    """CORS-safe rewritten HLS playlist for the in-page player."""
+    from camera_hls import fetch_playlist
+    from camera_models import cameras as cam_table
+
+    async with async_session() as session:
+        row = (await session.execute(
+            select(cam_table).where(cam_table.c.id == camera_id)
+        )).mappings().one_or_none()
+    if not row:
+        raise HTTPException(404, "Camera not found")
+    url = row["snapshot_url"] or row["source_url"] or ""
+    if ".m3u8" not in url.lower() and row.get("device_type") != "hls":
+        raise HTTPException(404, "Camera is not an HLS feed")
+    return await fetch_playlist(str(camera_id), url)
+
+
+@app.get("/api/cameras/{camera_id}/hlsseg")
+async def camera_hls_segment(camera_id: UUID, u: str = Query(..., min_length=8)):
+    """Proxy one HLS segment/playlist URI rewritten by hls.m3u8."""
+    from camera_hls import fetch_segment
+    from camera_models import cameras as cam_table
+
+    async with async_session() as session:
+        exists = (await session.execute(
+            select(cam_table.c.id).where(cam_table.c.id == camera_id)
+        )).scalar_one_or_none()
+    if not exists:
+        raise HTTPException(404, "Camera not found")
+    return await fetch_segment(str(camera_id), u)
+
+
 # ── News pipeline (scraper + summarizer) ──────────────────────────────────
 # Backing data for the frontend news panel. Written by the vendored
 # news-scraper (hourly Scrapy crawl) and news-summarizer (hourly Gemini

app/static/index.html

@@ -133,6 +133,7 @@
         /* ── Camera popup thumbnail ───────────────────────────────── */
         .cam-pop { min-width: 210px; max-width: 260px; }
         .cam-pop .thumb { width: 100%; height: 160px; object-fit: cover; border-radius: 6px; border: 1px solid var(--border); margin: 0.3rem 0; box-shadow: 0 0 10px rgba(56,189,248,0.25); background: #0b1220; }
+        .cam-pop video.thumb, .cam-pop iframe.thumb { height: 180px; background: #000; }
         .cam-pop .thumb.placeholder { display: flex; align-items: center; justify-content: center; color: var(--muted); font-size: 0.7rem; height: 80px; }
         .cam-pop table { width: 100%; font-size: 0.72rem; border-collapse: collapse; }
         .cam-pop td { padding: 0.12rem 0.2rem; vertical-align: top; }
@@ -415,6 +416,7 @@
 
 <script src="/static/vendor/leaflet/leaflet.js"></script>
 <script src="/static/vendor/leaflet/leaflet.heat.js"></script>
+<script src="/static/vendor/hls/hls.min.js"></script>
 <script>
 const API = '';
 
@@ -781,6 +783,37 @@ async function initMap() {
         });
         L.control.zoom({ position: 'topright' }).addTo(map);
         map.attributionControl.setPrefix('');
+        let activeHls = null;
+        map.on('popupopen', (e) => {
+            if (activeHls) { try { activeHls.destroy(); } catch (_) {} activeHls = null; }
+            const root = e.popup.getElement();
+            const v = root && root.querySelector('video[data-hls]');
+            if (!v) return;
+            const fallback = () => {
+                const img = document.createElement('img');
+                img.className = 'thumb';
+                img.src = v.dataset.fallback;
+                img.alt = 'camera preview';
+                v.replaceWith(img);
+            };
+            if (window.Hls && Hls.isSupported()) {
+                activeHls = new Hls({ enableWorker: true, lowLatencyMode: true });
+                activeHls.loadSource(v.dataset.hls);
+                activeHls.attachMedia(v);
+                v.play && v.play().catch(() => {});
+                activeHls.on(Hls.Events.ERROR, (_, data) => {
+                    if (data && data.fatal) fallback();
+                });
+            } else if (v.canPlayType && v.canPlayType('application/vnd.apple.mpegurl')) {
+                v.src = v.dataset.hls;
+                v.addEventListener('error', fallback, { once: true });
+            } else {
+                fallback();
+            }
+        });
+        map.on('popupclose', () => {
+            if (activeHls) { try { activeHls.destroy(); } catch (_) {} activeHls = null; }
+        });
         updateHeatLegend();
         // Reload overlays when the user pans/zooms. Skip when a popup is open:
         // opening a camera popup auto-pans the map to fit it, and that moveend
@@ -1035,16 +1068,29 @@ function camSourceLink(c) {
     }
     return `<a href="${esc(url)}" target="_blank" rel="noopener">source page ↗</a>`;
 }
+function youtubeId(url) {
+    const m = String(url || '').match(/(?:youtu\.be\/|v=)([A-Za-z0-9_-]{6,})/);
+    return m ? m[1] : '';
+}
+function camKind(c) {
+    const u = String(c.snapshot_url || c.source_url || '').toLowerCase();
+    if (c.device_type === 'youtube' || u.includes('youtube.com') || u.includes('youtu.be')) return 'youtube';
+    if (c.device_type === 'hls' || u.includes('.m3u8')) return 'hls';
+    if (c.device_type === 'rtsp' || u.startsWith('rtsp://')) return 'rtsp';
+    return 'http';
+}
 function camThumb(c) {
-    // Preview is always served by the dashboard (never a raw rtsp:// href —
-    // browsers hand those to VLC). HTTP cams stream live MJPEG; RTSP/masscan
-    // finds grab a still via /snapshot (ffmpeg / HTTP probe). Fallback:
-    //   live stream → still snapshot → placeholder.
     if (!c.id) return '<div class="thumb placeholder">no preview available</div>';
-    const isRtsp = (c.device_type === 'rtsp')
-        || String(c.source_url || '').toLowerCase().startsWith('rtsp://')
-        || String(c.snapshot_url || '').toLowerCase().startsWith('rtsp://');
-    const first = isRtsp
+    const kind = camKind(c);
+    if (kind === 'youtube') {
+        const vid = youtubeId(c.snapshot_url || c.source_url);
+        if (!vid) return '<div class="thumb placeholder">youtube id missing</div>';
+        return `<iframe class="thumb" src="https://www.youtube-nocookie.com/embed/${esc(vid)}?autoplay=1&mute=1" allow="autoplay; encrypted-media; picture-in-picture" allowfullscreen loading="lazy"></iframe>`;
+    }
+    if (kind === 'hls') {
+        return `<video class="thumb" data-hls="/api/cameras/${esc(c.id)}/hls.m3u8" data-fallback="/api/cameras/${esc(c.id)}/stream" muted autoplay playsinline controls></video>`;
+    }
+    const first = kind === 'rtsp'
         ? `/api/cameras/${esc(c.id)}/snapshot`
         : `/api/cameras/${esc(c.id)}/stream`;
     const onerr = (