diff --git a/.env.example b/.env.example
index e3dc5dd..d7a4f36 100644
--- a/.env.example
+++ b/.env.example
@@ -21,7 +21,8 @@ MINIO_SECURE=false
# ── Camera discovery scraper ───────────────────────────────────────────────
# Comma-separated public directory/list/API URLs (Insecam-style pages,
# plain-text lists, or the ALERTWest JSON API). Empty = built-in defaults
-# (public-ip-cams README + ALERTCalifornia/ALERTWest official JPEGs).
+# (public-ip-cams README + ALERTCalifornia/ALERTWest official JPEGs +
+# Live-Environment-Streams HLS/YouTube GeoJSON).
CAMERA_SOURCE_URLS=
CAMERA_SCRAPE_INTERVAL=3600
CAMERA_REQUEST_DELAY=2.0
diff --git a/app/camera_config.py b/app/camera_config.py
index 59b38d1..7774a53 100644
--- a/app/camera_config.py
+++ b/app/camera_config.py
@@ -17,6 +17,8 @@ _DEFAULT_SOURCE_URL = ",".join((
"https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md",
# ALERTCalifornia / ALERTWest official public JPEG API (wildfire + DOT + FAA).
"https://api.cdn.prod.alertwest.com/api/getCameraDataByLoc",
+ # Curated global outdoor streams (HLS / YouTube / JPEG) — VDOT, MDSHA, etc.
+ "https://raw.githubusercontent.com/willytop8/Live-Environment-Streams/main/streams.geojson",
))
CAMERA_SOURCE_URLS = [
u.strip()
diff --git a/app/camera_hls.py b/app/camera_hls.py
new file mode 100644
index 0000000..9ae9555
--- /dev/null
+++ b/app/camera_hls.py
@@ -0,0 +1,112 @@
+"""HLS playlist rewriter so the in-page player can play CORS-blocked feeds.
+
+The browser talks only to /api/cameras/{id}/hls.m3u8 and /hlsseg. We fetch
+the real playlist, rewrite every URI to our proxy, and remember the hosts
+that appeared so /hlsseg cannot be used as an open proxy.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from urllib.parse import quote, unquote, urljoin, urlparse
+
+import httpx
+from fastapi import HTTPException
+from fastapi.responses import Response
+
+from camera_scraper import is_public_url
+
+_UA = {"User-Agent": "osint-dashboard-hls/1.0"}
+# camera_id -> (expiry_epoch, allowed_hosts)
+_ALLOWED: dict[str, tuple[float, set[str]]] = {}
+_TTL = 600.0
+_URI_ATTR = re.compile(r'URI="([^"]+)"', re.I)
+
+
+def _remember(camera_id: str, url: str) -> None:
+ host = urlparse(url).hostname
+ if not host:
+ return
+ now = time.monotonic()
+ exp, hosts = _ALLOWED.get(camera_id, (now + _TTL, set()))
+ hosts.add(host.lower())
+ _ALLOWED[camera_id] = (now + _TTL, hosts)
+
+
+def _host_ok(camera_id: str, url: str) -> bool:
+ host = (urlparse(url).hostname or "").lower()
+ if not host:
+ return False
+ rec = _ALLOWED.get(camera_id)
+ if not rec or rec[0] < time.monotonic():
+ return False
+ return host in rec[1]
+
+
+def _proxied(camera_id: str, abs_url: str) -> str:
+ _remember(camera_id, abs_url)
+ return f"/api/cameras/{camera_id}/hlsseg?u={quote(abs_url, safe='')}"
+
+
+def rewrite_m3u8(text: str, base: str, camera_id: str) -> str:
+ out: list[str] = []
+ for line in text.splitlines():
+ stripped = line.strip()
+ if not stripped:
+ out.append(line)
+ continue
+ if stripped.startswith("#"):
+ def repl(m: re.Match[str]) -> str:
+ return f'URI="{_proxied(camera_id, urljoin(base, m.group(1)))}"'
+ out.append(_URI_ATTR.sub(repl, line))
+ continue
+ out.append(_proxied(camera_id, urljoin(base, stripped)))
+ return "\n".join(out) + "\n"
+
+
+async def fetch_playlist(camera_id: str, url: str) -> Response:
+ if not is_public_url(url):
+ raise HTTPException(400, "HLS URL is not public")
+ _remember(camera_id, url)
+ try:
+ async with httpx.AsyncClient(timeout=15, follow_redirects=True, headers=_UA) as c:
+ r = await c.get(url)
+ except Exception as exc: # noqa: BLE001
+ raise HTTPException(502, "HLS playlist unavailable") from exc
+ if r.status_code != 200:
+ raise HTTPException(502, "HLS playlist unavailable")
+ base = str(r.url)
+ body = rewrite_m3u8(r.text, base, camera_id)
+ return Response(
+ content=body,
+ media_type="application/vnd.apple.mpegurl",
+ headers={"Cache-Control": "no-store"},
+ )
+
+
+async def fetch_segment(camera_id: str, raw_url: str) -> Response:
+ url = unquote(raw_url)
+ if not url.lower().startswith(("http://", "https://")):
+ raise HTTPException(400, "invalid segment URL")
+ if not is_public_url(url):
+ raise HTTPException(400, "segment URL is not public")
+ if not _host_ok(camera_id, url):
+ raise HTTPException(400, "segment host not in playlist")
+ try:
+ async with httpx.AsyncClient(timeout=20, follow_redirects=True, headers=_UA) as c:
+ r = await c.get(url)
+ except Exception as exc: # noqa: BLE001
+ raise HTTPException(502, "HLS segment unavailable") from exc
+ if r.status_code != 200:
+ raise HTTPException(502, "HLS segment unavailable")
+ ctype = (r.headers.get("content-type") or "").lower()
+ if "mpegurl" in ctype or "m3u8" in url.lower().split("?")[0]:
+ body = rewrite_m3u8(r.text, str(r.url), camera_id)
+ return Response(content=body, media_type="application/vnd.apple.mpegurl",
+ headers={"Cache-Control": "no-store"})
+ return Response(
+ content=r.content,
+ media_type=r.headers.get("content-type") or "application/octet-stream",
+ headers={"Cache-Control": "no-store"},
+ )
diff --git a/app/camera_preview.py b/app/camera_preview.py
index 426caa5..b82dba0 100644
--- a/app/camera_preview.py
+++ b/app/camera_preview.py
@@ -177,8 +177,10 @@ async def ffmpeg_mjpeg_stream(url: str):
return
cmd = [
_FFMPEG, "-hide_banner", "-loglevel", "error", "-nostdin",
- "-rtsp_transport", "tcp",
- "-timeout", "4000000",
+ ]
+ if url.lower().startswith("rtsp://"):
+ cmd += ["-rtsp_transport", "tcp", "-timeout", "4000000"]
+ cmd += [
"-i", url,
"-an", "-c:v", "mjpeg", "-q:v", "8",
"-f", "mpjpeg", "pipe:1",
diff --git a/app/camera_scraper.py b/app/camera_scraper.py
index b097a03..444d226 100644
--- a/app/camera_scraper.py
+++ b/app/camera_scraper.py
@@ -303,6 +303,8 @@ def parse_alertwest_json(text: str, source_name: str) -> list[dict]:
f"https://img.cdn.prod.alertwest.com/data/img/"
f"{cid}/{dt:%Y}/{dt:%m}/{dt:%d}/{img}"
)
+ # Stable identity (the JPEG filename changes every capture).
+ ident = f"https://img.cdn.prod.alertwest.com/cam/{cid}"
loc = locs.get(cam.get("lid")) or {}
try:
lat = float(loc["lat"]) if loc.get("lat") is not None else None
@@ -312,7 +314,7 @@ def parse_alertwest_json(text: str, source_name: str) -> list[dict]:
bits = [cam.get("cn"), cam.get("co"), loc.get("st") or cam.get("st")]
name = ", ".join(str(b) for b in bits if b)
out.append({
- "source_url": snap,
+ "source_url": ident,
"snapshot_url": snap,
"discovery_source": source_name,
"location_lat": lat,
@@ -324,6 +326,49 @@ def parse_alertwest_json(text: str, source_name: str) -> list[dict]:
return out
+def parse_live_streams_geojson(text: str, source_name: str) -> list[dict]:
+ """Parse willytop8/Live-Environment-Streams GeoJSON.
+
+ Only direct, playable URLs: HLS, YouTube, HTTP stills. Skip html_page
+ feeds that need token extraction or a headless browser.
+ """
+ try:
+ payload = json.loads(text)
+ except (json.JSONDecodeError, ValueError):
+ return []
+ usable = {"hls", "youtube", "http_image"}
+ out: list[dict] = []
+ for feat in payload.get("features") or []:
+ props = feat.get("properties") or {}
+ ut = (props.get("url_type") or "").lower()
+ if ut not in usable:
+ continue
+ if props.get("source_url_requires"):
+ continue
+ url = (props.get("url") or "").strip()
+ if not url:
+ continue
+ coords = (feat.get("geometry") or {}).get("coordinates") or []
+ lon = lat = None
+ if len(coords) >= 2:
+ try:
+ lon, lat = float(coords[0]), float(coords[1])
+ except (TypeError, ValueError):
+ lon = lat = None
+ dtype = "ip-cam" if ut == "http_image" else ut
+ out.append({
+ "source_url": url,
+ "snapshot_url": url,
+ "discovery_source": source_name,
+ "location_lat": lat,
+ "location_lon": lon,
+ "location_name": props.get("display_name") or props.get("name"),
+ "vendor": props.get("source_family"),
+ "device_type": dtype,
+ })
+ return out
+
+
def parse_directory_html(html: str, base_url: str, source_name: str) -> list[dict]:
"""Generic Insecam-style directory parser."""
cams = []
@@ -393,6 +438,9 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder,
if ("getCameraDataByLoc" in src_url
or ("json" in ctype and '"locs"' in body[:4000] and '"cams"' in body[:8000])):
cams = parse_alertwest_json(body, name)
+ elif (src_url.endswith(".geojson") or src_url.endswith("/streams.geojson")
+ or '"FeatureCollection"' in body[:400]):
+ cams = parse_live_streams_geojson(body, name)
elif "html" in ctype:
cams = parse_directory_html(body, str(resp.url), name)
else:
diff --git a/app/main.py b/app/main.py
index 472d841..1d7474d 100644
--- a/app/main.py
+++ b/app/main.py
@@ -812,10 +812,13 @@ async def camera_stream(camera_id: UUID):
if not row:
raise HTTPException(404, "Camera not found")
url = row["snapshot_url"] or row["source_url"]
- if str(url or "").lower().startswith("rtsp://"):
+ low = str(url or "").lower()
+ if low.startswith("rtsp://") or ".m3u8" in low or row.get("device_type") == "hls":
from camera_preview import ffmpeg_mjpeg_stream, _FFMPEG
if not _FFMPEG:
- raise HTTPException(502, "RTSP preview requires ffmpeg")
+ raise HTTPException(502, "Live preview requires ffmpeg")
+ if not low.startswith("rtsp://") and not low.startswith(("http://", "https://")):
+ raise HTTPException(404, "Camera or snapshot not found")
return StreamingResponse(
ffmpeg_mjpeg_stream(url),
media_type="multipart/x-mixed-replace; boundary=ffmpeg",
@@ -858,6 +861,39 @@ async def camera_stream(camera_id: UUID):
return StreamingResponse(gen(), media_type=media)
+@app.get("/api/cameras/{camera_id}/hls.m3u8")
+async def camera_hls_playlist(camera_id: UUID):
+ """CORS-safe rewritten HLS playlist for the in-page player."""
+ from camera_hls import fetch_playlist
+ from camera_models import cameras as cam_table
+
+ async with async_session() as session:
+ row = (await session.execute(
+ select(cam_table).where(cam_table.c.id == camera_id)
+ )).mappings().one_or_none()
+ if not row:
+ raise HTTPException(404, "Camera not found")
+ url = row["snapshot_url"] or row["source_url"] or ""
+ if ".m3u8" not in url.lower() and row.get("device_type") != "hls":
+ raise HTTPException(404, "Camera is not an HLS feed")
+ return await fetch_playlist(str(camera_id), url)
+
+
+@app.get("/api/cameras/{camera_id}/hlsseg")
+async def camera_hls_segment(camera_id: UUID, u: str = Query(..., min_length=8)):
+ """Proxy one HLS segment/playlist URI rewritten by hls.m3u8."""
+ from camera_hls import fetch_segment
+ from camera_models import cameras as cam_table
+
+ async with async_session() as session:
+ exists = (await session.execute(
+ select(cam_table.c.id).where(cam_table.c.id == camera_id)
+ )).scalar_one_or_none()
+ if not exists:
+ raise HTTPException(404, "Camera not found")
+ return await fetch_segment(str(camera_id), u)
+
+
# ── News pipeline (scraper + summarizer) ──────────────────────────────────
# Backing data for the frontend news panel. Written by the vendored
# news-scraper (hourly Scrapy crawl) and news-summarizer (hourly Gemini
diff --git a/app/static/index.html b/app/static/index.html
index fdfba5d..6c6c88d 100644
--- a/app/static/index.html
+++ b/app/static/index.html
@@ -133,6 +133,7 @@
/* ── Camera popup thumbnail ───────────────────────────────── */
.cam-pop { min-width: 210px; max-width: 260px; }
.cam-pop .thumb { width: 100%; height: 160px; object-fit: cover; border-radius: 6px; border: 1px solid var(--border); margin: 0.3rem 0; box-shadow: 0 0 10px rgba(56,189,248,0.25); background: #0b1220; }
+ .cam-pop video.thumb, .cam-pop iframe.thumb { height: 180px; background: #000; }
.cam-pop .thumb.placeholder { display: flex; align-items: center; justify-content: center; color: var(--muted); font-size: 0.7rem; height: 80px; }
.cam-pop table { width: 100%; font-size: 0.72rem; border-collapse: collapse; }
.cam-pop td { padding: 0.12rem 0.2rem; vertical-align: top; }
@@ -415,6 +416,7 @@
+