diff --git a/.env.example b/.env.example index e3dc5dd..d7a4f36 100644 --- a/.env.example +++ b/.env.example @@ -21,7 +21,8 @@ MINIO_SECURE=false # ── Camera discovery scraper ─────────────────────────────────────────────── # Comma-separated public directory/list/API URLs (Insecam-style pages, # plain-text lists, or the ALERTWest JSON API). Empty = built-in defaults -# (public-ip-cams README + ALERTCalifornia/ALERTWest official JPEGs). +# (public-ip-cams README + ALERTCalifornia/ALERTWest official JPEGs + +# Live-Environment-Streams HLS/YouTube GeoJSON). CAMERA_SOURCE_URLS= CAMERA_SCRAPE_INTERVAL=3600 CAMERA_REQUEST_DELAY=2.0 diff --git a/app/camera_config.py b/app/camera_config.py index 59b38d1..7774a53 100644 --- a/app/camera_config.py +++ b/app/camera_config.py @@ -17,6 +17,8 @@ _DEFAULT_SOURCE_URL = ",".join(( "https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md", # ALERTCalifornia / ALERTWest official public JPEG API (wildfire + DOT + FAA). "https://api.cdn.prod.alertwest.com/api/getCameraDataByLoc", + # Curated global outdoor streams (HLS / YouTube / JPEG) — VDOT, MDSHA, etc. + "https://raw.githubusercontent.com/willytop8/Live-Environment-Streams/main/streams.geojson", )) CAMERA_SOURCE_URLS = [ u.strip() diff --git a/app/camera_hls.py b/app/camera_hls.py new file mode 100644 index 0000000..9ae9555 --- /dev/null +++ b/app/camera_hls.py @@ -0,0 +1,112 @@ +"""HLS playlist rewriter so the in-page player can play CORS-blocked feeds. + +The browser talks only to /api/cameras/{id}/hls.m3u8 and /hlsseg. We fetch +the real playlist, rewrite every URI to our proxy, and remember the hosts +that appeared so /hlsseg cannot be used as an open proxy. +""" + +from __future__ import annotations + +import re +import time +from urllib.parse import quote, unquote, urljoin, urlparse + +import httpx +from fastapi import HTTPException +from fastapi.responses import Response + +from camera_scraper import is_public_url + +_UA = {"User-Agent": "osint-dashboard-hls/1.0"} +# camera_id -> (expiry_epoch, allowed_hosts) +_ALLOWED: dict[str, tuple[float, set[str]]] = {} +_TTL = 600.0 +_URI_ATTR = re.compile(r'URI="([^"]+)"', re.I) + + +def _remember(camera_id: str, url: str) -> None: + host = urlparse(url).hostname + if not host: + return + now = time.monotonic() + exp, hosts = _ALLOWED.get(camera_id, (now + _TTL, set())) + hosts.add(host.lower()) + _ALLOWED[camera_id] = (now + _TTL, hosts) + + +def _host_ok(camera_id: str, url: str) -> bool: + host = (urlparse(url).hostname or "").lower() + if not host: + return False + rec = _ALLOWED.get(camera_id) + if not rec or rec[0] < time.monotonic(): + return False + return host in rec[1] + + +def _proxied(camera_id: str, abs_url: str) -> str: + _remember(camera_id, abs_url) + return f"/api/cameras/{camera_id}/hlsseg?u={quote(abs_url, safe='')}" + + +def rewrite_m3u8(text: str, base: str, camera_id: str) -> str: + out: list[str] = [] + for line in text.splitlines(): + stripped = line.strip() + if not stripped: + out.append(line) + continue + if stripped.startswith("#"): + def repl(m: re.Match[str]) -> str: + return f'URI="{_proxied(camera_id, urljoin(base, m.group(1)))}"' + out.append(_URI_ATTR.sub(repl, line)) + continue + out.append(_proxied(camera_id, urljoin(base, stripped))) + return "\n".join(out) + "\n" + + +async def fetch_playlist(camera_id: str, url: str) -> Response: + if not is_public_url(url): + raise HTTPException(400, "HLS URL is not public") + _remember(camera_id, url) + try: + async with httpx.AsyncClient(timeout=15, follow_redirects=True, headers=_UA) as c: + r = await c.get(url) + except Exception as exc: # noqa: BLE001 + raise HTTPException(502, "HLS playlist unavailable") from exc + if r.status_code != 200: + raise HTTPException(502, "HLS playlist unavailable") + base = str(r.url) + body = rewrite_m3u8(r.text, base, camera_id) + return Response( + content=body, + media_type="application/vnd.apple.mpegurl", + headers={"Cache-Control": "no-store"}, + ) + + +async def fetch_segment(camera_id: str, raw_url: str) -> Response: + url = unquote(raw_url) + if not url.lower().startswith(("http://", "https://")): + raise HTTPException(400, "invalid segment URL") + if not is_public_url(url): + raise HTTPException(400, "segment URL is not public") + if not _host_ok(camera_id, url): + raise HTTPException(400, "segment host not in playlist") + try: + async with httpx.AsyncClient(timeout=20, follow_redirects=True, headers=_UA) as c: + r = await c.get(url) + except Exception as exc: # noqa: BLE001 + raise HTTPException(502, "HLS segment unavailable") from exc + if r.status_code != 200: + raise HTTPException(502, "HLS segment unavailable") + ctype = (r.headers.get("content-type") or "").lower() + if "mpegurl" in ctype or "m3u8" in url.lower().split("?")[0]: + body = rewrite_m3u8(r.text, str(r.url), camera_id) + return Response(content=body, media_type="application/vnd.apple.mpegurl", + headers={"Cache-Control": "no-store"}) + return Response( + content=r.content, + media_type=r.headers.get("content-type") or "application/octet-stream", + headers={"Cache-Control": "no-store"}, + ) diff --git a/app/camera_preview.py b/app/camera_preview.py index 426caa5..b82dba0 100644 --- a/app/camera_preview.py +++ b/app/camera_preview.py @@ -177,8 +177,10 @@ async def ffmpeg_mjpeg_stream(url: str): return cmd = [ _FFMPEG, "-hide_banner", "-loglevel", "error", "-nostdin", - "-rtsp_transport", "tcp", - "-timeout", "4000000", + ] + if url.lower().startswith("rtsp://"): + cmd += ["-rtsp_transport", "tcp", "-timeout", "4000000"] + cmd += [ "-i", url, "-an", "-c:v", "mjpeg", "-q:v", "8", "-f", "mpjpeg", "pipe:1", diff --git a/app/camera_scraper.py b/app/camera_scraper.py index b097a03..444d226 100644 --- a/app/camera_scraper.py +++ b/app/camera_scraper.py @@ -303,6 +303,8 @@ def parse_alertwest_json(text: str, source_name: str) -> list[dict]: f"https://img.cdn.prod.alertwest.com/data/img/" f"{cid}/{dt:%Y}/{dt:%m}/{dt:%d}/{img}" ) + # Stable identity (the JPEG filename changes every capture). + ident = f"https://img.cdn.prod.alertwest.com/cam/{cid}" loc = locs.get(cam.get("lid")) or {} try: lat = float(loc["lat"]) if loc.get("lat") is not None else None @@ -312,7 +314,7 @@ def parse_alertwest_json(text: str, source_name: str) -> list[dict]: bits = [cam.get("cn"), cam.get("co"), loc.get("st") or cam.get("st")] name = ", ".join(str(b) for b in bits if b) out.append({ - "source_url": snap, + "source_url": ident, "snapshot_url": snap, "discovery_source": source_name, "location_lat": lat, @@ -324,6 +326,49 @@ def parse_alertwest_json(text: str, source_name: str) -> list[dict]: return out +def parse_live_streams_geojson(text: str, source_name: str) -> list[dict]: + """Parse willytop8/Live-Environment-Streams GeoJSON. + + Only direct, playable URLs: HLS, YouTube, HTTP stills. Skip html_page + feeds that need token extraction or a headless browser. + """ + try: + payload = json.loads(text) + except (json.JSONDecodeError, ValueError): + return [] + usable = {"hls", "youtube", "http_image"} + out: list[dict] = [] + for feat in payload.get("features") or []: + props = feat.get("properties") or {} + ut = (props.get("url_type") or "").lower() + if ut not in usable: + continue + if props.get("source_url_requires"): + continue + url = (props.get("url") or "").strip() + if not url: + continue + coords = (feat.get("geometry") or {}).get("coordinates") or [] + lon = lat = None + if len(coords) >= 2: + try: + lon, lat = float(coords[0]), float(coords[1]) + except (TypeError, ValueError): + lon = lat = None + dtype = "ip-cam" if ut == "http_image" else ut + out.append({ + "source_url": url, + "snapshot_url": url, + "discovery_source": source_name, + "location_lat": lat, + "location_lon": lon, + "location_name": props.get("display_name") or props.get("name"), + "vendor": props.get("source_family"), + "device_type": dtype, + }) + return out + + def parse_directory_html(html: str, base_url: str, source_name: str) -> list[dict]: """Generic Insecam-style directory parser.""" cams = [] @@ -393,6 +438,9 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder, if ("getCameraDataByLoc" in src_url or ("json" in ctype and '"locs"' in body[:4000] and '"cams"' in body[:8000])): cams = parse_alertwest_json(body, name) + elif (src_url.endswith(".geojson") or src_url.endswith("/streams.geojson") + or '"FeatureCollection"' in body[:400]): + cams = parse_live_streams_geojson(body, name) elif "html" in ctype: cams = parse_directory_html(body, str(resp.url), name) else: diff --git a/app/main.py b/app/main.py index 472d841..1d7474d 100644 --- a/app/main.py +++ b/app/main.py @@ -812,10 +812,13 @@ async def camera_stream(camera_id: UUID): if not row: raise HTTPException(404, "Camera not found") url = row["snapshot_url"] or row["source_url"] - if str(url or "").lower().startswith("rtsp://"): + low = str(url or "").lower() + if low.startswith("rtsp://") or ".m3u8" in low or row.get("device_type") == "hls": from camera_preview import ffmpeg_mjpeg_stream, _FFMPEG if not _FFMPEG: - raise HTTPException(502, "RTSP preview requires ffmpeg") + raise HTTPException(502, "Live preview requires ffmpeg") + if not low.startswith("rtsp://") and not low.startswith(("http://", "https://")): + raise HTTPException(404, "Camera or snapshot not found") return StreamingResponse( ffmpeg_mjpeg_stream(url), media_type="multipart/x-mixed-replace; boundary=ffmpeg", @@ -858,6 +861,39 @@ async def camera_stream(camera_id: UUID): return StreamingResponse(gen(), media_type=media) +@app.get("/api/cameras/{camera_id}/hls.m3u8") +async def camera_hls_playlist(camera_id: UUID): + """CORS-safe rewritten HLS playlist for the in-page player.""" + from camera_hls import fetch_playlist + from camera_models import cameras as cam_table + + async with async_session() as session: + row = (await session.execute( + select(cam_table).where(cam_table.c.id == camera_id) + )).mappings().one_or_none() + if not row: + raise HTTPException(404, "Camera not found") + url = row["snapshot_url"] or row["source_url"] or "" + if ".m3u8" not in url.lower() and row.get("device_type") != "hls": + raise HTTPException(404, "Camera is not an HLS feed") + return await fetch_playlist(str(camera_id), url) + + +@app.get("/api/cameras/{camera_id}/hlsseg") +async def camera_hls_segment(camera_id: UUID, u: str = Query(..., min_length=8)): + """Proxy one HLS segment/playlist URI rewritten by hls.m3u8.""" + from camera_hls import fetch_segment + from camera_models import cameras as cam_table + + async with async_session() as session: + exists = (await session.execute( + select(cam_table.c.id).where(cam_table.c.id == camera_id) + )).scalar_one_or_none() + if not exists: + raise HTTPException(404, "Camera not found") + return await fetch_segment(str(camera_id), u) + + # ── News pipeline (scraper + summarizer) ────────────────────────────────── # Backing data for the frontend news panel. Written by the vendored # news-scraper (hourly Scrapy crawl) and news-summarizer (hourly Gemini diff --git a/app/static/index.html b/app/static/index.html index fdfba5d..6c6c88d 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -133,6 +133,7 @@ /* ── Camera popup thumbnail ───────────────────────────────── */ .cam-pop { min-width: 210px; max-width: 260px; } .cam-pop .thumb { width: 100%; height: 160px; object-fit: cover; border-radius: 6px; border: 1px solid var(--border); margin: 0.3rem 0; box-shadow: 0 0 10px rgba(56,189,248,0.25); background: #0b1220; } + .cam-pop video.thumb, .cam-pop iframe.thumb { height: 180px; background: #000; } .cam-pop .thumb.placeholder { display: flex; align-items: center; justify-content: center; color: var(--muted); font-size: 0.7rem; height: 80px; } .cam-pop table { width: 100%; font-size: 0.72rem; border-collapse: collapse; } .cam-pop td { padding: 0.12rem 0.2rem; vertical-align: top; } @@ -415,6 +416,7 @@ +