cameras: RTSP preview via ffmpeg, stop opening VLC, drop masscan to 200pps

Masscan finds are rtsp:// with no snapshot_url, so the popup skipped the
<img> and the leftover source link handed the OS an rtsp:// URL (VLC).

- Popup always hits /api/cameras/{id}/snapshot (HTTP stills, then one
  ffmpeg frame grab). No credentials. 10s hard timeout.
- rtsp:// is rendered as text, never as an href.
- ffmpeg added to the app image for the RTSP still/MJPEG path.
- Default MASSCAN_RATE 200 (1k/10k saturated the home uplink).

.env.example

@@ -32,11 +32,11 @@ SNAPSHOT_TTL_SECONDS=300
 # Continuous rolling sweep for open RTSP port 554 across a range. Runs on the
 # Pi host via deploy/osint-masscan.service (needs root + raw sockets). Results
 # land in the same `cameras` table as the scraper (discovery_source=masscan).
-# NOTE: 1000 pps is the residential-safe default. 10k pps saturated a home
-# uplink. A full 0.0.0.0/0 sweep at 1000 pps takes ~50 days (rolling).
+# NOTE: 200 pps is the residential-safe default. 1k/10k pps saturated a home
+# uplink. A full 0.0.0.0/0 sweep at 200 pps takes ~8 months (rolling).
 MASSCAN_RANGE=0.0.0.0/0
 MASSCAN_PORTS=554
-MASSCAN_RATE=1000
+MASSCAN_RATE=200
 MASSCAN_RETRIES=1
 MASSCAN_WAIT=0
 MASSCAN_EXCLUDEFILE=/etc/osint-dashboard/masscan-excludes.txt

Dockerfile

@@ -3,7 +3,7 @@ FROM python:3.13-slim AS base
 WORKDIR /app
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    gcc libpq-dev \
+    gcc libpq-dev ffmpeg \
     && rm -rf /var/lib/apt/lists/*
 
 COPY app/requirements.txt .

app/camera_preview.py

@@ -0,0 +1,206 @@
+"""Resolve a browser-renderable preview for a camera.
+
+HTTP/MJPEG cameras already expose a snapshot_url the existing proxy can
+stream. masscan finds are stored as `rtsp://IP/` with no snapshot_url, so
+the map popup used to skip the <img> entirely and the leftover source link
+handed the browser an rtsp:// URL (which opens VLC).
+
+This module:
+  1. Tries a short list of unauthenticated HTTP snapshot paths (fast).
+  2. Falls back to grabbing one JPEG frame from RTSP via ffmpeg (no auth).
+  3. Remembers the first URL that worked on the camera row so the next
+     popup is a cache hit.
+
+No credentials are ever tried.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import shutil
+from urllib.parse import urlparse
+
+import httpx
+
+from camera_config import USER_AGENT
+from camera_models import cameras
+from camera_scraper import fetch_snapshot
+from database import async_session
+
+logger = logging.getLogger("osint.camera_preview")
+
+# Most common unauthenticated still-image endpoints on consumer NVRs/IP cams.
+# Keep this list SHORT — it runs on popup click.
+_HTTP_PATHS = (
+    "/snapshot.jpg",
+    "/cgi-bin/snapshot.cgi",
+    "/jpg/image.jpg",
+    "/image.jpg",
+    "/onvif/snapshot",
+    "/axis-cgi/jpg/image.cgi",
+    "/tmpfs/auto.jpg",
+)
+
+_FFMPEG = shutil.which("ffmpeg")
+
+
+def _host_from_url(url: str) -> str | None:
+    try:
+        return urlparse(url).hostname
+    except Exception:  # noqa: BLE001
+        return None
+
+
+def _looks_like_jpeg(data: bytes) -> bool:
+    return bool(data) and len(data) >= 64 and data[:2] == b"\xff\xd8"
+
+
+async def _http_get_image(url: str, timeout: float = 2.5) -> bytes | None:
+    try:
+        async with httpx.AsyncClient(
+            timeout=timeout, follow_redirects=True,
+            headers={"User-Agent": USER_AGENT},
+        ) as c:
+            r = await c.get(url)
+        if r.status_code != 200:
+            return None
+        ctype = (r.headers.get("content-type") or "").lower()
+        if "html" in ctype or "text/" in ctype:
+            return None
+        if not _looks_like_jpeg(r.content) and "image/" not in ctype:
+            return None
+        if len(r.content) < 64:
+            return None
+        return r.content
+    except Exception:  # noqa: BLE001
+        return None
+
+
+async def ffmpeg_snapshot(url: str, timeout: float = 8.0) -> bytes | None:
+    """Grab a single JPEG frame from an RTSP URL. None if ffmpeg missing/fails."""
+    if not _FFMPEG or not url.lower().startswith("rtsp://"):
+        return None
+    cmd = [
+        _FFMPEG, "-hide_banner", "-loglevel", "error", "-nostdin",
+        "-rtsp_transport", "tcp",
+        "-timeout", "4000000",          # 4s socket timeout, microseconds
+        "-i", url,
+        "-frames:v", "1",
+        "-f", "image2pipe", "-vcodec", "mjpeg",
+        "pipe:1",
+    ]
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            *cmd,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.DEVNULL,
+        )
+    except FileNotFoundError:
+        return None
+    try:
+        stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout)
+    except asyncio.TimeoutError:
+        proc.kill()
+        try:
+            await proc.wait()
+        except Exception:  # noqa: BLE001
+            pass
+        return None
+    if proc.returncode not in (0, None) or not _looks_like_jpeg(stdout or b""):
+        return None
+    return stdout
+
+
+async def ffmpeg_mjpeg_stream(url: str):
+    """Yield an MJPEG multipart body transcoded from RTSP. Caller streams it."""
+    if not _FFMPEG:
+        return
+    cmd = [
+        _FFMPEG, "-hide_banner", "-loglevel", "error", "-nostdin",
+        "-rtsp_transport", "tcp",
+        "-timeout", "4000000",
+        "-i", url,
+        "-an", "-c:v", "mjpeg", "-q:v", "8",
+        "-f", "mpjpeg", "pipe:1",
+    ]
+    proc = await asyncio.create_subprocess_exec(
+        *cmd,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.DEVNULL,
+    )
+    try:
+        assert proc.stdout is not None
+        while True:
+            chunk = await proc.stdout.read(64 * 1024)
+            if not chunk:
+                break
+            yield chunk
+    finally:
+        if proc.returncode is None:
+            proc.kill()
+            try:
+                await proc.wait()
+            except Exception:  # noqa: BLE001
+                pass
+
+
+async def _remember(camera_id, url: str) -> None:
+    try:
+        async with async_session() as session:
+            await session.execute(
+                cameras.update()
+                .where(cameras.c.id == camera_id)
+                .values(snapshot_url=url)
+            )
+            await session.commit()
+    except Exception:  # noqa: BLE001
+        logger.warning("failed to persist snapshot_url for %s", camera_id,
+                       exc_info=True)
+
+
+async def resolve_preview(row) -> tuple[bytes | None, str | None]:
+    """Return (jpeg_bytes, working_url) for a cameras-table row.
+
+    Tries, in order:
+      * existing HTTP snapshot_url (via the TTL cache)
+      * common HTTP still-image paths on the host
+      * ffmpeg frame grab from the stored RTSP URL, then common RTSP paths
+    """
+    snap = row.get("snapshot_url") or ""
+    source = row.get("source_url") or ""
+    host = _host_from_url(snap) or _host_from_url(source)
+    if not host:
+        return None, None
+
+    # 1. Known HTTP snapshot — go through the existing TTL cache.
+    if snap.lower().startswith(("http://", "https://")):
+        data = await fetch_snapshot(snap)
+        if data:
+            return data, snap
+
+    # 2. Probe unauthenticated HTTP stills in parallel (fast fail) BEFORE
+    # any ffmpeg — most open cams that preview at all do it over HTTP.
+    http_urls = [f"http://{host}{p}" for p in _HTTP_PATHS]
+    http_urls.append(f"http://{host}:8080/shot.jpg")
+    results = await asyncio.gather(
+        *(_http_get_image(u) for u in http_urls),
+        return_exceptions=True,
+    )
+    for url, data in zip(http_urls, results):
+        if isinstance(data, (bytes, bytearray)) and data:
+            await _remember(row["id"], url)
+            return bytes(data), url
+
+    # 3. One ffmpeg grab of the stored RTSP URL. Path-walking is too slow
+    # for a popup click; unauthenticated RTSP often needs a vendor path
+    # and/or credentials we will not try.
+    rtsp_url = snap if snap.lower().startswith("rtsp://") else source
+    if rtsp_url.lower().startswith("rtsp://"):
+        data = await ffmpeg_snapshot(rtsp_url, timeout=5.0)
+        if data:
+            if not snap:
+                await _remember(row["id"], rtsp_url)
+            return data, rtsp_url
+
+    return None, None

app/main.py

@@ -11,6 +11,7 @@ Real-time geospatial OSINT dashboard API:
 
 from __future__ import annotations
 
+import asyncio
 import json
 import logging
 from datetime import datetime, timedelta, timezone
@@ -752,20 +753,28 @@ async def list_cameras(
 
 @app.get("/api/cameras/{camera_id}/snapshot")
 async def camera_snapshot(camera_id: UUID):
-    """Snapshot image for one camera, served through the local TTL cache."""
+    """Still image for one camera.
+
+    HTTP cameras go through the TTL cache. masscan/RTSP finds have no HTTP
+    snapshot_url — we probe common still-image paths and, failing that, grab
+    one JPEG frame from RTSP via ffmpeg. No credentials are tried.
+    """
     from camera_models import cameras as cam_table
-    from camera_scraper import fetch_snapshot
+    from camera_preview import resolve_preview
+    from fastapi.responses import Response
 
     async with async_session() as session:
         row = (await session.execute(
             select(cam_table).where(cam_table.c.id == camera_id)
         )).mappings().one_or_none()
-    if not row or not row["snapshot_url"]:
-        raise HTTPException(404, "Camera or snapshot not found")
-    data = await fetch_snapshot(row["snapshot_url"])
+    if not row:
+        raise HTTPException(404, "Camera not found")
+    try:
+        data, _url = await asyncio.wait_for(resolve_preview(row), timeout=10)
+    except asyncio.TimeoutError:
+        raise HTTPException(502, "Snapshot unavailable")
     if not data:
         raise HTTPException(502, "Snapshot unavailable")
-    from fastapi.responses import Response
     return Response(content=data, media_type="image/jpeg")
 
 
@@ -783,13 +792,22 @@ async def camera_stream(camera_id: UUID):
     import httpx
 
     async with async_session() as session:
-        url = (await session.execute(
-            select(cam_table.c.snapshot_url).where(cam_table.c.id == camera_id)
-        )).scalar_one_or_none()
-    if not url:
+        row = (await session.execute(
+            select(cam_table).where(cam_table.c.id == camera_id)
+        )).mappings().one_or_none()
+    if not row:
+        raise HTTPException(404, "Camera not found")
+    url = row["snapshot_url"] or row["source_url"]
+    if str(url or "").lower().startswith("rtsp://"):
+        from camera_preview import ffmpeg_mjpeg_stream, _FFMPEG
+        if not _FFMPEG:
+            raise HTTPException(502, "RTSP preview requires ffmpeg")
+        return StreamingResponse(
+            ffmpeg_mjpeg_stream(url),
+            media_type="multipart/x-mixed-replace; boundary=ffmpeg",
+        )
+    if not str(url or "").lower().startswith(("http://", "https://")):
         raise HTTPException(404, "Camera or snapshot not found")
-    if not str(url).lower().startswith(("http://", "https://")):
-        raise HTTPException(400, "URL is not streamable over HTTP")
 
     # connect timeout short so dead cams fail fast; read timeout None because
     # an MJPEG stream legitimately idles between frames.

app/masscan_config.py

@@ -12,12 +12,12 @@ ETHICS / SCOPE (mirrors camera_scraper.py):
     scanner never probes RFC1918, loopback, link-local, multicast, or the
     bogons. Fail closed if the excludefile is missing.
 
-TIMING REALITY: at the residential-safe default of 1,000 pps a full IPv4
-sweep (0.0.0.0/0, ~4.29B addresses) takes ~50 days. This is therefore a
+TIMING REALITY: at the residential-safe default of 200 pps a full IPv4
+sweep (0.0.0.0/0, ~4.29B addresses) takes ~8 months. This is therefore a
 CONTINUOUS ROLLING SWEEP, not a "finish in a day" job: masscan streams
 open hosts to stdout and the runner ingests them incrementally, then
 restarts the sweep when a pass completes. New cameras are detected as they
-appear on each pass. 10k pps saturated a home uplink — do not raise the
+appear on each pass. 1k/10k pps saturated a home uplink — do not raise the
 rate unless you are on a VPS / unmetered link.
 """
 
@@ -34,9 +34,9 @@ MASSCAN_RANGE = os.getenv("MASSCAN_RANGE", "0.0.0.0/0")
 # Port(s) to probe. Default 554 = RTSP, the typical IP-camera port.
 MASSCAN_PORTS = os.getenv("MASSCAN_PORTS", "554")
 
-# Packets/sec. 1,000 is the residential-safe default — 10k pps saturated
+# Packets/sec. 200 is the residential-safe default — 1k/10k pps saturated
 # a home uplink. Raise only on a VPS / unmetered link.
-MASSCAN_RATE = int(os.getenv("MASSCAN_RATE", "1000"))
+MASSCAN_RATE = int(os.getenv("MASSCAN_RATE", "200"))
 
 # Retransmission count. 1 maximizes unique-host coverage at low rate; the
 # default (10) spends most of the budget re-probing the same hosts.

app/static/index.html

@@ -1025,16 +1025,33 @@ function esc(s) {
     return String(s == null ? '' : s).replace(/[&<>"']/g,
         c => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
 }
+function camSourceLink(c) {
+    const url = c.source_url || '';
+    if (!url) return '';
+    // Never emit an rtsp:// href — the OS opens VLC (or another player).
+    if (url.toLowerCase().startsWith('rtsp://')) {
+        const host = url.replace(/^rtsp:\/\//i, '').replace(/\/.*$/, '');
+        return `<span style="color:var(--muted)">RTSP ${esc(host)}:554</span>`;
+    }
+    return `<a href="${esc(url)}" target="_blank" rel="noopener">source page ↗</a>`;
+}
 function camThumb(c) {
-    // Live preview: stream endpoint proxies the camera's own MJPEG feed, which
-    // browsers render natively in an <img>. Fallback chain on error:
-    //   live stream → TTL-cached snapshot → placeholder.
-    if (!c.snapshot_url || !c.id) return '<div class="thumb placeholder">no preview available</div>';
+    // Preview is always served by the dashboard (never a raw rtsp:// href —
+    // browsers hand those to VLC). HTTP cams stream live MJPEG; RTSP/masscan
+    // finds grab a still via /snapshot (ffmpeg / HTTP probe). Fallback:
+    //   live stream → still snapshot → placeholder.
+    if (!c.id) return '<div class="thumb placeholder">no preview available</div>';
+    const isRtsp = (c.device_type === 'rtsp')
+        || String(c.source_url || '').toLowerCase().startsWith('rtsp://')
+        || String(c.snapshot_url || '').toLowerCase().startsWith('rtsp://');
+    const first = isRtsp
+        ? `/api/cameras/${esc(c.id)}/snapshot`
+        : `/api/cameras/${esc(c.id)}/stream`;
     const onerr = (
         "if (!this.dataset.f) { this.dataset.f='1'; this.src='/api/cameras/" + esc(c.id) + "/snapshot'; } " +
         "else { this.classList.add('placeholder'); this.onerror=null; this.alt='preview unavailable'; this.removeAttribute('src'); }"
     );
-    return `<img class="thumb" src="/api/cameras/${esc(c.id)}/stream" alt="live camera preview" loading="lazy" onerror="${onerr}">`;
+    return `<img class="thumb" src="${first}" alt="camera preview" loading="lazy" onerror="${onerr}">`;
 }
 async function loadCams() {
     if (!map) return;
@@ -1059,7 +1076,7 @@ async function loadCams() {
                     `<tr><td class="k">Last seen</td><td>${esc((c.last_seen||'').slice(0,16).replace('T',' '))}</td></tr>` +
                     `<tr><td class="k">Coords</td><td>${(c.lat!=null&&c.lon!=null) ? c.lat.toFixed(3)+', '+c.lon.toFixed(3) : 'unknown'}</td></tr>` +
                     `</table>` +
-                    (c.source_url ? `<a href="${esc(c.source_url)}" target="_blank" rel="noopener">source page ↗</a>` : '') +
+                    camSourceLink(c) +
                     `</div>`);
         }));
         camsGroup.addTo(map);