diff --git a/.env.example b/.env.example
index 7de0c53..f3f121d 100644
--- a/.env.example
+++ b/.env.example
@@ -32,11 +32,11 @@ SNAPSHOT_TTL_SECONDS=300
# Continuous rolling sweep for open RTSP port 554 across a range. Runs on the
# Pi host via deploy/osint-masscan.service (needs root + raw sockets). Results
# land in the same `cameras` table as the scraper (discovery_source=masscan).
-# NOTE: 1000 pps is the residential-safe default. 10k pps saturated a home
-# uplink. A full 0.0.0.0/0 sweep at 1000 pps takes ~50 days (rolling).
+# NOTE: 200 pps is the residential-safe default. 1k/10k pps saturated a home
+# uplink. A full 0.0.0.0/0 sweep at 200 pps takes ~8 months (rolling).
MASSCAN_RANGE=0.0.0.0/0
MASSCAN_PORTS=554
-MASSCAN_RATE=1000
+MASSCAN_RATE=200
MASSCAN_RETRIES=1
MASSCAN_WAIT=0
MASSCAN_EXCLUDEFILE=/etc/osint-dashboard/masscan-excludes.txt
diff --git a/Dockerfile b/Dockerfile
index c1d4a3a..51e29f1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,7 +3,7 @@ FROM python:3.13-slim AS base
WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends \
- gcc libpq-dev \
+ gcc libpq-dev ffmpeg \
&& rm -rf /var/lib/apt/lists/*
COPY app/requirements.txt .
diff --git a/app/camera_preview.py b/app/camera_preview.py
new file mode 100644
index 0000000..cbf2f94
--- /dev/null
+++ b/app/camera_preview.py
@@ -0,0 +1,206 @@
+"""Resolve a browser-renderable preview for a camera.
+
+HTTP/MJPEG cameras already expose a snapshot_url the existing proxy can
+stream. masscan finds are stored as `rtsp://IP/` with no snapshot_url, so
+the map popup used to skip the entirely and the leftover source link
+handed the browser an rtsp:// URL (which opens VLC).
+
+This module:
+ 1. Tries a short list of unauthenticated HTTP snapshot paths (fast).
+ 2. Falls back to grabbing one JPEG frame from RTSP via ffmpeg (no auth).
+ 3. Remembers the first URL that worked on the camera row so the next
+ popup is a cache hit.
+
+No credentials are ever tried.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import shutil
+from urllib.parse import urlparse
+
+import httpx
+
+from camera_config import USER_AGENT
+from camera_models import cameras
+from camera_scraper import fetch_snapshot
+from database import async_session
+
+logger = logging.getLogger("osint.camera_preview")
+
+# Most common unauthenticated still-image endpoints on consumer NVRs/IP cams.
+# Keep this list SHORT — it runs on popup click.
+_HTTP_PATHS = (
+ "/snapshot.jpg",
+ "/cgi-bin/snapshot.cgi",
+ "/jpg/image.jpg",
+ "/image.jpg",
+ "/onvif/snapshot",
+ "/axis-cgi/jpg/image.cgi",
+ "/tmpfs/auto.jpg",
+)
+
+_FFMPEG = shutil.which("ffmpeg")
+
+
+def _host_from_url(url: str) -> str | None:
+ try:
+ return urlparse(url).hostname
+ except Exception: # noqa: BLE001
+ return None
+
+
+def _looks_like_jpeg(data: bytes) -> bool:
+ return bool(data) and len(data) >= 64 and data[:2] == b"\xff\xd8"
+
+
+async def _http_get_image(url: str, timeout: float = 2.5) -> bytes | None:
+ try:
+ async with httpx.AsyncClient(
+ timeout=timeout, follow_redirects=True,
+ headers={"User-Agent": USER_AGENT},
+ ) as c:
+ r = await c.get(url)
+ if r.status_code != 200:
+ return None
+ ctype = (r.headers.get("content-type") or "").lower()
+ if "html" in ctype or "text/" in ctype:
+ return None
+ if not _looks_like_jpeg(r.content) and "image/" not in ctype:
+ return None
+ if len(r.content) < 64:
+ return None
+ return r.content
+ except Exception: # noqa: BLE001
+ return None
+
+
+async def ffmpeg_snapshot(url: str, timeout: float = 8.0) -> bytes | None:
+ """Grab a single JPEG frame from an RTSP URL. None if ffmpeg missing/fails."""
+ if not _FFMPEG or not url.lower().startswith("rtsp://"):
+ return None
+ cmd = [
+ _FFMPEG, "-hide_banner", "-loglevel", "error", "-nostdin",
+ "-rtsp_transport", "tcp",
+ "-timeout", "4000000", # 4s socket timeout, microseconds
+ "-i", url,
+ "-frames:v", "1",
+ "-f", "image2pipe", "-vcodec", "mjpeg",
+ "pipe:1",
+ ]
+ try:
+ proc = await asyncio.create_subprocess_exec(
+ *cmd,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.DEVNULL,
+ )
+ except FileNotFoundError:
+ return None
+ try:
+ stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout)
+ except asyncio.TimeoutError:
+ proc.kill()
+ try:
+ await proc.wait()
+ except Exception: # noqa: BLE001
+ pass
+ return None
+ if proc.returncode not in (0, None) or not _looks_like_jpeg(stdout or b""):
+ return None
+ return stdout
+
+
+async def ffmpeg_mjpeg_stream(url: str):
+ """Yield an MJPEG multipart body transcoded from RTSP. Caller streams it."""
+ if not _FFMPEG:
+ return
+ cmd = [
+ _FFMPEG, "-hide_banner", "-loglevel", "error", "-nostdin",
+ "-rtsp_transport", "tcp",
+ "-timeout", "4000000",
+ "-i", url,
+ "-an", "-c:v", "mjpeg", "-q:v", "8",
+ "-f", "mpjpeg", "pipe:1",
+ ]
+ proc = await asyncio.create_subprocess_exec(
+ *cmd,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.DEVNULL,
+ )
+ try:
+ assert proc.stdout is not None
+ while True:
+ chunk = await proc.stdout.read(64 * 1024)
+ if not chunk:
+ break
+ yield chunk
+ finally:
+ if proc.returncode is None:
+ proc.kill()
+ try:
+ await proc.wait()
+ except Exception: # noqa: BLE001
+ pass
+
+
+async def _remember(camera_id, url: str) -> None:
+ try:
+ async with async_session() as session:
+ await session.execute(
+ cameras.update()
+ .where(cameras.c.id == camera_id)
+ .values(snapshot_url=url)
+ )
+ await session.commit()
+ except Exception: # noqa: BLE001
+ logger.warning("failed to persist snapshot_url for %s", camera_id,
+ exc_info=True)
+
+
+async def resolve_preview(row) -> tuple[bytes | None, str | None]:
+ """Return (jpeg_bytes, working_url) for a cameras-table row.
+
+ Tries, in order:
+ * existing HTTP snapshot_url (via the TTL cache)
+ * common HTTP still-image paths on the host
+ * ffmpeg frame grab from the stored RTSP URL, then common RTSP paths
+ """
+ snap = row.get("snapshot_url") or ""
+ source = row.get("source_url") or ""
+ host = _host_from_url(snap) or _host_from_url(source)
+ if not host:
+ return None, None
+
+ # 1. Known HTTP snapshot — go through the existing TTL cache.
+ if snap.lower().startswith(("http://", "https://")):
+ data = await fetch_snapshot(snap)
+ if data:
+ return data, snap
+
+ # 2. Probe unauthenticated HTTP stills in parallel (fast fail) BEFORE
+ # any ffmpeg — most open cams that preview at all do it over HTTP.
+ http_urls = [f"http://{host}{p}" for p in _HTTP_PATHS]
+ http_urls.append(f"http://{host}:8080/shot.jpg")
+ results = await asyncio.gather(
+ *(_http_get_image(u) for u in http_urls),
+ return_exceptions=True,
+ )
+ for url, data in zip(http_urls, results):
+ if isinstance(data, (bytes, bytearray)) and data:
+ await _remember(row["id"], url)
+ return bytes(data), url
+
+ # 3. One ffmpeg grab of the stored RTSP URL. Path-walking is too slow
+ # for a popup click; unauthenticated RTSP often needs a vendor path
+ # and/or credentials we will not try.
+ rtsp_url = snap if snap.lower().startswith("rtsp://") else source
+ if rtsp_url.lower().startswith("rtsp://"):
+ data = await ffmpeg_snapshot(rtsp_url, timeout=5.0)
+ if data:
+ if not snap:
+ await _remember(row["id"], rtsp_url)
+ return data, rtsp_url
+
+ return None, None
diff --git a/app/main.py b/app/main.py
index e74cca1..8691df9 100644
--- a/app/main.py
+++ b/app/main.py
@@ -11,6 +11,7 @@ Real-time geospatial OSINT dashboard API:
from __future__ import annotations
+import asyncio
import json
import logging
from datetime import datetime, timedelta, timezone
@@ -752,20 +753,28 @@ async def list_cameras(
@app.get("/api/cameras/{camera_id}/snapshot")
async def camera_snapshot(camera_id: UUID):
- """Snapshot image for one camera, served through the local TTL cache."""
+ """Still image for one camera.
+
+ HTTP cameras go through the TTL cache. masscan/RTSP finds have no HTTP
+ snapshot_url — we probe common still-image paths and, failing that, grab
+ one JPEG frame from RTSP via ffmpeg. No credentials are tried.
+ """
from camera_models import cameras as cam_table
- from camera_scraper import fetch_snapshot
+ from camera_preview import resolve_preview
+ from fastapi.responses import Response
async with async_session() as session:
row = (await session.execute(
select(cam_table).where(cam_table.c.id == camera_id)
)).mappings().one_or_none()
- if not row or not row["snapshot_url"]:
- raise HTTPException(404, "Camera or snapshot not found")
- data = await fetch_snapshot(row["snapshot_url"])
+ if not row:
+ raise HTTPException(404, "Camera not found")
+ try:
+ data, _url = await asyncio.wait_for(resolve_preview(row), timeout=10)
+ except asyncio.TimeoutError:
+ raise HTTPException(502, "Snapshot unavailable")
if not data:
raise HTTPException(502, "Snapshot unavailable")
- from fastapi.responses import Response
return Response(content=data, media_type="image/jpeg")
@@ -783,13 +792,22 @@ async def camera_stream(camera_id: UUID):
import httpx
async with async_session() as session:
- url = (await session.execute(
- select(cam_table.c.snapshot_url).where(cam_table.c.id == camera_id)
- )).scalar_one_or_none()
- if not url:
+ row = (await session.execute(
+ select(cam_table).where(cam_table.c.id == camera_id)
+ )).mappings().one_or_none()
+ if not row:
+ raise HTTPException(404, "Camera not found")
+ url = row["snapshot_url"] or row["source_url"]
+ if str(url or "").lower().startswith("rtsp://"):
+ from camera_preview import ffmpeg_mjpeg_stream, _FFMPEG
+ if not _FFMPEG:
+ raise HTTPException(502, "RTSP preview requires ffmpeg")
+ return StreamingResponse(
+ ffmpeg_mjpeg_stream(url),
+ media_type="multipart/x-mixed-replace; boundary=ffmpeg",
+ )
+ if not str(url or "").lower().startswith(("http://", "https://")):
raise HTTPException(404, "Camera or snapshot not found")
- if not str(url).lower().startswith(("http://", "https://")):
- raise HTTPException(400, "URL is not streamable over HTTP")
# connect timeout short so dead cams fail fast; read timeout None because
# an MJPEG stream legitimately idles between frames.
diff --git a/app/masscan_config.py b/app/masscan_config.py
index f55c386..3606d3f 100644
--- a/app/masscan_config.py
+++ b/app/masscan_config.py
@@ -12,12 +12,12 @@ ETHICS / SCOPE (mirrors camera_scraper.py):
scanner never probes RFC1918, loopback, link-local, multicast, or the
bogons. Fail closed if the excludefile is missing.
-TIMING REALITY: at the residential-safe default of 1,000 pps a full IPv4
-sweep (0.0.0.0/0, ~4.29B addresses) takes ~50 days. This is therefore a
+TIMING REALITY: at the residential-safe default of 200 pps a full IPv4
+sweep (0.0.0.0/0, ~4.29B addresses) takes ~8 months. This is therefore a
CONTINUOUS ROLLING SWEEP, not a "finish in a day" job: masscan streams
open hosts to stdout and the runner ingests them incrementally, then
restarts the sweep when a pass completes. New cameras are detected as they
-appear on each pass. 10k pps saturated a home uplink — do not raise the
+appear on each pass. 1k/10k pps saturated a home uplink — do not raise the
rate unless you are on a VPS / unmetered link.
"""
@@ -34,9 +34,9 @@ MASSCAN_RANGE = os.getenv("MASSCAN_RANGE", "0.0.0.0/0")
# Port(s) to probe. Default 554 = RTSP, the typical IP-camera port.
MASSCAN_PORTS = os.getenv("MASSCAN_PORTS", "554")
-# Packets/sec. 1,000 is the residential-safe default — 10k pps saturated
+# Packets/sec. 200 is the residential-safe default — 1k/10k pps saturated
# a home uplink. Raise only on a VPS / unmetered link.
-MASSCAN_RATE = int(os.getenv("MASSCAN_RATE", "1000"))
+MASSCAN_RATE = int(os.getenv("MASSCAN_RATE", "200"))
# Retransmission count. 1 maximizes unique-host coverage at low rate; the
# default (10) spends most of the budget re-probing the same hosts.
diff --git a/app/static/index.html b/app/static/index.html
index 0dc38e2..5bd95b4 100644
--- a/app/static/index.html
+++ b/app/static/index.html
@@ -1025,16 +1025,33 @@ function esc(s) {
return String(s == null ? '' : s).replace(/[&<>"']/g,
c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
}
+function camSourceLink(c) {
+ const url = c.source_url || '';
+ if (!url) return '';
+ // Never emit an rtsp:// href — the OS opens VLC (or another player).
+ if (url.toLowerCase().startsWith('rtsp://')) {
+ const host = url.replace(/^rtsp:\/\//i, '').replace(/\/.*$/, '');
+ return `RTSP ${esc(host)}:554`;
+ }
+ return `source page ↗`;
+}
function camThumb(c) {
- // Live preview: stream endpoint proxies the camera's own MJPEG feed, which
- // browsers render natively in an
. Fallback chain on error:
- // live stream → TTL-cached snapshot → placeholder.
- if (!c.snapshot_url || !c.id) return '