diff --git a/.env.example b/.env.example index 7de0c53..f3f121d 100644 --- a/.env.example +++ b/.env.example @@ -32,11 +32,11 @@ SNAPSHOT_TTL_SECONDS=300 # Continuous rolling sweep for open RTSP port 554 across a range. Runs on the # Pi host via deploy/osint-masscan.service (needs root + raw sockets). Results # land in the same `cameras` table as the scraper (discovery_source=masscan). -# NOTE: 1000 pps is the residential-safe default. 10k pps saturated a home -# uplink. A full 0.0.0.0/0 sweep at 1000 pps takes ~50 days (rolling). +# NOTE: 200 pps is the residential-safe default. 1k/10k pps saturated a home +# uplink. A full 0.0.0.0/0 sweep at 200 pps takes ~8 months (rolling). MASSCAN_RANGE=0.0.0.0/0 MASSCAN_PORTS=554 -MASSCAN_RATE=1000 +MASSCAN_RATE=200 MASSCAN_RETRIES=1 MASSCAN_WAIT=0 MASSCAN_EXCLUDEFILE=/etc/osint-dashboard/masscan-excludes.txt diff --git a/Dockerfile b/Dockerfile index c1d4a3a..51e29f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM python:3.13-slim AS base WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc libpq-dev \ + gcc libpq-dev ffmpeg \ && rm -rf /var/lib/apt/lists/* COPY app/requirements.txt . diff --git a/app/camera_preview.py b/app/camera_preview.py new file mode 100644 index 0000000..cbf2f94 --- /dev/null +++ b/app/camera_preview.py @@ -0,0 +1,206 @@ +"""Resolve a browser-renderable preview for a camera. + +HTTP/MJPEG cameras already expose a snapshot_url the existing proxy can +stream. masscan finds are stored as `rtsp://IP/` with no snapshot_url, so +the map popup used to skip the entirely and the leftover source link +handed the browser an rtsp:// URL (which opens VLC). + +This module: + 1. Tries a short list of unauthenticated HTTP snapshot paths (fast). + 2. Falls back to grabbing one JPEG frame from RTSP via ffmpeg (no auth). + 3. Remembers the first URL that worked on the camera row so the next + popup is a cache hit. + +No credentials are ever tried. +""" + +from __future__ import annotations + +import asyncio +import logging +import shutil +from urllib.parse import urlparse + +import httpx + +from camera_config import USER_AGENT +from camera_models import cameras +from camera_scraper import fetch_snapshot +from database import async_session + +logger = logging.getLogger("osint.camera_preview") + +# Most common unauthenticated still-image endpoints on consumer NVRs/IP cams. +# Keep this list SHORT — it runs on popup click. +_HTTP_PATHS = ( + "/snapshot.jpg", + "/cgi-bin/snapshot.cgi", + "/jpg/image.jpg", + "/image.jpg", + "/onvif/snapshot", + "/axis-cgi/jpg/image.cgi", + "/tmpfs/auto.jpg", +) + +_FFMPEG = shutil.which("ffmpeg") + + +def _host_from_url(url: str) -> str | None: + try: + return urlparse(url).hostname + except Exception: # noqa: BLE001 + return None + + +def _looks_like_jpeg(data: bytes) -> bool: + return bool(data) and len(data) >= 64 and data[:2] == b"\xff\xd8" + + +async def _http_get_image(url: str, timeout: float = 2.5) -> bytes | None: + try: + async with httpx.AsyncClient( + timeout=timeout, follow_redirects=True, + headers={"User-Agent": USER_AGENT}, + ) as c: + r = await c.get(url) + if r.status_code != 200: + return None + ctype = (r.headers.get("content-type") or "").lower() + if "html" in ctype or "text/" in ctype: + return None + if not _looks_like_jpeg(r.content) and "image/" not in ctype: + return None + if len(r.content) < 64: + return None + return r.content + except Exception: # noqa: BLE001 + return None + + +async def ffmpeg_snapshot(url: str, timeout: float = 8.0) -> bytes | None: + """Grab a single JPEG frame from an RTSP URL. None if ffmpeg missing/fails.""" + if not _FFMPEG or not url.lower().startswith("rtsp://"): + return None + cmd = [ + _FFMPEG, "-hide_banner", "-loglevel", "error", "-nostdin", + "-rtsp_transport", "tcp", + "-timeout", "4000000", # 4s socket timeout, microseconds + "-i", url, + "-frames:v", "1", + "-f", "image2pipe", "-vcodec", "mjpeg", + "pipe:1", + ] + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + except FileNotFoundError: + return None + try: + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except asyncio.TimeoutError: + proc.kill() + try: + await proc.wait() + except Exception: # noqa: BLE001 + pass + return None + if proc.returncode not in (0, None) or not _looks_like_jpeg(stdout or b""): + return None + return stdout + + +async def ffmpeg_mjpeg_stream(url: str): + """Yield an MJPEG multipart body transcoded from RTSP. Caller streams it.""" + if not _FFMPEG: + return + cmd = [ + _FFMPEG, "-hide_banner", "-loglevel", "error", "-nostdin", + "-rtsp_transport", "tcp", + "-timeout", "4000000", + "-i", url, + "-an", "-c:v", "mjpeg", "-q:v", "8", + "-f", "mpjpeg", "pipe:1", + ] + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + try: + assert proc.stdout is not None + while True: + chunk = await proc.stdout.read(64 * 1024) + if not chunk: + break + yield chunk + finally: + if proc.returncode is None: + proc.kill() + try: + await proc.wait() + except Exception: # noqa: BLE001 + pass + + +async def _remember(camera_id, url: str) -> None: + try: + async with async_session() as session: + await session.execute( + cameras.update() + .where(cameras.c.id == camera_id) + .values(snapshot_url=url) + ) + await session.commit() + except Exception: # noqa: BLE001 + logger.warning("failed to persist snapshot_url for %s", camera_id, + exc_info=True) + + +async def resolve_preview(row) -> tuple[bytes | None, str | None]: + """Return (jpeg_bytes, working_url) for a cameras-table row. + + Tries, in order: + * existing HTTP snapshot_url (via the TTL cache) + * common HTTP still-image paths on the host + * ffmpeg frame grab from the stored RTSP URL, then common RTSP paths + """ + snap = row.get("snapshot_url") or "" + source = row.get("source_url") or "" + host = _host_from_url(snap) or _host_from_url(source) + if not host: + return None, None + + # 1. Known HTTP snapshot — go through the existing TTL cache. + if snap.lower().startswith(("http://", "https://")): + data = await fetch_snapshot(snap) + if data: + return data, snap + + # 2. Probe unauthenticated HTTP stills in parallel (fast fail) BEFORE + # any ffmpeg — most open cams that preview at all do it over HTTP. + http_urls = [f"http://{host}{p}" for p in _HTTP_PATHS] + http_urls.append(f"http://{host}:8080/shot.jpg") + results = await asyncio.gather( + *(_http_get_image(u) for u in http_urls), + return_exceptions=True, + ) + for url, data in zip(http_urls, results): + if isinstance(data, (bytes, bytearray)) and data: + await _remember(row["id"], url) + return bytes(data), url + + # 3. One ffmpeg grab of the stored RTSP URL. Path-walking is too slow + # for a popup click; unauthenticated RTSP often needs a vendor path + # and/or credentials we will not try. + rtsp_url = snap if snap.lower().startswith("rtsp://") else source + if rtsp_url.lower().startswith("rtsp://"): + data = await ffmpeg_snapshot(rtsp_url, timeout=5.0) + if data: + if not snap: + await _remember(row["id"], rtsp_url) + return data, rtsp_url + + return None, None diff --git a/app/main.py b/app/main.py index e74cca1..8691df9 100644 --- a/app/main.py +++ b/app/main.py @@ -11,6 +11,7 @@ Real-time geospatial OSINT dashboard API: from __future__ import annotations +import asyncio import json import logging from datetime import datetime, timedelta, timezone @@ -752,20 +753,28 @@ async def list_cameras( @app.get("/api/cameras/{camera_id}/snapshot") async def camera_snapshot(camera_id: UUID): - """Snapshot image for one camera, served through the local TTL cache.""" + """Still image for one camera. + + HTTP cameras go through the TTL cache. masscan/RTSP finds have no HTTP + snapshot_url — we probe common still-image paths and, failing that, grab + one JPEG frame from RTSP via ffmpeg. No credentials are tried. + """ from camera_models import cameras as cam_table - from camera_scraper import fetch_snapshot + from camera_preview import resolve_preview + from fastapi.responses import Response async with async_session() as session: row = (await session.execute( select(cam_table).where(cam_table.c.id == camera_id) )).mappings().one_or_none() - if not row or not row["snapshot_url"]: - raise HTTPException(404, "Camera or snapshot not found") - data = await fetch_snapshot(row["snapshot_url"]) + if not row: + raise HTTPException(404, "Camera not found") + try: + data, _url = await asyncio.wait_for(resolve_preview(row), timeout=10) + except asyncio.TimeoutError: + raise HTTPException(502, "Snapshot unavailable") if not data: raise HTTPException(502, "Snapshot unavailable") - from fastapi.responses import Response return Response(content=data, media_type="image/jpeg") @@ -783,13 +792,22 @@ async def camera_stream(camera_id: UUID): import httpx async with async_session() as session: - url = (await session.execute( - select(cam_table.c.snapshot_url).where(cam_table.c.id == camera_id) - )).scalar_one_or_none() - if not url: + row = (await session.execute( + select(cam_table).where(cam_table.c.id == camera_id) + )).mappings().one_or_none() + if not row: + raise HTTPException(404, "Camera not found") + url = row["snapshot_url"] or row["source_url"] + if str(url or "").lower().startswith("rtsp://"): + from camera_preview import ffmpeg_mjpeg_stream, _FFMPEG + if not _FFMPEG: + raise HTTPException(502, "RTSP preview requires ffmpeg") + return StreamingResponse( + ffmpeg_mjpeg_stream(url), + media_type="multipart/x-mixed-replace; boundary=ffmpeg", + ) + if not str(url or "").lower().startswith(("http://", "https://")): raise HTTPException(404, "Camera or snapshot not found") - if not str(url).lower().startswith(("http://", "https://")): - raise HTTPException(400, "URL is not streamable over HTTP") # connect timeout short so dead cams fail fast; read timeout None because # an MJPEG stream legitimately idles between frames. diff --git a/app/masscan_config.py b/app/masscan_config.py index f55c386..3606d3f 100644 --- a/app/masscan_config.py +++ b/app/masscan_config.py @@ -12,12 +12,12 @@ ETHICS / SCOPE (mirrors camera_scraper.py): scanner never probes RFC1918, loopback, link-local, multicast, or the bogons. Fail closed if the excludefile is missing. -TIMING REALITY: at the residential-safe default of 1,000 pps a full IPv4 -sweep (0.0.0.0/0, ~4.29B addresses) takes ~50 days. This is therefore a +TIMING REALITY: at the residential-safe default of 200 pps a full IPv4 +sweep (0.0.0.0/0, ~4.29B addresses) takes ~8 months. This is therefore a CONTINUOUS ROLLING SWEEP, not a "finish in a day" job: masscan streams open hosts to stdout and the runner ingests them incrementally, then restarts the sweep when a pass completes. New cameras are detected as they -appear on each pass. 10k pps saturated a home uplink — do not raise the +appear on each pass. 1k/10k pps saturated a home uplink — do not raise the rate unless you are on a VPS / unmetered link. """ @@ -34,9 +34,9 @@ MASSCAN_RANGE = os.getenv("MASSCAN_RANGE", "0.0.0.0/0") # Port(s) to probe. Default 554 = RTSP, the typical IP-camera port. MASSCAN_PORTS = os.getenv("MASSCAN_PORTS", "554") -# Packets/sec. 1,000 is the residential-safe default — 10k pps saturated +# Packets/sec. 200 is the residential-safe default — 1k/10k pps saturated # a home uplink. Raise only on a VPS / unmetered link. -MASSCAN_RATE = int(os.getenv("MASSCAN_RATE", "1000")) +MASSCAN_RATE = int(os.getenv("MASSCAN_RATE", "200")) # Retransmission count. 1 maximizes unique-host coverage at low rate; the # default (10) spends most of the budget re-probing the same hosts. diff --git a/app/static/index.html b/app/static/index.html index 0dc38e2..5bd95b4 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -1025,16 +1025,33 @@ function esc(s) { return String(s == null ? '' : s).replace(/[&<>"']/g, c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); } +function camSourceLink(c) { + const url = c.source_url || ''; + if (!url) return ''; + // Never emit an rtsp:// href — the OS opens VLC (or another player). + if (url.toLowerCase().startsWith('rtsp://')) { + const host = url.replace(/^rtsp:\/\//i, '').replace(/\/.*$/, ''); + return `RTSP ${esc(host)}:554`; + } + return `source page ↗`; +} function camThumb(c) { - // Live preview: stream endpoint proxies the camera's own MJPEG feed, which - // browsers render natively in an . Fallback chain on error: - // live stream → TTL-cached snapshot → placeholder. - if (!c.snapshot_url || !c.id) return '
no preview available
'; + // Preview is always served by the dashboard (never a raw rtsp:// href — + // browsers hand those to VLC). HTTP cams stream live MJPEG; RTSP/masscan + // finds grab a still via /snapshot (ffmpeg / HTTP probe). Fallback: + // live stream → still snapshot → placeholder. + if (!c.id) return '
no preview available
'; + const isRtsp = (c.device_type === 'rtsp') + || String(c.source_url || '').toLowerCase().startsWith('rtsp://') + || String(c.snapshot_url || '').toLowerCase().startsWith('rtsp://'); + const first = isRtsp + ? `/api/cameras/${esc(c.id)}/snapshot` + : `/api/cameras/${esc(c.id)}/stream`; const onerr = ( "if (!this.dataset.f) { this.dataset.f='1'; this.src='/api/cameras/" + esc(c.id) + "/snapshot'; } " + "else { this.classList.add('placeholder'); this.onerror=null; this.alt='preview unavailable'; this.removeAttribute('src'); }" ); - return `live camera preview`; + return `camera preview`; } async function loadCams() { if (!map) return; @@ -1059,7 +1076,7 @@ async function loadCams() { `Last seen${esc((c.last_seen||'').slice(0,16).replace('T',' '))}` + `Coords${(c.lat!=null&&c.lon!=null) ? c.lat.toFixed(3)+', '+c.lon.toFixed(3) : 'unknown'}` + `` + - (c.source_url ? `source page ↗` : '') + + camSourceLink(c) + ``); })); camsGroup.addTo(map);