osint-dashboard/app/masscan_scanner.py

"""masscan result parsing + ingestion for the OSINT dashboard.

Turns a stream of masscan JSON-lines (open port 554 hosts) into rows in the
`cameras` table with discovery_source='masscan', deduped by URL hash against
whatever the passive scraper already found. Newly discovered hosts are also
published to NATS (`events.camera`) so the shared ingester pipeline persists
them exactly like scraper finds.

Scope: detection of OPEN hosts only. No credentials, no banners, no feed
access. Private/reserved ranges never enter masscan (see excludefile).
"""

from __future__ import annotations

import asyncio
import json
import logging
from datetime import datetime, timezone

from camera_models import cameras
from camera_scraper import url_hash, geolocate_ips
from database import async_session

from masscan_config import (
    MASSCAN_NATS_SUBJECT, MASSCAN_DISCOVERY_SOURCE,
)

logger = logging.getLogger("osint.masscan_scanner")


# ── URL building ──────────────────────────────────────────────────────────

def build_rtsp_url(ip: str) -> str:
    """Canonical URL for an open-RTSP host. Used as the dedupe key."""
    return f"rtsp://{ip}/"


# ── masscan JSON parsing ──────────────────────────────────────────────────
# masscan --output-format=json --output-file=- emits line-delimited JSON on a
# pipe (a bare object per open host), not the array form used for seekable
# files. We parse per-line and tolerate an accidental leading '['.

def parse_masscan_line(line: str) -> list[dict]:
    """Parse one masscan stdout line into a list of host records.

    A line may contain one JSON object or, defensively, be wrapped in an
    array. Returns [] on anything unparseable (harmless — the sweep repeats).
    """
    s = line.strip()
    if not s:
        return []
    s = s.lstrip("[").rstrip("]").strip()
    if not s:
        return []
    # Multiple records may share a line separated by '},{'.
    if s.endswith(","):
        s = s[:-1].rstrip()
    out: list[dict] = []
    for cand in _split_records(s):
        try:
            obj = json.loads(cand)
        except (json.JSONDecodeError, ValueError):
            continue
        if isinstance(obj, dict) and obj.get("ip"):
            out.append(obj)
    return out


def _split_records(s: str) -> list[str]:
    """Split a buffer into individual JSON object strings, honoring nesting."""
    records, depth, start = [], 0, 0
    for i, ch in enumerate(s):
        if ch == "{":
            if depth == 0:
                start = i
            depth += 1
        elif ch == "}":
            depth -= 1
            if depth == 0:
                records.append(s[start:i + 1])
    return records


def extract_open_ips(records: list[dict], port: int) -> list[str]:
    """Return the list of IPs from records that have `port` open."""
    ips: list[str] = []
    for rec in records:
        for p in rec.get("ports", []):
            if p.get("port") == port and p.get("status") == "open":
                ips.append(rec["ip"])
                break
    return ips


# ── Persistence ───────────────────────────────────────────────────────────

async def ingest_open_hosts(ips: list[str]) -> tuple[int, list[str]]:
    """Insert-or-refresh camera rows for open RTSP hosts that have a public feed.

    A host only lands in the table (and therefore on the map) if an
    unauthenticated HTTP still or MJPEG URL responds. Port-554-only hosts
    are skipped. Returns (newly_inserted, hosts_with_working_feed).
    """
    if not ips:
        return 0, []
    from camera_preview import probe_public_feed

    unique = list(dict.fromkeys(ips))
    sem = asyncio.Semaphore(20)

    async def _probe(ip: str) -> tuple[str, str | None]:
        async with sem:
            return ip, await probe_public_feed(ip)

    probed = await asyncio.gather(*(_probe(ip) for ip in unique))
    live = [(ip, feed) for ip, feed in probed if feed]
    if not live:
        logger.info("masscan ingest: 0 working feeds of %d open-554 hosts",
                    len(unique))
        return 0, []

    now = datetime.now(timezone.utc)
    coords = await geolocate_ips([ip for ip, _ in live])
    new = 0
    async with async_session() as session:
        for ip, feed in live:
            url = build_rtsp_url(ip)
            h = url_hash(url)
            lat, lon = coords.get(ip, (None, None))
            existing = (await session.execute(
                cameras.select().where(cameras.c.url_hash == h)
            )).one_or_none()
            if existing is None:
                await session.execute(cameras.insert().values(
                    url_hash=h,
                    source_url=url,
                    snapshot_url=feed,
                    discovery_source=MASSCAN_DISCOVERY_SOURCE,
                    location_lat=lat,
                    location_lon=lon,
                    location_name=f"{ip} (IP-geo)" if lat is not None else None,
                    vendor=None,
                    device_type="rtsp",
                    first_seen=now,
                    last_seen=now,
                    raw={"discovered_via": "masscan", "port": 554,
                         "public_feed": feed},
                ))
                new += 1
            else:
                await session.execute(cameras.update().where(
                    cameras.c.url_hash == h
                ).values(
                    last_seen=now,
                    snapshot_url=feed,
                    location_lat=lat,
                    location_lon=lon,
                    location_name=f"{ip} (IP-geo)" if lat is not None else None,
                ))
        await session.commit()
    logger.info("masscan ingest: %d new working feeds (%d probed, %d open-554)",
                new, len(live), len(unique))
    return new, [ip for ip, _ in live]


# ── NATS publish ──────────────────────────────────────────────────────────

async def publish_new_hosts(ips: list[str]) -> int:
    """Publish newly-found open hosts to NATS for the shared ingester.

    Returns the number of messages published (0 if NATS is down).
    """
    import json as _json
    import nats
    from config import NATS_URL

    if not ips:
        return 0
    try:
        nc = await nats.connect(NATS_URL)
    except Exception:  # noqa: BLE001
        logger.warning("NATS unavailable — skipping publish pass")
        return 0
    published = 0
    try:
        js = nc.jetstream()
        for ip in dict.fromkeys(ips):
            url = build_rtsp_url(ip)
            msg = {
                "source_type": "camera",
                "title": f"Open RTSP camera ({ip})",
                "url": url,
                "location_lat": None,
                "location_lon": None,
                "location_name": None,
                "tags": ["osint", "camera", MASSCAN_DISCOVERY_SOURCE],
                "raw": {
                    "url_hash": url_hash(url),
                    "source_url": url,
                    "snapshot_url": None,
                    "vendor": None,
                    "device_type": "rtsp",
                    "discovered_via": "masscan",
                    "port": 554,
                },
                "source_timestamp": datetime.now(timezone.utc).isoformat(),
            }
            await js.publish(MASSCAN_NATS_SUBJECT, _json.dumps(msg).encode())
            published += 1
    finally:
        await nc.close()
    logger.info("published %d masscan finds to %s", published, MASSCAN_NATS_SUBJECT)
    return published


# ── Batch drain helper used by the runner ─────────────────────────────────

async def flush(seen: set[str], new_accum: int) -> tuple[int, int]:
    """Ingest + publish the accumulated host set; return (new, published)."""
    if not seen:
        return 0, 0
    ips = list(seen)
    new, live = await ingest_open_hosts(ips)
    published = await publish_new_hosts(live)
    seen.clear()
    return new, published