osint-dashboard/app/masscan_scanner.py

"""masscan result parsing + ingestion for the OSINT dashboard.

Turns a stream of masscan JSON-lines (open port 554 hosts) into rows in the
`cameras` table with discovery_source='masscan', deduped by URL hash against
whatever the passive scraper already found. Newly discovered hosts are also
published to NATS (`events.camera`) so the shared ingester pipeline persists
them exactly like scraper finds.

Scope: detection of OPEN hosts only. No credentials, no banners, no feed
access. Private/reserved ranges never enter masscan (see excludefile).
"""

from __future__ import annotations

import asyncio
import json
import logging
from datetime import datetime, timezone

from camera_models import cameras
from camera_scraper import url_hash, geolocate_ips
from database import async_session

from masscan_config import (
    MASSCAN_NATS_SUBJECT, MASSCAN_DISCOVERY_SOURCE,
)

logger = logging.getLogger("osint.masscan_scanner")


# ── URL building ──────────────────────────────────────────────────────────

def build_rtsp_url(ip: str) -> str:
    """Canonical URL for an open-RTSP host. Used as the dedupe key."""
    return f"rtsp://{ip}/"


# ── masscan JSON parsing ──────────────────────────────────────────────────
# masscan --output-format=json --output-file=- emits line-delimited JSON on a
# pipe (a bare object per open host), not the array form used for seekable
# files. We parse per-line and tolerate an accidental leading '['.

def parse_masscan_line(line: str) -> list[dict]:
    """Parse one masscan stdout line into a list of host records.

    A line may contain one JSON object or, defensively, be wrapped in an
    array. Returns [] on anything unparseable (harmless — the sweep repeats).
    """
    s = line.strip()
    if not s:
        return []
    s = s.lstrip("[").rstrip("]").strip()
    if not s:
        return []
    # Multiple records may share a line separated by '},{'.
    if s.endswith(","):
        s = s[:-1].rstrip()
    out: list[dict] = []
    for cand in _split_records(s):
        try:
            obj = json.loads(cand)
        except (json.JSONDecodeError, ValueError):
            continue
        if isinstance(obj, dict) and obj.get("ip"):
            out.append(obj)
    return out


def _split_records(s: str) -> list[str]:
    """Split a buffer into individual JSON object strings, honoring nesting."""
    records, depth, start = [], 0, 0
    for i, ch in enumerate(s):
        if ch == "{":
            if depth == 0:
                start = i
            depth += 1
        elif ch == "}":
            depth -= 1
            if depth == 0:
                records.append(s[start:i + 1])
    return records


def extract_open_ips(records: list[dict], port: int) -> list[str]:
    """Return the list of IPs from records that have `port` open."""
    ips: list[str] = []
    for rec in records:
        for p in rec.get("ports", []):
            if p.get("port") == port and p.get("status") == "open":
                ips.append(rec["ip"])
                break
    return ips


# ── Persistence ───────────────────────────────────────────────────────────

async def ingest_open_hosts(ips: list[str]) -> tuple[int, int]:
    """Insert-or-refresh camera rows for open RTSP hosts.

    Returns (newly_inserted, total_hosts_seen_this_batch). Geolocates each
    host via the shared ip-api batch resolver. Already-known hosts (matching
    url_hash) have last_seen/coords refreshed and are NOT counted as new.
    """
    if not ips:
        return 0, 0
    now = datetime.now(timezone.utc)
    coords = await geolocate_ips(list(dict.fromkeys(ips)))
    new = 0
    async with async_session() as session:
        for ip in dict.fromkeys(ips):
            url = build_rtsp_url(ip)
            h = url_hash(url)
            lat, lon = coords.get(ip, (None, None))
            existing = (await session.execute(
                cameras.select().where(cameras.c.url_hash == h)
            )).one_or_none()
            if existing is None:
                await session.execute(cameras.insert().values(
                    url_hash=h,
                    source_url=url,
                    snapshot_url=None,          # RTSP-only; no HTTP snapshot
                    discovery_source=MASSCAN_DISCOVERY_SOURCE,
                    location_lat=lat,
                    location_lon=lon,
                    location_name=f"{ip} (IP-geo)" if lat is not None else None,
                    vendor=None,
                    device_type="rtsp",
                    first_seen=now,
                    last_seen=now,
                    raw={"discovered_via": "masscan", "port": 554},
                ))
                new += 1
            else:
                await session.execute(cameras.update().where(
                    cameras.c.url_hash == h
                ).values(
                    last_seen=now,
                    location_lat=lat,
                    location_lon=lon,
                    location_name=f"{ip} (IP-geo)" if lat is not None else None,
                ))
        await session.commit()
    logger.info("masscan ingest: %d new, %d refreshed", new, len(set(ips)))
    return new, len(set(ips))


# ── NATS publish ──────────────────────────────────────────────────────────

async def publish_new_hosts(ips: list[str]) -> int:
    """Publish newly-found open hosts to NATS for the shared ingester.

    Returns the number of messages published (0 if NATS is down).
    """
    import json as _json
    import nats
    from config import NATS_URL

    if not ips:
        return 0
    try:
        nc = await nats.connect(NATS_URL)
    except Exception:  # noqa: BLE001
        logger.warning("NATS unavailable — skipping publish pass")
        return 0
    published = 0
    try:
        js = nc.jetstream()
        for ip in dict.fromkeys(ips):
            url = build_rtsp_url(ip)
            msg = {
                "source_type": "camera",
                "title": f"Open RTSP camera ({ip})",
                "url": url,
                "location_lat": None,
                "location_lon": None,
                "location_name": None,
                "tags": ["osint", "camera", MASSCAN_DISCOVERY_SOURCE],
                "raw": {
                    "url_hash": url_hash(url),
                    "source_url": url,
                    "snapshot_url": None,
                    "vendor": None,
                    "device_type": "rtsp",
                    "discovered_via": "masscan",
                    "port": 554,
                },
                "source_timestamp": datetime.now(timezone.utc).isoformat(),
            }
            await js.publish(MASSCAN_NATS_SUBJECT, _json.dumps(msg).encode())
            published += 1
    finally:
        await nc.close()
    logger.info("published %d masscan finds to %s", published, MASSCAN_NATS_SUBJECT)
    return published


# ── Batch drain helper used by the runner ─────────────────────────────────

async def flush(seen: set[str], new_accum: int) -> tuple[int, int]:
    """Ingest + publish the accumulated host set; return (new, published)."""
    if not seen:
        return 0, 0
    ips = list(seen)
    new, _ = await ingest_open_hosts(ips)
    published = await publish_new_hosts(ips)
    seen.clear()
    return new, published