osint-dashboard/app/run_masscan_service.py

"""Continuous masscan rolling-sweep service for the OSINT dashboard.

Runs masscan against the configured range for open port 554 (RTSP), streams
the JSON-lines output, and ingests open hosts into the `cameras` table (new
finds only) plus publishes them to NATS — exactly like the passive scraper.

Because a full IPv4 sweep at a conservative rate takes days, this runs
masscan CONTINUOUSLY: each pass streams results in as they're found, and when
a pass completes the sweep restarts from the top. New cameras are picked up
on every pass.

Ethics: detection-only (open-port SYN scan). Private/reserved ranges are
excluded and the service REFUSES to start if the excludefile is missing, so
we never probe private space by accident.

Run once (for a manual/test pass):  python app/run_masscan_service.py --once
Run forever (systemd):            python app/run_masscan_service.py
"""

from __future__ import annotations

import asyncio
import logging
import os
import sys
from pathlib import Path

sys_path = str(Path(__file__).parent)
sys.path.insert(0, sys_path)

import masscan_config as cfg  # noqa: E402
from database import init_extensions  # noqa: E402
from masscan_scanner import (  # noqa: E402
    parse_masscan_line, extract_open_ips, flush,
)

logging.basicConfig(level=logging.INFO,
                    format="%(asctime)s %(levelname)s %(name)s: %(message)s")
logger = logging.getLogger("osint.masscan_service")

ONCE = "--once" in sys.argv[1:]


def _verify_excludefile() -> None:
    """Fail closed: refuse to sweep the wide range without an excludefile."""
    if not cfg.MASSCAN_EXCLUDEFILE:
        raise SystemExit("MASSCAN_EXCLUDEFILE is empty — refusing to run")
    if not Path(cfg.MASSCAN_EXCLUDEFILE).is_file():
        raise SystemExit(
            f"excludefile {cfg.MASSCAN_EXCLUDEFILE!r} missing — refusing to "
            f"run (would risk probing private ranges). Install the excludefile "
            f"first (see deploy/masscan-excludes.txt)."
        )


def build_command() -> list[str]:
    cmd = [
        cfg.MASSCAN_BIN,
        cfg.MASSCAN_RANGE,
        f"-p{cfg.MASSCAN_PORTS}",
        f"--rate={cfg.MASSCAN_RATE}",
        f"--retries={cfg.MASSCAN_RETRIES}",
        f"--wait={cfg.MASSCAN_WAIT}",
        "--output-format=json",
        "--output-file=-",
    ]
    if cfg.MASSCAN_EXCLUDEFILE:
        cmd.append(f"--excludefile={cfg.MASSCAN_EXCLUDEFILE}")
    return cmd


async def _drain_stderr(stream: asyncio.StreamReader) -> None:
    """Consume masscan's progress chatter so its stderr pipe never fills."""
    while True:
        line = await stream.readline()
        if not line:
            break
        text = line.decode(errors="ignore").strip()
        if text and not text.startswith("rate:"):
            logger.debug("masscan: %s", text)


async def run_pass() -> tuple[int, int]:
    """Run one full sweep pass, ingesting incrementally.

    Returns (new_hosts, total_hosts_seen) for the whole pass.
    """
    cmd = build_command()
    logger.info("starting masscan pass: %s", " ".join(cmd))
    proc = await asyncio.create_subprocess_exec(
        *cmd,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    if proc.stderr is not None:
        asyncio.ensure_future(_drain_stderr(proc.stderr))

    seen: set[str] = set()
    total_seen = 0
    total_new = 0
    try:
        while True:
            raw = await proc.stdout.readline()
            if not raw:
                break
            records = parse_masscan_line(raw.decode(errors="ignore"))
            for ip in extract_open_ips(records, 554):
                if ip in seen:
                    continue
                seen.add(ip)
                if len(seen) >= cfg.MASSCAN_FLUSH_EVERY:
                    new, _published = await flush(seen, total_new)
                    total_new += new
                    total_seen += new
        # Drain the final partial batch.
        if seen:
            new, _published = await flush(seen, total_new)
            total_new += new
        rc = await proc.wait()
    except asyncio.CancelledError:
        proc.kill()
        raise
    logger.info("masscan pass finished (rc=%s): %d new hosts ingested",
                rc, total_new)
    return total_new, total_seen


async def main() -> None:
    _verify_excludefile()
    await init_extensions()
    logger.info(
        "masscan service starting: range=%s ports=%s rate=%s pps (full sweep "
        "~%.0fh at this rate)",
        cfg.MASSCAN_RANGE, cfg.MASSCAN_PORTS, cfg.MASSCAN_RATE,
        4.29e9 / cfg.MASSCAN_RATE / 3600,
    )
    while True:
        try:
            await run_pass()
        except Exception:  # noqa: BLE001
            logger.exception("masscan pass error")
        if ONCE:
            return
        # Small gap between passes so the restart is visible in logs.
        await asyncio.sleep(5)


if __name__ == "__main__":
    asyncio.run(main())