diff --git a/.env.example b/.env.example index f3f121d..e3dc5dd 100644 --- a/.env.example +++ b/.env.example @@ -19,11 +19,13 @@ MINIO_SECRET_KEY= MINIO_SECURE=false # ── Camera discovery scraper ─────────────────────────────────────────────── -# Comma-separated public directory/list URLs (Insecam-style pages or -# plain-text lists: url[|lat,lon|vendor|location]). +# Comma-separated public directory/list/API URLs (Insecam-style pages, +# plain-text lists, or the ALERTWest JSON API). Empty = built-in defaults +# (public-ip-cams README + ALERTCalifornia/ALERTWest official JPEGs). CAMERA_SOURCE_URLS= CAMERA_SCRAPE_INTERVAL=3600 CAMERA_REQUEST_DELAY=2.0 +CAMERA_MAX_PER_SOURCE=20000 NOMINATIM_URL=https://nominatim.openstreetmap.org NOMINATIM_MIN_INTERVAL=1.1 SNAPSHOT_TTL_SECONDS=300 diff --git a/app/camera_config.py b/app/camera_config.py index 428221a..59b38d1 100644 --- a/app/camera_config.py +++ b/app/camera_config.py @@ -8,16 +8,16 @@ from __future__ import annotations import os # ── Camera scraper ───────────────────────────────────────────────────────── -# Comma-separated public directory/list URLs to scrape. Two formats supported: -# * Insecam-style HTML directory pages (lat/lon embedded per camera) -# * Plain-text lists, one camera per line: -# [|,||] +# Comma-separated public directory/list/API URLs to scrape. +# Formats: Insecam-style HTML, plain-text URL lists, ALERTWest JSON. # NOTE: docker-compose always defines CAMERA_SOURCE_URLS (empty when no .env), # so os.getenv()'s default would never apply — use `or` semantics instead. -_DEFAULT_SOURCE_URL = ( +_DEFAULT_SOURCE_URL = ",".join(( # Publicly published open-camera list (markdown bullets of stream URLs). - "https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md" -) + "https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md", + # ALERTCalifornia / ALERTWest official public JPEG API (wildfire + DOT + FAA). + "https://api.cdn.prod.alertwest.com/api/getCameraDataByLoc", +)) CAMERA_SOURCE_URLS = [ u.strip() for u in (os.getenv("CAMERA_SOURCE_URLS") or _DEFAULT_SOURCE_URL).split(",") @@ -31,7 +31,7 @@ CAMERA_SCRAPE_INTERVAL = int(os.getenv("CAMERA_SCRAPE_INTERVAL", "3600")) CAMERA_REQUEST_DELAY = float(os.getenv("CAMERA_REQUEST_DELAY", "2.0")) # Max cameras accepted per scrape cycle per source (safety cap). -CAMERA_MAX_PER_SOURCE = int(os.getenv("CAMERA_MAX_PER_SOURCE", "5000")) +CAMERA_MAX_PER_SOURCE = int(os.getenv("CAMERA_MAX_PER_SOURCE", "20000")) # ── Geocoding (Nominatim — free, 1 req/s hard politeness limit) ──────────── NOMINATIM_URL = os.getenv("NOMINATIM_URL", "https://nominatim.openstreetmap.org") diff --git a/app/camera_scraper.py b/app/camera_scraper.py index e10391e..b097a03 100644 --- a/app/camera_scraper.py +++ b/app/camera_scraper.py @@ -45,6 +45,9 @@ logger = logging.getLogger("osint.camera_scraper") # ── Private-range guard ──────────────────────────────────────────────────── +_public_host_cache: dict[str, bool] = {} + + def is_public_url(url: str) -> bool: """True only if the URL host resolves to a globally-routable address. @@ -63,16 +66,22 @@ def is_public_url(url: str) -> bool: return addr.is_global and not addr.is_private except ValueError: pass + if host in _public_host_cache: + return _public_host_cache[host] # Private-name quick rejects before any DNS work. if (host.endswith(".local") or host.endswith(".internal") or host == "localhost" or re.fullmatch(r"10\..*|172\.(1[6-9]|2\d|3[01])\..*|192\.168\..*", host)): + _public_host_cache[host] = False return False import socket + ok = True for info in socket.getaddrinfo(host, None): addr = ipaddress.ip_address(info[4][0]) if not (addr.is_global and not addr.is_private): - return False - return True + ok = False + break + _public_host_cache[host] = ok + return ok except Exception: # noqa: BLE001 return False @@ -92,7 +101,7 @@ class RateLimitedClient: self._delay = min_delay self._last: dict[str, float] = {} self.client = httpx.AsyncClient( - timeout=20, follow_redirects=True, + timeout=60, follow_redirects=True, headers={"User-Agent": USER_AGENT}, ) @@ -257,6 +266,64 @@ def parse_plain_list(text: str, source_name: str) -> list[dict]: return cams +def parse_alertwest_json(text: str, source_name: str) -> list[dict]: + """Parse ALERTWest / ALERTCalifornia getCameraDataByLoc JSON. + + Official public JPEG stills (wildfire, DOT, FAA). Skip private + offline. + Image URL formula from their docs: + https://img.cdn.prod.alertwest.com/data/img/{cid}/{yyyy}/{mm}/{dd}/{img} + with yyyy/mm/dd taken from the epoch embedded in the filename. + """ + try: + payload = json.loads(text) + except (json.JSONDecodeError, ValueError): + return [] + data = (payload or {}).get("data") or {} + locs = {loc.get("id"): loc for loc in (data.get("locs") or {}).get("data") or []} + cams_in = (data.get("cams") or {}).get("data") or [] + epoch_re = re.compile(r"_(\d{10,})_\d+") + out: list[dict] = [] + for cam in cams_in: + if cam.get("pv") not in (0, "0", None): + continue + if cam.get("off") not in (0, "0", None): + continue + img = cam.get("img") or "" + cid = cam.get("id") + if not img or cid is None: + continue + m = epoch_re.search(img) + if not m: + continue + try: + dt = datetime.fromtimestamp(int(m.group(1)), timezone.utc) + except (OSError, ValueError, OverflowError): + continue + snap = ( + f"https://img.cdn.prod.alertwest.com/data/img/" + f"{cid}/{dt:%Y}/{dt:%m}/{dt:%d}/{img}" + ) + loc = locs.get(cam.get("lid")) or {} + try: + lat = float(loc["lat"]) if loc.get("lat") is not None else None + lon = float(loc["lon"]) if loc.get("lon") is not None else None + except (TypeError, ValueError): + lat = lon = None + bits = [cam.get("cn"), cam.get("co"), loc.get("st") or cam.get("st")] + name = ", ".join(str(b) for b in bits if b) + out.append({ + "source_url": snap, + "snapshot_url": snap, + "discovery_source": source_name, + "location_lat": lat, + "location_lon": lon, + "location_name": name or None, + "vendor": cam.get("pr") or cam.get("cc"), + "device_type": cam.get("cc") or "cctv", + }) + return out + + def parse_directory_html(html: str, base_url: str, source_name: str) -> list[dict]: """Generic Insecam-style directory parser.""" cams = [] @@ -322,10 +389,14 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder, name = urlparse(src_url).netloc ctype = resp.headers.get("content-type", "") - if "html" in ctype: - cams = parse_directory_html(resp.text, str(resp.url), name) + body = resp.text + if ("getCameraDataByLoc" in src_url + or ("json" in ctype and '"locs"' in body[:4000] and '"cams"' in body[:8000])): + cams = parse_alertwest_json(body, name) + elif "html" in ctype: + cams = parse_directory_html(body, str(resp.url), name) else: - cams = parse_plain_list(resp.text, name) + cams = parse_plain_list(body, name) out: list[dict] = [] seen_in_batch: set[str] = set() @@ -365,8 +436,11 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder, cam.setdefault("vendor", None) if not cam.get("vendor"): cam["vendor"] = _vendor_from_url(cam["snapshot_url"] or cam["source_url"]) - cam["device_type"] = "rtsp" if (cam["snapshot_url"] or "").startswith("rtsp") else "ip-cam" - cam["raw"] = {"discovered_via": src_url} + if not cam.get("device_type"): + cam["device_type"] = ( + "rtsp" if (cam["snapshot_url"] or "").startswith("rtsp") else "ip-cam" + ) + cam.setdefault("raw", {"discovered_via": src_url}) out.append(cam) logger.info("source %s yielded %d public cameras", src_url, len(out)) return out diff --git a/app/static/index.html b/app/static/index.html index 5bd95b4..fdfba5d 100644 --- a/app/static/index.html +++ b/app/static/index.html @@ -1056,7 +1056,7 @@ function camThumb(c) { async function loadCams() { if (!map) return; try { - const r = await fetch(`${API}/api/cameras?bbox=${currentBBox()}&limit=500`); + const r = await fetch(`${API}/api/cameras?bbox=${currentBBox()}&limit=2000`); const cams = await r.json(); if (camsGroup) map.removeLayer(camsGroup); camsGroup = L.layerGroup(cams.map(c => {