cameras: ingest ALERTCalifornia/ALERTWest official public JPEGs

Adds the documented getCameraDataByLoc API (~10k public wildfire, DOT,
and FAA stills with lat/lon). Masscan will not produce viewable feeds;
this will. Map bbox fetch raised to 2000 markers.

.env.example

@@ -19,11 +19,13 @@ MINIO_SECRET_KEY=
 MINIO_SECURE=false
 
 # ── Camera discovery scraper ───────────────────────────────────────────────
-# Comma-separated public directory/list URLs (Insecam-style pages or
-# plain-text lists: url[|lat,lon|vendor|location]).
+# Comma-separated public directory/list/API URLs (Insecam-style pages,
+# plain-text lists, or the ALERTWest JSON API). Empty = built-in defaults
+# (public-ip-cams README + ALERTCalifornia/ALERTWest official JPEGs).
 CAMERA_SOURCE_URLS=
 CAMERA_SCRAPE_INTERVAL=3600
 CAMERA_REQUEST_DELAY=2.0
+CAMERA_MAX_PER_SOURCE=20000
 NOMINATIM_URL=https://nominatim.openstreetmap.org
 NOMINATIM_MIN_INTERVAL=1.1
 SNAPSHOT_TTL_SECONDS=300

app/camera_config.py

@@ -8,16 +8,16 @@ from __future__ import annotations
 import os
 
 # ── Camera scraper ─────────────────────────────────────────────────────────
-# Comma-separated public directory/list URLs to scrape. Two formats supported:
-#   * Insecam-style HTML directory pages (lat/lon embedded per camera)
-#   * Plain-text lists, one camera per line:
-#       <url>[|<lat>,<lon>|<vendor>|<location_name>]
+# Comma-separated public directory/list/API URLs to scrape.
+# Formats: Insecam-style HTML, plain-text URL lists, ALERTWest JSON.
 # NOTE: docker-compose always defines CAMERA_SOURCE_URLS (empty when no .env),
 # so os.getenv()'s default would never apply — use `or` semantics instead.
-_DEFAULT_SOURCE_URL = (
+_DEFAULT_SOURCE_URL = ",".join((
     # Publicly published open-camera list (markdown bullets of stream URLs).
-    "https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md"
-)
+    "https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md",
+    # ALERTCalifornia / ALERTWest official public JPEG API (wildfire + DOT + FAA).
+    "https://api.cdn.prod.alertwest.com/api/getCameraDataByLoc",
+))
 CAMERA_SOURCE_URLS = [
     u.strip()
     for u in (os.getenv("CAMERA_SOURCE_URLS") or _DEFAULT_SOURCE_URL).split(",")
@@ -31,7 +31,7 @@ CAMERA_SCRAPE_INTERVAL = int(os.getenv("CAMERA_SCRAPE_INTERVAL", "3600"))
 CAMERA_REQUEST_DELAY = float(os.getenv("CAMERA_REQUEST_DELAY", "2.0"))
 
 # Max cameras accepted per scrape cycle per source (safety cap).
-CAMERA_MAX_PER_SOURCE = int(os.getenv("CAMERA_MAX_PER_SOURCE", "5000"))
+CAMERA_MAX_PER_SOURCE = int(os.getenv("CAMERA_MAX_PER_SOURCE", "20000"))
 
 # ── Geocoding (Nominatim — free, 1 req/s hard politeness limit) ────────────
 NOMINATIM_URL = os.getenv("NOMINATIM_URL", "https://nominatim.openstreetmap.org")

app/camera_scraper.py

@@ -45,6 +45,9 @@ logger = logging.getLogger("osint.camera_scraper")
 
 # ── Private-range guard ────────────────────────────────────────────────────
 
+_public_host_cache: dict[str, bool] = {}
+
+
 def is_public_url(url: str) -> bool:
     """True only if the URL host resolves to a globally-routable address.
 
@@ -63,16 +66,22 @@ def is_public_url(url: str) -> bool:
             return addr.is_global and not addr.is_private
         except ValueError:
             pass
+        if host in _public_host_cache:
+            return _public_host_cache[host]
         # Private-name quick rejects before any DNS work.
         if (host.endswith(".local") or host.endswith(".internal")
                 or host == "localhost" or re.fullmatch(r"10\..*|172\.(1[6-9]|2\d|3[01])\..*|192\.168\..*", host)):
+            _public_host_cache[host] = False
             return False
         import socket
+        ok = True
         for info in socket.getaddrinfo(host, None):
             addr = ipaddress.ip_address(info[4][0])
             if not (addr.is_global and not addr.is_private):
-                return False
-        return True
+                ok = False
+                break
+        _public_host_cache[host] = ok
+        return ok
     except Exception:  # noqa: BLE001
         return False
 
@@ -92,7 +101,7 @@ class RateLimitedClient:
         self._delay = min_delay
         self._last: dict[str, float] = {}
         self.client = httpx.AsyncClient(
-            timeout=20, follow_redirects=True,
+            timeout=60, follow_redirects=True,
             headers={"User-Agent": USER_AGENT},
         )
 
@@ -257,6 +266,64 @@ def parse_plain_list(text: str, source_name: str) -> list[dict]:
     return cams
 
 
+def parse_alertwest_json(text: str, source_name: str) -> list[dict]:
+    """Parse ALERTWest / ALERTCalifornia getCameraDataByLoc JSON.
+
+    Official public JPEG stills (wildfire, DOT, FAA). Skip private + offline.
+    Image URL formula from their docs:
+      https://img.cdn.prod.alertwest.com/data/img/{cid}/{yyyy}/{mm}/{dd}/{img}
+    with yyyy/mm/dd taken from the epoch embedded in the filename.
+    """
+    try:
+        payload = json.loads(text)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    data = (payload or {}).get("data") or {}
+    locs = {loc.get("id"): loc for loc in (data.get("locs") or {}).get("data") or []}
+    cams_in = (data.get("cams") or {}).get("data") or []
+    epoch_re = re.compile(r"_(\d{10,})_\d+")
+    out: list[dict] = []
+    for cam in cams_in:
+        if cam.get("pv") not in (0, "0", None):
+            continue
+        if cam.get("off") not in (0, "0", None):
+            continue
+        img = cam.get("img") or ""
+        cid = cam.get("id")
+        if not img or cid is None:
+            continue
+        m = epoch_re.search(img)
+        if not m:
+            continue
+        try:
+            dt = datetime.fromtimestamp(int(m.group(1)), timezone.utc)
+        except (OSError, ValueError, OverflowError):
+            continue
+        snap = (
+            f"https://img.cdn.prod.alertwest.com/data/img/"
+            f"{cid}/{dt:%Y}/{dt:%m}/{dt:%d}/{img}"
+        )
+        loc = locs.get(cam.get("lid")) or {}
+        try:
+            lat = float(loc["lat"]) if loc.get("lat") is not None else None
+            lon = float(loc["lon"]) if loc.get("lon") is not None else None
+        except (TypeError, ValueError):
+            lat = lon = None
+        bits = [cam.get("cn"), cam.get("co"), loc.get("st") or cam.get("st")]
+        name = ", ".join(str(b) for b in bits if b)
+        out.append({
+            "source_url": snap,
+            "snapshot_url": snap,
+            "discovery_source": source_name,
+            "location_lat": lat,
+            "location_lon": lon,
+            "location_name": name or None,
+            "vendor": cam.get("pr") or cam.get("cc"),
+            "device_type": cam.get("cc") or "cctv",
+        })
+    return out
+
+
 def parse_directory_html(html: str, base_url: str, source_name: str) -> list[dict]:
     """Generic Insecam-style directory parser."""
     cams = []
@@ -322,10 +389,14 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder,
 
     name = urlparse(src_url).netloc
     ctype = resp.headers.get("content-type", "")
-    if "html" in ctype:
-        cams = parse_directory_html(resp.text, str(resp.url), name)
+    body = resp.text
+    if ("getCameraDataByLoc" in src_url
+            or ("json" in ctype and '"locs"' in body[:4000] and '"cams"' in body[:8000])):
+        cams = parse_alertwest_json(body, name)
+    elif "html" in ctype:
+        cams = parse_directory_html(body, str(resp.url), name)
     else:
-        cams = parse_plain_list(resp.text, name)
+        cams = parse_plain_list(body, name)
 
     out: list[dict] = []
     seen_in_batch: set[str] = set()
@@ -365,8 +436,11 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder,
         cam.setdefault("vendor", None)
         if not cam.get("vendor"):
             cam["vendor"] = _vendor_from_url(cam["snapshot_url"] or cam["source_url"])
-        cam["device_type"] = "rtsp" if (cam["snapshot_url"] or "").startswith("rtsp") else "ip-cam"
-        cam["raw"] = {"discovered_via": src_url}
+        if not cam.get("device_type"):
+            cam["device_type"] = (
+                "rtsp" if (cam["snapshot_url"] or "").startswith("rtsp") else "ip-cam"
+            )
+        cam.setdefault("raw", {"discovered_via": src_url})
         out.append(cam)
     logger.info("source %s yielded %d public cameras", src_url, len(out))
     return out

app/static/index.html

@@ -1056,7 +1056,7 @@ function camThumb(c) {
 async function loadCams() {
     if (!map) return;
     try {
-        const r = await fetch(`${API}/api/cameras?bbox=${currentBBox()}&limit=500`);
+        const r = await fetch(`${API}/api/cameras?bbox=${currentBBox()}&limit=2000`);
         const cams = await r.json();
         if (camsGroup) map.removeLayer(camsGroup);
         camsGroup = L.layerGroup(cams.map(c => {