From 347ee3a6426bfbae854f95efbed48eff5c74a8c2 Mon Sep 17 00:00:00 2001 From: Sirius DevOps Date: Mon, 24 Aug 2026 21:03:33 -0400 Subject: [PATCH] Camera scraper: live public source (fury999io/public-ip-cams), markdown-list parser, batch IP geolocation via ip-api; fix empty-env fallback so default source applies --- app/camera_config.py | 14 +++--- app/camera_scraper.py | 101 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 90 insertions(+), 25 deletions(-) diff --git a/app/camera_config.py b/app/camera_config.py index 96020ea..428221a 100644 --- a/app/camera_config.py +++ b/app/camera_config.py @@ -12,14 +12,16 @@ import os # * Insecam-style HTML directory pages (lat/lon embedded per camera) # * Plain-text lists, one camera per line: # [|,||] +# NOTE: docker-compose always defines CAMERA_SOURCE_URLS (empty when no .env), +# so os.getenv()'s default would never apply — use `or` semantics instead. +_DEFAULT_SOURCE_URL = ( + # Publicly published open-camera list (markdown bullets of stream URLs). + "https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md" +) CAMERA_SOURCE_URLS = [ u.strip() - for u in os.getenv( - "CAMERA_SOURCE_URLS", - # Default: publicly published open-camera lists (free, no paid API). - "https://raw.githubusercontent.com/neo23x0/CameraHacks/main/camera_list.txt", - ).split(",") - if u.strip() + for u in (os.getenv("CAMERA_SOURCE_URLS") or _DEFAULT_SOURCE_URL).split(",") + if u.strip() # empty env var → fall back to the default above ] # Seconds between full scrape cycles of every source. diff --git a/app/camera_scraper.py b/app/camera_scraper.py index bb6ecf9..e10391e 100644 --- a/app/camera_scraper.py +++ b/app/camera_scraper.py @@ -212,27 +212,41 @@ def _vendor_from_url(url: str) -> str | None: def parse_plain_list(text: str, source_name: str) -> list[dict]: - """Parse lines of: url[|lat,lon|vendor|location_name]""" + """Parse camera URLs from a plain-text or markdown list. + + Accepted line shapes: + * `url[|lat,lon|vendor|location_name]` (strict form) + * markdown bullets / bare lines that merely CONTAIN a URL + (e.g. `* http://1.2.3.4/mjpg/video.mjpg`) — coords/vendor unknown. + """ cams = [] + url_re = re.compile(r'(?:https?|rtsp)://[^\s\)\]>"\']+', re.I) for line in text.splitlines(): line = line.strip() if not line or line.startswith("#"): continue - parts = [p.strip() for p in line.split("|")] - url = parts[0] - lat = lon = vendor = loc = None - if len(parts) > 1 and "," in parts[1]: - try: - la, lo = parts[1].split(",", 1) - lat, lon = float(la), float(lo) - except ValueError: - pass - if len(parts) > 2 and parts[2]: - vendor = parts[2] - if len(parts) > 3 and parts[3]: - loc = parts[3] - if not url.startswith(("http://", "https://", "rtsp://")): + m = url_re.search(line) + if not m: continue + # Reuse strict parsing when the URL is pipe-delimited with metadata; + # otherwise take just the URL and leave metadata empty. + if "|" in line: + parts = [p.strip() for p in line.split("|")] + url = parts[0] + lat = lon = vendor = loc = None + if len(parts) > 1 and "," in parts[1]: + try: + la, lo = parts[1].split(",", 1) + lat, lon = float(la), float(lo) + except ValueError: + pass + if len(parts) > 2 and parts[2]: + vendor = parts[2] + if len(parts) > 3 and parts[3]: + loc = parts[3] + else: + url = m.group(0) + lat = lon = vendor = loc = None cams.append({ "source_url": url, "snapshot_url": url, @@ -266,6 +280,36 @@ def parse_directory_html(html: str, base_url: str, source_name: str) -> list[dic return cams +# ── Batch IP geolocation (ip-api.com — free, 100 IPs per batch call) ─────── + +IPAPI_BATCH_URL = "http://ip-api.com/batch" +IPAPI_BATCH_SIZE = 100 + + +async def geolocate_ips(ips: list[str]) -> dict[str, tuple[float, float]]: + """Resolve public IPs → (lat, lon). Unresolvable IPs are simply absent.""" + out: dict[str, tuple[float, float]] = {} + async with httpx.AsyncClient(timeout=15) as c: + for i in range(0, len(ips), IPAPI_BATCH_SIZE): + chunk = ips[i: i + IPAPI_BATCH_SIZE] + try: + fields = "status,country,city,lat,lon,query" + r = await c.post(IPAPI_BATCH_URL, json=[ + {"query": ip, "fields": fields} for ip in chunk + ]) + r.raise_for_status() + for row in r.json(): + if (row.get("status") == "success" + and row.get("lat") is not None): + out[row["query"]] = ( + float(row["lat"]), float(row["lon"])) + except Exception: # noqa: BLE001 + logger.warning("ip-api batch failed", exc_info=True) + if i + IPAPI_BATCH_SIZE < len(ips): + await asyncio.sleep(2) # stay well under the free rate limit + return out + + async def scrape_source(client: RateLimitedClient, geo: Geocoder, src_url: str) -> list[dict]: """Fetch one source and return normalized camera dicts (public only).""" @@ -285,23 +329,42 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder, out: list[dict] = [] seen_in_batch: set[str] = set() + # Batch-geolocate coordinate-less camera hosts via ip-api (one call per + # 100 IPs) instead of dropping them — raw-IP lists have no embedded coords. + need_geo: dict[str, str] = {} for cam in cams[:CAMERA_MAX_PER_SOURCE]: url = cam["source_url"] h = url_hash(url) if h in seen_in_batch: continue seen_in_batch.add(h) - # Hard scope guard: drop anything not publicly routable. if not is_public_url(url): continue + if (cam["location_lat"] is None and cam["location_name"] is None): + host = urlparse(url).hostname or "" + try: + ipaddress.ip_address(host) + need_geo.setdefault(host, url) + except ValueError: + pass # hostname-only: Nominatim fallback below handles it + geo_by_ip = await geolocate_ips(list(need_geo)) if need_geo else {} + + for cam in cams: + if url_hash(cam["source_url"]) not in seen_in_batch: + continue # didn't survive dedupe/public-range filtering above if cam["location_lat"] is None and cam["location_name"]: cam["location_lat"], cam["location_lon"] = await geo.geocode( cam["location_name"]) - if cam["location_lat"] is None and cam["location_lon"] is None: - continue # map display requires coordinates + if (cam["location_lat"] is None and cam["location_lon"] is None): + host = urlparse(cam["source_url"]).hostname or "" + coords = geo_by_ip.get(host) + if coords is None: + continue # map display requires coordinates + cam["location_lat"], cam["location_lon"] = coords + cam["location_name"] = f"{host} (IP-geo)" cam.setdefault("vendor", None) if not cam.get("vendor"): - cam["vendor"] = _vendor_from_url(cam["snapshot_url"] or url) + cam["vendor"] = _vendor_from_url(cam["snapshot_url"] or cam["source_url"]) cam["device_type"] = "rtsp" if (cam["snapshot_url"] or "").startswith("rtsp") else "ip-cam" cam["raw"] = {"discovered_via": src_url} out.append(cam)