Camera scraper: live public source (fury999io/public-ip-cams), markdown-list parser, batch IP geolocation via ip-api; fix empty-env fallback so default source applies

app/camera_config.py

@@ -12,14 +12,16 @@ import os
 #   * Insecam-style HTML directory pages (lat/lon embedded per camera)
 #   * Plain-text lists, one camera per line:
 #       <url>[|<lat>,<lon>|<vendor>|<location_name>]
+# NOTE: docker-compose always defines CAMERA_SOURCE_URLS (empty when no .env),
+# so os.getenv()'s default would never apply — use `or` semantics instead.
+_DEFAULT_SOURCE_URL = (
+    # Publicly published open-camera list (markdown bullets of stream URLs).
+    "https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md"
+)
 CAMERA_SOURCE_URLS = [
     u.strip()
-    for u in os.getenv(
-        "CAMERA_SOURCE_URLS",
-        # Default: publicly published open-camera lists (free, no paid API).
-        "https://raw.githubusercontent.com/neo23x0/CameraHacks/main/camera_list.txt",
-    ).split(",")
-    if u.strip()
+    for u in (os.getenv("CAMERA_SOURCE_URLS") or _DEFAULT_SOURCE_URL).split(",")
+    if u.strip()  # empty env var → fall back to the default above
 ]
 
 # Seconds between full scrape cycles of every source.

app/camera_scraper.py

@@ -212,27 +212,41 @@ def _vendor_from_url(url: str) -> str | None:
 
 
 def parse_plain_list(text: str, source_name: str) -> list[dict]:
-    """Parse lines of: url[|lat,lon|vendor|location_name]"""
+    """Parse camera URLs from a plain-text or markdown list.
+
+    Accepted line shapes:
+      * `url[|lat,lon|vendor|location_name]` (strict form)
+      * markdown bullets / bare lines that merely CONTAIN a URL
+        (e.g. `* http://1.2.3.4/mjpg/video.mjpg`) — coords/vendor unknown.
+    """
     cams = []
+    url_re = re.compile(r'(?:https?|rtsp)://[^\s\)\]>"\']+', re.I)
     for line in text.splitlines():
         line = line.strip()
         if not line or line.startswith("#"):
             continue
-        parts = [p.strip() for p in line.split("|")]
-        url = parts[0]
-        lat = lon = vendor = loc = None
-        if len(parts) > 1 and "," in parts[1]:
-            try:
-                la, lo = parts[1].split(",", 1)
-                lat, lon = float(la), float(lo)
-            except ValueError:
-                pass
-        if len(parts) > 2 and parts[2]:
-            vendor = parts[2]
-        if len(parts) > 3 and parts[3]:
-            loc = parts[3]
-        if not url.startswith(("http://", "https://", "rtsp://")):
+        m = url_re.search(line)
+        if not m:
             continue
+        # Reuse strict parsing when the URL is pipe-delimited with metadata;
+        # otherwise take just the URL and leave metadata empty.
+        if "|" in line:
+            parts = [p.strip() for p in line.split("|")]
+            url = parts[0]
+            lat = lon = vendor = loc = None
+            if len(parts) > 1 and "," in parts[1]:
+                try:
+                    la, lo = parts[1].split(",", 1)
+                    lat, lon = float(la), float(lo)
+                except ValueError:
+                    pass
+            if len(parts) > 2 and parts[2]:
+                vendor = parts[2]
+            if len(parts) > 3 and parts[3]:
+                loc = parts[3]
+        else:
+            url = m.group(0)
+            lat = lon = vendor = loc = None
         cams.append({
             "source_url": url,
             "snapshot_url": url,
@@ -266,6 +280,36 @@ def parse_directory_html(html: str, base_url: str, source_name: str) -> list[dic
     return cams
 
 
+# ── Batch IP geolocation (ip-api.com — free, 100 IPs per batch call) ───────
+
+IPAPI_BATCH_URL = "http://ip-api.com/batch"
+IPAPI_BATCH_SIZE = 100
+
+
+async def geolocate_ips(ips: list[str]) -> dict[str, tuple[float, float]]:
+    """Resolve public IPs → (lat, lon). Unresolvable IPs are simply absent."""
+    out: dict[str, tuple[float, float]] = {}
+    async with httpx.AsyncClient(timeout=15) as c:
+        for i in range(0, len(ips), IPAPI_BATCH_SIZE):
+            chunk = ips[i: i + IPAPI_BATCH_SIZE]
+            try:
+                fields = "status,country,city,lat,lon,query"
+                r = await c.post(IPAPI_BATCH_URL, json=[
+                    {"query": ip, "fields": fields} for ip in chunk
+                ])
+                r.raise_for_status()
+                for row in r.json():
+                    if (row.get("status") == "success"
+                            and row.get("lat") is not None):
+                        out[row["query"]] = (
+                            float(row["lat"]), float(row["lon"]))
+            except Exception:  # noqa: BLE001
+                logger.warning("ip-api batch failed", exc_info=True)
+            if i + IPAPI_BATCH_SIZE < len(ips):
+                await asyncio.sleep(2)  # stay well under the free rate limit
+    return out
+
+
 async def scrape_source(client: RateLimitedClient, geo: Geocoder,
                         src_url: str) -> list[dict]:
     """Fetch one source and return normalized camera dicts (public only)."""
@@ -285,23 +329,42 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder,
 
     out: list[dict] = []
     seen_in_batch: set[str] = set()
+    # Batch-geolocate coordinate-less camera hosts via ip-api (one call per
+    # 100 IPs) instead of dropping them — raw-IP lists have no embedded coords.
+    need_geo: dict[str, str] = {}
     for cam in cams[:CAMERA_MAX_PER_SOURCE]:
         url = cam["source_url"]
         h = url_hash(url)
         if h in seen_in_batch:
             continue
         seen_in_batch.add(h)
-        # Hard scope guard: drop anything not publicly routable.
         if not is_public_url(url):
             continue
+        if (cam["location_lat"] is None and cam["location_name"] is None):
+            host = urlparse(url).hostname or ""
+            try:
+                ipaddress.ip_address(host)
+                need_geo.setdefault(host, url)
+            except ValueError:
+                pass  # hostname-only: Nominatim fallback below handles it
+    geo_by_ip = await geolocate_ips(list(need_geo)) if need_geo else {}
+
+    for cam in cams:
+        if url_hash(cam["source_url"]) not in seen_in_batch:
+            continue  # didn't survive dedupe/public-range filtering above
         if cam["location_lat"] is None and cam["location_name"]:
             cam["location_lat"], cam["location_lon"] = await geo.geocode(
                 cam["location_name"])
-        if cam["location_lat"] is None and cam["location_lon"] is None:
-            continue  # map display requires coordinates
+        if (cam["location_lat"] is None and cam["location_lon"] is None):
+            host = urlparse(cam["source_url"]).hostname or ""
+            coords = geo_by_ip.get(host)
+            if coords is None:
+                continue  # map display requires coordinates
+            cam["location_lat"], cam["location_lon"] = coords
+            cam["location_name"] = f"{host} (IP-geo)"
         cam.setdefault("vendor", None)
         if not cam.get("vendor"):
-            cam["vendor"] = _vendor_from_url(cam["snapshot_url"] or url)
+            cam["vendor"] = _vendor_from_url(cam["snapshot_url"] or cam["source_url"])
         cam["device_type"] = "rtsp" if (cam["snapshot_url"] or "").startswith("rtsp") else "ip-cam"
         cam["raw"] = {"discovered_via": src_url}
         out.append(cam)