Merge pull request 'feat(cameras): MDOT MiDrive parser -> cameras table' (#34) from osint-dashboard/t_8be8d860-osint-mdot-midrive-cameras-cameras-table into master

Reviewed-on: #34

app/camera_config.py

@@ -16,6 +16,8 @@ CALTRANS_CCTV_URLS = tuple(
     f"https://cwwp2.dot.ca.gov/data/d{n}/cctv/cctvStatusD{n:02d}.json"
     for n in range(1, 13)
 )
+# MDOT MiDrive official DOT CCTV list (fields carry rendered HTML).
+MDOT_CAMERA_URL = "https://mdotjboss.state.mi.us/MiDrive/camera/list"
 _DEFAULT_SOURCE_URL = ",".join((
     # Publicly published open-camera list (markdown bullets of stream URLs).
     "https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md",
@@ -25,6 +27,8 @@ _DEFAULT_SOURCE_URL = ",".join((
     "https://raw.githubusercontent.com/willytop8/Live-Environment-Streams/main/streams.geojson",
     # Official Caltrans CWWP2 JPEG + HLS CCTV (districts 1–12).
     *CALTRANS_CCTV_URLS,
+    # Official MDOT MiDrive CCTV (JPEG stills, Michigan).
+    MDOT_CAMERA_URL,
 ))
 CAMERA_SOURCE_URLS = [
     u.strip()

app/camera_scraper.py

@@ -379,6 +379,79 @@ def parse_caltrans_json(text: str, source_name: str) -> list[dict]:
     return out
 
 
+# MDOT MiDrive field extractors (fields carry rendered HTML).
+_MDOT_LAT_RE = re.compile(r"lat=(-?\d+(?:\.\d+)?)", re.I)
+_MDOT_LON_RE = re.compile(r"lon=(-?\d+(?:\.\d+)?)", re.I)
+_MDOT_ID_RE = re.compile(r"[?&]id=(\d+)", re.I)
+_MDOT_IMG_RE = re.compile(r'<img[^>]+src=["\']([^"\']+)["\']', re.I)
+
+# Michigan bbox (docs/osiris-ideas.md §3.2): lat 41.6–48.3, lon -90.5–-82.1.
+MDOT_LAT_RANGE = (41.6, 48.3)
+MDOT_LON_RANGE = (-90.5, -82.1)
+
+
+def parse_mdot_json(text: str, source_name: str) -> list[dict]:
+    """Parse MDOT MiDrive `camera/list` JSON (fields carry rendered HTML).
+
+    Coordinates and the stable id live in the `county` field's map link
+    (`/MiDrive/map?...lat=&lon=&id=`); the `image` field carries an `<img>`
+    whose src is the JPEG still. Out-of-bbox and coord-less rows are dropped.
+    """
+    try:
+        payload = json.loads(text)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    if not isinstance(payload, list):
+        return []
+    out: list[dict] = []
+    for row in payload:
+        if not isinstance(row, dict):
+            continue
+        county_html = row.get("county") or ""
+        m_lat = _MDOT_LAT_RE.search(county_html)
+        m_lon = _MDOT_LON_RE.search(county_html)
+        m_id = _MDOT_ID_RE.search(county_html)
+        if not (m_lat and m_lon and m_id):
+            continue  # missing coordinates / stable id → drop
+        try:
+            lat = float(m_lat.group(1))
+            lon = float(m_lon.group(1))
+        except ValueError:
+            continue
+        if not (MDOT_LAT_RANGE[0] <= lat <= MDOT_LAT_RANGE[1]
+                and MDOT_LON_RANGE[0] <= lon <= MDOT_LON_RANGE[1]):
+            continue  # out of Michigan bbox → drop
+        img_m = _MDOT_IMG_RE.search(row.get("image") or "")
+        if not img_m:
+            continue
+        snap = img_m.group(1).strip()
+        low = snap.lower()
+        if not (low.startswith("http://") or low.startswith("https://")):
+            continue
+        if low.startswith("rtsp"):
+            continue
+        cam_id = m_id.group(1)
+        route = (row.get("route") or "").strip()
+        loc = (row.get("location") or "").strip().lstrip("@").strip()
+        county_name = county_html.split("<a", 1)[0].strip()
+        bits = [
+            f"{route} @ {loc}" if (route and loc) else (route or loc or None),
+            county_name or None,
+        ]
+        name = ", ".join(b for b in bits if b) or None
+        out.append({
+            "source_url": f"https://mdotjboss.state.mi.us/MiDrive/camera/{cam_id}",
+            "snapshot_url": snap,
+            "discovery_source": "mdot",
+            "location_lat": lat,
+            "location_lon": lon,
+            "location_name": name,
+            "vendor": "MDOT",
+            "device_type": "http",
+        })
+    return out
+
+
 def parse_live_streams_geojson(text: str, source_name: str) -> list[dict]:
     """Parse willytop8/Live-Environment-Streams GeoJSON.
 
@@ -490,6 +563,8 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder,
     body = resp.text
     if "cwwp2.dot.ca.gov" in src_url or "cctvStatus" in src_url:
         cams = parse_caltrans_json(body, name)
+    elif "mdotjboss.state.mi.us" in src_url or "/MiDrive/camera/list" in src_url:
+        cams = parse_mdot_json(body, name)
     elif ("getCameraDataByLoc" in src_url
             or ("json" in ctype and '"locs"' in body[:4000] and '"cams"' in body[:8000])):
         cams = parse_alertwest_json(body, name)

tests/test_live_layers.py

@@ -1,5 +1,7 @@
 """Unit tests for live map-layer mappers (aircraft, trains, AIS, WFIGS, Caltrans)."""
 
+import json
+
 from live_layers import (
     MARKER_FIELDS,
     bbox_center_radius_nm,
@@ -25,7 +27,7 @@ from live_layers import (
     _wfigs_params,
 )
 
-from camera_scraper import parse_caltrans_json
+from camera_scraper import parse_caltrans_json, parse_mdot_json
 
 
 def test_parse_bbox_and_radius_clamps_to_150_nm():
@@ -257,6 +259,65 @@ def test_parse_caltrans_skips_oos_and_maps_jpeg_hls():
     assert "rtsp://" not in cam["snapshot_url"].lower()
 
 
+def test_parse_mdot_extracts_html_fields_and_bbox_filters():
+    rows = [
+        # In-bbox, full fields.
+        {
+            "route": "11 Mile",
+            "county": 'Wayne County <a href="/MiDrive/map?cameras=true&lat=42.491304&lon=-83.04479&zoom=15&id=1129"target="_blank">Go to</a>',
+            "location": " @ Mound NB",
+            "direction": "Traffic closest to camera is traveling north.",
+            "image": '<img alt="x" class="cameraImageForActivePane" id="1129Img" src="https://micamerasimages.net/thumbs/semtoc_cam_253.flv.jpg?item=1" height="170" width="250" onerror="cameraImageBroken(this)">',
+        },
+        # Out of bbox (lat 50) → drop.
+        {
+            "route": "Far",
+            "county": 'Nowhere <a href="/MiDrive/map?lat=50.0&lon=-83.0&zoom=15&id=9999">Go to</a>',
+            "location": "",
+            "image": '<img src="https://micamerasimages.net/thumbs/x.jpg">',
+        },
+        # Missing coordinates → drop.
+        {
+            "route": "NoCoords",
+            "county": 'Somewhere <a href="/MiDrive/map?zoom=15&id=8888">Go to</a>',
+            "location": "",
+            "image": '<img src="https://micamerasimages.net/thumbs/y.jpg">',
+        },
+        # Missing image → drop.
+        {
+            "route": "NoImage",
+            "county": 'Kent <a href="/MiDrive/map?lat=42.8841&lon=-85.6646&zoom=15&id=2113">Go to</a>',
+            "location": " @ Division",
+            "image": "",
+        },
+        # RTSP image src → drop.
+        {
+            "route": "Rtsp",
+            "county": 'Wayne <a href="/MiDrive/map?lat=42.4&lon=-83.1&zoom=15&id=1234">Go to</a>',
+            "location": "",
+            "image": '<img src="rtsp://10.0.0.1/stream">',
+        },
+    ]
+    cams = parse_mdot_json(json.dumps(rows), "mdotjboss.state.mi.us")
+    assert len(cams) == 1
+    cam = cams[0]
+    assert cam["discovery_source"] == "mdot"
+    assert cam["location_lat"] == 42.491304
+    assert cam["location_lon"] == -83.04479
+    assert cam["snapshot_url"] == "https://micamerasimages.net/thumbs/semtoc_cam_253.flv.jpg?item=1"
+    assert cam["source_url"] == "https://mdotjboss.state.mi.us/MiDrive/camera/1129"
+    assert cam["device_type"] == "http"
+    assert cam["vendor"] == "MDOT"
+    assert "11 Mile @ Mound NB" in cam["location_name"]
+    assert "Wayne County" in cam["location_name"]
+
+
+def test_parse_mdot_handles_malformed_payload():
+    assert parse_mdot_json("not json", "mdot") == []
+    assert parse_mdot_json('{"not": "a list"}', "mdot") == []
+    assert parse_mdot_json("[]", "mdot") == []
+
+
 def test_quantize_bbox_stable_under_jitter():
     a = quantize_bbox(*parse_bbox("-78.7912,35.7711,-78.6101,35.9102"))
     b = quantize_bbox(*parse_bbox("-78.7900,35.7700,-78.6110,35.9090"))