diff --git a/app/camera_config.py b/app/camera_config.py index fd12ab6..d83fc06 100644 --- a/app/camera_config.py +++ b/app/camera_config.py @@ -16,6 +16,8 @@ CALTRANS_CCTV_URLS = tuple( f"https://cwwp2.dot.ca.gov/data/d{n}/cctv/cctvStatusD{n:02d}.json" for n in range(1, 13) ) +# MDOT MiDrive official DOT CCTV list (fields carry rendered HTML). +MDOT_CAMERA_URL = "https://mdotjboss.state.mi.us/MiDrive/camera/list" _DEFAULT_SOURCE_URL = ",".join(( # Publicly published open-camera list (markdown bullets of stream URLs). "https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md", @@ -25,6 +27,8 @@ _DEFAULT_SOURCE_URL = ",".join(( "https://raw.githubusercontent.com/willytop8/Live-Environment-Streams/main/streams.geojson", # Official Caltrans CWWP2 JPEG + HLS CCTV (districts 1–12). *CALTRANS_CCTV_URLS, + # Official MDOT MiDrive CCTV (JPEG stills, Michigan). + MDOT_CAMERA_URL, )) CAMERA_SOURCE_URLS = [ u.strip() diff --git a/app/camera_scraper.py b/app/camera_scraper.py index ad5dc91..bed3aea 100644 --- a/app/camera_scraper.py +++ b/app/camera_scraper.py @@ -379,6 +379,79 @@ def parse_caltrans_json(text: str, source_name: str) -> list[dict]: return out +# MDOT MiDrive field extractors (fields carry rendered HTML). +_MDOT_LAT_RE = re.compile(r"lat=(-?\d+(?:\.\d+)?)", re.I) +_MDOT_LON_RE = re.compile(r"lon=(-?\d+(?:\.\d+)?)", re.I) +_MDOT_ID_RE = re.compile(r"[?&]id=(\d+)", re.I) +_MDOT_IMG_RE = re.compile(r']+src=["\']([^"\']+)["\']', re.I) + +# Michigan bbox (docs/osiris-ideas.md §3.2): lat 41.6–48.3, lon -90.5–-82.1. +MDOT_LAT_RANGE = (41.6, 48.3) +MDOT_LON_RANGE = (-90.5, -82.1) + + +def parse_mdot_json(text: str, source_name: str) -> list[dict]: + """Parse MDOT MiDrive `camera/list` JSON (fields carry rendered HTML). + + Coordinates and the stable id live in the `county` field's map link + (`/MiDrive/map?...lat=&lon=&id=`); the `image` field carries an `` + whose src is the JPEG still. Out-of-bbox and coord-less rows are dropped. + """ + try: + payload = json.loads(text) + except (json.JSONDecodeError, ValueError): + return [] + if not isinstance(payload, list): + return [] + out: list[dict] = [] + for row in payload: + if not isinstance(row, dict): + continue + county_html = row.get("county") or "" + m_lat = _MDOT_LAT_RE.search(county_html) + m_lon = _MDOT_LON_RE.search(county_html) + m_id = _MDOT_ID_RE.search(county_html) + if not (m_lat and m_lon and m_id): + continue # missing coordinates / stable id → drop + try: + lat = float(m_lat.group(1)) + lon = float(m_lon.group(1)) + except ValueError: + continue + if not (MDOT_LAT_RANGE[0] <= lat <= MDOT_LAT_RANGE[1] + and MDOT_LON_RANGE[0] <= lon <= MDOT_LON_RANGE[1]): + continue # out of Michigan bbox → drop + img_m = _MDOT_IMG_RE.search(row.get("image") or "") + if not img_m: + continue + snap = img_m.group(1).strip() + low = snap.lower() + if not (low.startswith("http://") or low.startswith("https://")): + continue + if low.startswith("rtsp"): + continue + cam_id = m_id.group(1) + route = (row.get("route") or "").strip() + loc = (row.get("location") or "").strip().lstrip("@").strip() + county_name = county_html.split(" list[dict]: """Parse willytop8/Live-Environment-Streams GeoJSON. @@ -490,6 +563,8 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder, body = resp.text if "cwwp2.dot.ca.gov" in src_url or "cctvStatus" in src_url: cams = parse_caltrans_json(body, name) + elif "mdotjboss.state.mi.us" in src_url or "/MiDrive/camera/list" in src_url: + cams = parse_mdot_json(body, name) elif ("getCameraDataByLoc" in src_url or ("json" in ctype and '"locs"' in body[:4000] and '"cams"' in body[:8000])): cams = parse_alertwest_json(body, name) diff --git a/tests/test_live_layers.py b/tests/test_live_layers.py index ae0f68e..a646deb 100644 --- a/tests/test_live_layers.py +++ b/tests/test_live_layers.py @@ -1,5 +1,7 @@ """Unit tests for live map-layer mappers (aircraft, trains, AIS, WFIGS, Caltrans).""" +import json + from live_layers import ( MARKER_FIELDS, bbox_center_radius_nm, @@ -25,7 +27,7 @@ from live_layers import ( _wfigs_params, ) -from camera_scraper import parse_caltrans_json +from camera_scraper import parse_caltrans_json, parse_mdot_json def test_parse_bbox_and_radius_clamps_to_150_nm(): @@ -257,6 +259,65 @@ def test_parse_caltrans_skips_oos_and_maps_jpeg_hls(): assert "rtsp://" not in cam["snapshot_url"].lower() +def test_parse_mdot_extracts_html_fields_and_bbox_filters(): + rows = [ + # In-bbox, full fields. + { + "route": "11 Mile", + "county": 'Wayne County Go to', + "location": " @ Mound NB", + "direction": "Traffic closest to camera is traveling north.", + "image": 'x', + }, + # Out of bbox (lat 50) → drop. + { + "route": "Far", + "county": 'Nowhere Go to', + "location": "", + "image": '', + }, + # Missing coordinates → drop. + { + "route": "NoCoords", + "county": 'Somewhere Go to', + "location": "", + "image": '', + }, + # Missing image → drop. + { + "route": "NoImage", + "county": 'Kent Go to', + "location": " @ Division", + "image": "", + }, + # RTSP image src → drop. + { + "route": "Rtsp", + "county": 'Wayne Go to', + "location": "", + "image": '', + }, + ] + cams = parse_mdot_json(json.dumps(rows), "mdotjboss.state.mi.us") + assert len(cams) == 1 + cam = cams[0] + assert cam["discovery_source"] == "mdot" + assert cam["location_lat"] == 42.491304 + assert cam["location_lon"] == -83.04479 + assert cam["snapshot_url"] == "https://micamerasimages.net/thumbs/semtoc_cam_253.flv.jpg?item=1" + assert cam["source_url"] == "https://mdotjboss.state.mi.us/MiDrive/camera/1129" + assert cam["device_type"] == "http" + assert cam["vendor"] == "MDOT" + assert "11 Mile @ Mound NB" in cam["location_name"] + assert "Wayne County" in cam["location_name"] + + +def test_parse_mdot_handles_malformed_payload(): + assert parse_mdot_json("not json", "mdot") == [] + assert parse_mdot_json('{"not": "a list"}', "mdot") == [] + assert parse_mdot_json("[]", "mdot") == [] + + def test_quantize_bbox_stable_under_jitter(): a = quantize_bbox(*parse_bbox("-78.7912,35.7711,-78.6101,35.9102")) b = quantize_bbox(*parse_bbox("-78.7900,35.7700,-78.6110,35.9090"))