diff --git a/app/camera_config.py b/app/camera_config.py index fd12ab6..d83fc06 100644 --- a/app/camera_config.py +++ b/app/camera_config.py @@ -16,6 +16,8 @@ CALTRANS_CCTV_URLS = tuple( f"https://cwwp2.dot.ca.gov/data/d{n}/cctv/cctvStatusD{n:02d}.json" for n in range(1, 13) ) +# MDOT MiDrive official DOT CCTV list (fields carry rendered HTML). +MDOT_CAMERA_URL = "https://mdotjboss.state.mi.us/MiDrive/camera/list" _DEFAULT_SOURCE_URL = ",".join(( # Publicly published open-camera list (markdown bullets of stream URLs). "https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md", @@ -25,6 +27,8 @@ _DEFAULT_SOURCE_URL = ",".join(( "https://raw.githubusercontent.com/willytop8/Live-Environment-Streams/main/streams.geojson", # Official Caltrans CWWP2 JPEG + HLS CCTV (districts 1–12). *CALTRANS_CCTV_URLS, + # Official MDOT MiDrive CCTV (JPEG stills, Michigan). + MDOT_CAMERA_URL, )) CAMERA_SOURCE_URLS = [ u.strip() diff --git a/app/camera_scraper.py b/app/camera_scraper.py index ad5dc91..bed3aea 100644 --- a/app/camera_scraper.py +++ b/app/camera_scraper.py @@ -379,6 +379,79 @@ def parse_caltrans_json(text: str, source_name: str) -> list[dict]: return out +# MDOT MiDrive field extractors (fields carry rendered HTML). +_MDOT_LAT_RE = re.compile(r"lat=(-?\d+(?:\.\d+)?)", re.I) +_MDOT_LON_RE = re.compile(r"lon=(-?\d+(?:\.\d+)?)", re.I) +_MDOT_ID_RE = re.compile(r"[?&]id=(\d+)", re.I) +_MDOT_IMG_RE = re.compile(r']+src=["\']([^"\']+)["\']', re.I) + +# Michigan bbox (docs/osiris-ideas.md §3.2): lat 41.6–48.3, lon -90.5–-82.1. +MDOT_LAT_RANGE = (41.6, 48.3) +MDOT_LON_RANGE = (-90.5, -82.1) + + +def parse_mdot_json(text: str, source_name: str) -> list[dict]: + """Parse MDOT MiDrive `camera/list` JSON (fields carry rendered HTML). + + Coordinates and the stable id live in the `county` field's map link + (`/MiDrive/map?...lat=&lon=&id=`); the `image` field carries an `` + whose src is the JPEG still. Out-of-bbox and coord-less rows are dropped. + """ + try: + payload = json.loads(text) + except (json.JSONDecodeError, ValueError): + return [] + if not isinstance(payload, list): + return [] + out: list[dict] = [] + for row in payload: + if not isinstance(row, dict): + continue + county_html = row.get("county") or "" + m_lat = _MDOT_LAT_RE.search(county_html) + m_lon = _MDOT_LON_RE.search(county_html) + m_id = _MDOT_ID_RE.search(county_html) + if not (m_lat and m_lon and m_id): + continue # missing coordinates / stable id → drop + try: + lat = float(m_lat.group(1)) + lon = float(m_lon.group(1)) + except ValueError: + continue + if not (MDOT_LAT_RANGE[0] <= lat <= MDOT_LAT_RANGE[1] + and MDOT_LON_RANGE[0] <= lon <= MDOT_LON_RANGE[1]): + continue # out of Michigan bbox → drop + img_m = _MDOT_IMG_RE.search(row.get("image") or "") + if not img_m: + continue + snap = img_m.group(1).strip() + low = snap.lower() + if not (low.startswith("http://") or low.startswith("https://")): + continue + if low.startswith("rtsp"): + continue + cam_id = m_id.group(1) + route = (row.get("route") or "").strip() + loc = (row.get("location") or "").strip().lstrip("@").strip() + county_name = county_html.split(" list[dict]: """Parse willytop8/Live-Environment-Streams GeoJSON. @@ -490,6 +563,8 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder, body = resp.text if "cwwp2.dot.ca.gov" in src_url or "cctvStatus" in src_url: cams = parse_caltrans_json(body, name) + elif "mdotjboss.state.mi.us" in src_url or "/MiDrive/camera/list" in src_url: + cams = parse_mdot_json(body, name) elif ("getCameraDataByLoc" in src_url or ("json" in ctype and '"locs"' in body[:4000] and '"cams"' in body[:8000])): cams = parse_alertwest_json(body, name) diff --git a/app/conflicts.py b/app/conflicts.py new file mode 100644 index 0000000..bfc48cd --- /dev/null +++ b/app/conflicts.py @@ -0,0 +1,168 @@ +"""Curated OSINT conflict-zone catalog + point-in-bbox event counting. + +A static, human-curated list of active conflict theatres (war / high / +elevated). Purely descriptive — this is a catalog, not a live feed and not a +scrape of LiveUAMap or any other source. Severity and descriptions are +editorial judgement kept short and factual. + +Each zone carries an internal ``bbox`` (``min_lat, min_lon, max_lat, max_lon``) +used only to count pre-existing geocoded news/GDELT/``/api/news/map`` rows that +fall inside it. The bbox is not part of the API response; callers get the +``eventCount`` roll-up instead. + +Never call an upstream API from here — event counts come from rows already in +the local database (``events`` with geocoords + ``news_items`` map pins). +""" + +from __future__ import annotations + +from datetime import datetime + +# id → zone. ``lat``/``lon`` is the fly-to anchor; ``bbox`` is the internal +# count window in ``min_lat, min_lon, max_lat, max_lon`` order. +_ZONES: tuple[dict, ...] = ( + { + "id": "ukraine", + "label": "Ukraine", + "severity": "war", + "lat": 48.5, + "lon": 31.0, + "description": "Full-scale Russian invasion since 2022; active front lines in the east and south.", + "bbox": (44.3, 22.1, 52.4, 40.2), + }, + { + "id": "gaza", + "label": "Gaza", + "severity": "war", + "lat": 31.4, + "lon": 34.4, + "description": "Israel–Hamas war; sustained fighting and a severe humanitarian crisis in the Gaza Strip.", + "bbox": (31.0, 34.1, 31.8, 34.7), + }, + { + "id": "sudan", + "label": "Sudan", + "severity": "war", + "lat": 15.5, + "lon": 30.0, + "description": "Civil war between the SAF and RSF since 2023, with mass displacement across the country.", + "bbox": (8.7, 21.8, 22.0, 38.6), + }, + { + "id": "myanmar", + "label": "Myanmar", + "severity": "war", + "lat": 21.5, + "lon": 96.0, + "description": "Post-2021 coup conflict pitting the junta against resistance and ethnic armed groups.", + "bbox": (9.5, 92.2, 28.5, 101.2), + }, + { + "id": "drc", + "label": "DR Congo", + "severity": "war", + "lat": -1.5, + "lon": 28.0, + "description": "Eastern DRC conflict involving M23 and other armed groups; heavy displacement around Goma.", + "bbox": (-5.0, 26.0, 3.0, 31.0), + }, + { + "id": "yemen", + "label": "Yemen", + "severity": "war", + "lat": 15.5, + "lon": 47.5, + "description": "Protracted Houthi–government/coalition war with one of the world's worst humanitarian emergencies.", + "bbox": (12.6, 42.5, 19.0, 54.0), + }, + { + "id": "syria", + "label": "Syria", + "severity": "war", + "lat": 34.5, + "lon": 38.5, + "description": "Multi-sided civil war; government, opposition, and external actors continue to engage.", + "bbox": (32.3, 35.7, 37.3, 42.4), + }, + { + "id": "lebanon", + "label": "Lebanon", + "severity": "high", + "lat": 33.9, + "lon": 35.9, + "description": "Israel–Hezbollah hostilities with periodic escalation along the southern border.", + "bbox": (33.0, 35.0, 34.7, 36.6), + }, + { + "id": "sahel", + "label": "Sahel", + "severity": "high", + "lat": 14.5, + "lon": 0.0, + "description": "Jihadist insurgencies across Mali, Burkina Faso, and Niger destabilising the central Sahel.", + "bbox": (10.0, -10.0, 20.0, 12.0), + }, + { + "id": "somalia", + "label": "Somalia", + "severity": "high", + "lat": 6.0, + "lon": 45.0, + "description": "Al-Shabaab insurgency against the federal government and security forces.", + "bbox": (-2.0, 41.0, 12.0, 51.5), + }, + { + "id": "red_sea", + "label": "Red Sea", + "severity": "high", + "lat": 18.0, + "lon": 40.0, + "description": "Houthi attacks on commercial shipping transiting the Red Sea corridor.", + "bbox": (12.0, 34.0, 22.0, 44.0), + }, + { + "id": "taiwan_strait", + "label": "Taiwan Strait", + "severity": "elevated", + "lat": 24.5, + "lon": 119.5, + "description": "Heightened military standoff between China and Taiwan, including deterrence patrols.", + "bbox": (21.9, 117.0, 26.5, 122.0), + }, + { + "id": "korean_dmz", + "label": "Korean DMZ", + "severity": "elevated", + "lat": 38.3, + "lon": 127.0, + "description": "Heavily fortified inter-Korean border with periodic tensions and military drills.", + "bbox": (37.5, 126.0, 39.0, 128.5), + }, +) + +SEVERITIES: frozenset[str] = frozenset({"war", "high", "elevated"}) + + +def conflict_zones() -> list[dict]: + """Return a fresh shallow copy of the catalog (callers must not mutate).""" + return [dict(z) for z in _ZONES] + + +def zone_event_stats( + points: list[tuple[float, float, datetime | None]], + bbox: tuple[float, float, float, float], +) -> tuple[int, datetime | None]: + """Count points inside ``bbox`` and return (count, latest timestamp). + + ``points`` is an iterable of ``(lat, lon, ts)``; ``ts`` may be ``None``. + ``bbox`` is ``(min_lat, min_lon, max_lat, max_lon)``. + """ + min_lat, min_lon, max_lat, max_lon = bbox + count = 0 + latest: datetime | None = None + for lat, lon, ts in points: + if min_lat <= lat <= max_lat and min_lon <= lon <= max_lon: + count += 1 + if ts is not None and (latest is None or ts > latest): + latest = ts + return count, latest diff --git a/app/live_layers.py b/app/live_layers.py index fa632b1..1d8a0f1 100644 --- a/app/live_layers.py +++ b/app/live_layers.py @@ -150,6 +150,12 @@ def overlay_catalog() -> dict: "endpoint": "/api/map/gpsjam", "attribution": "GPSJAM / John Wiseman / ADS-B Exchange", }, + "conflicts": { + "id": "conflicts", + "kind": "points", + "endpoint": "/api/conflicts", + "attribution": "Curated OSINT conflict catalog", + }, } diff --git a/app/main.py b/app/main.py index 2d12e25..f659811 100644 --- a/app/main.py +++ b/app/main.py @@ -38,6 +38,7 @@ from models import ( ) from schemas import ( AlertCreate, AlertOut, AlertSeverity, AlertType, AlertUpdate, + ConflictZoneOut, ConflictsOut, DashboardSummary, EntityCreate, EntityKind, EntityOut, EventCreate, EventOut, FireOut, NewsArticleOut, NewsMapItemOut, NewsSummaryOut, NewsTickerItemOut, @@ -1432,6 +1433,58 @@ async def map_chokepoints(): return {"chokepoints": chokepoints()} +async def _fetch_geocoded_points() -> list[tuple[float, float, datetime | None]]: + """Collect geocoded ``(lat, lon, ts)`` rows from the local DB. + + Sources are the flagged map pins (``news_items`` kind=map) and geocoded + news/GDELT events (``events`` with ``location_lat/lon``). This is the + pre-existing geocoded corpus the conflict-zone counters roll up — no + upstream scraping and no generated/jittered coordinates. + """ + async with async_session() as session: + map_rows = ( + await session.execute( + select(news_items.c.lat, news_items.c.lon, news_items.c.created_at) + .where( + news_items.c.kind == "map", + news_items.c.lat.isnot(None), + news_items.c.lon.isnot(None), + ) + ) + ).all() + event_rows = ( + await session.execute( + select(events.c.location_lat, events.c.location_lon, events.c.source_timestamp) + .where( + events.c.source_type.in_(["rss", "gdel-t2"]), + events.c.location_lat.isnot(None), + events.c.location_lon.isnot(None), + ) + ) + ).all() + return [tuple(r) for r in map_rows] + [tuple(r) for r in event_rows] + + +@app.get("/api/conflicts", response_model=ConflictsOut) +async def list_conflicts(): + """Curated conflict-zone catalog with per-zone event counts. + + Static catalogue (severity + short factual description) merged with a live + ``eventCount`` roll-up of pre-existing geocoded news/GDELT//api/news/map + rows inside each zone bbox. Empty DB → ``eventCount=0`` (never 502). + """ + from conflicts import conflict_zones, zone_event_stats + + points = await _fetch_geocoded_points() + timestamp = datetime.now(timezone.utc) + zones = [] + for z in conflict_zones(): + bbox = z.pop("bbox") + count, latest = zone_event_stats(points, bbox) + zones.append({**z, "eventCount": count, "lastUpdated": latest}) + return {"zones": zones, "timestamp": timestamp} + + def _upstream_or_502(exc: Exception, name: str) -> NoReturn: logger.warning("live_layer_upstream_failed", layer=name, error=str(exc)) raise HTTPException(502, f"{name} upstream unavailable: {exc}") from exc diff --git a/app/schemas.py b/app/schemas.py index 799aeb3..64b86c5 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -4,7 +4,7 @@ from __future__ import annotations from datetime import datetime from enum import Enum -from typing import Optional +from typing import Literal, Optional from uuid import UUID from pydantic import BaseModel, Field, field_validator @@ -386,3 +386,24 @@ class GeofenceUpdate(BaseModel): geojson: Optional[dict] = None active: Optional[bool] = None + +class ConflictZoneOut(BaseModel): + """One curated conflict theatre as exposed by GET /api/conflicts.""" + + id: str + label: str + severity: Literal["war", "high", "elevated"] + lat: float + lon: float + description: str + eventCount: int + lastUpdated: Optional[datetime] = None + + +class ConflictsOut(BaseModel): + """Response envelope for GET /api/conflicts.""" + + zones: list[ConflictZoneOut] + timestamp: datetime + + diff --git a/tests/test_conflicts.py b/tests/test_conflicts.py new file mode 100644 index 0000000..eb60d68 --- /dev/null +++ b/tests/test_conflicts.py @@ -0,0 +1,132 @@ +"""GET /api/conflicts — curated conflict-zone catalog + event-count roll-up. + +No outbound HTTP: event counts come from geocoded rows already (or not) in the +DB, and the API tests monkeypatch ``main._fetch_geocoded_points`` so no database +is required for the contract checks. +""" + +from datetime import datetime, timezone + +import httpx + +from conflicts import SEVERITIES, conflict_zones, zone_event_stats +from live_layers import overlay_catalog +from main import app + +BASE = "http://test" + + +def _get(path: str, monkeypatch=None, points=None) -> httpx.Response: + import asyncio + + async def run() -> httpx.Response: + if monkeypatch is not None: + async def fake(): + return points or [] + + monkeypatch.setattr("main._fetch_geocoded_points", fake) + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url=BASE) as client: + return await client.get(path) + + return asyncio.run(run()) + + +# ── Catalog shape ────────────────────────────────────────────────────── + + +def test_catalog_length(): + zones = conflict_zones() + assert len(zones) == 13 + + +def test_catalog_severity_enum(): + zones = conflict_zones() + sevs = {z["severity"] for z in zones} + assert sevs.issubset(SEVERITIES) + # All three tiers are represented. + assert sevs == SEVERITIES + + +def test_catalog_fields_factual_and_complete(): + zones = conflict_zones() + ids = [z["id"] for z in zones] + assert len(set(ids)) == len(ids) # unique ids + for z in zones: + assert z["label"] + assert z["description"].strip() + assert -90.0 <= z["lat"] <= 90.0 + assert -180.0 <= z["lon"] <= 180.0 + # internal-only bbox is well-formed: (min_lat, min_lon, max_lat, max_lon) + min_lat, min_lon, max_lat, max_lon = z["bbox"] + assert min_lat <= max_lat and min_lon <= max_lon + assert min_lat <= z["lat"] <= max_lat and min_lon <= z["lon"] <= max_lon + + +def test_overlay_catalog_has_conflicts(): + entry = overlay_catalog()["conflicts"] + assert entry["kind"] == "points" + assert entry["endpoint"] == "/api/conflicts" + + +# ── Pure counting ────────────────────────────────────────────────────── + +TS1 = datetime(2026, 8, 30, 12, 0, tzinfo=timezone.utc) +TS2 = datetime(2026, 8, 30, 13, 0, tzinfo=timezone.utc) + + +def test_zone_event_stats_counts_and_picks_latest(): + bbox = (40.0, 20.0, 52.0, 40.0) # roughly Ukraine + points = [ + (50.45, 30.52, TS1), # inside + (48.0, 25.0, TS2), # inside, later + (0.0, -60.0, TS1), # outside + (15.0, 45.0, TS2), # outside (lat ok, lon out) + ] + count, latest = zone_event_stats(points, bbox) + assert count == 2 + assert latest == TS2 + + +def test_zone_event_stats_empty_bbox(): + count, latest = zone_event_stats([], (0.0, 0.0, 1.0, 1.0)) + assert count == 0 + assert latest is None + + +# ── API contract (mocked map items, no DB) ───────────────────────────── + + +def test_conflicts_returns_catalog_with_mocked_counts(monkeypatch): + points = [ + (50.45, 30.52, TS1), # Ukraine + (25.03, 121.56, TS2), # Taiwan Strait + (0.0, -60.0, TS1), # nowhere + ] + resp = _get("/api/conflicts", monkeypatch=monkeypatch, points=points) + assert resp.status_code == 200 + body = resp.json() + assert "zones" in body and "timestamp" in body + by_id = {z["id"]: z for z in body["zones"]} + assert len(body["zones"]) == 13 + + zone = by_id["ukraine"] + assert zone["eventCount"] == 1 + assert zone["lastUpdated"] == TS1.isoformat().replace("+00:00", "Z") + assert zone["severity"] == "war" + + assert by_id["taiwan_strait"]["eventCount"] == 1 + assert by_id["gaza"]["eventCount"] == 0 + # exact per-zone key contract the frontend consumes + assert set(zone.keys()) == { + "id", "label", "severity", "lat", "lon", + "description", "eventCount", "lastUpdated", + } + + +def test_conflicts_empty_db_yields_zero_counts(monkeypatch): + resp = _get("/api/conflicts", monkeypatch=monkeypatch, points=[]) + assert resp.status_code == 200 + body = resp.json() + assert all(z["eventCount"] == 0 for z in body["zones"]) + assert all(z["lastUpdated"] is None for z in body["zones"]) diff --git a/tests/test_live_layers.py b/tests/test_live_layers.py index ae0f68e..a646deb 100644 --- a/tests/test_live_layers.py +++ b/tests/test_live_layers.py @@ -1,5 +1,7 @@ """Unit tests for live map-layer mappers (aircraft, trains, AIS, WFIGS, Caltrans).""" +import json + from live_layers import ( MARKER_FIELDS, bbox_center_radius_nm, @@ -25,7 +27,7 @@ from live_layers import ( _wfigs_params, ) -from camera_scraper import parse_caltrans_json +from camera_scraper import parse_caltrans_json, parse_mdot_json def test_parse_bbox_and_radius_clamps_to_150_nm(): @@ -257,6 +259,65 @@ def test_parse_caltrans_skips_oos_and_maps_jpeg_hls(): assert "rtsp://" not in cam["snapshot_url"].lower() +def test_parse_mdot_extracts_html_fields_and_bbox_filters(): + rows = [ + # In-bbox, full fields. + { + "route": "11 Mile", + "county": 'Wayne County Go to', + "location": " @ Mound NB", + "direction": "Traffic closest to camera is traveling north.", + "image": 'x', + }, + # Out of bbox (lat 50) → drop. + { + "route": "Far", + "county": 'Nowhere Go to', + "location": "", + "image": '', + }, + # Missing coordinates → drop. + { + "route": "NoCoords", + "county": 'Somewhere Go to', + "location": "", + "image": '', + }, + # Missing image → drop. + { + "route": "NoImage", + "county": 'Kent Go to', + "location": " @ Division", + "image": "", + }, + # RTSP image src → drop. + { + "route": "Rtsp", + "county": 'Wayne Go to', + "location": "", + "image": '', + }, + ] + cams = parse_mdot_json(json.dumps(rows), "mdotjboss.state.mi.us") + assert len(cams) == 1 + cam = cams[0] + assert cam["discovery_source"] == "mdot" + assert cam["location_lat"] == 42.491304 + assert cam["location_lon"] == -83.04479 + assert cam["snapshot_url"] == "https://micamerasimages.net/thumbs/semtoc_cam_253.flv.jpg?item=1" + assert cam["source_url"] == "https://mdotjboss.state.mi.us/MiDrive/camera/1129" + assert cam["device_type"] == "http" + assert cam["vendor"] == "MDOT" + assert "11 Mile @ Mound NB" in cam["location_name"] + assert "Wayne County" in cam["location_name"] + + +def test_parse_mdot_handles_malformed_payload(): + assert parse_mdot_json("not json", "mdot") == [] + assert parse_mdot_json('{"not": "a list"}', "mdot") == [] + assert parse_mdot_json("[]", "mdot") == [] + + def test_quantize_bbox_stable_under_jitter(): a = quantize_bbox(*parse_bbox("-78.7912,35.7711,-78.6101,35.9102")) b = quantize_bbox(*parse_bbox("-78.7900,35.7700,-78.6110,35.9090"))