Merge branch 'master' into osint-dashboard/t_1d9c5839-osint-map-aircraft-popup-enrichment-emer

This commit is contained in:
sirius 2026-08-31 21:54:54 -04:00
commit 71b6f589a5
8 changed files with 522 additions and 2 deletions

View file

@ -16,6 +16,8 @@ CALTRANS_CCTV_URLS = tuple(
f"https://cwwp2.dot.ca.gov/data/d{n}/cctv/cctvStatusD{n:02d}.json" f"https://cwwp2.dot.ca.gov/data/d{n}/cctv/cctvStatusD{n:02d}.json"
for n in range(1, 13) for n in range(1, 13)
) )
# MDOT MiDrive official DOT CCTV list (fields carry rendered HTML).
MDOT_CAMERA_URL = "https://mdotjboss.state.mi.us/MiDrive/camera/list"
_DEFAULT_SOURCE_URL = ",".join(( _DEFAULT_SOURCE_URL = ",".join((
# Publicly published open-camera list (markdown bullets of stream URLs). # Publicly published open-camera list (markdown bullets of stream URLs).
"https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md", "https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md",
@ -25,6 +27,8 @@ _DEFAULT_SOURCE_URL = ",".join((
"https://raw.githubusercontent.com/willytop8/Live-Environment-Streams/main/streams.geojson", "https://raw.githubusercontent.com/willytop8/Live-Environment-Streams/main/streams.geojson",
# Official Caltrans CWWP2 JPEG + HLS CCTV (districts 112). # Official Caltrans CWWP2 JPEG + HLS CCTV (districts 112).
*CALTRANS_CCTV_URLS, *CALTRANS_CCTV_URLS,
# Official MDOT MiDrive CCTV (JPEG stills, Michigan).
MDOT_CAMERA_URL,
)) ))
CAMERA_SOURCE_URLS = [ CAMERA_SOURCE_URLS = [
u.strip() u.strip()

View file

@ -379,6 +379,79 @@ def parse_caltrans_json(text: str, source_name: str) -> list[dict]:
return out return out
# MDOT MiDrive field extractors (fields carry rendered HTML).
_MDOT_LAT_RE = re.compile(r"lat=(-?\d+(?:\.\d+)?)", re.I)
_MDOT_LON_RE = re.compile(r"lon=(-?\d+(?:\.\d+)?)", re.I)
_MDOT_ID_RE = re.compile(r"[?&]id=(\d+)", re.I)
_MDOT_IMG_RE = re.compile(r'<img[^>]+src=["\']([^"\']+)["\']', re.I)
# Michigan bbox (docs/osiris-ideas.md §3.2): lat 41.648.3, lon -90.5-82.1.
MDOT_LAT_RANGE = (41.6, 48.3)
MDOT_LON_RANGE = (-90.5, -82.1)
def parse_mdot_json(text: str, source_name: str) -> list[dict]:
"""Parse MDOT MiDrive `camera/list` JSON (fields carry rendered HTML).
Coordinates and the stable id live in the `county` field's map link
(`/MiDrive/map?...lat=&lon=&id=`); the `image` field carries an `<img>`
whose src is the JPEG still. Out-of-bbox and coord-less rows are dropped.
"""
try:
payload = json.loads(text)
except (json.JSONDecodeError, ValueError):
return []
if not isinstance(payload, list):
return []
out: list[dict] = []
for row in payload:
if not isinstance(row, dict):
continue
county_html = row.get("county") or ""
m_lat = _MDOT_LAT_RE.search(county_html)
m_lon = _MDOT_LON_RE.search(county_html)
m_id = _MDOT_ID_RE.search(county_html)
if not (m_lat and m_lon and m_id):
continue # missing coordinates / stable id → drop
try:
lat = float(m_lat.group(1))
lon = float(m_lon.group(1))
except ValueError:
continue
if not (MDOT_LAT_RANGE[0] <= lat <= MDOT_LAT_RANGE[1]
and MDOT_LON_RANGE[0] <= lon <= MDOT_LON_RANGE[1]):
continue # out of Michigan bbox → drop
img_m = _MDOT_IMG_RE.search(row.get("image") or "")
if not img_m:
continue
snap = img_m.group(1).strip()
low = snap.lower()
if not (low.startswith("http://") or low.startswith("https://")):
continue
if low.startswith("rtsp"):
continue
cam_id = m_id.group(1)
route = (row.get("route") or "").strip()
loc = (row.get("location") or "").strip().lstrip("@").strip()
county_name = county_html.split("<a", 1)[0].strip()
bits = [
f"{route} @ {loc}" if (route and loc) else (route or loc or None),
county_name or None,
]
name = ", ".join(b for b in bits if b) or None
out.append({
"source_url": f"https://mdotjboss.state.mi.us/MiDrive/camera/{cam_id}",
"snapshot_url": snap,
"discovery_source": "mdot",
"location_lat": lat,
"location_lon": lon,
"location_name": name,
"vendor": "MDOT",
"device_type": "http",
})
return out
def parse_live_streams_geojson(text: str, source_name: str) -> list[dict]: def parse_live_streams_geojson(text: str, source_name: str) -> list[dict]:
"""Parse willytop8/Live-Environment-Streams GeoJSON. """Parse willytop8/Live-Environment-Streams GeoJSON.
@ -490,6 +563,8 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder,
body = resp.text body = resp.text
if "cwwp2.dot.ca.gov" in src_url or "cctvStatus" in src_url: if "cwwp2.dot.ca.gov" in src_url or "cctvStatus" in src_url:
cams = parse_caltrans_json(body, name) cams = parse_caltrans_json(body, name)
elif "mdotjboss.state.mi.us" in src_url or "/MiDrive/camera/list" in src_url:
cams = parse_mdot_json(body, name)
elif ("getCameraDataByLoc" in src_url elif ("getCameraDataByLoc" in src_url
or ("json" in ctype and '"locs"' in body[:4000] and '"cams"' in body[:8000])): or ("json" in ctype and '"locs"' in body[:4000] and '"cams"' in body[:8000])):
cams = parse_alertwest_json(body, name) cams = parse_alertwest_json(body, name)

168
app/conflicts.py Normal file
View file

@ -0,0 +1,168 @@
"""Curated OSINT conflict-zone catalog + point-in-bbox event counting.
A static, human-curated list of active conflict theatres (war / high /
elevated). Purely descriptive this is a catalog, not a live feed and not a
scrape of LiveUAMap or any other source. Severity and descriptions are
editorial judgement kept short and factual.
Each zone carries an internal ``bbox`` (``min_lat, min_lon, max_lat, max_lon``)
used only to count pre-existing geocoded news/GDELT/``/api/news/map`` rows that
fall inside it. The bbox is not part of the API response; callers get the
``eventCount`` roll-up instead.
Never call an upstream API from here event counts come from rows already in
the local database (``events`` with geocoords + ``news_items`` map pins).
"""
from __future__ import annotations
from datetime import datetime
# id → zone. ``lat``/``lon`` is the fly-to anchor; ``bbox`` is the internal
# count window in ``min_lat, min_lon, max_lat, max_lon`` order.
_ZONES: tuple[dict, ...] = (
{
"id": "ukraine",
"label": "Ukraine",
"severity": "war",
"lat": 48.5,
"lon": 31.0,
"description": "Full-scale Russian invasion since 2022; active front lines in the east and south.",
"bbox": (44.3, 22.1, 52.4, 40.2),
},
{
"id": "gaza",
"label": "Gaza",
"severity": "war",
"lat": 31.4,
"lon": 34.4,
"description": "IsraelHamas war; sustained fighting and a severe humanitarian crisis in the Gaza Strip.",
"bbox": (31.0, 34.1, 31.8, 34.7),
},
{
"id": "sudan",
"label": "Sudan",
"severity": "war",
"lat": 15.5,
"lon": 30.0,
"description": "Civil war between the SAF and RSF since 2023, with mass displacement across the country.",
"bbox": (8.7, 21.8, 22.0, 38.6),
},
{
"id": "myanmar",
"label": "Myanmar",
"severity": "war",
"lat": 21.5,
"lon": 96.0,
"description": "Post-2021 coup conflict pitting the junta against resistance and ethnic armed groups.",
"bbox": (9.5, 92.2, 28.5, 101.2),
},
{
"id": "drc",
"label": "DR Congo",
"severity": "war",
"lat": -1.5,
"lon": 28.0,
"description": "Eastern DRC conflict involving M23 and other armed groups; heavy displacement around Goma.",
"bbox": (-5.0, 26.0, 3.0, 31.0),
},
{
"id": "yemen",
"label": "Yemen",
"severity": "war",
"lat": 15.5,
"lon": 47.5,
"description": "Protracted Houthigovernment/coalition war with one of the world's worst humanitarian emergencies.",
"bbox": (12.6, 42.5, 19.0, 54.0),
},
{
"id": "syria",
"label": "Syria",
"severity": "war",
"lat": 34.5,
"lon": 38.5,
"description": "Multi-sided civil war; government, opposition, and external actors continue to engage.",
"bbox": (32.3, 35.7, 37.3, 42.4),
},
{
"id": "lebanon",
"label": "Lebanon",
"severity": "high",
"lat": 33.9,
"lon": 35.9,
"description": "IsraelHezbollah hostilities with periodic escalation along the southern border.",
"bbox": (33.0, 35.0, 34.7, 36.6),
},
{
"id": "sahel",
"label": "Sahel",
"severity": "high",
"lat": 14.5,
"lon": 0.0,
"description": "Jihadist insurgencies across Mali, Burkina Faso, and Niger destabilising the central Sahel.",
"bbox": (10.0, -10.0, 20.0, 12.0),
},
{
"id": "somalia",
"label": "Somalia",
"severity": "high",
"lat": 6.0,
"lon": 45.0,
"description": "Al-Shabaab insurgency against the federal government and security forces.",
"bbox": (-2.0, 41.0, 12.0, 51.5),
},
{
"id": "red_sea",
"label": "Red Sea",
"severity": "high",
"lat": 18.0,
"lon": 40.0,
"description": "Houthi attacks on commercial shipping transiting the Red Sea corridor.",
"bbox": (12.0, 34.0, 22.0, 44.0),
},
{
"id": "taiwan_strait",
"label": "Taiwan Strait",
"severity": "elevated",
"lat": 24.5,
"lon": 119.5,
"description": "Heightened military standoff between China and Taiwan, including deterrence patrols.",
"bbox": (21.9, 117.0, 26.5, 122.0),
},
{
"id": "korean_dmz",
"label": "Korean DMZ",
"severity": "elevated",
"lat": 38.3,
"lon": 127.0,
"description": "Heavily fortified inter-Korean border with periodic tensions and military drills.",
"bbox": (37.5, 126.0, 39.0, 128.5),
},
)
SEVERITIES: frozenset[str] = frozenset({"war", "high", "elevated"})
def conflict_zones() -> list[dict]:
"""Return a fresh shallow copy of the catalog (callers must not mutate)."""
return [dict(z) for z in _ZONES]
def zone_event_stats(
points: list[tuple[float, float, datetime | None]],
bbox: tuple[float, float, float, float],
) -> tuple[int, datetime | None]:
"""Count points inside ``bbox`` and return (count, latest timestamp).
``points`` is an iterable of ``(lat, lon, ts)``; ``ts`` may be ``None``.
``bbox`` is ``(min_lat, min_lon, max_lat, max_lon)``.
"""
min_lat, min_lon, max_lat, max_lon = bbox
count = 0
latest: datetime | None = None
for lat, lon, ts in points:
if min_lat <= lat <= max_lat and min_lon <= lon <= max_lon:
count += 1
if ts is not None and (latest is None or ts > latest):
latest = ts
return count, latest

View file

@ -150,6 +150,12 @@ def overlay_catalog() -> dict:
"endpoint": "/api/map/gpsjam", "endpoint": "/api/map/gpsjam",
"attribution": "GPSJAM / John Wiseman / ADS-B Exchange", "attribution": "GPSJAM / John Wiseman / ADS-B Exchange",
}, },
"conflicts": {
"id": "conflicts",
"kind": "points",
"endpoint": "/api/conflicts",
"attribution": "Curated OSINT conflict catalog",
},
} }

View file

@ -38,6 +38,7 @@ from models import (
) )
from schemas import ( from schemas import (
AlertCreate, AlertOut, AlertSeverity, AlertType, AlertUpdate, AlertCreate, AlertOut, AlertSeverity, AlertType, AlertUpdate,
ConflictZoneOut, ConflictsOut,
DashboardSummary, EntityCreate, EntityKind, EntityOut, DashboardSummary, EntityCreate, EntityKind, EntityOut,
EventCreate, EventOut, FireOut, NewsArticleOut, NewsMapItemOut, EventCreate, EventOut, FireOut, NewsArticleOut, NewsMapItemOut,
NewsSummaryOut, NewsTickerItemOut, NewsSummaryOut, NewsTickerItemOut,
@ -1432,6 +1433,58 @@ async def map_chokepoints():
return {"chokepoints": chokepoints()} return {"chokepoints": chokepoints()}
async def _fetch_geocoded_points() -> list[tuple[float, float, datetime | None]]:
"""Collect geocoded ``(lat, lon, ts)`` rows from the local DB.
Sources are the flagged map pins (``news_items`` kind=map) and geocoded
news/GDELT events (``events`` with ``location_lat/lon``). This is the
pre-existing geocoded corpus the conflict-zone counters roll up no
upstream scraping and no generated/jittered coordinates.
"""
async with async_session() as session:
map_rows = (
await session.execute(
select(news_items.c.lat, news_items.c.lon, news_items.c.created_at)
.where(
news_items.c.kind == "map",
news_items.c.lat.isnot(None),
news_items.c.lon.isnot(None),
)
)
).all()
event_rows = (
await session.execute(
select(events.c.location_lat, events.c.location_lon, events.c.source_timestamp)
.where(
events.c.source_type.in_(["rss", "gdel-t2"]),
events.c.location_lat.isnot(None),
events.c.location_lon.isnot(None),
)
)
).all()
return [tuple(r) for r in map_rows] + [tuple(r) for r in event_rows]
@app.get("/api/conflicts", response_model=ConflictsOut)
async def list_conflicts():
"""Curated conflict-zone catalog with per-zone event counts.
Static catalogue (severity + short factual description) merged with a live
``eventCount`` roll-up of pre-existing geocoded news/GDELT//api/news/map
rows inside each zone bbox. Empty DB ``eventCount=0`` (never 502).
"""
from conflicts import conflict_zones, zone_event_stats
points = await _fetch_geocoded_points()
timestamp = datetime.now(timezone.utc)
zones = []
for z in conflict_zones():
bbox = z.pop("bbox")
count, latest = zone_event_stats(points, bbox)
zones.append({**z, "eventCount": count, "lastUpdated": latest})
return {"zones": zones, "timestamp": timestamp}
def _upstream_or_502(exc: Exception, name: str) -> NoReturn: def _upstream_or_502(exc: Exception, name: str) -> NoReturn:
logger.warning("live_layer_upstream_failed", layer=name, error=str(exc)) logger.warning("live_layer_upstream_failed", layer=name, error=str(exc))
raise HTTPException(502, f"{name} upstream unavailable: {exc}") from exc raise HTTPException(502, f"{name} upstream unavailable: {exc}") from exc

View file

@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import datetime from datetime import datetime
from enum import Enum from enum import Enum
from typing import Optional from typing import Literal, Optional
from uuid import UUID from uuid import UUID
from pydantic import BaseModel, Field, field_validator from pydantic import BaseModel, Field, field_validator
@ -386,3 +386,24 @@ class GeofenceUpdate(BaseModel):
geojson: Optional[dict] = None geojson: Optional[dict] = None
active: Optional[bool] = None active: Optional[bool] = None
class ConflictZoneOut(BaseModel):
"""One curated conflict theatre as exposed by GET /api/conflicts."""
id: str
label: str
severity: Literal["war", "high", "elevated"]
lat: float
lon: float
description: str
eventCount: int
lastUpdated: Optional[datetime] = None
class ConflictsOut(BaseModel):
"""Response envelope for GET /api/conflicts."""
zones: list[ConflictZoneOut]
timestamp: datetime

132
tests/test_conflicts.py Normal file
View file

@ -0,0 +1,132 @@
"""GET /api/conflicts — curated conflict-zone catalog + event-count roll-up.
No outbound HTTP: event counts come from geocoded rows already (or not) in the
DB, and the API tests monkeypatch ``main._fetch_geocoded_points`` so no database
is required for the contract checks.
"""
from datetime import datetime, timezone
import httpx
from conflicts import SEVERITIES, conflict_zones, zone_event_stats
from live_layers import overlay_catalog
from main import app
BASE = "http://test"
def _get(path: str, monkeypatch=None, points=None) -> httpx.Response:
import asyncio
async def run() -> httpx.Response:
if monkeypatch is not None:
async def fake():
return points or []
monkeypatch.setattr("main._fetch_geocoded_points", fake)
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
return await client.get(path)
return asyncio.run(run())
# ── Catalog shape ──────────────────────────────────────────────────────
def test_catalog_length():
zones = conflict_zones()
assert len(zones) == 13
def test_catalog_severity_enum():
zones = conflict_zones()
sevs = {z["severity"] for z in zones}
assert sevs.issubset(SEVERITIES)
# All three tiers are represented.
assert sevs == SEVERITIES
def test_catalog_fields_factual_and_complete():
zones = conflict_zones()
ids = [z["id"] for z in zones]
assert len(set(ids)) == len(ids) # unique ids
for z in zones:
assert z["label"]
assert z["description"].strip()
assert -90.0 <= z["lat"] <= 90.0
assert -180.0 <= z["lon"] <= 180.0
# internal-only bbox is well-formed: (min_lat, min_lon, max_lat, max_lon)
min_lat, min_lon, max_lat, max_lon = z["bbox"]
assert min_lat <= max_lat and min_lon <= max_lon
assert min_lat <= z["lat"] <= max_lat and min_lon <= z["lon"] <= max_lon
def test_overlay_catalog_has_conflicts():
entry = overlay_catalog()["conflicts"]
assert entry["kind"] == "points"
assert entry["endpoint"] == "/api/conflicts"
# ── Pure counting ──────────────────────────────────────────────────────
TS1 = datetime(2026, 8, 30, 12, 0, tzinfo=timezone.utc)
TS2 = datetime(2026, 8, 30, 13, 0, tzinfo=timezone.utc)
def test_zone_event_stats_counts_and_picks_latest():
bbox = (40.0, 20.0, 52.0, 40.0) # roughly Ukraine
points = [
(50.45, 30.52, TS1), # inside
(48.0, 25.0, TS2), # inside, later
(0.0, -60.0, TS1), # outside
(15.0, 45.0, TS2), # outside (lat ok, lon out)
]
count, latest = zone_event_stats(points, bbox)
assert count == 2
assert latest == TS2
def test_zone_event_stats_empty_bbox():
count, latest = zone_event_stats([], (0.0, 0.0, 1.0, 1.0))
assert count == 0
assert latest is None
# ── API contract (mocked map items, no DB) ─────────────────────────────
def test_conflicts_returns_catalog_with_mocked_counts(monkeypatch):
points = [
(50.45, 30.52, TS1), # Ukraine
(25.03, 121.56, TS2), # Taiwan Strait
(0.0, -60.0, TS1), # nowhere
]
resp = _get("/api/conflicts", monkeypatch=monkeypatch, points=points)
assert resp.status_code == 200
body = resp.json()
assert "zones" in body and "timestamp" in body
by_id = {z["id"]: z for z in body["zones"]}
assert len(body["zones"]) == 13
zone = by_id["ukraine"]
assert zone["eventCount"] == 1
assert zone["lastUpdated"] == TS1.isoformat().replace("+00:00", "Z")
assert zone["severity"] == "war"
assert by_id["taiwan_strait"]["eventCount"] == 1
assert by_id["gaza"]["eventCount"] == 0
# exact per-zone key contract the frontend consumes
assert set(zone.keys()) == {
"id", "label", "severity", "lat", "lon",
"description", "eventCount", "lastUpdated",
}
def test_conflicts_empty_db_yields_zero_counts(monkeypatch):
resp = _get("/api/conflicts", monkeypatch=monkeypatch, points=[])
assert resp.status_code == 200
body = resp.json()
assert all(z["eventCount"] == 0 for z in body["zones"])
assert all(z["lastUpdated"] is None for z in body["zones"])

View file

@ -1,5 +1,7 @@
"""Unit tests for live map-layer mappers (aircraft, trains, AIS, WFIGS, Caltrans).""" """Unit tests for live map-layer mappers (aircraft, trains, AIS, WFIGS, Caltrans)."""
import json
from live_layers import ( from live_layers import (
MARKER_FIELDS, MARKER_FIELDS,
bbox_center_radius_nm, bbox_center_radius_nm,
@ -25,7 +27,7 @@ from live_layers import (
_wfigs_params, _wfigs_params,
) )
from camera_scraper import parse_caltrans_json from camera_scraper import parse_caltrans_json, parse_mdot_json
def test_parse_bbox_and_radius_clamps_to_150_nm(): def test_parse_bbox_and_radius_clamps_to_150_nm():
@ -257,6 +259,65 @@ def test_parse_caltrans_skips_oos_and_maps_jpeg_hls():
assert "rtsp://" not in cam["snapshot_url"].lower() assert "rtsp://" not in cam["snapshot_url"].lower()
def test_parse_mdot_extracts_html_fields_and_bbox_filters():
rows = [
# In-bbox, full fields.
{
"route": "11 Mile",
"county": 'Wayne County <a href="/MiDrive/map?cameras=true&lat=42.491304&lon=-83.04479&zoom=15&id=1129"target="_blank">Go to</a>',
"location": " @ Mound NB",
"direction": "Traffic closest to camera is traveling north.",
"image": '<img alt="x" class="cameraImageForActivePane" id="1129Img" src="https://micamerasimages.net/thumbs/semtoc_cam_253.flv.jpg?item=1" height="170" width="250" onerror="cameraImageBroken(this)">',
},
# Out of bbox (lat 50) → drop.
{
"route": "Far",
"county": 'Nowhere <a href="/MiDrive/map?lat=50.0&lon=-83.0&zoom=15&id=9999">Go to</a>',
"location": "",
"image": '<img src="https://micamerasimages.net/thumbs/x.jpg">',
},
# Missing coordinates → drop.
{
"route": "NoCoords",
"county": 'Somewhere <a href="/MiDrive/map?zoom=15&id=8888">Go to</a>',
"location": "",
"image": '<img src="https://micamerasimages.net/thumbs/y.jpg">',
},
# Missing image → drop.
{
"route": "NoImage",
"county": 'Kent <a href="/MiDrive/map?lat=42.8841&lon=-85.6646&zoom=15&id=2113">Go to</a>',
"location": " @ Division",
"image": "",
},
# RTSP image src → drop.
{
"route": "Rtsp",
"county": 'Wayne <a href="/MiDrive/map?lat=42.4&lon=-83.1&zoom=15&id=1234">Go to</a>',
"location": "",
"image": '<img src="rtsp://10.0.0.1/stream">',
},
]
cams = parse_mdot_json(json.dumps(rows), "mdotjboss.state.mi.us")
assert len(cams) == 1
cam = cams[0]
assert cam["discovery_source"] == "mdot"
assert cam["location_lat"] == 42.491304
assert cam["location_lon"] == -83.04479
assert cam["snapshot_url"] == "https://micamerasimages.net/thumbs/semtoc_cam_253.flv.jpg?item=1"
assert cam["source_url"] == "https://mdotjboss.state.mi.us/MiDrive/camera/1129"
assert cam["device_type"] == "http"
assert cam["vendor"] == "MDOT"
assert "11 Mile @ Mound NB" in cam["location_name"]
assert "Wayne County" in cam["location_name"]
def test_parse_mdot_handles_malformed_payload():
assert parse_mdot_json("not json", "mdot") == []
assert parse_mdot_json('{"not": "a list"}', "mdot") == []
assert parse_mdot_json("[]", "mdot") == []
def test_quantize_bbox_stable_under_jitter(): def test_quantize_bbox_stable_under_jitter():
a = quantize_bbox(*parse_bbox("-78.7912,35.7711,-78.6101,35.9102")) a = quantize_bbox(*parse_bbox("-78.7912,35.7711,-78.6101,35.9102"))
b = quantize_bbox(*parse_bbox("-78.7900,35.7700,-78.6110,35.9090")) b = quantize_bbox(*parse_bbox("-78.7900,35.7700,-78.6110,35.9090"))