sirius/osint-dashboard
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into osint-dashboard/t_1d9c5839-osint-map-aircraft-popup-enrichment-emer

Browse source
This commit is contained in:
sirius 2026-08-31 21:54:54 -04:00
commit 71b6f589a5
8 changed files with 522 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
app/camera_config.py
View file

@ -16,6 +16,8 @@ CALTRANS_CCTV_URLS = tuple(
f"https://cwwp2.dot.ca.gov/data/d{n}/cctv/cctvStatusD{n:02d}.json"
for n in range(1, 13)
)
# MDOT MiDrive official DOT CCTV list (fields carry rendered HTML).
MDOT_CAMERA_URL = "https://mdotjboss.state.mi.us/MiDrive/camera/list"
_DEFAULT_SOURCE_URL = ",".join((
# Publicly published open-camera list (markdown bullets of stream URLs).
"https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md",
@ -25,6 +27,8 @@ _DEFAULT_SOURCE_URL = ",".join((
"https://raw.githubusercontent.com/willytop8/Live-Environment-Streams/main/streams.geojson",
# Official Caltrans CWWP2 JPEG + HLS CCTV (districts 1–12).
*CALTRANS_CCTV_URLS,
# Official MDOT MiDrive CCTV (JPEG stills, Michigan).
MDOT_CAMERA_URL,
))
CAMERA_SOURCE_URLS = [
u.strip()

75
app/camera_scraper.py
View file

@ -379,6 +379,79 @@ def parse_caltrans_json(text: str, source_name: str) -> list[dict]:
return out
# MDOT MiDrive field extractors (fields carry rendered HTML).
_MDOT_LAT_RE = re.compile(r"lat=(-?\d+(?:\.\d+)?)", re.I)
_MDOT_LON_RE = re.compile(r"lon=(-?\d+(?:\.\d+)?)", re.I)
_MDOT_ID_RE = re.compile(r"[?&]id=(\d+)", re.I)
_MDOT_IMG_RE = re.compile(r'<img[^>]+src=["\']([^"\']+)["\']', re.I)
# Michigan bbox (docs/osiris-ideas.md §3.2): lat 41.6–48.3, lon -90.5–-82.1.
MDOT_LAT_RANGE = (41.6, 48.3)
MDOT_LON_RANGE = (-90.5, -82.1)
def parse_mdot_json(text: str, source_name: str) -> list[dict]:
"""Parse MDOT MiDrive `camera/list` JSON (fields carry rendered HTML).
Coordinates and the stable id live in the `county` field's map link
(`/MiDrive/map?...lat=&lon=&id=`); the `image` field carries an `<img>`
whose src is the JPEG still. Out-of-bbox and coord-less rows are dropped.
"""
try:
payload = json.loads(text)
except (json.JSONDecodeError, ValueError):
return []
if not isinstance(payload, list):
return []
out: list[dict] = []
for row in payload:
if not isinstance(row, dict):
continue
county_html = row.get("county") or ""
m_lat = _MDOT_LAT_RE.search(county_html)
m_lon = _MDOT_LON_RE.search(county_html)
m_id = _MDOT_ID_RE.search(county_html)
if not (m_lat and m_lon and m_id):
continue # missing coordinates / stable id → drop
try:
lat = float(m_lat.group(1))
lon = float(m_lon.group(1))
except ValueError:
continue
if not (MDOT_LAT_RANGE[0] <= lat <= MDOT_LAT_RANGE[1]
and MDOT_LON_RANGE[0] <= lon <= MDOT_LON_RANGE[1]):
continue # out of Michigan bbox → drop
img_m = _MDOT_IMG_RE.search(row.get("image") or "")
if not img_m:
continue
snap = img_m.group(1).strip()
low = snap.lower()
if not (low.startswith("http://") or low.startswith("https://")):
continue
if low.startswith("rtsp"):
continue
cam_id = m_id.group(1)
route = (row.get("route") or "").strip()
loc = (row.get("location") or "").strip().lstrip("@").strip()
county_name = county_html.split("<a", 1)[0].strip()
bits = [
f"{route} @ {loc}" if (route and loc) else (route or loc or None),
county_name or None,
]
name = ", ".join(b for b in bits if b) or None
out.append({
"source_url": f"https://mdotjboss.state.mi.us/MiDrive/camera/{cam_id}",
"snapshot_url": snap,
"discovery_source": "mdot",
"location_lat": lat,
"location_lon": lon,
"location_name": name,
"vendor": "MDOT",
"device_type": "http",
})
return out
def parse_live_streams_geojson(text: str, source_name: str) -> list[dict]:
"""Parse willytop8/Live-Environment-Streams GeoJSON.
@ -490,6 +563,8 @@ async def scrape_source(client: RateLimitedClient, geo: Geocoder,
body = resp.text
if "cwwp2.dot.ca.gov" in src_url or "cctvStatus" in src_url:
cams = parse_caltrans_json(body, name)
elif "mdotjboss.state.mi.us" in src_url or "/MiDrive/camera/list" in src_url:
cams = parse_mdot_json(body, name)
elif ("getCameraDataByLoc" in src_url
or ("json" in ctype and '"locs"' in body[:4000] and '"cams"' in body[:8000])):
cams = parse_alertwest_json(body, name)

168
app/conflicts.py Normal file
View file

@ -0,0 +1,168 @@
"""Curated OSINT conflict-zone catalog + point-in-bbox event counting.
A static, human-curated list of active conflict theatres (war / high /
elevated). Purely descriptive — this is a catalog, not a live feed and not a
scrape of LiveUAMap or any other source. Severity and descriptions are
editorial judgement kept short and factual.
Each zone carries an internal ``bbox`` (``min_lat, min_lon, max_lat, max_lon``)
used only to count pre-existing geocoded news/GDELT/``/api/news/map`` rows that
fall inside it. The bbox is not part of the API response; callers get the
``eventCount`` roll-up instead.
Never call an upstream API from here — event counts come from rows already in
the local database (``events`` with geocoords + ``news_items`` map pins).
"""
from __future__ import annotations
from datetime import datetime
# id → zone. ``lat``/``lon`` is the fly-to anchor; ``bbox`` is the internal
# count window in ``min_lat, min_lon, max_lat, max_lon`` order.
_ZONES: tuple[dict, ...] = (
{
"id": "ukraine",
"label": "Ukraine",
"severity": "war",
"lat": 48.5,
"lon": 31.0,
"description": "Full-scale Russian invasion since 2022; active front lines in the east and south.",
"bbox": (44.3, 22.1, 52.4, 40.2),
},
{
"id": "gaza",
"label": "Gaza",
"severity": "war",
"lat": 31.4,
"lon": 34.4,
"description": "Israel–Hamas war; sustained fighting and a severe humanitarian crisis in the Gaza Strip.",
"bbox": (31.0, 34.1, 31.8, 34.7),
},
{
"id": "sudan",
"label": "Sudan",
"severity": "war",
"lat": 15.5,
"lon": 30.0,
"description": "Civil war between the SAF and RSF since 2023, with mass displacement across the country.",
"bbox": (8.7, 21.8, 22.0, 38.6),
},
{
"id": "myanmar",
"label": "Myanmar",
"severity": "war",
"lat": 21.5,
"lon": 96.0,
"description": "Post-2021 coup conflict pitting the junta against resistance and ethnic armed groups.",
"bbox": (9.5, 92.2, 28.5, 101.2),
},
{
"id": "drc",
"label": "DR Congo",
"severity": "war",
"lat": -1.5,
"lon": 28.0,
"description": "Eastern DRC conflict involving M23 and other armed groups; heavy displacement around Goma.",
"bbox": (-5.0, 26.0, 3.0, 31.0),
},
{
"id": "yemen",
"label": "Yemen",
"severity": "war",
"lat": 15.5,
"lon": 47.5,
"description": "Protracted Houthi–government/coalition war with one of the world's worst humanitarian emergencies.",
"bbox": (12.6, 42.5, 19.0, 54.0),
},
{
"id": "syria",
"label": "Syria",
"severity": "war",
"lat": 34.5,
"lon": 38.5,
"description": "Multi-sided civil war; government, opposition, and external actors continue to engage.",
"bbox": (32.3, 35.7, 37.3, 42.4),
},
{
"id": "lebanon",
"label": "Lebanon",
"severity": "high",
"lat": 33.9,
"lon": 35.9,
"description": "Israel–Hezbollah hostilities with periodic escalation along the southern border.",
"bbox": (33.0, 35.0, 34.7, 36.6),
},
{
"id": "sahel",
"label": "Sahel",
"severity": "high",
"lat": 14.5,
"lon": 0.0,
"description": "Jihadist insurgencies across Mali, Burkina Faso, and Niger destabilising the central Sahel.",
"bbox": (10.0, -10.0, 20.0, 12.0),
},
{
"id": "somalia",
"label": "Somalia",
"severity": "high",
"lat": 6.0,
"lon": 45.0,
"description": "Al-Shabaab insurgency against the federal government and security forces.",
"bbox": (-2.0, 41.0, 12.0, 51.5),
},
{
"id": "red_sea",
"label": "Red Sea",
"severity": "high",
"lat": 18.0,
"lon": 40.0,
"description": "Houthi attacks on commercial shipping transiting the Red Sea corridor.",
"bbox": (12.0, 34.0, 22.0, 44.0),
},
{
"id": "taiwan_strait",
"label": "Taiwan Strait",
"severity": "elevated",
"lat": 24.5,
"lon": 119.5,
"description": "Heightened military standoff between China and Taiwan, including deterrence patrols.",
"bbox": (21.9, 117.0, 26.5, 122.0),
},
{
"id": "korean_dmz",
"label": "Korean DMZ",
"severity": "elevated",
"lat": 38.3,
"lon": 127.0,
"description": "Heavily fortified inter-Korean border with periodic tensions and military drills.",
"bbox": (37.5, 126.0, 39.0, 128.5),
},
)
SEVERITIES: frozenset[str] = frozenset({"war", "high", "elevated"})
def conflict_zones() -> list[dict]:
"""Return a fresh shallow copy of the catalog (callers must not mutate)."""
return [dict(z) for z in _ZONES]
def zone_event_stats(
points: list[tuple[float, float, datetime | None]],
bbox: tuple[float, float, float, float],
) -> tuple[int, datetime | None]:
"""Count points inside ``bbox`` and return (count, latest timestamp).
``points`` is an iterable of ``(lat, lon, ts)``; ``ts`` may be ``None``.
``bbox`` is ``(min_lat, min_lon, max_lat, max_lon)``.
"""
min_lat, min_lon, max_lat, max_lon = bbox
count = 0
latest: datetime | None = None
for lat, lon, ts in points:
if min_lat <= lat <= max_lat and min_lon <= lon <= max_lon:
count += 1
if ts is not None and (latest is None or ts > latest):
latest = ts
return count, latest

6
app/live_layers.py
View file

@ -150,6 +150,12 @@ def overlay_catalog() -> dict:
"endpoint": "/api/map/gpsjam",
"attribution": "GPSJAM / John Wiseman / ADS-B Exchange",
},
"conflicts": {
"id": "conflicts",
"kind": "points",
"endpoint": "/api/conflicts",
"attribution": "Curated OSINT conflict catalog",
},
}

53
app/main.py
View file

@ -38,6 +38,7 @@ from models import (
)
from schemas import (
AlertCreate, AlertOut, AlertSeverity, AlertType, AlertUpdate,
ConflictZoneOut, ConflictsOut,
DashboardSummary, EntityCreate, EntityKind, EntityOut,
EventCreate, EventOut, FireOut, NewsArticleOut, NewsMapItemOut,
NewsSummaryOut, NewsTickerItemOut,
@ -1432,6 +1433,58 @@ async def map_chokepoints():
return {"chokepoints": chokepoints()}
async def _fetch_geocoded_points() -> list[tuple[float, float, datetime | None]]:
"""Collect geocoded ``(lat, lon, ts)`` rows from the local DB.
Sources are the flagged map pins (``news_items`` kind=map) and geocoded
news/GDELT events (``events`` with ``location_lat/lon``). This is the
pre-existing geocoded corpus the conflict-zone counters roll up — no
upstream scraping and no generated/jittered coordinates.
"""
async with async_session() as session:
map_rows = (
await session.execute(
select(news_items.c.lat, news_items.c.lon, news_items.c.created_at)
.where(
news_items.c.kind == "map",
news_items.c.lat.isnot(None),
news_items.c.lon.isnot(None),
)
)
).all()
event_rows = (
await session.execute(
select(events.c.location_lat, events.c.location_lon, events.c.source_timestamp)
.where(
events.c.source_type.in_(["rss", "gdel-t2"]),
events.c.location_lat.isnot(None),
events.c.location_lon.isnot(None),
)
)
).all()
return [tuple(r) for r in map_rows] + [tuple(r) for r in event_rows]
@app.get("/api/conflicts", response_model=ConflictsOut)
async def list_conflicts():
"""Curated conflict-zone catalog with per-zone event counts.
Static catalogue (severity + short factual description) merged with a live
``eventCount`` roll-up of pre-existing geocoded news/GDELT//api/news/map
rows inside each zone bbox. Empty DB → ``eventCount=0`` (never 502).
"""
from conflicts import conflict_zones, zone_event_stats
points = await _fetch_geocoded_points()
timestamp = datetime.now(timezone.utc)
zones = []
for z in conflict_zones():
bbox = z.pop("bbox")
count, latest = zone_event_stats(points, bbox)
zones.append({**z, "eventCount": count, "lastUpdated": latest})
return {"zones": zones, "timestamp": timestamp}
def _upstream_or_502(exc: Exception, name: str) -> NoReturn:
logger.warning("live_layer_upstream_failed", layer=name, error=str(exc))
raise HTTPException(502, f"{name} upstream unavailable: {exc}") from exc

23
app/schemas.py
View file

@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import datetime
from enum import Enum
from typing import Optional
from typing import Literal, Optional
from uuid import UUID
from pydantic import BaseModel, Field, field_validator
@ -386,3 +386,24 @@ class GeofenceUpdate(BaseModel):
geojson: Optional[dict] = None
active: Optional[bool] = None
class ConflictZoneOut(BaseModel):
"""One curated conflict theatre as exposed by GET /api/conflicts."""
id: str
label: str
severity: Literal["war", "high", "elevated"]
lat: float
lon: float
description: str
eventCount: int
lastUpdated: Optional[datetime] = None
class ConflictsOut(BaseModel):
"""Response envelope for GET /api/conflicts."""
zones: list[ConflictZoneOut]
timestamp: datetime

132
tests/test_conflicts.py Normal file
View file

@ -0,0 +1,132 @@
"""GET /api/conflicts — curated conflict-zone catalog + event-count roll-up.
No outbound HTTP: event counts come from geocoded rows already (or not) in the
DB, and the API tests monkeypatch ``main._fetch_geocoded_points`` so no database
is required for the contract checks.
"""
from datetime import datetime, timezone
import httpx
from conflicts import SEVERITIES, conflict_zones, zone_event_stats
from live_layers import overlay_catalog
from main import app
BASE = "http://test"
def _get(path: str, monkeypatch=None, points=None) -> httpx.Response:
import asyncio
async def run() -> httpx.Response:
if monkeypatch is not None:
async def fake():
return points or []
monkeypatch.setattr("main._fetch_geocoded_points", fake)
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
return await client.get(path)
return asyncio.run(run())
# ── Catalog shape ──────────────────────────────────────────────────────
def test_catalog_length():
zones = conflict_zones()
assert len(zones) == 13
def test_catalog_severity_enum():
zones = conflict_zones()
sevs = {z["severity"] for z in zones}
assert sevs.issubset(SEVERITIES)
# All three tiers are represented.
assert sevs == SEVERITIES
def test_catalog_fields_factual_and_complete():
zones = conflict_zones()
ids = [z["id"] for z in zones]
assert len(set(ids)) == len(ids) # unique ids
for z in zones:
assert z["label"]
assert z["description"].strip()
assert -90.0 <= z["lat"] <= 90.0
assert -180.0 <= z["lon"] <= 180.0
# internal-only bbox is well-formed: (min_lat, min_lon, max_lat, max_lon)
min_lat, min_lon, max_lat, max_lon = z["bbox"]
assert min_lat <= max_lat and min_lon <= max_lon
assert min_lat <= z["lat"] <= max_lat and min_lon <= z["lon"] <= max_lon
def test_overlay_catalog_has_conflicts():
entry = overlay_catalog()["conflicts"]
assert entry["kind"] == "points"
assert entry["endpoint"] == "/api/conflicts"
# ── Pure counting ──────────────────────────────────────────────────────
TS1 = datetime(2026, 8, 30, 12, 0, tzinfo=timezone.utc)
TS2 = datetime(2026, 8, 30, 13, 0, tzinfo=timezone.utc)
def test_zone_event_stats_counts_and_picks_latest():
bbox = (40.0, 20.0, 52.0, 40.0) # roughly Ukraine
points = [
(50.45, 30.52, TS1), # inside
(48.0, 25.0, TS2), # inside, later
(0.0, -60.0, TS1), # outside
(15.0, 45.0, TS2), # outside (lat ok, lon out)
]
count, latest = zone_event_stats(points, bbox)
assert count == 2
assert latest == TS2
def test_zone_event_stats_empty_bbox():
count, latest = zone_event_stats([], (0.0, 0.0, 1.0, 1.0))
assert count == 0
assert latest is None
# ── API contract (mocked map items, no DB) ─────────────────────────────
def test_conflicts_returns_catalog_with_mocked_counts(monkeypatch):
points = [
(50.45, 30.52, TS1), # Ukraine
(25.03, 121.56, TS2), # Taiwan Strait
(0.0, -60.0, TS1), # nowhere
]
resp = _get("/api/conflicts", monkeypatch=monkeypatch, points=points)
assert resp.status_code == 200
body = resp.json()
assert "zones" in body and "timestamp" in body
by_id = {z["id"]: z for z in body["zones"]}
assert len(body["zones"]) == 13
zone = by_id["ukraine"]
assert zone["eventCount"] == 1
assert zone["lastUpdated"] == TS1.isoformat().replace("+00:00", "Z")
assert zone["severity"] == "war"
assert by_id["taiwan_strait"]["eventCount"] == 1
assert by_id["gaza"]["eventCount"] == 0
# exact per-zone key contract the frontend consumes
assert set(zone.keys()) == {
"id", "label", "severity", "lat", "lon",
"description", "eventCount", "lastUpdated",
}
def test_conflicts_empty_db_yields_zero_counts(monkeypatch):
resp = _get("/api/conflicts", monkeypatch=monkeypatch, points=[])
assert resp.status_code == 200
body = resp.json()
assert all(z["eventCount"] == 0 for z in body["zones"])
assert all(z["lastUpdated"] is None for z in body["zones"])

63
tests/test_live_layers.py
View file

@ -1,5 +1,7 @@
"""Unit tests for live map-layer mappers (aircraft, trains, AIS, WFIGS, Caltrans)."""
import json
from live_layers import (
MARKER_FIELDS,
bbox_center_radius_nm,
@ -25,7 +27,7 @@ from live_layers import (
_wfigs_params,
)
from camera_scraper import parse_caltrans_json
from camera_scraper import parse_caltrans_json, parse_mdot_json
def test_parse_bbox_and_radius_clamps_to_150_nm():
@ -257,6 +259,65 @@ def test_parse_caltrans_skips_oos_and_maps_jpeg_hls():
assert "rtsp://" not in cam["snapshot_url"].lower()
def test_parse_mdot_extracts_html_fields_and_bbox_filters():
rows = [
# In-bbox, full fields.
{
"route": "11 Mile",
"county": 'Wayne County <a href="/MiDrive/map?cameras=true&lat=42.491304&lon=-83.04479&zoom=15&id=1129"target="_blank">Go to</a>',
"location": " @ Mound NB",
"direction": "Traffic closest to camera is traveling north.",
"image": '<img alt="x" class="cameraImageForActivePane" id="1129Img" src="https://micamerasimages.net/thumbs/semtoc_cam_253.flv.jpg?item=1" height="170" width="250" onerror="cameraImageBroken(this)">',
},
# Out of bbox (lat 50) → drop.
{
"route": "Far",
"county": 'Nowhere <a href="/MiDrive/map?lat=50.0&lon=-83.0&zoom=15&id=9999">Go to</a>',
"location": "",
"image": '<img src="https://micamerasimages.net/thumbs/x.jpg">',
},
# Missing coordinates → drop.
{
"route": "NoCoords",
"county": 'Somewhere <a href="/MiDrive/map?zoom=15&id=8888">Go to</a>',
"location": "",
"image": '<img src="https://micamerasimages.net/thumbs/y.jpg">',
},
# Missing image → drop.
{
"route": "NoImage",
"county": 'Kent <a href="/MiDrive/map?lat=42.8841&lon=-85.6646&zoom=15&id=2113">Go to</a>',
"location": " @ Division",
"image": "",
},
# RTSP image src → drop.
{
"route": "Rtsp",
"county": 'Wayne <a href="/MiDrive/map?lat=42.4&lon=-83.1&zoom=15&id=1234">Go to</a>',
"location": "",
"image": '<img src="rtsp://10.0.0.1/stream">',
},
]
cams = parse_mdot_json(json.dumps(rows), "mdotjboss.state.mi.us")
assert len(cams) == 1
cam = cams[0]
assert cam["discovery_source"] == "mdot"
assert cam["location_lat"] == 42.491304
assert cam["location_lon"] == -83.04479
assert cam["snapshot_url"] == "https://micamerasimages.net/thumbs/semtoc_cam_253.flv.jpg?item=1"
assert cam["source_url"] == "https://mdotjboss.state.mi.us/MiDrive/camera/1129"
assert cam["device_type"] == "http"
assert cam["vendor"] == "MDOT"
assert "11 Mile @ Mound NB" in cam["location_name"]
assert "Wayne County" in cam["location_name"]
def test_parse_mdot_handles_malformed_payload():
assert parse_mdot_json("not json", "mdot") == []
assert parse_mdot_json('{"not": "a list"}', "mdot") == []
assert parse_mdot_json("[]", "mdot") == []
def test_quantize_bbox_stable_under_jitter():
a = quantize_bbox(*parse_bbox("-78.7912,35.7711,-78.6101,35.9102"))
b = quantize_bbox(*parse_bbox("-78.7900,35.7700,-78.6110,35.9090"))