a9ca98e791
Add parse_udot_ibi_page + scrape_udot_ibi for the UDOT IBI 511 traffic
camera feed (prod-ut.ibi511.com, no key). DataTables endpoint is POST
form-encoded and caps at 100 rows/page regardless of `length`; page walk
uses recordsTotal with a UDOT_IBI_MAX_PAGES (default 40) runaway cap.
Skip images[0].blocked/disabled, parse WKT POINT(lng lat) from
latLng.geography.wellKnownText, drop outside the Utah bbox
(lat 36.9-42.1, lon -114.2--108.9). discovery_source=udot, stable
source_url == snapshot_url == /map/Cctv/{id} (never scrape frames),
url_hash dedupe, OSINT_USER_AGENT + X-Requested-With header.
Unit tests: WKT lng/lat order, blocked/disabled skip, bbox drop,
malformed payload, missing WKT.
69 lines
3.6 KiB
Python
69 lines
3.6 KiB
Python
"""Camera discovery configuration (12-factor, env-driven).
|
||
|
||
All knobs read from the environment with container-friendly defaults.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
|
||
# ── Camera scraper ─────────────────────────────────────────────────────────
|
||
# Comma-separated public directory/list/API URLs to scrape.
|
||
# Formats: Insecam-style HTML, plain-text URL lists, ALERTWest JSON.
|
||
# NOTE: docker-compose always defines CAMERA_SOURCE_URLS (empty when no .env),
|
||
# so os.getenv()'s default would never apply — use `or` semantics instead.
|
||
CALTRANS_CCTV_URLS = tuple(
|
||
f"https://cwwp2.dot.ca.gov/data/d{n}/cctv/cctvStatusD{n:02d}.json"
|
||
for n in range(1, 13)
|
||
)
|
||
_DEFAULT_SOURCE_URL = ",".join((
|
||
# Publicly published open-camera list (markdown bullets of stream URLs).
|
||
"https://raw.githubusercontent.com/fury999io/public-ip-cams/main/README.md",
|
||
# ALERTCalifornia / ALERTWest official public JPEG API (wildfire + DOT + FAA).
|
||
"https://api.cdn.prod.alertwest.com/api/getCameraDataByLoc",
|
||
# Curated global outdoor streams (HLS / YouTube / JPEG) — VDOT, MDSHA, etc.
|
||
"https://raw.githubusercontent.com/willytop8/Live-Environment-Streams/main/streams.geojson",
|
||
# Official Caltrans CWWP2 JPEG + HLS CCTV (districts 1–12).
|
||
*CALTRANS_CCTV_URLS,
|
||
))
|
||
CAMERA_SOURCE_URLS = [
|
||
u.strip()
|
||
for u in (os.getenv("CAMERA_SOURCE_URLS") or _DEFAULT_SOURCE_URL).split(",")
|
||
if u.strip() # empty env var → fall back to the default above
|
||
]
|
||
|
||
# Seconds between full scrape cycles of every source.
|
||
CAMERA_SCRAPE_INTERVAL = int(os.getenv("CAMERA_SCRAPE_INTERVAL", "3600"))
|
||
|
||
# Politeness: minimum seconds between consecutive requests to the SAME host.
|
||
CAMERA_REQUEST_DELAY = float(os.getenv("CAMERA_REQUEST_DELAY", "2.0"))
|
||
|
||
# Max cameras accepted per scrape cycle per source (safety cap).
|
||
CAMERA_MAX_PER_SOURCE = int(os.getenv("CAMERA_MAX_PER_SOURCE", "20000"))
|
||
|
||
# ── Geocoding (Nominatim — free, 1 req/s hard politeness limit) ────────────
|
||
NOMINATIM_URL = os.getenv("NOMINATIM_URL", "https://nominatim.openstreetmap.org")
|
||
NOMINATIM_MIN_INTERVAL = float(os.getenv("NOMINATIM_MIN_INTERVAL", "1.0"))
|
||
USER_AGENT = os.getenv(
|
||
"OSINT_USER_AGENT", "osint-dashboard-camera-scraper/1.0 (self-hosted)"
|
||
)
|
||
|
||
# ── Snapshot cache ─────────────────────────────────────────────────────────
|
||
SNAPSHOT_CACHE_DIR = os.getenv("SNAPSHOT_CACHE_DIR", "/data/snapshots")
|
||
SNAPSHOT_TTL_SECONDS = int(os.getenv("SNAPSHOT_TTL_SECONDS", "300"))
|
||
SNAPSHOT_TIMEOUT = float(os.getenv("SNAPSHOT_TIMEOUT", "8.0"))
|
||
|
||
# NATS subject cameras are published on (consumed by the shared ingester).
|
||
CAMERA_NATS_SUBJECT = os.getenv("CAMERA_NATS_SUBJECT", "events.camera")
|
||
|
||
|
||
# ── UDOT IBI 511 traffic cameras ──────────────────────────────────────────
|
||
# DataTables endpoint (POST form-encoded; server caps at 100 rows/page no
|
||
# matter what `length` is sent). No API key. Snapshot stills live at a stable
|
||
# /map/Cctv/{id} URL — same URL always serves the latest frame, so we store
|
||
# the URL and never scrape every frame ourselves.
|
||
UDOT_IBI_URL = "https://prod-ut.ibi511.com/List/GetData/Cameras"
|
||
UDOT_IBI_BASE = "https://prod-ut.ibi511.com"
|
||
UDOT_IBI_PAGE_SIZE = 100
|
||
# Safety cap on pages per cycle so a runaway recordsTotal cannot fan out.
|
||
UDOT_IBI_MAX_PAGES = int(os.getenv("UDOT_IBI_MAX_PAGES", "40"))
|