69a30115cc
Some checks failed
build-and-deploy / build (push) Failing after 5s
- camera_scraper: public-directory-only discovery (Insecam-style HTML + plain-text lists), hard private-range guard (fail closed), per-host rate limiting, Nominatim geocoding at <=1 req/s, TTL'd local snapshot cache, sha256 url_hash dedupe with Postgres upsert - cameras table (migration 002) + /api/cameras?bbox= + snapshot endpoint with cache passthrough in main.py - run_camera_service: long-running cycle worker following NATS->ingester pattern; publishes events.camera for shared ingester - docker-compose camera-service profile, .env.example knobs Verified E2E against TimescaleDB+PostGIS: private-range entries dropped, dedupe across cycles holds, bbox query returns expected rows.
47 lines
2.2 KiB
Python
47 lines
2.2 KiB
Python
"""Camera discovery configuration (12-factor, env-driven).
|
|
|
|
All knobs read from the environment with container-friendly defaults.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
# ── Camera scraper ─────────────────────────────────────────────────────────
|
|
# Comma-separated public directory/list URLs to scrape. Two formats supported:
|
|
# * Insecam-style HTML directory pages (lat/lon embedded per camera)
|
|
# * Plain-text lists, one camera per line:
|
|
# <url>[|<lat>,<lon>|<vendor>|<location_name>]
|
|
CAMERA_SOURCE_URLS = [
|
|
u.strip()
|
|
for u in os.getenv(
|
|
"CAMERA_SOURCE_URLS",
|
|
# Default: publicly published open-camera lists (free, no paid API).
|
|
"https://raw.githubusercontent.com/neo23x0/CameraHacks/main/camera_list.txt",
|
|
).split(",")
|
|
if u.strip()
|
|
]
|
|
|
|
# Seconds between full scrape cycles of every source.
|
|
CAMERA_SCRAPE_INTERVAL = int(os.getenv("CAMERA_SCRAPE_INTERVAL", "3600"))
|
|
|
|
# Politeness: minimum seconds between consecutive requests to the SAME host.
|
|
CAMERA_REQUEST_DELAY = float(os.getenv("CAMERA_REQUEST_DELAY", "2.0"))
|
|
|
|
# Max cameras accepted per scrape cycle per source (safety cap).
|
|
CAMERA_MAX_PER_SOURCE = int(os.getenv("CAMERA_MAX_PER_SOURCE", "5000"))
|
|
|
|
# ── Geocoding (Nominatim — free, 1 req/s hard politeness limit) ────────────
|
|
NOMINATIM_URL = os.getenv("NOMINATIM_URL", "https://nominatim.openstreetmap.org")
|
|
NOMINATIM_MIN_INTERVAL = float(os.getenv("NOMINATIM_MIN_INTERVAL", "1.0"))
|
|
USER_AGENT = os.getenv(
|
|
"OSINT_USER_AGENT", "osint-dashboard-camera-scraper/1.0 (self-hosted)"
|
|
)
|
|
|
|
# ── Snapshot cache ─────────────────────────────────────────────────────────
|
|
SNAPSHOT_CACHE_DIR = os.getenv("SNAPSHOT_CACHE_DIR", "/data/snapshots")
|
|
SNAPSHOT_TTL_SECONDS = int(os.getenv("SNAPSHOT_TTL_SECONDS", "300"))
|
|
SNAPSHOT_TIMEOUT = float(os.getenv("SNAPSHOT_TIMEOUT", "8.0"))
|
|
|
|
# NATS subject cameras are published on (consumed by the shared ingester).
|
|
CAMERA_NATS_SUBJECT = os.getenv("CAMERA_NATS_SUBJECT", "events.camera")
|