1f23083351
Add a server-side REST poller for VesselAPI (free tier 150 calls/mo) that upserts last-known positions into the shared vessel_last_known store when AISStream is unset. Default box is the Strait of Hormuz (span 3.6 <= 4 deg), never CONUS/NC (AISStream owns US coasts). - app/vesselapi.py: worker loop, 4deg span validator, position->marker transform (skip suspected_glitch), durable Postgres daily-quota table. - Local hard cap 5 successful 2xx/UTC day (VESSELAPI_MAX_CALLS_PER_DAY), monthly floor from X-RateLimit-Remaining, single request limit=50, no nextToken, no filter.sat, no retry-storm. - keystore VESSELAPI_API_KEY registry entry; config + compose env passthrough (app + ingest); wired next to AISStream in main.py lifespan + run_ingester. - GET /api/vessels docstring notes AISStream and/or VesselAPI cache. - tests/test_vesselapi.py: 20 unit tests (no network/DB).
99 lines
5.6 KiB
Python
99 lines
5.6 KiB
Python
"""OSINT Dashboard — centralized configuration (12-factor, env-driven).
|
|
|
|
All infrastructure endpoints are read from the environment with
|
|
container-friendly defaults. No secrets, hostnames, or cluster-specific
|
|
addresses are hardcoded anywhere in the codebase.
|
|
|
|
Override per environment via environment variables (docker-compose,
|
|
k8s ConfigMap/Secret, or systemd):
|
|
|
|
DB_USER, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME
|
|
NATS_URL
|
|
MINIO_ENDPOINT, MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_SECURE
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from urllib.parse import quote_plus
|
|
|
|
# ── PostgreSQL ────────────────────────────────────────────────────────────
|
|
# Defaults assume docker-compose/k8s service names. Override for your stack.
|
|
DB_USER = os.getenv("DB_USER", "osint")
|
|
DB_PASSWORD = os.getenv("DB_PASSWORD", "")
|
|
DB_HOST = os.getenv("DB_HOST", "postgres")
|
|
DB_PORT = os.getenv("DB_PORT", "5432")
|
|
DB_NAME = os.getenv("DB_NAME", "osint_data")
|
|
|
|
# Async SQLAlchemy URL. Password is URL-encoded so special chars are safe.
|
|
DATABASE_URL = (
|
|
f"postgresql+asyncpg://"
|
|
f"{DB_USER}:{quote_plus(DB_PASSWORD)}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
|
|
)
|
|
|
|
# ── NATS JetStream ─────────────────────────────────────────────────────────
|
|
NATS_URL = os.getenv("NATS_URL", "nats://nats:4222")
|
|
|
|
# ── MinIO (document storage) ────────────────────────────────────────────────
|
|
MINIO_ENDPOINT = os.getenv("MINIO_ENDPOINT", "minio:9000")
|
|
MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY", "")
|
|
MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY", "")
|
|
# "false" / "0" / "no" (case-insensitive) → plain HTTP (e.g. local compose).
|
|
MINIO_SECURE = os.getenv("MINIO_SECURE", "false").lower() not in ("false", "0", "no")
|
|
|
|
# ── NASA FIRMS (active fire / hotspot ingest) ───────────────────────────────
|
|
# MAP_KEY is free; obtain one at https://firms.modaps.eosdis.nasa.gov/api/map_key_info/
|
|
# and set FIRMS_MAP_KEY in .env. Until it is set, the fire ingestor logs a
|
|
# warning and stays idle (no crash).
|
|
FIRMS_MAP_KEY = os.getenv("FIRMS_MAP_KEY", "")
|
|
# NRT VIIRS S-NPP active fire/hotspot detection (375m).
|
|
# NASA stops Suomi NPP product delivery on 2026-11-01 — default to NOAA-20.
|
|
FIRMS_DATASET = os.getenv("FIRMS_DATASET", "VIIRS_NOAA20_NRT")
|
|
# Area bounding box as "minlon,minlat,maxlon,maxlat". Default covers most of
|
|
# the inhabited globe; narrow it (e.g. CONUS "-125,24,-66,50") to shrink
|
|
# payloads and the Postgres write volume.
|
|
FIRMS_BBOX = os.getenv("FIRMS_BBOX", "-180,-60,180,75")
|
|
# Poll cadence in seconds. FIRMS NRT updates every ~5-10 min; 900 = 15 min.
|
|
FIRMS_INTERVAL = int(os.getenv("FIRMS_INTERVAL", "900"))
|
|
# Day range for the area-CSV request. FIRMS accepts [1..5] days of NRT
|
|
# detections; the API 400s when this parameter is omitted. Keep >=2: at
|
|
# some hours "1" returns zero detections due to NRT data latency.
|
|
FIRMS_DAYS = int(os.getenv("FIRMS_DAYS", "2"))
|
|
# Outbound HTTP timeout for the FIRMS CSV download.
|
|
FIRMS_TIMEOUT = float(os.getenv("FIRMS_TIMEOUT", "60"))
|
|
# Comma-separated FIRMS products to dual-write. S-NPP delivery ends 2026-11-01;
|
|
# default to NOAA-20 + NOAA-21 NRT. FIRMS_DATASET is still honored when
|
|
# FIRMS_DATASETS is unset (empty string means "use FIRMS_DATASET only").
|
|
_FIRMS_DATASETS_RAW = os.getenv("FIRMS_DATASETS", "VIIRS_NOAA20_NRT,VIIRS_NOAA21_NRT")
|
|
FIRMS_DATASETS = [d.strip() for d in _FIRMS_DATASETS_RAW.split(",") if d.strip()] or [FIRMS_DATASET]
|
|
|
|
# Identifying User-Agent for NWS / Amtraker / Nominatim (mandatory on some APIs).
|
|
OSINT_USER_AGENT = os.getenv(
|
|
"OSINT_USER_AGENT", "osint-dashboard/1.0 (self-hosted; lancewalters94@gmail.com)"
|
|
)
|
|
|
|
# AISStream (server-side WebSocket only). Idle when unset.
|
|
AISSTREAM_API_KEY = os.getenv("AISSTREAM_API_KEY", "")
|
|
# Bounding box(es) as minlat,minlon,maxlat,maxlon — note lat/lon order (AISStream).
|
|
# Default: CONUS coasts + Great Lakes, not the world.
|
|
AISSTREAM_BBOX = os.getenv("AISSTREAM_BBOX", "24,-125,50,-66")
|
|
# Run the AIS worker inside the dashboard process (default on so vessels work
|
|
# without the ingest profile). Set 0 if the ingester owns the only connection.
|
|
AISSTREAM_IN_APP = os.getenv("AISSTREAM_IN_APP", "1").lower() in ("1", "true", "yes")
|
|
AISSTREAM_IN_INGEST = os.getenv("AISSTREAM_IN_INGEST", "0").lower() in ("1", "true", "yes")
|
|
|
|
# VesselAPI (quota-capped REST AIS poller — free tier 150 calls/mo).
|
|
# AISStream keeps US coasts; VesselAPI fills the Middle East blind spot. The
|
|
# poller idles when VESSELAPI_API_KEY is unset (never from GET /api/vessels).
|
|
VESSELAPI_API_KEY = os.getenv("VESSELAPI_API_KEY", "")
|
|
# Bounding box(es) as minlat,minlon,maxlat,maxlon — note lat/lon order (same as
|
|
# AISSTREAM_BBOX). Semicolon-separated for multiple boxes. Default: Strait of
|
|
# Hormuz (|dLat|+|dLon| = 3.6 ≤ 4° span cap). VesselAPI 400s any box over 4°.
|
|
VESSELAPI_BBOX = os.getenv("VESSELAPI_BBOX", "25.5,55.4,27.3,57.2")
|
|
# Poll cadence in seconds. 17280 = 4.8h → 5 polls/day (150/mo free tier).
|
|
VESSELAPI_INTERVAL = int(os.getenv("VESSELAPI_INTERVAL", "17280"))
|
|
# Local hard cap on successful 2xx calls per UTC day (persisted in Postgres).
|
|
VESSELAPI_MAX_CALLS_PER_DAY = int(os.getenv("VESSELAPI_MAX_CALLS_PER_DAY", "5"))
|
|
# Run the VesselAPI poller inside the dashboard process (default on, like AIS).
|
|
VESSELAPI_IN_APP = os.getenv("VESSELAPI_IN_APP", "1").lower() in ("1", "true", "yes")
|
|
VESSELAPI_IN_INGEST = os.getenv("VESSELAPI_IN_INGEST", "0").lower() in ("1", "true", "yes")
|