osint-dashboard/docker-compose.yml

# OSINT Dashboard — container stack (all env-driven, 12-factor).
# Single combined TimescaleDB+PostGIS image is pinned to pg13 because that is
# the only arm64 build Timescale publishes with PostGIS bundled. The app uses
# only standard SQL types, so pg13 is fully sufficient.
#
# Bring up:   docker compose up -d
# With NATS:  docker compose --profile ingest up -d
# Env:        copy .env.example to .env and adjust.
#
# Project name is pinned here (NOT via COMPOSE_PROJECT_NAME in CI) so the
# named volumes stay osint-dashboard_osint-pgdata / _camera-snapshots and
# match the live stack on the Pi.
name: osint-dashboard

services:
  db:
    # NO build: block here on purpose. The DB image (TimescaleDB+PostGIS) is
    # built once via Dockerfile.pg and pinned as localhost/osint-dashboard-pg.
    # Dockerfile.pg installs TimescaleDB from packagecloud.io, which some
    # networks (and ISP abuse-mitigation blackholes) block, so rebuilding it
    # on every CI deploy made the pipeline flaky. Rebuild manually when the
    # base image or extensions need bumping:
    #   docker compose build db && docker compose up -d db
    image: localhost/osint-dashboard-pg:latest
    container_name: osint-db
    restart: unless-stopped
    environment:
      POSTGRES_USER: ${DB_USER:-osint}
      POSTGRES_PASSWORD: ${DB_PASSWORD:-osint}
      POSTGRES_DB: ${DB_NAME:-osint_data}
    # Ensure TimescaleDB is preloaded (conf.d drop-in may be ignored by the
    # official image's runtime-generated postgresql.conf, so pass it explicitly).
    # shared_buffers capped at 2GB for Pi 5 8GB / 4 cores.
    command:
      [
        "-c", "shared_preload_libraries=timescaledb",
        "-c", "shared_buffers=2GB",
      ]
    deploy:
      resources:
        limits:
          memory: 3G
    ports:
      - "127.0.0.1:5432:5432"
    volumes:
      - osint-pgdata:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-osint} -d ${DB_NAME:-osint_data}"]
      interval: 10s
      timeout: 5s
      retries: 10

  nats:
    image: nats:2.10
    platform: linux/arm64
    container_name: osint-nats
    restart: unless-stopped
    profiles: ["ingest"]
    ports:
      - "127.0.0.1:4222:4222"
      - "127.0.0.1:8222:8222"
    command: ["-js"]

  ingester:
    build:
      context: .
      dockerfile: Dockerfile
      platforms: ["linux/arm64"]
    image: localhost/osint-dashboard:latest
    container_name: osint-ingester
    restart: unless-stopped
    profiles: ["ingest"]
    depends_on:
      nats:
        condition: service_started
      db:
        condition: service_healthy
    environment:
      DB_USER: ${DB_USER:-osint}
      DB_PASSWORD: ${DB_PASSWORD:-osint}
      DB_HOST: db
      DB_PORT: ${DB_PORT:-5432}
      DB_NAME: ${DB_NAME:-osint_data}
      NATS_URL: ${NATS_URL:-nats://nats:4222}
      RSS_URL: ${RSS_URL:-}
      GDELT_QUERY: ${GDELT_QUERY:-}
      INGEST_INTERVAL: ${INGEST_INTERVAL:-300}
      INGEST_EARTHQUAKES: ${INGEST_EARTHQUAKES:-1}
      # ── NASA FIRMS active fires ──
      INGEST_FIRES: ${INGEST_FIRES:-1}
      FIRMS_MAP_KEY: ${FIRMS_MAP_KEY:-}
      FIRMS_DATASET: ${FIRMS_DATASET:-VIIRS_NOAA20_NRT}
      FIRMS_DATASETS: ${FIRMS_DATASETS:-VIIRS_NOAA20_NRT,VIIRS_NOAA21_NRT}
      FIRMS_BBOX: ${FIRMS_BBOX:--180,-60,180,75}
      FIRMS_INTERVAL: ${FIRMS_INTERVAL:-900}
      OSINT_USER_AGENT: ${OSINT_USER_AGENT:-osint-dashboard/1.0 (self-hosted; lancewalters94@gmail.com)}
      AISSTREAM_API_KEY: ${AISSTREAM_API_KEY:-}
      AISSTREAM_BBOX: ${AISSTREAM_BBOX:-24,-125,50,-66}
      AISSTREAM_IN_INGEST: ${AISSTREAM_IN_INGEST:-0}
      VESSELAPI_API_KEY: ${VESSELAPI_API_KEY:-}
      VESSELAPI_BBOX: ${VESSELAPI_BBOX:-25.5,55.4,27.3,57.2}
      VESSELAPI_INTERVAL: ${VESSELAPI_INTERVAL:-17280}
      VESSELAPI_MAX_CALLS_PER_DAY: ${VESSELAPI_MAX_CALLS_PER_DAY:-5}
      VESSELAPI_IN_INGEST: ${VESSELAPI_IN_INGEST:-0}
    command: ["python", "app/run_ingester.py"]
    entrypoint: ["python", "app/run_ingester.py"]

  app:
    build:
      context: .
      dockerfile: Dockerfile
      platforms: ["linux/arm64"]
    image: localhost/osint-dashboard:latest
    container_name: osint-dashboard
    restart: unless-stopped
    depends_on:
      db:
        condition: service_healthy
    environment:
      DB_USER: ${DB_USER:-osint}
      DB_PASSWORD: ${DB_PASSWORD:-osint}
      DB_HOST: db
      DB_PORT: ${DB_PORT:-5432}
      DB_NAME: ${DB_NAME:-osint_data}
      NATS_URL: ${NATS_URL:-nats://nats:4222}
      MINIO_ENDPOINT: ${MINIO_ENDPOINT:-minio:9000}
      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-}
      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-}
      MINIO_SECURE: ${MINIO_SECURE:-false}
      # ── NASA FIRMS (for the /api/ingest/fires trigger endpoint) ──
      FIRMS_MAP_KEY: ${FIRMS_MAP_KEY:-}
      FIRMS_DATASET: ${FIRMS_DATASET:-VIIRS_NOAA20_NRT}
      FIRMS_DATASETS: ${FIRMS_DATASETS:-VIIRS_NOAA20_NRT,VIIRS_NOAA21_NRT}
      FIRMS_BBOX: ${FIRMS_BBOX:--180,-60,180,75}
      OSINT_USER_AGENT: ${OSINT_USER_AGENT:-osint-dashboard/1.0 (self-hosted; lancewalters94@gmail.com)}
      AISSTREAM_API_KEY: ${AISSTREAM_API_KEY:-}
      AISSTREAM_BBOX: ${AISSTREAM_BBOX:-24,-125,50,-66}
      AISSTREAM_IN_APP: ${AISSTREAM_IN_APP:-1}
      VESSELAPI_API_KEY: ${VESSELAPI_API_KEY:-}
      VESSELAPI_BBOX: ${VESSELAPI_BBOX:-25.5,55.4,27.3,57.2}
      VESSELAPI_INTERVAL: ${VESSELAPI_INTERVAL:-17280}
      VESSELAPI_MAX_CALLS_PER_DAY: ${VESSELAPI_MAX_CALLS_PER_DAY:-5}
      VESSELAPI_IN_APP: ${VESSELAPI_IN_APP:-1}
      # ── Self-hosted TiTiler (Sentinel-1 SAR tiles) ──
      TITILER_PUBLIC_BASE: ${TITILER_PUBLIC_BASE:-/titiler}
      TITILER_INTERNAL_URL: ${TITILER_INTERNAL_URL:-http://titiler:8000}
    ports:
      - "127.0.0.1:8000:8000"
    deploy:
      resources:
        limits:
          memory: 2G
    healthcheck:
      test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/api/health').status==200 else 1)\""]
      interval: 30s
      timeout: 5s
      retries: 5

  # ── Self-hosted TiTiler (Sentinel-1 SAR COG → XYZ tiles) ────────────────
  # Warps the signed Planetary Computer COG into WebMercator XYZ tiles so the
  # browser never loads a multi-GB GeoTIFF. The FastAPI app signs the COG URL
  # and returns a /titiler/... template; nginx routes /titiler/ here.
  # Listens on 8000 INSIDE the container (the app already owns host 8000);
  # published on host loopback 127.0.0.1:8001 only.
  titiler:
    image: ghcr.io/developmentseed/titiler:latest
    container_name: osint-titiler
    platform: linux/arm64
    restart: unless-stopped
    environment:
      - PORT=8000
      - WORKERS_PER_CORE=1
    ports:
      - "127.0.0.1:8001:8000"
    deploy:
      resources:
        limits:
          memory: 1G

  camera-service:
    build:
      context: .
      dockerfile: Dockerfile
      platforms: ["linux/arm64"]
    image: localhost/osint-dashboard:latest
    container_name: osint-camera-scraper
    restart: unless-stopped
    profiles: ["ingest"]
    depends_on:
      nats:
        condition: service_started
      db:
        condition: service_healthy
    environment:
      DB_USER: ${DB_USER:-osint}
      DB_PASSWORD: ${DB_PASSWORD:-osint}
      DB_HOST: db
      DB_PORT: ${DB_PORT:-5432}
      DB_NAME: ${DB_NAME:-osint_data}
      NATS_URL: ${NATS_URL:-nats://nats:4222}
      CAMERA_SOURCE_URLS: ${CAMERA_SOURCE_URLS:-}
      CAMERA_SCRAPE_INTERVAL: ${CAMERA_SCRAPE_INTERVAL:-3600}
      CAMERA_REQUEST_DELAY: ${CAMERA_REQUEST_DELAY:-2.0}
      NOMINATIM_URL: ${NOMINATIM_URL:-https://nominatim.openstreetmap.org}
      NOMINATIM_MIN_INTERVAL: ${NOMINATIM_MIN_INTERVAL:-1.1}
      SNAPSHOT_CACHE_DIR: /data/snapshots
      SNAPSHOT_TTL_SECONDS: ${SNAPSHOT_TTL_SECONDS:-300}
    command: ["python", "app/run_camera_service.py"]
    entrypoint: []
    volumes:
      - camera-snapshots:/data/snapshots

  # ── News pipeline: continuous scraper + 15-min summarizer ───────────────
  # Both services point at the EXISTING osint-db (tables articles +
  # article_summaries, created by idempotent alembic migration 003_news).
  # Scheduling replaces the upstream k8s CronJobs with in-compose wall-clock
  # loops (run_news_scraper.py / run_news_summarizer.py).
  news-scraper:
    build:
      context: ./news/scraper
      dockerfile: Dockerfile
      platforms: ["linux/arm64"]
    image: localhost/osint-news-scraper:latest
    container_name: osint-news-scraper
    restart: unless-stopped
    profiles: ["ingest"]
    depends_on:
      db:
        condition: service_healthy
    environment:
      DB_USER: ${DB_USER:-osint}
      DB_PASSWORD: ${DB_PASSWORD:-osint}
      DB_HOST: db
      DB_PORT: ${DB_PORT:-5432}
      DB_NAME: ${DB_NAME:-osint_data}
      LOG_LEVEL: ${NEWS_LOG_LEVEL:-INFO}
      NEWS_SCRAPE_INTERVAL_S: ${NEWS_SCRAPE_INTERVAL_S:-10}
      NEWS_SCRAPE_RUN_ON_START: ${NEWS_SCRAPE_RUN_ON_START:-1}
    # Override the image ENTRYPOINT ["scrapy"] with the scheduler loop.
    entrypoint: []
    command: ["python", "run_news_scraper.py"]

  news-summarizer:
    build:
      context: ./news/summerizer
      dockerfile: Dockerfile
      platforms: ["linux/arm64"]
    image: localhost/osint-news-summarizer:latest
    container_name: osint-news-summarizer
    restart: unless-stopped
    profiles: ["ingest"]
    depends_on:
      db:
        condition: service_healthy
    environment:
      DB_USER: ${DB_USER:-osint}
      DB_PASSWORD: ${DB_PASSWORD:-osint}
      DB_HOST: db
      DB_PORT: ${DB_PORT:-5432}
      DB_NAME: ${DB_NAME:-osint_data}
      NOUS_API_KEY: ${NOUS_API_KEY:-}
      NOUS_BASE_URL: ${NOUS_BASE_URL:-https://inference-api.nousresearch.com/v1}
      SUMMARY_MODEL: ${SUMMARY_MODEL:-}
      OSINT_USER_AGENT: ${OSINT_USER_AGENT:-osint-dashboard-news-summarizer}
      BATCH_SIZE: ${NEWS_BATCH_SIZE:-50}
      SUMMARY_WINDOW_MINUTES: ${SUMMARY_WINDOW_MINUTES:-15}
      INCLUDE_FUTURES: ${INCLUDE_FUTURES:-0}
      NEWS_SUMMARIZE_INTERVAL_S: ${NEWS_SUMMARIZE_INTERVAL_S:-900}
      NEWS_SUMMARIZE_RUN_ON_START: ${NEWS_SUMMARIZE_RUN_ON_START:-1}
      NEWS_SUMMARIZE_FORCE: ${NEWS_SUMMARIZE_FORCE:-0}
      TZ: ${TZ:-America/New_York}
      NEWS_RECAP_HOUR: ${NEWS_RECAP_HOUR:-23}
      NEWS_RECAP_MINUTE: ${NEWS_RECAP_MINUTE:-0}
    command: ["python", "run_news_summarizer.py"]

volumes:
  osint-pgdata:
  camera-snapshots: