osint-dashboard/docker-compose.yml

# OSINT Dashboard — container stack (all env-driven, 12-factor).
# Single combined TimescaleDB+PostGIS image is pinned to pg13 because that is
# the only arm64 build Timescale publishes with PostGIS bundled. The app uses
# only standard SQL types, so pg13 is fully sufficient.
#
# Bring up:   docker compose up -d
# With NATS:  docker compose --profile ingest up -d
# Env:        copy .env.example to .env and adjust.
#
# Project name is pinned here (NOT via COMPOSE_PROJECT_NAME in CI) so the
# named volumes stay osint-dashboard_osint-pgdata / _camera-snapshots and
# match the live stack on the Pi.
name: osint-dashboard

services:
  db:
    # NO build: block here on purpose. The DB image (TimescaleDB+PostGIS) is
    # built once via Dockerfile.pg and pinned as localhost/osint-dashboard-pg.
    # Dockerfile.pg installs TimescaleDB from packagecloud.io, which some
    # networks (and ISP abuse-mitigation blackholes) block, so rebuilding it
    # on every CI deploy made the pipeline flaky. Rebuild manually when the
    # base image or extensions need bumping:
    #   docker compose build db && docker compose up -d db
    image: localhost/osint-dashboard-pg:latest
    container_name: osint-db
    restart: unless-stopped
    environment:
      POSTGRES_USER: ${DB_USER:-osint}
      POSTGRES_PASSWORD: ${DB_PASSWORD:-osint}
      POSTGRES_DB: ${DB_NAME:-osint_data}
    # Ensure TimescaleDB is preloaded (conf.d drop-in may be ignored by the
    # official image's runtime-generated postgresql.conf, so pass it explicitly).
    command: ["-c", "shared_preload_libraries=timescaledb"]
    ports:
      - "127.0.0.1:5432:5432"
    volumes:
      - osint-pgdata:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-osint} -d ${DB_NAME:-osint_data}"]
      interval: 10s
      timeout: 5s
      retries: 10

  nats:
    image: nats:2.10
    platform: linux/arm64
    container_name: osint-nats
    restart: unless-stopped
    profiles: ["ingest"]
    ports:
      - "127.0.0.1:4222:4222"
      - "127.0.0.1:8222:8222"
    command: ["-js"]

  ingester:
    build:
      context: .
      dockerfile: Dockerfile
      platforms: ["linux/arm64"]
    image: localhost/osint-dashboard:latest
    container_name: osint-ingester
    restart: unless-stopped
    profiles: ["ingest"]
    depends_on:
      nats:
        condition: service_started
      db:
        condition: service_healthy
    environment:
      DB_USER: ${DB_USER:-osint}
      DB_PASSWORD: ${DB_PASSWORD:-osint}
      DB_HOST: db
      DB_PORT: ${DB_PORT:-5432}
      DB_NAME: ${DB_NAME:-osint_data}
      NATS_URL: ${NATS_URL:-nats://nats:4222}
      RSS_URL: ${RSS_URL:-}
      GDELT_QUERY: ${GDELT_QUERY:-}
      INGEST_INTERVAL: ${INGEST_INTERVAL:-300}
      INGEST_EARTHQUAKES: ${INGEST_EARTHQUAKES:-1}
      # ── NASA FIRMS active fires ──
      INGEST_FIRES: ${INGEST_FIRES:-1}
      FIRMS_MAP_KEY: ${FIRMS_MAP_KEY:-}
      FIRMS_DATASET: ${FIRMS_DATASET:-VIIRS_SNPP_NRT}
      FIRMS_BBOX: ${FIRMS_BBOX:--180,-60,180,75}
      FIRMS_INTERVAL: ${FIRMS_INTERVAL:-900}
    command: ["python", "app/run_ingester.py"]
    entrypoint: ["python", "app/run_ingester.py"]

  app:
    build:
      context: .
      dockerfile: Dockerfile
      platforms: ["linux/arm64"]
    image: localhost/osint-dashboard:latest
    container_name: osint-dashboard
    restart: unless-stopped
    depends_on:
      db:
        condition: service_healthy
    environment:
      DB_USER: ${DB_USER:-osint}
      DB_PASSWORD: ${DB_PASSWORD:-osint}
      DB_HOST: db
      DB_PORT: ${DB_PORT:-5432}
      DB_NAME: ${DB_NAME:-osint_data}
      NATS_URL: ${NATS_URL:-nats://nats:4222}
      MINIO_ENDPOINT: ${MINIO_ENDPOINT:-minio:9000}
      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-}
      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-}
      MINIO_SECURE: ${MINIO_SECURE:-false}
      # ── NASA FIRMS (for the /api/ingest/fires trigger endpoint) ──
      FIRMS_MAP_KEY: ${FIRMS_MAP_KEY:-}
      FIRMS_DATASET: ${FIRMS_DATASET:-VIIRS_SNPP_NRT}
      FIRMS_BBOX: ${FIRMS_BBOX:--180,-60,180,75}
    ports:
      - "127.0.0.1:8000:8000"
    healthcheck:
      test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/api/health').status==200 else 1)\""]
      interval: 30s
      timeout: 5s
      retries: 5

  camera-service:
    build:
      context: .
      dockerfile: Dockerfile
      platforms: ["linux/arm64"]
    image: localhost/osint-dashboard:latest
    container_name: osint-camera-scraper
    restart: unless-stopped
    profiles: ["ingest"]
    depends_on:
      nats:
        condition: service_started
      db:
        condition: service_healthy
    environment:
      DB_USER: ${DB_USER:-osint}
      DB_PASSWORD: ${DB_PASSWORD:-osint}
      DB_HOST: db
      DB_PORT: ${DB_PORT:-5432}
      DB_NAME: ${DB_NAME:-osint_data}
      NATS_URL: ${NATS_URL:-nats://nats:4222}
      CAMERA_SOURCE_URLS: ${CAMERA_SOURCE_URLS:-}
      CAMERA_SCRAPE_INTERVAL: ${CAMERA_SCRAPE_INTERVAL:-3600}
      CAMERA_REQUEST_DELAY: ${CAMERA_REQUEST_DELAY:-2.0}
      NOMINATIM_URL: ${NOMINATIM_URL:-https://nominatim.openstreetmap.org}
      NOMINATIM_MIN_INTERVAL: ${NOMINATIM_MIN_INTERVAL:-1.1}
      SNAPSHOT_CACHE_DIR: /data/snapshots
      SNAPSHOT_TTL_SECONDS: ${SNAPSHOT_TTL_SECONDS:-300}
    command: ["python", "app/run_camera_service.py"]
    entrypoint: []
    volumes:
      - camera-snapshots:/data/snapshots

  # ── News pipeline: hourly scraper (:00) + summarizer (:05) ───────────────
  # Both services point at the EXISTING osint-db (tables articles +
  # article_summaries, created by idempotent alembic migration 003_news).
  # Scheduling replaces the upstream k8s CronJobs with in-compose wall-clock
  # loops (run_news_scraper.py / run_news_summarizer.py).
  news-scraper:
    build:
      context: ./news/scraper
      dockerfile: Dockerfile
      platforms: ["linux/arm64"]
    image: localhost/osint-news-scraper:latest
    container_name: osint-news-scraper
    restart: unless-stopped
    profiles: ["ingest"]
    depends_on:
      db:
        condition: service_healthy
    environment:
      DB_USER: ${DB_USER:-osint}
      DB_PASSWORD: ${DB_PASSWORD:-osint}
      DB_HOST: db
      DB_PORT: ${DB_PORT:-5432}
      DB_NAME: ${DB_NAME:-osint_data}
      LOG_LEVEL: ${NEWS_LOG_LEVEL:-INFO}
      NEWS_SCRAPE_MINUTE: ${NEWS_SCRAPE_MINUTE:-0}
      NEWS_SCRAPE_RUN_ON_START: ${NEWS_SCRAPE_RUN_ON_START:-1}
    # Override the image ENTRYPOINT ["scrapy"] with the scheduler loop.
    entrypoint: []
    command: ["python", "run_news_scraper.py"]

  news-summarizer:
    build:
      context: ./news/summerizer
      dockerfile: Dockerfile
      platforms: ["linux/arm64"]
    image: localhost/osint-news-summarizer:latest
    container_name: osint-news-summarizer
    restart: unless-stopped
    profiles: ["ingest"]
    depends_on:
      db:
        condition: service_healthy
    environment:
      DB_USER: ${DB_USER:-osint}
      DB_PASSWORD: ${DB_PASSWORD:-osint}
      DB_HOST: db
      DB_PORT: ${DB_PORT:-5432}
      DB_NAME: ${DB_NAME:-osint_data}
      # Required to do real work; unset → the loop logs and idles.
      GEMINI_API_KEY: ${GEMINI_API_KEY:-}
      SUMMARY_MODEL: ${SUMMARY_MODEL:-gemini-2.0-flash}
      BATCH_SIZE: ${NEWS_BATCH_SIZE:-50}
      SUMMARY_WINDOW_HOURS: ${SUMMARY_WINDOW_HOURS:-1}
      INCLUDE_FUTURES: ${INCLUDE_FUTURES:-0}
      NEWS_SUMMARIZE_MINUTE: ${NEWS_SUMMARIZE_MINUTE:-5}
      NEWS_SUMMARIZE_RUN_ON_START: ${NEWS_SUMMARIZE_RUN_ON_START:-1}
    command: ["python", "run_news_summarizer.py"]

volumes:
  osint-pgdata:
  camera-snapshots: