Containerize for Pi: env-var config, compose stack, CI workflow

- Add app/config.py (12-factor, env-driven: DB/NATS/MinIO, URL-encoded pw)
- Route database/ingestor/sources/alembic through config; kill hardcoded
  svc.cluster.local hostnames
- Drop unused redis dependency
- Add docker-compose.yml (timescaledb-postgis pg13 arm64 + optional nats)
- Add .env.example, .dockerignore, .forgejo/workflows/build.yml (arm64 build+SSH deploy)

.dockerignore

@@ -0,0 +1,9 @@
+__pycache__/
+*.pyc
+.git/
+.gitignore
+.env
+.env.example
+.forgejo/
+README.md
+*.md

.env.example

@@ -0,0 +1,19 @@
+# OSINT Dashboard — environment configuration
+# Copy to `.env` and adjust. All values have safe defaults for local compose.
+
+# ── PostgreSQL / TimescaleDB ───────────────────────────────────────────────
+DB_USER=osint
+DB_PASSWORD=osint
+DB_HOST=db
+DB_PORT=5432
+DB_NAME=osint_data
+
+# ── NATS JetStream (optional — only needed when profile `ingest` is used) ──
+NATS_URL=nats://nats:4222
+
+# ── MinIO (optional — document storage; endpoint used if wired later) ──────
+MINIO_ENDPOINT=minio:9000
+MINIO_ACCESS_KEY=
+MINIO_SECRET_KEY=
+# "true" for TLS endpoints (e.g. S3-compatible prod); "false" for local HTTP.
+MINIO_SECURE=false

.forgejo/workflows/build.yml

@@ -0,0 +1,25 @@
+name: build-and-deploy
+on:
+  push:
+    branches: [main, master]
+jobs:
+  build:
+    runs-on: docker
+    steps:
+      - name: Checkout
+        uses: https://code.forgejo.org/actions/checkout@v4
+      - name: Deploy to Pi via SSH
+        env:
+          PI_SSH_KEY: ${{ secrets.PI_SSH_KEY }}
+        run: |
+          set -e
+          mkdir -p ~/.ssh
+          echo "$PI_SSH_KEY" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+          ssh-keyscan -H rpi.tail14a963.ts.net >> ~/.ssh/known_hosts 2>/dev/null || true
+          # ship source to the Pi build dir
+          ssh sirius@rpi.tail14a963.ts.net 'rm -rf /opt/siriusdevops/build/osint-dashboard && mkdir -p /opt/siriusdevops/build/osint-dashboard'
+          scp -r . sirius@rpi.tail14a963.ts.net:/opt/siriusdevops/build/osint-dashboard/
+          # build arm64 image on the Pi's native docker, then bring up the stack
+          ssh sirius@rpi.tail14a963.ts.net 'cd /opt/siriusdevops/build/osint-dashboard && docker build --platform linux/arm64 -t localhost/osint-dashboard:latest . && docker compose up -d --force-recreate && docker image prune -f'
+          echo 'osint-dashboard deployed'

alembic.ini

@@ -1,7 +1,6 @@
 [alembic]
 script_location = alembic
-# Override at runtime via DATABASE_URL environment variable (set in ConfigMap/Deployment)
-sqlalchemy.url = driver://user:pass@localhost/dbname
+sqlalchemy.url = postgresql+asyncpg://user:pass@host:5432/dbname
 
 [loggers]
 keys = root,sqlalchemy,alembic

alembic/env.py

@@ -13,6 +13,9 @@ sys.path.insert(0, str(Path(__file__).parent.parent / "app"))
 from database import metadata, DATABASE_URL
 
 config = context.config
+# Prefer the app's env-driven DATABASE_URL over the ini placeholder so
+# migrations always target the same database the app connects to.
+config.set_main_option("sqlalchemy.url", DATABASE_URL)
 if config.config_file_name is not None:
     fileConfig(config.config_file_name)
 target_metadata = metadata

app/config.py

@@ -0,0 +1,42 @@
+"""OSINT Dashboard — centralized configuration (12-factor, env-driven).
+
+All infrastructure endpoints are read from the environment with
+container-friendly defaults. No secrets, hostnames, or cluster-specific
+addresses are hardcoded anywhere in the codebase.
+
+Override per environment via environment variables (docker-compose,
+k8s ConfigMap/Secret, or systemd):
+
+  DB_USER, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME
+  NATS_URL
+  MINIO_ENDPOINT, MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_SECURE
+"""
+
+from __future__ import annotations
+
+import os
+from urllib.parse import quote_plus
+
+# ── PostgreSQL ────────────────────────────────────────────────────────────
+# Defaults assume docker-compose/k8s service names. Override for your stack.
+DB_USER = os.getenv("DB_USER", "osint")
+DB_PASSWORD = os.getenv("DB_PASSWORD", "")
+DB_HOST = os.getenv("DB_HOST", "postgres")
+DB_PORT = os.getenv("DB_PORT", "5432")
+DB_NAME = os.getenv("DB_NAME", "osint_data")
+
+# Async SQLAlchemy URL. Password is URL-encoded so special chars are safe.
+DATABASE_URL = (
+    f"postgresql+asyncpg://"
+    f"{DB_USER}:{quote_plus(DB_PASSWORD)}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
+)
+
+# ── NATS JetStream ─────────────────────────────────────────────────────────
+NATS_URL = os.getenv("NATS_URL", "nats://nats:4222")
+
+# ── MinIO (document storage) ────────────────────────────────────────────────
+MINIO_ENDPOINT = os.getenv("MINIO_ENDPOINT", "minio:9000")
+MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY", "")
+MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY", "")
+# "false" / "0" / "no" (case-insensitive) → plain HTTP (e.g. local compose).
+MINIO_SECURE = os.getenv("MINIO_SECURE", "false").lower() not in ("false", "0", "no")

app/database.py

@@ -1,15 +1,7 @@
-import os
-
 from sqlalchemy import MetaData, event, text
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 
-DB_USER = os.getenv("DB_USER", "osint")
-DB_PASS = os.getenv("DB_PASSWORD", "")
-DB_HOST = os.getenv("DB_HOST", "osint-pgdb-rw.customer1.svc.cluster.local")
-DB_PORT = os.getenv("DB_PORT", "5432")
-DB_NAME = os.getenv("DB_NAME", "osint_data")
-
-DATABASE_URL = f"postgresql+asyncpg://{DB_USER}:{DB_PASS}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
+from config import DATABASE_URL
 
 engine = create_async_engine(
     DATABASE_URL, echo=False, pool_size=5, max_overflow=10, pool_recycle=300

app/ingestor.py

@@ -11,17 +11,15 @@ from nats.errors import TimeoutError
 
 from database import async_session
 from models import events as events_table
+from config import NATS_URL
 
 logger = logging.getLogger("osint.ingestor")
 
 # NATS connection settings
-NATS_URLS = "nats://osint-nats.customer1.svc.cluster.local:4222"
+NATS_URLS = NATS_URL
 NATS_STREAM = "events"
 NATS_DURABLE = "osint-ingestor"
 
-# Redis cache settings
-REDIS_URL = "redis://osint-redis-sentinel.customer1.svc.cluster.local:26379/0"
-
 
 async def ingest_event(msg: dict):
     """Ingest a single event from NATS into PostgreSQL."""

app/requirements.txt

@@ -3,7 +3,6 @@ uvicorn[standard]>=0.34
 sqlalchemy[asyncio]>=2.0
 asyncpg>=0.30
 nats-py>=2.9
-redis[hiredis]>=5.2
 minio>=7.2
 pydantic>=2.10
 alembic>=1.14

app/sources.py

@@ -11,6 +11,8 @@ import httpx
 import feedparser
 import nats
 
+from config import NATS_URL
+
 logger = logging.getLogger("osint.sources")
 
 
@@ -24,7 +26,7 @@ def _parse_rfc822(date_str: object) -> str | None:
         return None
 
 # NATS connection
-NATS_URLS = "nats://osint-nats.customer1.svc.cluster.local:4222"
+NATS_URLS = NATS_URL
 
 
 async def publish_event(subject: str, event: dict):

docker-compose.yml

@@ -0,0 +1,70 @@
+# OSINT Dashboard — container stack (all env-driven, 12-factor).
+# Single combined TimescaleDB+PostGIS image is pinned to pg13 because that is
+# the only arm64 build Timescale publishes with PostGIS bundled. The app uses
+# only standard SQL types, so pg13 is fully sufficient.
+#
+# Bring up:   docker compose up -d
+# With NATS:  docker compose --profile ingest up -d
+# Env:        copy .env.example to .env and adjust.
+
+services:
+  db:
+    image: timescale/timescaledb-postgis:latest-pg13
+    container_name: osint-db
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: ${DB_USER:-osint}
+      POSTGRES_PASSWORD: ${DB_PASSWORD:-osint}
+      POSTGRES_DB: ${DB_NAME:-osint_data}
+    ports:
+      - "127.0.0.1:5432:5432"
+    volumes:
+      - osint-pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-osint} -d ${DB_NAME:-osint_data}"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+
+  nats:
+    image: nats:2.10
+    container_name: osint-nats
+    restart: unless-stopped
+    profiles: ["ingest"]
+    ports:
+      - "127.0.0.1:4222:4222"
+      - "127.0.0.1:8222:8222"
+    command: ["-js"]
+
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      platforms: ["linux/arm64"]
+    image: localhost/osint-dashboard:latest
+    container_name: osint-dashboard
+    restart: unless-stopped
+    depends_on:
+      db:
+        condition: service_healthy
+    environment:
+      DB_USER: ${DB_USER:-osint}
+      DB_PASSWORD: ${DB_PASSWORD:-osint}
+      DB_HOST: db
+      DB_PORT: ${DB_PORT:-5432}
+      DB_NAME: ${DB_NAME:-osint_data}
+      NATS_URL: ${NATS_URL:-nats://nats:4222}
+      MINIO_ENDPOINT: ${MINIO_ENDPOINT:-minio:9000}
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-}
+      MINIO_SECURE: ${MINIO_SECURE:-false}
+    ports:
+      - "127.0.0.1:8000:8000"
+    healthcheck:
+      test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/api/health').status==200 else 1)\""]
+      interval: 30s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  osint-pgdata: