diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cbacc74 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +__pycache__/ +*.pyc +.git/ +.gitignore +.env +.env.example +.forgejo/ +README.md +*.md diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..04e1658 --- /dev/null +++ b/.env.example @@ -0,0 +1,19 @@ +# OSINT Dashboard — environment configuration +# Copy to `.env` and adjust. All values have safe defaults for local compose. + +# ── PostgreSQL / TimescaleDB ─────────────────────────────────────────────── +DB_USER=osint +DB_PASSWORD=osint +DB_HOST=db +DB_PORT=5432 +DB_NAME=osint_data + +# ── NATS JetStream (optional — only needed when profile `ingest` is used) ── +NATS_URL=nats://nats:4222 + +# ── MinIO (optional — document storage; endpoint used if wired later) ────── +MINIO_ENDPOINT=minio:9000 +MINIO_ACCESS_KEY= +MINIO_SECRET_KEY= +# "true" for TLS endpoints (e.g. S3-compatible prod); "false" for local HTTP. +MINIO_SECURE=false diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml new file mode 100644 index 0000000..80f4aa9 --- /dev/null +++ b/.forgejo/workflows/build.yml @@ -0,0 +1,25 @@ +name: build-and-deploy +on: + push: + branches: [main, master] +jobs: + build: + runs-on: docker + steps: + - name: Checkout + uses: https://code.forgejo.org/actions/checkout@v4 + - name: Deploy to Pi via SSH + env: + PI_SSH_KEY: ${{ secrets.PI_SSH_KEY }} + run: | + set -e + mkdir -p ~/.ssh + echo "$PI_SSH_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + ssh-keyscan -H rpi.tail14a963.ts.net >> ~/.ssh/known_hosts 2>/dev/null || true + # ship source to the Pi build dir + ssh sirius@rpi.tail14a963.ts.net 'rm -rf /opt/siriusdevops/build/osint-dashboard && mkdir -p /opt/siriusdevops/build/osint-dashboard' + scp -r . sirius@rpi.tail14a963.ts.net:/opt/siriusdevops/build/osint-dashboard/ + # build arm64 image on the Pi's native docker, then bring up the stack + ssh sirius@rpi.tail14a963.ts.net 'cd /opt/siriusdevops/build/osint-dashboard && docker build --platform linux/arm64 -t localhost/osint-dashboard:latest . && docker compose up -d --force-recreate && docker image prune -f' + echo 'osint-dashboard deployed' diff --git a/alembic.ini b/alembic.ini index a470ca8..d8d3074 100644 --- a/alembic.ini +++ b/alembic.ini @@ -1,7 +1,6 @@ [alembic] script_location = alembic -# Override at runtime via DATABASE_URL environment variable (set in ConfigMap/Deployment) -sqlalchemy.url = driver://user:pass@localhost/dbname +sqlalchemy.url = postgresql+asyncpg://user:pass@host:5432/dbname [loggers] keys = root,sqlalchemy,alembic diff --git a/alembic/env.py b/alembic/env.py index 9b2fb7e..926478a 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -13,6 +13,9 @@ sys.path.insert(0, str(Path(__file__).parent.parent / "app")) from database import metadata, DATABASE_URL config = context.config +# Prefer the app's env-driven DATABASE_URL over the ini placeholder so +# migrations always target the same database the app connects to. +config.set_main_option("sqlalchemy.url", DATABASE_URL) if config.config_file_name is not None: fileConfig(config.config_file_name) target_metadata = metadata diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..da2684e --- /dev/null +++ b/app/config.py @@ -0,0 +1,42 @@ +"""OSINT Dashboard — centralized configuration (12-factor, env-driven). + +All infrastructure endpoints are read from the environment with +container-friendly defaults. No secrets, hostnames, or cluster-specific +addresses are hardcoded anywhere in the codebase. + +Override per environment via environment variables (docker-compose, +k8s ConfigMap/Secret, or systemd): + + DB_USER, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME + NATS_URL + MINIO_ENDPOINT, MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_SECURE +""" + +from __future__ import annotations + +import os +from urllib.parse import quote_plus + +# ── PostgreSQL ──────────────────────────────────────────────────────────── +# Defaults assume docker-compose/k8s service names. Override for your stack. +DB_USER = os.getenv("DB_USER", "osint") +DB_PASSWORD = os.getenv("DB_PASSWORD", "") +DB_HOST = os.getenv("DB_HOST", "postgres") +DB_PORT = os.getenv("DB_PORT", "5432") +DB_NAME = os.getenv("DB_NAME", "osint_data") + +# Async SQLAlchemy URL. Password is URL-encoded so special chars are safe. +DATABASE_URL = ( + f"postgresql+asyncpg://" + f"{DB_USER}:{quote_plus(DB_PASSWORD)}@{DB_HOST}:{DB_PORT}/{DB_NAME}" +) + +# ── NATS JetStream ───────────────────────────────────────────────────────── +NATS_URL = os.getenv("NATS_URL", "nats://nats:4222") + +# ── MinIO (document storage) ──────────────────────────────────────────────── +MINIO_ENDPOINT = os.getenv("MINIO_ENDPOINT", "minio:9000") +MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY", "") +MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY", "") +# "false" / "0" / "no" (case-insensitive) → plain HTTP (e.g. local compose). +MINIO_SECURE = os.getenv("MINIO_SECURE", "false").lower() not in ("false", "0", "no") diff --git a/app/database.py b/app/database.py index 767c26a..262de31 100644 --- a/app/database.py +++ b/app/database.py @@ -1,15 +1,7 @@ -import os - from sqlalchemy import MetaData, event, text from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine -DB_USER = os.getenv("DB_USER", "osint") -DB_PASS = os.getenv("DB_PASSWORD", "") -DB_HOST = os.getenv("DB_HOST", "osint-pgdb-rw.customer1.svc.cluster.local") -DB_PORT = os.getenv("DB_PORT", "5432") -DB_NAME = os.getenv("DB_NAME", "osint_data") - -DATABASE_URL = f"postgresql+asyncpg://{DB_USER}:{DB_PASS}@{DB_HOST}:{DB_PORT}/{DB_NAME}" +from config import DATABASE_URL engine = create_async_engine( DATABASE_URL, echo=False, pool_size=5, max_overflow=10, pool_recycle=300 diff --git a/app/ingestor.py b/app/ingestor.py index d74d4f0..a44ffd4 100644 --- a/app/ingestor.py +++ b/app/ingestor.py @@ -11,17 +11,15 @@ from nats.errors import TimeoutError from database import async_session from models import events as events_table +from config import NATS_URL logger = logging.getLogger("osint.ingestor") # NATS connection settings -NATS_URLS = "nats://osint-nats.customer1.svc.cluster.local:4222" +NATS_URLS = NATS_URL NATS_STREAM = "events" NATS_DURABLE = "osint-ingestor" -# Redis cache settings -REDIS_URL = "redis://osint-redis-sentinel.customer1.svc.cluster.local:26379/0" - async def ingest_event(msg: dict): """Ingest a single event from NATS into PostgreSQL.""" diff --git a/app/requirements.txt b/app/requirements.txt index 3edff72..e5ee103 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -3,7 +3,6 @@ uvicorn[standard]>=0.34 sqlalchemy[asyncio]>=2.0 asyncpg>=0.30 nats-py>=2.9 -redis[hiredis]>=5.2 minio>=7.2 pydantic>=2.10 alembic>=1.14 diff --git a/app/sources.py b/app/sources.py index 8ca87ff..e7609bd 100644 --- a/app/sources.py +++ b/app/sources.py @@ -11,6 +11,8 @@ import httpx import feedparser import nats +from config import NATS_URL + logger = logging.getLogger("osint.sources") @@ -24,7 +26,7 @@ def _parse_rfc822(date_str: object) -> str | None: return None # NATS connection -NATS_URLS = "nats://osint-nats.customer1.svc.cluster.local:4222" +NATS_URLS = NATS_URL async def publish_event(subject: str, event: dict): diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..dac36b4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,70 @@ +# OSINT Dashboard — container stack (all env-driven, 12-factor). +# Single combined TimescaleDB+PostGIS image is pinned to pg13 because that is +# the only arm64 build Timescale publishes with PostGIS bundled. The app uses +# only standard SQL types, so pg13 is fully sufficient. +# +# Bring up: docker compose up -d +# With NATS: docker compose --profile ingest up -d +# Env: copy .env.example to .env and adjust. + +services: + db: + image: timescale/timescaledb-postgis:latest-pg13 + container_name: osint-db + restart: unless-stopped + environment: + POSTGRES_USER: ${DB_USER:-osint} + POSTGRES_PASSWORD: ${DB_PASSWORD:-osint} + POSTGRES_DB: ${DB_NAME:-osint_data} + ports: + - "127.0.0.1:5432:5432" + volumes: + - osint-pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-osint} -d ${DB_NAME:-osint_data}"] + interval: 10s + timeout: 5s + retries: 10 + + nats: + image: nats:2.10 + container_name: osint-nats + restart: unless-stopped + profiles: ["ingest"] + ports: + - "127.0.0.1:4222:4222" + - "127.0.0.1:8222:8222" + command: ["-js"] + + app: + build: + context: . + dockerfile: Dockerfile + platforms: ["linux/arm64"] + image: localhost/osint-dashboard:latest + container_name: osint-dashboard + restart: unless-stopped + depends_on: + db: + condition: service_healthy + environment: + DB_USER: ${DB_USER:-osint} + DB_PASSWORD: ${DB_PASSWORD:-osint} + DB_HOST: db + DB_PORT: ${DB_PORT:-5432} + DB_NAME: ${DB_NAME:-osint_data} + NATS_URL: ${NATS_URL:-nats://nats:4222} + MINIO_ENDPOINT: ${MINIO_ENDPOINT:-minio:9000} + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-} + MINIO_SECURE: ${MINIO_SECURE:-false} + ports: + - "127.0.0.1:8000:8000" + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/api/health').status==200 else 1)\""] + interval: 30s + timeout: 5s + retries: 5 + +volumes: + osint-pgdata: