5651d251d0
CISA KEV republished 1685 NIST URLs every 5 min; FIRMS re-inserted ~325k global hotspots every 15 min. Producer now skips URLs already in event_dedup and FIRMS CSVs that are unchanged (delta-only persist). CI rebuilds only images whose paths changed and never pulls/rebuilds Timescale or bounces osint-db unless Dockerfile.pg changes.
249 lines
9.2 KiB
YAML
249 lines
9.2 KiB
YAML
# Build changed OSINT images, publish to the Forgejo container registry, then
|
|
# redeploy on the Pi runner (docker.sock mounted).
|
|
#
|
|
# Unchanged images are skipped. Dockerfile.pg / osint-dashboard-pg is NOT
|
|
# rebuilt or pulled on a normal merge — Postgres stays up. Rebuild it only
|
|
# when Dockerfile.pg changes, or via workflow_dispatch rebuild_pg.
|
|
#
|
|
# Public pull host: forgejo.siriusdevops.com (NOT ghcr.io)
|
|
# CI push host: 127.0.0.1:3000 — Cloudflare 413s layers ≳100MB on the public
|
|
# hostname, even from the Pi (hairpins out through the tunnel).
|
|
# Images (public names):
|
|
# forgejo.siriusdevops.com/sirius/osint-dashboard
|
|
# forgejo.siriusdevops.com/sirius/osint-dashboard-pg
|
|
# forgejo.siriusdevops.com/sirius/osint-news-scraper
|
|
# forgejo.siriusdevops.com/sirius/osint-news-summarizer
|
|
#
|
|
# Optional repo variable FORGEJO_REGISTRY overrides the *public* pull host.
|
|
# Deploy still uses the local docker socket on the runner host (rpi).
|
|
|
|
name: build-and-deploy
|
|
|
|
on:
|
|
push:
|
|
branches: [main, master]
|
|
workflow_dispatch:
|
|
inputs:
|
|
rebuild_pg:
|
|
description: Rebuild Timescale+PostGIS (Dockerfile.pg)
|
|
type: boolean
|
|
default: false
|
|
rebuild_all:
|
|
description: Rebuild every app image (ignore path filter)
|
|
type: boolean
|
|
default: false
|
|
|
|
env:
|
|
PUBLIC_REGISTRY: ${{ vars.FORGEJO_REGISTRY || 'forgejo.siriusdevops.com' }}
|
|
PUSH_REGISTRY: 127.0.0.1:3000
|
|
OWNER: sirius
|
|
# Keep compose project/volumes stable on the Pi
|
|
COMPOSE_PROJECT_NAME: osint-dashboard
|
|
|
|
jobs:
|
|
build-push-deploy:
|
|
runs-on: docker
|
|
permissions:
|
|
contents: read
|
|
packages: write
|
|
steps:
|
|
- name: Checkout
|
|
uses: https://code.forgejo.org/actions/checkout@v4
|
|
with:
|
|
fetch-depth: 50
|
|
|
|
- name: Plan image builds
|
|
id: plan
|
|
run: |
|
|
set -euo pipefail
|
|
APP=0
|
|
SCRAPER=0
|
|
SUM=0
|
|
PG=0
|
|
COMPOSE=0
|
|
|
|
mark() {
|
|
case "$1" in
|
|
Dockerfile.pg)
|
|
PG=1 ;;
|
|
Dockerfile|app/*|alembic/*|alembic.ini)
|
|
APP=1 ;;
|
|
news/scraper/*)
|
|
SCRAPER=1 ;;
|
|
news/summerizer/*)
|
|
SUM=1 ;;
|
|
docker-compose.yml|scripts/compose-reup.sh)
|
|
COMPOSE=1 ;;
|
|
esac
|
|
}
|
|
|
|
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
|
APP=1; SCRAPER=1; SUM=1
|
|
if [ "${{ github.event.inputs.rebuild_all }}" = "true" ]; then
|
|
APP=1; SCRAPER=1; SUM=1; PG=1
|
|
fi
|
|
if [ "${{ github.event.inputs.rebuild_pg }}" = "true" ]; then
|
|
PG=1
|
|
fi
|
|
else
|
|
BEFORE="${{ github.event.before }}"
|
|
SHA="${GITHUB_SHA}"
|
|
ZEROS="0000000000000000000000000000000000000000"
|
|
if [ -z "$BEFORE" ] || [ "$BEFORE" = "$ZEROS" ]; then
|
|
echo "No previous SHA — build app images, skip pg"
|
|
APP=1; SCRAPER=1; SUM=1
|
|
elif ! git cat-file -e "${BEFORE}^{commit}" 2>/dev/null; then
|
|
echo "Previous SHA $BEFORE not in history — build app images, skip pg"
|
|
APP=1; SCRAPER=1; SUM=1
|
|
else
|
|
while IFS= read -r f; do
|
|
[ -z "$f" ] && continue
|
|
mark "$f"
|
|
done < <(git diff --name-only "$BEFORE" "$SHA")
|
|
fi
|
|
fi
|
|
|
|
{
|
|
echo "app=$APP"
|
|
echo "scraper=$SCRAPER"
|
|
echo "summarizer=$SUM"
|
|
echo "pg=$PG"
|
|
echo "compose=$COMPOSE"
|
|
} >> "$GITHUB_OUTPUT"
|
|
echo "plan app=$APP scraper=$SCRAPER summarizer=$SUM pg=$PG compose=$COMPOSE"
|
|
|
|
- name: Image refs
|
|
id: img
|
|
run: |
|
|
set -euo pipefail
|
|
PUSH="${PUSH_REGISTRY}"
|
|
PUB="${PUBLIC_REGISTRY}"
|
|
OWN="${OWNER}"
|
|
SHA="${GITHUB_SHA::12}"
|
|
{
|
|
echo "reg=$PUSH"
|
|
echo "pub=$PUB"
|
|
echo "sha=$SHA"
|
|
echo "app=$PUSH/$OWN/osint-dashboard"
|
|
echo "pg=$PUSH/$OWN/osint-dashboard-pg"
|
|
echo "scraper=$PUSH/$OWN/osint-news-scraper"
|
|
echo "summarizer=$PUSH/$OWN/osint-news-summarizer"
|
|
} >> "$GITHUB_OUTPUT"
|
|
echo "Push registry: $PUSH"
|
|
echo "Public pull: $PUB"
|
|
echo "SHA tag: $SHA"
|
|
|
|
- name: Login to Forgejo registry
|
|
if: steps.plan.outputs.app == '1' || steps.plan.outputs.scraper == '1' || steps.plan.outputs.summarizer == '1' || steps.plan.outputs.pg == '1'
|
|
run: |
|
|
set -euo pipefail
|
|
# GITHUB_TOKEN login "succeeds" but blob uploads 401 (Forgejo packages
|
|
# reject the Actions token). Use a user PAT with write:package.
|
|
if [ -z "${{ secrets.FORGEJO_TOKEN }}" ]; then
|
|
echo "Missing repo secret FORGEJO_TOKEN (user PAT, not GITHUB_TOKEN)"
|
|
exit 1
|
|
fi
|
|
echo "${{ secrets.FORGEJO_TOKEN }}" | docker login "${{ steps.img.outputs.reg }}" \
|
|
-u sirius --password-stdin
|
|
|
|
- name: Build application image (api / ingester / cameras)
|
|
if: steps.plan.outputs.app == '1'
|
|
run: |
|
|
set -ex
|
|
APP="${{ steps.img.outputs.app }}"
|
|
SHA="${{ steps.img.outputs.sha }}"
|
|
docker build -f Dockerfile -t "${APP}:latest" -t "${APP}:${SHA}" \
|
|
-t "localhost/osint-dashboard:latest" .
|
|
docker push "${APP}:latest"
|
|
docker push "${APP}:${SHA}"
|
|
|
|
- name: Build news-scraper image
|
|
if: steps.plan.outputs.scraper == '1'
|
|
run: |
|
|
set -ex
|
|
IMG="${{ steps.img.outputs.scraper }}"
|
|
SHA="${{ steps.img.outputs.sha }}"
|
|
docker build -f news/scraper/Dockerfile -t "${IMG}:latest" -t "${IMG}:${SHA}" \
|
|
-t "localhost/osint-news-scraper:latest" news/scraper
|
|
docker push "${IMG}:latest"
|
|
docker push "${IMG}:${SHA}"
|
|
|
|
- name: Build news-summarizer image
|
|
if: steps.plan.outputs.summarizer == '1'
|
|
run: |
|
|
set -ex
|
|
IMG="${{ steps.img.outputs.summarizer }}"
|
|
SHA="${{ steps.img.outputs.sha }}"
|
|
docker build -f news/summerizer/Dockerfile -t "${IMG}:latest" -t "${IMG}:${SHA}" \
|
|
-t "localhost/osint-news-summarizer:latest" news/summerizer
|
|
docker push "${IMG}:latest"
|
|
docker push "${IMG}:${SHA}"
|
|
|
|
- name: Build / refresh Timescale+PostGIS image
|
|
if: steps.plan.outputs.pg == '1'
|
|
run: |
|
|
set -ex
|
|
PG="${{ steps.img.outputs.pg }}"
|
|
SHA="${{ steps.img.outputs.sha }}"
|
|
if docker build -f Dockerfile.pg -t "${PG}:latest" -t "${PG}:${SHA}" \
|
|
-t "localhost/osint-dashboard-pg:latest" .; then
|
|
docker push "${PG}:latest"
|
|
docker push "${PG}:${SHA}"
|
|
elif docker image inspect "localhost/osint-dashboard-pg:latest" >/dev/null 2>&1; then
|
|
echo "WARN: Dockerfile.pg build failed; keeping existing local pg image"
|
|
else
|
|
echo "ERROR: cannot build or find osint-dashboard-pg image"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Deploy on runner host (compose)
|
|
run: |
|
|
set -ex
|
|
cd "${GITHUB_WORKSPACE}"
|
|
APP="${{ steps.plan.outputs.app }}"
|
|
SCRAPER="${{ steps.plan.outputs.scraper }}"
|
|
SUM="${{ steps.plan.outputs.summarizer }}"
|
|
PG="${{ steps.plan.outputs.pg }}"
|
|
COMPOSE="${{ steps.plan.outputs.compose }}"
|
|
|
|
SVCS=()
|
|
[ "$APP" = "1" ] && SVCS+=(app ingester camera-service)
|
|
[ "$SCRAPER" = "1" ] && SVCS+=(news-scraper)
|
|
[ "$SUM" = "1" ] && SVCS+=(news-summarizer)
|
|
if [ "$COMPOSE" = "1" ]; then
|
|
# compose/script change: bounce workers so env/command updates apply.
|
|
# Still do not bounce Postgres.
|
|
for s in app ingester camera-service news-scraper news-summarizer; do
|
|
case " ${SVCS[*]} " in
|
|
*" $s "*) ;;
|
|
*) SVCS+=("$s") ;;
|
|
esac
|
|
done
|
|
fi
|
|
|
|
chmod +x scripts/compose-reup.sh
|
|
if [ "$PG" = "1" ]; then
|
|
FORCE_RECREATE_DB=1 COMPOSE_PROJECT_NAME=osint-dashboard COMPOSE_PROFILES=ingest \
|
|
scripts/compose-reup.sh "${SVCS[@]}" db
|
|
elif [ "${#SVCS[@]}" -gt 0 ]; then
|
|
COMPOSE_PROJECT_NAME=osint-dashboard COMPOSE_PROFILES=ingest \
|
|
scripts/compose-reup.sh "${SVCS[@]}"
|
|
else
|
|
echo "No image or compose changes — leave running containers alone"
|
|
docker compose --profile ingest ps
|
|
fi
|
|
docker image prune -f
|
|
echo "osint-dashboard deploy done; db image left in place unless pg=1"
|
|
|
|
- name: Summary
|
|
if: always()
|
|
run: |
|
|
{
|
|
echo "## Image plan"
|
|
echo "- app: \`${{ steps.plan.outputs.app }}\`"
|
|
echo "- news-scraper: \`${{ steps.plan.outputs.scraper }}\`"
|
|
echo "- news-summarizer: \`${{ steps.plan.outputs.summarizer }}\`"
|
|
echo "- pg (Timescale): \`${{ steps.plan.outputs.pg }}\`"
|
|
echo
|
|
echo "Postgres is rebuilt/pulled only when \`Dockerfile.pg\` changes (or workflow_dispatch rebuild_pg)."
|
|
} >> "$GITHUB_STEP_SUMMARY"
|