Sirius Devops
93754dcdf6
Add FastAPI backend for real-time geospatial OSINT dashboard: - Full-text search via PostgreSQL tsvector (parameterized queries) - Entity tracking, alert management, sentiment analytics - Data ingestion: RSS feeds, GDELT, USGS earthquakes, social signals - NATS JetStream consumer for event ingestion - MinIO document storage integration - Redis caching layer - Alembic migrations with PostGIS + TimescaleDB extensions - Single-page dashboard UI with live polling - OpenTelemetry distributed tracing Helm chart with infrastructure: - CNPG PostgreSQL cluster (PostGIS + TimescaleDB) - NATS JetStream with persistent streams - MinIO distributed object storage (3 buckets) - Redis Sentinel (1 primary + 2 replicas) - NGINX Ingress with TLS and WebSocket support - Prometheus + Grafana + Alertmanager monitoring stack - Network policies with default deny - ConfigMap, CronJob, Deployment, Service templates Fixes applied during review: - SQL injection in search endpoint (parameterized :q binding) - Dockerfile PYTHONPATH mismatch (/app/app -> /app) - Hardcoded DB credentials in alembic.ini - RSS timestamp parsing (feedparser published_parsed -> parsedate_to_datetime) - Removed dead PGVECTOR import
207 lines
7.1 KiB
YAML
207 lines
7.1 KiB
YAML
{{- if .Values.api.enabled }}
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: {{ include "osint-dashboard.fullname" . }}-rss-ingestor
|
|
namespace: {{ .Values.namespace }}
|
|
labels:
|
|
{{- include "osint-dashboard.labels" . | nindent 4 }}
|
|
app.kubernetes.io/component: ingestor
|
|
spec:
|
|
schedule: "*/5 * * * *"
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
template:
|
|
metadata:
|
|
labels:
|
|
{{- include "osint-dashboard.selectorLabels" . | nindent 12 }}
|
|
app.kubernetes.io/component: ingestor
|
|
spec:
|
|
restartPolicy: OnFailure
|
|
securityContext:
|
|
{{- toYaml .Values.securityContext | nindent 12 }}
|
|
containers:
|
|
- name: rss-ingestor
|
|
image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag }}"
|
|
imagePullPolicy: {{ .Values.api.image.pullPolicy | default "Always" }}
|
|
command:
|
|
- python
|
|
- /app/app/ingest_cron.py
|
|
envFrom:
|
|
- configMapRef:
|
|
name: {{ include "osint-dashboard.fullname" . }}-api-config
|
|
env:
|
|
- name: DB_USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: {{ .Values.postgresql.credentialsSecret }}
|
|
key: username
|
|
- name: DB_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: {{ .Values.postgresql.credentialsSecret }}
|
|
key: password
|
|
- name: INGESTOR_TYPE
|
|
value: "rss"
|
|
resources:
|
|
{{- toYaml .Values.api.resources | nindent 16 }}
|
|
{{- end }}
|
|
---
|
|
{{- if .Values.api.enabled }}
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: {{ include "osint-dashboard.fullname" . }}-gdelt-ingestor
|
|
namespace: {{ .Values.namespace }}
|
|
labels:
|
|
{{- include "osint-dashboard.labels" . | nindent 4 }}
|
|
app.kubernetes.io/component: ingestor
|
|
spec:
|
|
schedule: "*/15 * * * *"
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
template:
|
|
metadata:
|
|
labels:
|
|
{{- include "osint-dashboard.selectorLabels" . | nindent 12 }}
|
|
app.kubernetes.io/component: ingestor
|
|
spec:
|
|
restartPolicy: OnFailure
|
|
securityContext:
|
|
{{- toYaml .Values.securityContext | nindent 12 }}
|
|
containers:
|
|
- name: gdelt-ingestor
|
|
image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag }}"
|
|
imagePullPolicy: {{ .Values.api.image.pullPolicy | default "Always" }}
|
|
command:
|
|
- python
|
|
- /app/app/ingest_cron.py
|
|
envFrom:
|
|
- configMapRef:
|
|
name: {{ include "osint-dashboard.fullname" . }}-api-config
|
|
env:
|
|
- name: DB_USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: {{ .Values.postgresql.credentialsSecret }}
|
|
key: username
|
|
- name: DB_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: {{ .Values.postgresql.credentialsSecret }}
|
|
key: password
|
|
- name: INGESTOR_TYPE
|
|
value: "gdelt"
|
|
resources:
|
|
{{- toYaml .Values.api.resources | nindent 16 }}
|
|
{{- end }}
|
|
---
|
|
{{- if .Values.api.enabled }}
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: {{ include "osint-dashboard.fullname" . }}-earthquake-ingestor
|
|
namespace: {{ .Values.namespace }}
|
|
labels:
|
|
{{- include "osint-dashboard.labels" . | nindent 4 }}
|
|
app.kubernetes.io/component: ingestor
|
|
spec:
|
|
schedule: "0 * * * *"
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
template:
|
|
metadata:
|
|
labels:
|
|
{{- include "osint-dashboard.selectorLabels" . | nindent 12 }}
|
|
app.kubernetes.io/component: ingestor
|
|
spec:
|
|
restartPolicy: OnFailure
|
|
securityContext:
|
|
{{- toYaml .Values.securityContext | nindent 12 }}
|
|
containers:
|
|
- name: earthquake-ingestor
|
|
image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag }}"
|
|
imagePullPolicy: {{ .Values.api.image.pullPolicy | default "Always" }}
|
|
command:
|
|
- python
|
|
- /app/app/ingest_cron.py
|
|
envFrom:
|
|
- configMapRef:
|
|
name: {{ include "osint-dashboard.fullname" . }}-api-config
|
|
env:
|
|
- name: DB_USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: {{ .Values.postgresql.credentialsSecret }}
|
|
key: username
|
|
- name: DB_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: {{ .Values.postgresql.credentialsSecret }}
|
|
key: password
|
|
- name: INGESTOR_TYPE
|
|
value: "earthquake"
|
|
resources:
|
|
{{- toYaml .Values.api.resources | nindent 16 }}
|
|
{{- end }}
|
|
---
|
|
{{- if .Values.api.enabled }}
|
|
apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: {{ include "osint-dashboard.fullname" . }}-nats-processor
|
|
namespace: {{ .Values.namespace }}
|
|
labels:
|
|
{{- include "osint-dashboard.labels" . | nindent 4 }}
|
|
app.kubernetes.io/component: ingestor
|
|
spec:
|
|
schedule: "*/2 * * * *"
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
template:
|
|
metadata:
|
|
labels:
|
|
{{- include "osint-dashboard.selectorLabels" . | nindent 12 }}
|
|
app.kubernetes.io/component: ingestor
|
|
spec:
|
|
restartPolicy: OnFailure
|
|
securityContext:
|
|
{{- toYaml .Values.securityContext | nindent 12 }}
|
|
containers:
|
|
- name: nats-processor
|
|
image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag }}"
|
|
imagePullPolicy: {{ .Values.api.image.pullPolicy | default "Always" }}
|
|
command:
|
|
- python
|
|
- /app/app/ingest_cron.py
|
|
envFrom:
|
|
- configMapRef:
|
|
name: {{ include "osint-dashboard.fullname" . }}-api-config
|
|
env:
|
|
- name: DB_USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: {{ .Values.postgresql.credentialsSecret }}
|
|
key: username
|
|
- name: DB_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: {{ .Values.postgresql.credentialsSecret }}
|
|
key: password
|
|
- name: INGESTOR_TYPE
|
|
value: "nats"
|
|
resources:
|
|
{{- toYaml .Values.api.resources | nindent 16 }}
|
|
{{- end }}
|