085061492e
All checks were successful
build-and-deploy / build (push) Successful in 2m0s
Continuous whole-IPv4 rolling sweep for open TCP 554, feeding the same cameras table as the passive scraper (discovery_source=masscan). - masscan_config.py: env-driven knobs (range, ports, rate, retries, excludes) - masscan_scanner.py: JSON-lines parser, rtsp://IP/ URL + url_hash dedupe, ip-api geolocation, insert/refresh, NATS publish for new finds - run_masscan_service.py: long-lived rolling-sweep runner (streams results in, restarts on pass completion); fails closed without an excludefile - deploy/: systemd unit + README + excludes file for the Pi host - .env.example: masscan section Verified end-to-end against a local Postgres: parse, insert, and dedupe (0 new on re-ingest) all pass.
90 lines
4.1 KiB
Text
90 lines
4.1 KiB
Text
# OSINT Dashboard — environment configuration
|
|
# Copy to `.env` and adjust. All values have safe defaults for local compose.
|
|
|
|
# ── PostgreSQL / TimescaleDB ───────────────────────────────────────────────
|
|
DB_USER=osint
|
|
DB_PASSWORD=osint
|
|
DB_HOST=db
|
|
DB_PORT=5432
|
|
DB_NAME=osint_data
|
|
|
|
# ── NATS JetStream (optional — only needed when profile `ingest` is used) ──
|
|
NATS_URL=nats://nats:4222
|
|
|
|
# ── MinIO (optional — document storage; endpoint used if wired later) ──────
|
|
MINIO_ENDPOINT=minio:9000
|
|
MINIO_ACCESS_KEY=
|
|
MINIO_SECRET_KEY=
|
|
# "true" for TLS endpoints (e.g. S3-compatible prod); "false" for local HTTP.
|
|
MINIO_SECURE=false
|
|
|
|
# ── Camera discovery scraper ───────────────────────────────────────────────
|
|
# Comma-separated public directory/list URLs (Insecam-style pages or
|
|
# plain-text lists: url[|lat,lon|vendor|location]).
|
|
CAMERA_SOURCE_URLS=
|
|
CAMERA_SCRAPE_INTERVAL=3600
|
|
CAMERA_REQUEST_DELAY=2.0
|
|
NOMINATIM_URL=https://nominatim.openstreetmap.org
|
|
NOMINATIM_MIN_INTERVAL=1.1
|
|
SNAPSHOT_TTL_SECONDS=300
|
|
|
|
# ── masscan active camera discovery (host-level systemd service, NOT compose) ─
|
|
# Continuous rolling sweep for open RTSP port 554 across a range. Runs on the
|
|
# Pi host via deploy/osint-masscan.service (needs root + raw sockets). Results
|
|
# land in the same `cameras` table as the scraper (discovery_source=masscan).
|
|
# NOTE: at a conservative 10000 pps a full 0.0.0.0/0 sweep takes ~5 days, so
|
|
# this is a continuous rolling sweep, not a daily job.
|
|
MASSCAN_RANGE=0.0.0.0/0
|
|
MASSCAN_PORTS=554
|
|
MASSCAN_RATE=10000
|
|
MASSCAN_RETRIES=1
|
|
MASSCAN_WAIT=0
|
|
MASSCAN_EXCLUDEFILE=/etc/osint-dashboard/masscan-excludes.txt
|
|
MASSCAN_FLUSH_EVERY=250
|
|
|
|
# ── NASA FIRMS (active fire / hotspot ingest) ──────────────────────────────
|
|
# MAP_KEY is FREE — get one at https://firms.modaps.eosdis.nasa.gov/api/map_key_info/
|
|
# (1-minute signup, no payment). Leave blank to keep fire ingest idle.
|
|
FIRMS_MAP_KEY=
|
|
# NRT VIIRS S-NPP 375m active fire detection (near-real-time).
|
|
FIRMS_DATASET=VIIRS_SNPP_NRT
|
|
# Area to poll as "minlon,minlat,maxlon,maxlat". Narrow it to reduce payload
|
|
# (e.g. CONUS "-125,24,-66,50"). Default covers most of the inhabited globe.
|
|
FIRMS_BBOX=-180,-60,180,75
|
|
# Poll cadence in seconds (~15 min). FIRMS NRT updates every ~5-10 min.
|
|
FIRMS_INTERVAL=900
|
|
# Set to 0 to disable the fire loop entirely.
|
|
INGEST_FIRES=1
|
|
|
|
# ── API keys (managed from the dashboard UI) ──────────────────────────────
|
|
# Keys such as GEMINI_API_KEY and TELEGRAM_TOKEN are stored in the Postgres
|
|
# `api_keys` table and managed from the dashboard's "Keys" tab
|
|
# (GET/POST/DELETE /api/keys/{name}) — see app/keystore.py. The FIRMS ingestor
|
|
# currently reads FIRMS_MAP_KEY from .env (above); wiring the Keys-UI store as
|
|
# its lookup/fallback is a planned follow-up.
|
|
|
|
# ── News pipeline (scraper + summarizer, profile `ingest`) ────────────────
|
|
# Hourly: the scraper crawls 257 RSS sources at minute :00 and the summarizer
|
|
# runs the Gemini map-reduce at minute :05, both writing to the shared osint-db
|
|
# (tables `articles` + `article_summaries`, created by alembic 003_news).
|
|
# Consume via GET /api/news and GET /api/news/summaries.
|
|
# GEMINI_API_KEY is REQUIRED for summarization; unset = summarizer idles.
|
|
GEMINI_API_KEY=
|
|
# Optional LLM knobs
|
|
SUMMARY_MODEL=gemini-2.0-flash
|
|
NEWS_BATCH_SIZE=50
|
|
SUMMARY_WINDOW_HOURS=1
|
|
# Futures/markets coupling from the upstream pipeline is OFF by default
|
|
# (irrelevant to OSINT). Set INCLUDE_FUTURES=1 + install yfinance to enable.
|
|
INCLUDE_FUTURES=0
|
|
# Wall-clock scheduling (k8s CronJob replacement): scrape minute, summarize minute
|
|
NEWS_SCRAPE_MINUTE=0
|
|
NEWS_SUMMARIZE_MINUTE=5
|
|
# Run once immediately on container start (seeds data fast), then align to the
|
|
# scheduled minute.
|
|
NEWS_SCRAPE_RUN_ON_START=1
|
|
NEWS_SUMMARIZE_RUN_ON_START=1
|
|
NEWS_LOG_LEVEL=INFO
|
|
# Reserved for the (out-of-scope) Telegram delivery bot.
|
|
TELEGRAM_TOKEN=
|
|
TELEGRAM_CHAT_ID=
|