91f436390b
Some checks failed
build-and-deploy / build (push) Failing after 4s
Vendor the newsPipeline scraper + summarizer into the repo and wire them into
docker-compose against the EXISTING osint-db (no second Postgres), replacing
the upstream k8s CronJobs with in-compose wall-clock loops (:00 scrape, :05
summarize).
- news/scraper: vendored Scrapy project (257 RSS feeds) + hourly loop
scheduler (run_news_scraper.py)
- news/summerizer: vendored Gemini map-reduce summarizer, cleaned:
* fix broken google-genai response handling (_extract_text, defensive)
* fix malformed INSERT/GRANT query in save_summary_to_db
* OSINT-neutral default MAP_PROMPT; futures/markets language gated behind
INCLUDE_FUTURES=0 (yfinance lazy-imported)
* env-configurable model, batch size, lookback window
+ hourly loop scheduler (run_news_summarizer.py, :05)
- alembic 003_news: idempotent articles + article_summaries tables
- API: GET /api/news and GET /api/news/summaries (+ models, schemas)
- tests/test_api_news.py: 5 DB-backed contract tests (all pass vs real PG)
- docs/news.md + .env.example updates
Both services run under the `ingest` compose profile (matching the
ingester/camera-scraper pattern) and build arm64 on the Pi via the existing
Forgejo CI workflow. telebot left out of scope (reserved env only).
214 lines
8.6 KiB
Python
214 lines
8.6 KiB
Python
"""OSINT Dashboard — SQLAlchemy models (async, declarative)."""
|
|
|
|
from datetime import datetime, timezone
|
|
from uuid import uuid4
|
|
|
|
from sqlalchemy import (
|
|
Column, Enum, Float, Index, Integer, String, Text,
|
|
DateTime, JSON, func, Table, PrimaryKeyConstraint,
|
|
)
|
|
from sqlalchemy.dialects.postgresql import UUID, TSVECTOR
|
|
|
|
from database import metadata
|
|
|
|
|
|
# ── Feed Sources ──────────────────────────────────────────────────────────
|
|
|
|
feed_sources = Table(
|
|
"feed_sources",
|
|
metadata,
|
|
Column("id", UUID(as_uuid=True), primary_key=True, default=uuid4),
|
|
Column("name", String(256), nullable=False),
|
|
Column("source_type", Enum(
|
|
"rss", "gdel-t2", "social", "earthquake", "disaster",
|
|
"weather", "fire", "satellite", name="feed_source_type"
|
|
), nullable=False),
|
|
Column("url", Text),
|
|
Column("config", JSON),
|
|
Column("enabled", Integer, server_default="1", nullable=False),
|
|
Column("created_at", DateTime(timezone=True), server_default=func.now(), nullable=False),
|
|
Column("updated_at", DateTime(timezone=True), server_default=func.now(), onupdate=func.now()),
|
|
)
|
|
|
|
|
|
# ── Events (hypertable via TimescaleDB) ──────────────────────────────────
|
|
|
|
events = Table(
|
|
"events",
|
|
metadata,
|
|
Column("id", UUID(as_uuid=True), primary_key=True, default=uuid4),
|
|
Column("source_type", Enum(
|
|
"rss", "gdel-t2", "social", "earthquake", "disaster",
|
|
"weather", "fire", "satellite", name="event_source_type"
|
|
), nullable=False, index=True),
|
|
Column("source_id", UUID(as_uuid=True)),
|
|
Column("title", Text),
|
|
Column("body", Text),
|
|
Column("url", Text),
|
|
Column("sentiment_score", Float),
|
|
Column("sentiment_label", Enum("positive", "neutral", "negative", name="sentiment_label")),
|
|
Column("location_lat", Float),
|
|
Column("location_lon", Float),
|
|
Column("location_name", String(512)),
|
|
Column("entities", JSON),
|
|
Column("tags", JSON),
|
|
Column("raw", JSON),
|
|
Column("ingested_at", DateTime(timezone=True), server_default=func.now(), nullable=False),
|
|
Column("source_timestamp", DateTime(timezone=True), nullable=False),
|
|
# Full-text search vector
|
|
Column(
|
|
"search_vector",
|
|
TSVECTOR,
|
|
nullable=True,
|
|
),
|
|
)
|
|
|
|
# GIN index for full-text search
|
|
Index("ix_events_search_vector", events.c.search_vector, postgresql_using="gin")
|
|
# Spatial index on location
|
|
Index("ix_events_location", events.c.location_lat, events.c.location_lon)
|
|
|
|
|
|
# ── Entities (people, organizations, locations of interest) ──────────────
|
|
|
|
entities = Table(
|
|
"entities",
|
|
metadata,
|
|
Column("id", UUID(as_uuid=True), primary_key=True, default=uuid4),
|
|
Column("name", String(512), nullable=False, index=True),
|
|
Column("entity_type", Enum(
|
|
"person", "organization", "location", "topic", "asset",
|
|
name="entity_type"
|
|
), nullable=False),
|
|
Column("aliases", JSON),
|
|
Column("description", Text),
|
|
Column("metadata", JSON),
|
|
Column("location_lat", Float),
|
|
Column("location_lon", Float),
|
|
Column("event_count", Integer, server_default="0"),
|
|
Column("first_seen", DateTime(timezone=True), server_default=func.now()),
|
|
Column("last_seen", DateTime(timezone=True), server_default=func.now()),
|
|
)
|
|
|
|
|
|
# ── Entity-Event Link ────────────────────────────────────────────────────
|
|
|
|
entity_events = Table(
|
|
"entity_events",
|
|
metadata,
|
|
Column("entity_id", UUID(as_uuid=True), primary_key=True),
|
|
Column("event_id", UUID(as_uuid=True), primary_key=True),
|
|
Column("relevance_score", Float),
|
|
Column("linked_at", DateTime(timezone=True), server_default=func.now()),
|
|
)
|
|
|
|
|
|
# ── Alerts ───────────────────────────────────────────────────────────────
|
|
|
|
alerts = Table(
|
|
"alerts",
|
|
metadata,
|
|
Column("id", UUID(as_uuid=True), primary_key=True, default=uuid4),
|
|
Column("alert_type", Enum(
|
|
"entity_mention", "sentiment_shift", "geo_proximity",
|
|
"keyword_match", "threshold", "anomaly",
|
|
name="alert_type"
|
|
), nullable=False),
|
|
Column("entity_id", UUID(as_uuid=True)),
|
|
Column("event_id", UUID(as_uuid=True)),
|
|
Column("severity", Enum("low", "medium", "high", "critical", name="alert_severity"), nullable=False),
|
|
Column("title", Text, nullable=False),
|
|
Column("message", Text),
|
|
Column("context", JSON),
|
|
Column("acknowledged", Integer, server_default="0"),
|
|
Column("acknowledged_by", String(256)),
|
|
Column("created_at", DateTime(timezone=True), server_default=func.now(), nullable=False),
|
|
Column("resolved_at", DateTime(timezone=True)),
|
|
)
|
|
|
|
Index("ix_alerts_severity_created", alerts.c.severity, alerts.c.created_at.desc())
|
|
Index("ix_alerts_entity", alerts.c.entity_id)
|
|
|
|
|
|
# ── Documents (stored in MinIO, indexed here) ────────────────────────────
|
|
|
|
documents = Table(
|
|
"documents",
|
|
metadata,
|
|
Column("id", UUID(as_uuid=True), primary_key=True, default=uuid4),
|
|
Column("bucket", String(256), nullable=False),
|
|
Column("object_key", String(1024), nullable=False),
|
|
Column("content_type", String(256)),
|
|
Column("size_bytes", Integer),
|
|
Column("description", Text),
|
|
Column("tags", JSON),
|
|
Column("event_id", UUID(as_uuid=True)),
|
|
Column("uploaded_at", DateTime(timezone=True), server_default=func.now()),
|
|
)
|
|
|
|
|
|
# ── Active Fires / Hotspots (NASA FIRMS) ──────────────────────────────────
|
|
|
|
# VIIRS active fire/hotspot detections from NASA FIRMS. The primary key IS the
|
|
# natural key (latitude, longitude, acq_time, satellite) so repeated 15-minute
|
|
# polls of the same detection are idempotent (INSERT ... ON CONFLICT DO NOTHING
|
|
# silently ignores duplicates). acq_time is the partitioning column of the
|
|
# TimescaleDB hypertable, and since it is part of the PK, TimescaleDB's
|
|
# "all unique indexes must include the partitioning column" rule is satisfied.
|
|
# No surrogate id is needed — the natural key is the identity of a detection.
|
|
fires = Table(
|
|
"fires",
|
|
metadata,
|
|
Column("latitude", Float, nullable=False),
|
|
Column("longitude", Float, nullable=False),
|
|
Column("brightness", Float, nullable=False), # bright_ti4, Kelvin
|
|
Column("confidence", String(10), nullable=False), # VIIRS: n/l/h, MODIS: %
|
|
Column("acq_time", DateTime(timezone=True), nullable=False), # UTC acquisition
|
|
Column("satellite", String(16), nullable=False), # e.g. N (S-NPP), N20, N21
|
|
Column("instrument", String(16)),
|
|
Column("bright_ti5", Float), # 12µm brightness, Kelvin
|
|
Column("frp", Float), # fire radiative power, MW
|
|
Column("daynight", String(1)), # D / N
|
|
Column("scan", Float),
|
|
Column("track", Float),
|
|
Column("version", String(32)), # e.g. 2.0NRT / 2.0URT
|
|
Column("raw", JSON),
|
|
Column("ingested_at", DateTime(timezone=True), server_default=func.now(), nullable=False),
|
|
PrimaryKeyConstraint(
|
|
"latitude", "longitude", "acq_time", "satellite",
|
|
name="pk_fires_natural_key",
|
|
),
|
|
)
|
|
|
|
# Bounding-box index for `bbox=` filtering (lon first for max/min-lon scans).
|
|
Index("ix_fires_bbox", fires.c.longitude, fires.c.latitude)
|
|
|
|
|
|
# ── News pipeline (scraper + summarizer) ──────────────────────────────────
|
|
# Written by the vendored news-scraper (Scrapy) / news-summarizer (Gemini)
|
|
# services; schema must match the idempotent alembic migration 003_news.
|
|
|
|
articles = Table(
|
|
"articles",
|
|
metadata,
|
|
Column("id", Integer, primary_key=True, autoincrement=True),
|
|
Column("title", Text),
|
|
Column("url", Text, unique=True), # dedup key
|
|
Column("content", Text), # full extracted article text
|
|
Column("domain", Text), # source domain
|
|
Column("timestamp", DateTime(timezone=True)), # capture time
|
|
)
|
|
|
|
Index("ix_articles_timestamp", articles.c.timestamp)
|
|
|
|
|
|
article_summaries = Table(
|
|
"article_summaries",
|
|
metadata,
|
|
Column("id", Integer, primary_key=True, autoincrement=True),
|
|
Column("summary_text", Text, nullable=False),
|
|
Column("batch_timestamp", DateTime(timezone=True),
|
|
server_default=func.now(), nullable=False),
|
|
)
|
|
|
|
Index("ix_article_summaries_batch_timestamp", article_summaries.c.batch_timestamp)
|