osint-dashboard/app/tracks.py

"""Timescale 1-minute track rollups for DVR playback.

Live overlays stay in memory. Historical `?timestamp=` reads the 1-minute
continuous aggregates (or an in-process downsample when the DB is down).
"""

from __future__ import annotations

import json
from datetime import datetime, timedelta, timezone
from typing import Any, Literal

from sqlalchemy import text

from database import async_session
from live_layers import parse_bbox, to_marker


TRACK_BUCKET = "1 minute"
Kind = Literal["vessel", "aircraft"]

_RAW_TABLE = {
    "vessel": "vessel_positions",
    "aircraft": "aircraft_positions",
}
_CAGG = {
    "vessel": "vessel_tracks_1min",
    "aircraft": "aircraft_tracks_1min",
}
_ID_COL = {
    "vessel": "mmsi",
    "aircraft": "hex",
}

# Last persist time per entity so AIS/ADS-B does not write every frame.
_last_write: dict[tuple[str, str], datetime] = {}
_MIN_WRITE_GAP = timedelta(seconds=20)


def minute_bucket(ts: datetime) -> datetime:
    if ts.tzinfo is None:
        ts = ts.replace(tzinfo=timezone.utc)
    return ts.replace(second=0, microsecond=0)


def downsample_tracks(rows: list[dict]) -> list[dict]:
    """Last sample per id per 1-minute bucket (mirrors the CAGG)."""
    last: dict[tuple[str, datetime], dict] = {}
    for row in rows:
        rid = str(row.get("id") or "")
        ts = row.get("ts")
        if not rid or not isinstance(ts, datetime):
            continue
        bucket = minute_bucket(ts)
        key = (rid, bucket)
        prev = last.get(key)
        if prev is None or ts >= prev["ts"]:
            last[key] = {**row, "id": rid, "bucket": bucket, "ts": ts}
    out = []
    for (_id, bucket), row in last.items():
        out.append({
            "id": row["id"],
            "bucket": bucket,
            "lat": row.get("lat"),
            "lon": row.get("lon"),
            "heading": row.get("heading"),
            "speed": row.get("speed"),
            "label": row.get("label"),
        })
    return out


def positions_at_timestamp(rows: list[dict], ts: datetime) -> list[dict]:
    """Positions whose 1-minute bucket equals floor(ts)."""
    want = minute_bucket(ts)
    picked = [r for r in downsample_tracks(rows) if r["bucket"] == want]
    return [
        to_marker(
            r["id"], r.get("lat"), r.get("lon"),
            heading=r.get("heading"), speed=r.get("speed"),
            label=r.get("label") or r["id"],
        )
        for r in picked
        if r.get("lat") is not None and r.get("lon") is not None
    ]


def parse_timestamp(value: str | datetime | None) -> datetime | None:
    if value is None or value == "":
        return None
    if isinstance(value, datetime):
        ts = value
    else:
        raw = str(value).strip().replace("Z", "+00:00")
        ts = datetime.fromisoformat(raw)
    if ts.tzinfo is None:
        ts = ts.replace(tzinfo=timezone.utc)
    return ts


async def record_position(kind: Kind, marker: dict, ts: datetime | None = None) -> bool:
    """Insert one sample into the raw hypertable (rate-limited)."""
    vid = str(marker.get("id") or "")
    lat, lon = marker.get("lat"), marker.get("lon")
    if not vid or lat is None or lon is None:
        return False
    now = ts or datetime.now(timezone.utc)
    key = (kind, vid)
    prev = _last_write.get(key)
    if prev is not None and now - prev < _MIN_WRITE_GAP:
        return False
    _last_write[key] = now
    table = _RAW_TABLE[kind]
    id_col = _ID_COL[kind]
    extra = marker.get("extra") or {}
    try:
        async with async_session() as session:
            await session.execute(
                text(
                    f"""
                    INSERT INTO {table} ({id_col}, ts, lat, lon, heading, speed, label, extra)
                    VALUES (:id, :ts, :lat, :lon, :heading, :speed, :label, CAST(:extra AS jsonb))
                    ON CONFLICT ({id_col}, ts) DO NOTHING
                    """
                ),
                {
                    "id": vid,
                    "ts": now,
                    "lat": float(lat),
                    "lon": float(lon),
                    "heading": marker.get("heading"),
                    "speed": marker.get("speed"),
                    "label": marker.get("label") or vid,
                    "extra": json.dumps(extra),
                },
            )
            await session.commit()
        return True
    except Exception:
        return False


async def fetch_positions_at(
    kind: Kind,
    ts: datetime,
    bbox: str | None = None,
    limit: int = 2000,
) -> list[dict]:
    """Read the 1-minute CAGG for the bucket containing ``ts``."""
    bucket = minute_bucket(ts)
    table = _CAGG[kind]
    id_col = _ID_COL[kind]
    where = "bucket = :bucket"
    params: dict[str, Any] = {"bucket": bucket, "limit": limit}
    if bbox:
        minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
        where += " AND lon BETWEEN :minlon AND :maxlon AND lat BETWEEN :minlat AND :maxlat"
        params.update(minlon=minlon, minlat=minlat, maxlon=maxlon, maxlat=maxlat)
    sql = f"""
        SELECT {id_col} AS id, lat, lon, heading, speed, label, bucket
        FROM {table}
        WHERE {where}
        LIMIT :limit
    """
    try:
        async with async_session() as session:
            rows = (await session.execute(text(sql), params)).mappings().all()
        points = [
            to_marker(
                r["id"], r["lat"], r["lon"],
                heading=r["heading"], speed=r["speed"],
                label=r["label"] or r["id"],
                extra={"bucket": r["bucket"].isoformat() if r["bucket"] else None, "dvr": True},
            )
            for r in rows
            if r["lat"] is not None and r["lon"] is not None
        ]
        return points
    except Exception:
        return []


async def track_range() -> dict:
    """Earliest/latest buckets across both CAGGs — slider bounds."""
    try:
        async with async_session() as session:
            row = (await session.execute(text(
                """
                SELECT min(t) AS tmin, max(t) AS tmax FROM (
                  SELECT min(bucket) AS t FROM vessel_tracks_1min
                  UNION ALL SELECT max(bucket) FROM vessel_tracks_1min
                  UNION ALL SELECT min(bucket) FROM aircraft_tracks_1min
                  UNION ALL SELECT max(bucket) FROM aircraft_tracks_1min
                  UNION ALL SELECT min(poll_at) FROM vessels
                  UNION ALL SELECT max(poll_at) FROM vessels
                ) s
                """
            ))).mappings().first()
        if not row or row["tmin"] is None:
            now = datetime.now(timezone.utc).replace(second=0, microsecond=0)
            return {"min": (now - timedelta(hours=6)).isoformat(), "max": now.isoformat()}
        return {
            "min": row["tmin"].isoformat(),
            "max": row["tmax"].isoformat(),
        }
    except Exception:
        now = datetime.now(timezone.utc).replace(second=0, microsecond=0)
        return {"min": (now - timedelta(hours=6)).isoformat(), "max": now.isoformat()}


async def recent_markers(kind: Kind, limit: int = 2000) -> list[dict]:
    """Latest raw sample per id — used when in-process last-known is empty."""
    table = _RAW_TABLE[kind]
    id_col = _ID_COL[kind]
    sql = f"""
        SELECT DISTINCT ON ({id_col})
               {id_col} AS id, lat, lon, heading, speed, label, extra
        FROM {table}
        WHERE ts > now() - interval '15 minutes'
        ORDER BY {id_col}, ts DESC
        LIMIT :limit
    """
    try:
        async with async_session() as session:
            rows = (await session.execute(text(sql), {"limit": limit})).mappings().all()
        out = []
        for r in rows:
            extra = r.get("extra") or {}
            if isinstance(extra, str):
                try:
                    extra = json.loads(extra)
                except (TypeError, ValueError):
                    extra = {}
            m = to_marker(
                r["id"], r["lat"], r["lon"],
                heading=r["heading"], speed=r["speed"],
                label=r["label"] or r["id"],
                extra=extra if isinstance(extra, dict) else {},
            )
            if m.get("lat") is not None and m.get("lon") is not None:
                out.append(m)
        return out
    except Exception:
        return []