5ea9a4e879
Phase 1: in-memory ConnectionManager viewport fan-out, 500ms map debounce, cachetools TTLCache, background masscan/ffmpeg, compose memory caps. Phase 2: PostGIS geofences + ST_Intersects alerts, Timescale 1-min CAGGs and timestamp playback, FIRMS/WFIGS x firefighting ADS-B within 20 miles. No Redis/Kafka/Celery.
245 lines
8.1 KiB
Python
245 lines
8.1 KiB
Python
"""Timescale 1-minute track rollups for DVR playback.
|
|
|
|
Live overlays stay in memory. Historical `?timestamp=` reads the 1-minute
|
|
continuous aggregates (or an in-process downsample when the DB is down).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any, Literal
|
|
|
|
from sqlalchemy import text
|
|
|
|
from database import async_session
|
|
from live_layers import parse_bbox, to_marker
|
|
|
|
|
|
TRACK_BUCKET = "1 minute"
|
|
Kind = Literal["vessel", "aircraft"]
|
|
|
|
_RAW_TABLE = {
|
|
"vessel": "vessel_positions",
|
|
"aircraft": "aircraft_positions",
|
|
}
|
|
_CAGG = {
|
|
"vessel": "vessel_tracks_1min",
|
|
"aircraft": "aircraft_tracks_1min",
|
|
}
|
|
_ID_COL = {
|
|
"vessel": "mmsi",
|
|
"aircraft": "hex",
|
|
}
|
|
|
|
# Last persist time per entity so AIS/ADS-B does not write every frame.
|
|
_last_write: dict[tuple[str, str], datetime] = {}
|
|
_MIN_WRITE_GAP = timedelta(seconds=20)
|
|
|
|
|
|
def minute_bucket(ts: datetime) -> datetime:
|
|
if ts.tzinfo is None:
|
|
ts = ts.replace(tzinfo=timezone.utc)
|
|
return ts.replace(second=0, microsecond=0)
|
|
|
|
|
|
def downsample_tracks(rows: list[dict]) -> list[dict]:
|
|
"""Last sample per id per 1-minute bucket (mirrors the CAGG)."""
|
|
last: dict[tuple[str, datetime], dict] = {}
|
|
for row in rows:
|
|
rid = str(row.get("id") or "")
|
|
ts = row.get("ts")
|
|
if not rid or not isinstance(ts, datetime):
|
|
continue
|
|
bucket = minute_bucket(ts)
|
|
key = (rid, bucket)
|
|
prev = last.get(key)
|
|
if prev is None or ts >= prev["ts"]:
|
|
last[key] = {**row, "id": rid, "bucket": bucket, "ts": ts}
|
|
out = []
|
|
for (_id, bucket), row in last.items():
|
|
out.append({
|
|
"id": row["id"],
|
|
"bucket": bucket,
|
|
"lat": row.get("lat"),
|
|
"lon": row.get("lon"),
|
|
"heading": row.get("heading"),
|
|
"speed": row.get("speed"),
|
|
"label": row.get("label"),
|
|
})
|
|
return out
|
|
|
|
|
|
def positions_at_timestamp(rows: list[dict], ts: datetime) -> list[dict]:
|
|
"""Positions whose 1-minute bucket equals floor(ts)."""
|
|
want = minute_bucket(ts)
|
|
picked = [r for r in downsample_tracks(rows) if r["bucket"] == want]
|
|
return [
|
|
to_marker(
|
|
r["id"], r.get("lat"), r.get("lon"),
|
|
heading=r.get("heading"), speed=r.get("speed"),
|
|
label=r.get("label") or r["id"],
|
|
)
|
|
for r in picked
|
|
if r.get("lat") is not None and r.get("lon") is not None
|
|
]
|
|
|
|
|
|
def parse_timestamp(value: str | datetime | None) -> datetime | None:
|
|
if value is None or value == "":
|
|
return None
|
|
if isinstance(value, datetime):
|
|
ts = value
|
|
else:
|
|
raw = str(value).strip().replace("Z", "+00:00")
|
|
ts = datetime.fromisoformat(raw)
|
|
if ts.tzinfo is None:
|
|
ts = ts.replace(tzinfo=timezone.utc)
|
|
return ts
|
|
|
|
|
|
async def record_position(kind: Kind, marker: dict, ts: datetime | None = None) -> bool:
|
|
"""Insert one sample into the raw hypertable (rate-limited)."""
|
|
vid = str(marker.get("id") or "")
|
|
lat, lon = marker.get("lat"), marker.get("lon")
|
|
if not vid or lat is None or lon is None:
|
|
return False
|
|
now = ts or datetime.now(timezone.utc)
|
|
key = (kind, vid)
|
|
prev = _last_write.get(key)
|
|
if prev is not None and now - prev < _MIN_WRITE_GAP:
|
|
return False
|
|
_last_write[key] = now
|
|
table = _RAW_TABLE[kind]
|
|
id_col = _ID_COL[kind]
|
|
extra = marker.get("extra") or {}
|
|
try:
|
|
async with async_session() as session:
|
|
await session.execute(
|
|
text(
|
|
f"""
|
|
INSERT INTO {table} ({id_col}, ts, lat, lon, heading, speed, label, extra)
|
|
VALUES (:id, :ts, :lat, :lon, :heading, :speed, :label, CAST(:extra AS jsonb))
|
|
ON CONFLICT ({id_col}, ts) DO NOTHING
|
|
"""
|
|
),
|
|
{
|
|
"id": vid,
|
|
"ts": now,
|
|
"lat": float(lat),
|
|
"lon": float(lon),
|
|
"heading": marker.get("heading"),
|
|
"speed": marker.get("speed"),
|
|
"label": marker.get("label") or vid,
|
|
"extra": json.dumps(extra),
|
|
},
|
|
)
|
|
await session.commit()
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
async def fetch_positions_at(
|
|
kind: Kind,
|
|
ts: datetime,
|
|
bbox: str | None = None,
|
|
limit: int = 2000,
|
|
) -> list[dict]:
|
|
"""Read the 1-minute CAGG for the bucket containing ``ts``."""
|
|
bucket = minute_bucket(ts)
|
|
table = _CAGG[kind]
|
|
id_col = _ID_COL[kind]
|
|
where = "bucket = :bucket"
|
|
params: dict[str, Any] = {"bucket": bucket, "limit": limit}
|
|
if bbox:
|
|
minlon, minlat, maxlon, maxlat = parse_bbox(bbox)
|
|
where += " AND lon BETWEEN :minlon AND :maxlon AND lat BETWEEN :minlat AND :maxlat"
|
|
params.update(minlon=minlon, minlat=minlat, maxlon=maxlon, maxlat=maxlat)
|
|
sql = f"""
|
|
SELECT {id_col} AS id, lat, lon, heading, speed, label, bucket
|
|
FROM {table}
|
|
WHERE {where}
|
|
LIMIT :limit
|
|
"""
|
|
try:
|
|
async with async_session() as session:
|
|
rows = (await session.execute(text(sql), params)).mappings().all()
|
|
points = [
|
|
to_marker(
|
|
r["id"], r["lat"], r["lon"],
|
|
heading=r["heading"], speed=r["speed"],
|
|
label=r["label"] or r["id"],
|
|
extra={"bucket": r["bucket"].isoformat() if r["bucket"] else None, "dvr": True},
|
|
)
|
|
for r in rows
|
|
if r["lat"] is not None and r["lon"] is not None
|
|
]
|
|
return points
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
async def track_range() -> dict:
|
|
"""Earliest/latest buckets across both CAGGs — slider bounds."""
|
|
try:
|
|
async with async_session() as session:
|
|
row = (await session.execute(text(
|
|
"""
|
|
SELECT
|
|
LEAST(
|
|
(SELECT min(bucket) FROM vessel_tracks_1min),
|
|
(SELECT min(bucket) FROM aircraft_tracks_1min)
|
|
) AS tmin,
|
|
GREATEST(
|
|
(SELECT max(bucket) FROM vessel_tracks_1min),
|
|
(SELECT max(bucket) FROM aircraft_tracks_1min)
|
|
) AS tmax
|
|
"""
|
|
))).mappings().first()
|
|
if not row or row["tmin"] is None:
|
|
now = datetime.now(timezone.utc).replace(second=0, microsecond=0)
|
|
return {"min": (now - timedelta(hours=6)).isoformat(), "max": now.isoformat()}
|
|
return {
|
|
"min": row["tmin"].isoformat(),
|
|
"max": row["tmax"].isoformat(),
|
|
}
|
|
except Exception:
|
|
now = datetime.now(timezone.utc).replace(second=0, microsecond=0)
|
|
return {"min": (now - timedelta(hours=6)).isoformat(), "max": now.isoformat()}
|
|
|
|
|
|
async def recent_markers(kind: Kind, limit: int = 2000) -> list[dict]:
|
|
"""Latest raw sample per id — used when in-process last-known is empty."""
|
|
table = _RAW_TABLE[kind]
|
|
id_col = _ID_COL[kind]
|
|
sql = f"""
|
|
SELECT DISTINCT ON ({id_col})
|
|
{id_col} AS id, lat, lon, heading, speed, label, extra
|
|
FROM {table}
|
|
WHERE ts > now() - interval '15 minutes'
|
|
ORDER BY {id_col}, ts DESC
|
|
LIMIT :limit
|
|
"""
|
|
try:
|
|
async with async_session() as session:
|
|
rows = (await session.execute(text(sql), {"limit": limit})).mappings().all()
|
|
out = []
|
|
for r in rows:
|
|
extra = r.get("extra") or {}
|
|
if isinstance(extra, str):
|
|
try:
|
|
extra = json.loads(extra)
|
|
except (TypeError, ValueError):
|
|
extra = {}
|
|
m = to_marker(
|
|
r["id"], r["lat"], r["lon"],
|
|
heading=r["heading"], speed=r["speed"],
|
|
label=r["label"] or r["id"],
|
|
extra=extra if isinstance(extra, dict) else {},
|
|
)
|
|
if m.get("lat") is not None and m.get("lon") is not None:
|
|
out.append(m)
|
|
return out
|
|
except Exception:
|
|
return []
|