sirius/osint-dashboard
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

perf: slim fires/cameras/events/news map payloads

Browse source 
Heatmap uses {lat,lon,i,c}. Camera list drops URLs (detail via GET
/api/cameras/{id}). Event blips skip body. News omits content unless
include_content=true.
This commit is contained in:
Sirius DevOps 2026-08-27 21:19:42 -04:00
parent 9c14b899ee
commit 8f89994201
3 changed files with 131 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

96
app/main.py
View file

@ -69,17 +69,17 @@ def event_to_out(row: dict) -> EventOut:
return EventOut(
id=row["id"],
source_type=row["source_type"],
source_id=row["source_id"],
title=row["title"],
body=row["body"],
url=row["url"],
sentiment_score=row["sentiment_score"],
sentiment_label=row["sentiment_label"],
location_lat=row["location_lat"],
location_lon=row["location_lon"],
location_name=row["location_name"],
entities=row["entities"],
tags=row["tags"],
source_id=row.get("source_id"),
title=row.get("title"),
body=row.get("body"),
url=row.get("url"),
sentiment_score=row.get("sentiment_score"),
sentiment_label=row.get("sentiment_label"),
location_lat=row.get("location_lat"),
location_lon=row.get("location_lon"),
location_name=row.get("location_name"),
entities=row.get("entities"),
tags=row.get("tags"),
ingested_at=row["ingested_at"],
source_timestamp=row["source_timestamp"],
)
@ -218,7 +218,15 @@ async def list_events(
):
"""List recent ingested events."""
async with async_session() as session:
stmt = select(events).order_by(events.c.ingested_at.desc())
if has_coords:
stmt = select(
events.c.id, events.c.source_type, events.c.source_id,
events.c.title, events.c.url,
events.c.location_lat, events.c.location_lon, events.c.location_name,
events.c.ingested_at, events.c.source_timestamp,
).order_by(events.c.ingested_at.desc())
else:
stmt = select(events).order_by(events.c.ingested_at.desc())
if source_type:
stmt = stmt.where(events.c.source_type == source_type.value)
if since:
@ -272,7 +280,29 @@ async def create_event(payload: EventCreate):
# ── Active Fires / Hotspots (NASA FIRMS) ─────────────────────────────────
@app.get("/api/fires", response_model=list[FireOut])
def fire_heat_row(r) -> dict:
"""Minimal FIRMS point for the heatmap overlay."""
return {
"lat": r["latitude"],
"lon": r["longitude"],
"i": r["brightness"],
"c": r["confidence"],
}
def camera_map_row(r) -> dict:
"""Minimal camera pin — URLs stay off the list payload."""
return {
"id": str(r["id"]),
"lat": r["location_lat"],
"lon": r["location_lon"],
"device_type": r["device_type"],
"discovery_source": r["discovery_source"],
"location_name": r["location_name"],
}
@app.get("/api/fires", response_model=None)
async def list_fires(
bbox: str | None = Query(
None,
@ -286,12 +316,16 @@ async def list_fires(
"(ISO 8601, e.g. '2026-08-24T12:00:00Z').",
),
limit: int = Query(2000, ge=1, le=10000),
format: str = Query("full", description="'full' FireOut rows or 'heat' {lat,lon,i,c}"),
):
"""List stored FIRMS active fire/hotspot detections as JSON.
This is the data contract for the map's fire heatmap overlay: the frontend
calls `GET /api/fires?bbox=...&since=...` and renders the returned points.
calls `GET /api/fires?bbox=...&since=...&format=heat` and renders the points.
"""
fmt = (format or "full").lower()
if fmt not in ("full", "heat"):
raise HTTPException(422, "format must be 'full' or 'heat'")
async with async_session() as session:
stmt = select(fires).order_by(fires.c.acq_time.desc())
if since:
@ -316,6 +350,8 @@ async def list_fires(
)
stmt = stmt.limit(limit)
rows = (await session.execute(stmt)).mappings().all()
if fmt == "heat":
return [fire_heat_row(r) for r in rows]
return [
FireOut(
latitude=r["latitude"], longitude=r["longitude"],
@ -798,7 +834,11 @@ async def list_cameras(
stmt = stmt.where(cam_table.c.discovery_source == source)
rows = (await session.execute(stmt.limit(limit))).mappings().all()
return [{
return [camera_map_row(r) for r in rows]
def camera_detail_row(r) -> dict:
return {
"id": str(r["id"]),
"source_url": r["source_url"],
"snapshot_url": r["snapshot_url"],
@ -808,9 +848,23 @@ async def list_cameras(
"location_name": r["location_name"],
"vendor": r["vendor"],
"device_type": r["device_type"],
"first_seen": r["first_seen"].isoformat(),
"last_seen": r["last_seen"].isoformat(),
} for r in rows]
"first_seen": r["first_seen"].isoformat() if r["first_seen"] else None,
"last_seen": r["last_seen"].isoformat() if r["last_seen"] else None,
}
@app.get("/api/cameras/{camera_id}")
async def get_camera(camera_id: UUID):
"""Full camera row for a map popup. List endpoint stays slim."""
from camera_models import cameras as cam_table
async with async_session() as session:
row = (await session.execute(
select(cam_table).where(cam_table.c.id == camera_id)
)).mappings().one_or_none()
if not row:
raise HTTPException(404, "Camera not found")
return camera_detail_row(row)
@app.get("/api/cameras/{camera_id}/snapshot")
@ -959,6 +1013,10 @@ async def list_news(
),
limit: int = Query(50, ge=1, le=500),
offset: int = Query(0, ge=0),
include_content: bool = Query(
False,
description="Include full article body. Default false — ticker/list only need title/url.",
),
):
"""Most recent scraped news articles (newest first)."""
async with async_session() as session:
@ -974,7 +1032,7 @@ async def list_news(
return [
NewsArticleOut(
id=r["id"], title=r["title"], url=r["url"],
content=r["content"], domain=r["domain"],
content=r["content"] if include_content else None, domain=r["domain"],
timestamp=r["timestamp"],
)
for r in rows

12
app/static/index.html
View file

@ -1921,10 +1921,12 @@ function setBaseOpacity(v) {
/* ── FIRMS fire heatmap ── */
function firesIntensity(f) {
const confidence = f.confidence || f.c;
const brightness = f.brightness != null ? f.brightness : f.i;
if (firesColor === 'confidence') {
return f.confidence === 'h' ? 1.0 : f.confidence === 'l' ? 0.35 : 0.6;
return confidence === 'h' ? 1.0 : confidence === 'l' ? 0.35 : 0.6;
}
const b = f.brightness || 300;
const b = brightness || 300;
return Math.max(0.05, Math.min(1, (b - 290) / 110));
}
function firesGradient() {
@ -1972,14 +1974,14 @@ async function loadFires() {
if (!map) return;
const req = ++fireReq;
try {
let url = `${API}/api/fires?bbox=${currentBBox()}&limit=2000`;
let url = `${API}/api/fires?bbox=${currentBBox()}&limit=2000&format=heat`;
const since = sinceToISO(firesSince);
if (since) url += `&since=${encodeURIComponent(since)}`;
const r = await fetch(url);
const fires = await r.json();
if (req !== fireReq) return; // superseded by a newer pan/zoom
if (firesHeat) map.removeLayer(firesHeat);
const pts = fires.map(f => [f.latitude, f.longitude, firesIntensity(f)]);
const pts = fires.map(f => [f.lat ?? f.latitude, f.lon ?? f.longitude, firesIntensity(f)]);
firesHeat = L.heatLayer(pts, {
radius: 22, blur: 20, maxZoom: 9, max: 1.0, minOpacity: 0.2,
gradient: firesGradient(),
@ -2066,7 +2068,7 @@ async function loadCams() {
}
const req = ++camReq;
try {
const r = await fetch(`${API}/api/cameras?bbox=${currentBBox()}&limit=5000`);
const r = await fetch(`${API}/api/cameras?bbox=${currentBBox()}&limit=2000`);
const cams = await r.json();
if (req !== camReq) return; // superseded by a newer pan/zoom
if (camsGroup) map.removeLayer(camsGroup);

47
tests/test_map_payloads.py Normal file
View file

@ -0,0 +1,47 @@
"""Slim map-payload helpers (no DB)."""
from uuid import uuid4
from main import camera_map_row, fire_heat_row
def test_fire_heat_row_is_tiny():
row = {
"latitude": 35.0,
"longitude": -78.0,
"brightness": 340.1,
"confidence": "h",
"satellite": "N21",
"acq_time": "2026-08-27T00:00:00Z",
"instrument": "VIIRS",
"frp": 12.4,
}
out = fire_heat_row(row)
assert out == {"lat": 35.0, "lon": -78.0, "i": 340.1, "c": "h"}
assert "satellite" not in out
assert "frp" not in out
def test_camera_map_row_omits_urls():
cid = uuid4()
row = {
"id": cid,
"location_lat": 37.8,
"location_lon": -122.4,
"device_type": "hls",
"discovery_source": "caltrans",
"location_name": "I-80 WB",
"source_url": "https://example.invalid/playlist.m3u8",
"snapshot_url": "https://example.invalid/cam.jpg",
"vendor": "Caltrans",
"first_seen": None,
"last_seen": None,
}
out = camera_map_row(row)
assert out["id"] == str(cid)
assert out["lat"] == 37.8
assert out["lon"] == -122.4
assert out["device_type"] == "hls"
assert "source_url" not in out
assert "snapshot_url" not in out
assert "vendor" not in out