627990efde
Coherent merge of two coordinated features on the shared working tree: FIRMS fire heatmap (backend, t_6e404c14): - app/fire_sources.py: fetch FIRMS VIIRS area CSV (free MAP_KEY) -> NATS events.fire - fires hypertable (TimescaleDB, 1-day chunks) with natural-key PK (latitude, longitude, acq_time, satellite); idempotent ON CONFLICT DO NOTHING - alembic/versions/002_fires.py; GET /api/fires?bbox=&since= (JSON only) - POST /api/ingest/fires; ~15 min poll loop (FIRMS_INTERVAL=900) in ingester - env-driven config (FIRMS_MAP_KEY/DATASET/BBOX/INTERVAL); docs/firms.md covers the zero-cost GIBS VIIRS_SNPP_Thermal_Anomalies_375m_All tile alternative - 18 tests (parser, mapping, idempotency, API contract) verified vs real TimescaleDB+PostGIS (localhost/osint-dashboard-pg image) API keys page (frontend, t_4433cff2): - app/keystore.py: api_keys table (self-creating), FIRMS/GEMINI/TELEGRAM registry with format validation, ****last4 masking, get_api_key() - GET/POST/DELETE /api/keys (never returns full values); Keys tab in index.html DB_NULL_POOL env switch in app/database.py enables a NullPool for tests / short-lived processes that open a fresh event loop per unit.
78 lines
2.9 KiB
Python
78 lines
2.9 KiB
Python
"""Unit tests for the NASA FIRMS CSV parser / timestamp normalization."""
|
|
|
|
import asyncio
|
|
from datetime import datetime, timezone
|
|
|
|
from fire_sources import normalize_acq_time, parse_firms_csv, ingest_fires
|
|
|
|
# Grounded against real FIRMS VIIRS area-CSV output.
|
|
SAMPLE_CSV = """latitude,longitude,bright_ti4,scan,track,acq_date,acq_time,satellite,instrument,confidence,version,bright_ti5,frp,daynight
|
|
-16.28359,29.40531,295.78,0.50,0.66,2025-06-06,1,N20,VIIRS,n,2.0NRT,284.11,1.17,N
|
|
-16.28190,29.40279,303.31,0.50,0.66,2025-06-06,1,N20,VIIRS,n,2.0NRT,284.43,0.67,N
|
|
-14.98900,28.36286,341.04,0.41,0.60,2025-06-06,1,N20,VIIRS,n,2.0NRT,279.77,4.59,N
|
|
15.17397,-11.28343,341.40,0.45,0.47,2025-06-06,1425,N20,VIIRS,l,2.0NRT,315.18,7.91,D
|
|
15.45870,-11.13616,339.25,0.46,0.47,2025-06-06,1425,N20,VIIRS,l,2.0NRT,312.05,9.24,D
|
|
"""
|
|
|
|
|
|
def test_normalize_acq_time_single_digit_hour():
|
|
# acq_time "1" (HHMM int) -> 00:01 UTC
|
|
dt = normalize_acq_time("2025-06-06", "1")
|
|
assert dt == datetime(2025, 6, 6, 0, 1, tzinfo=timezone.utc)
|
|
|
|
|
|
def test_normalize_acq_time_four_digit():
|
|
dt = normalize_acq_time("2025-06-06", 1425)
|
|
assert dt == datetime(2025, 6, 6, 14, 25, tzinfo=timezone.utc)
|
|
|
|
|
|
def test_normalize_acq_time_bad_values():
|
|
assert normalize_acq_time(None, 100) is None
|
|
assert normalize_acq_time("2025-06-06", None) is None
|
|
assert normalize_acq_time("2025-06-06", "") is None
|
|
assert normalize_acq_time("not-a-date", 100) is None
|
|
|
|
|
|
def test_parse_firms_csv_happy_path():
|
|
points = parse_firms_csv(SAMPLE_CSV)
|
|
assert len(points) == 5
|
|
|
|
first = points[0]
|
|
assert first["latitude"] == -16.28359
|
|
assert first["longitude"] == 29.40531
|
|
assert first["brightness"] == 295.78
|
|
assert first["confidence"] == "n"
|
|
assert first["satellite"] == "N20"
|
|
assert first["instrument"] == "VIIRS"
|
|
assert first["frp"] == 1.17
|
|
assert first["daynight"] == "N"
|
|
assert first["acq_time"] == "2025-06-06T00:01:00+00:00"
|
|
|
|
# late-day acquisition (acq_time 1425) parses to 14:25 UTC
|
|
assert points[3]["acq_time"] == "2025-06-06T14:25:00+00:00"
|
|
assert points[3]["confidence"] == "l"
|
|
|
|
|
|
def test_parse_firms_csv_skips_legend_line():
|
|
# FIRMS occasionally prepends a legend/info line before the real header.
|
|
with_legend = (
|
|
"Active Fire Data from VIIRS (S-NPP) — near real time\n"
|
|
+ SAMPLE_CSV
|
|
)
|
|
points = parse_firms_csv(with_legend)
|
|
assert len(points) == 5
|
|
assert points[0]["latitude"] == -16.28359
|
|
|
|
|
|
def test_parse_firms_csv_empty_and_garbage():
|
|
assert parse_firms_csv("") == []
|
|
assert parse_firms_csv("not a csv at all\njust text\n") == []
|
|
# Header present but a data row that is too short / has junk lat-lon
|
|
junk = SAMPLE_CSV.splitlines()[0] + "\n1,2\n"
|
|
assert parse_firms_csv(junk) == []
|
|
|
|
|
|
def test_ingest_fires_idles_without_map_key(monkeypatch):
|
|
# No key -> returns 0 without attempting a network call.
|
|
monkeypatch.setattr("fire_sources.FIRMS_MAP_KEY", "")
|
|
assert asyncio.run(ingest_fires()) == 0
|