d81ea68dec
Stop the live HUD reconnect storm (nginx WS snippet + backoff), copy intel/nous_client into the summarizer image, and make event ingest idempotent on URL. GDELT uses the DOC API; NWS no longer sends bbox; FIRMS is one ON CONFLICT batch; GET /api/aircraft serves last-known. Health reports freshness without 503ing docker. EONET + CISA KEV added.
32 lines
1 KiB
Python
32 lines
1 KiB
Python
"""Feed helpers shared by the news spider (no Scrapy import)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime
|
|
from email.utils import parsedate_to_datetime
|
|
from urllib.parse import urlparse
|
|
|
|
AUDIO_EXT = (".mp3", ".m4a", ".ogg", ".wav", ".aac", ".flac", ".opus")
|
|
|
|
|
|
def is_audio_url(url: str) -> bool:
|
|
path = urlparse(url or "").path.lower()
|
|
return any(path.endswith(ext) for ext in AUDIO_EXT)
|
|
|
|
|
|
def article_timestamp(pub_date: str | None) -> datetime.datetime:
|
|
"""Prefer the feed's pubDate/published; fall back to now (UTC)."""
|
|
if pub_date:
|
|
try:
|
|
return parsedate_to_datetime(pub_date).astimezone(datetime.timezone.utc)
|
|
except (TypeError, ValueError, IndexError):
|
|
pass
|
|
try:
|
|
raw = pub_date.replace("Z", "+00:00")
|
|
ts = datetime.datetime.fromisoformat(raw)
|
|
if ts.tzinfo is None:
|
|
ts = ts.replace(tzinfo=datetime.timezone.utc)
|
|
return ts
|
|
except ValueError:
|
|
pass
|
|
return datetime.datetime.now(datetime.timezone.utc)
|