d81ea68dec
Stop the live HUD reconnect storm (nginx WS snippet + backoff), copy intel/nous_client into the summarizer image, and make event ingest idempotent on URL. GDELT uses the DOC API; NWS no longer sends bbox; FIRMS is one ON CONFLICT batch; GET /api/aircraft serves last-known. Health reports freshness without 503ing docker. EONET + CISA KEV added.
42 lines
1.3 KiB
Python
42 lines
1.3 KiB
Python
"""News spider/pipeline: skip audio, use pubDate, don't log dupes as errors."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
_SCRAPER = ROOT / "news/scraper"
|
|
if str(_SCRAPER) not in sys.path:
|
|
sys.path.insert(0, str(_SCRAPER))
|
|
|
|
|
|
def test_is_audio_enclosure():
|
|
from newsScraper.feed_util import is_audio_url
|
|
|
|
assert is_audio_url("https://cdn.example/podcast.mp3") is True
|
|
assert is_audio_url("https://cdn.example/show.m4a?x=1") is True
|
|
assert is_audio_url("https://www.example.com/world/story") is False
|
|
|
|
|
|
def test_article_timestamp_prefers_pubdate():
|
|
from newsScraper.feed_util import article_timestamp
|
|
|
|
ts = article_timestamp("Tue, 01 Apr 2025 12:00:00 GMT")
|
|
assert ts.tzinfo is not None
|
|
assert ts.year == 2025
|
|
assert ts.month == 4
|
|
assert ts.day == 1
|
|
|
|
|
|
def test_pipeline_does_not_wrap_dropitem_as_error():
|
|
src = (ROOT / "news/scraper/newsScraper/pipelines.py").read_text()
|
|
assert "except DropItem" in src
|
|
assert "seen_urls.add" in src or "self.seen_urls.add" in src
|
|
|
|
|
|
def test_spider_skips_audio_before_request():
|
|
src = (ROOT / "news/scraper/newsScraper/spiders/news_spider.py").read_text()
|
|
assert "is_audio_url" in src
|
|
assert "article_timestamp" in src
|
|
assert "datetime.datetime.now()" not in src
|