Fix news scraper feed discovery + summarizer startup race

The vendored spider only started on urls.txt lines containing '/rss' or
'/feed' — but the curated urls.txt holds 200 homepages, so the crawler
matched ZERO feeds and silently did nothing (2ms, 0 items). Rewrite the
spider with feed autodiscovery: fetch each homepage, find its
<link rel="alternate" type="application/rss+xml"> (or /feed|/rss link),
parse the feed, then follow each item to extract the article. Bump
DEPTH_LIMIT 1->3 (homepage -> feed -> article).

Also make the summarizer resilient to booting before the app container has
run alembic (docker-compose only guarantees `db` is up): add idempotent
ensure_tables() mirroring the scraper's CREATE TABLE IF NOT EXISTS.

news/scraper/newsScraper/settings.py

@@ -60,7 +60,8 @@ DNSCACHE_ENABLED = True
 DNSCACHE_SIZE = 20000
 DNS_TIMEOUT = 20 
 DNS_RESOLVER = 'scrapy.resolver.CachingThreadedResolver'
-DEPTH_LIMIT = os.getenv('DEPTH_LIMIT', 1)
+# Depth must allow homepage(0) -> discovered feed(1) -> article(2).
+DEPTH_LIMIT = os.getenv('DEPTH_LIMIT', 3)
 
 TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
 

news/scraper/newsScraper/spiders/news_spider.py

@@ -1,42 +1,108 @@
 import scrapy
-from scrapy.spiders import XMLFeedSpider
-from urllib.parse import urlparse
+from scrapy.spiders import Spider
+from urllib.parse import urljoin, urlparse
 import datetime
 import re
 
-class NewsRSSSpider(XMLFeedSpider):
+
+class NewsRSSSpider(Spider):
+    """Crawl the curated news sources in urls.txt and extract articles.
+
+    urls.txt contains 257 news HOMEPAGES (not feed URLs), so this spider
+    implements feed autodiscovery: it fetches each start URL, finds the
+    RSS/Atom feed link (`<link rel="alternate" type="application/rss+xml">`
+    or a visible /rss|/feed link), follows it, and then follows each feed
+    item's article link to extract the full text.
+
+    Upstream filtered urls.txt for '/rss'/'/feed' lines, which matched ZERO
+    of the 257 homepage URLs — the scraper silently did nothing. This version
+    makes the curated homepage list actually work.
+    """
+
     name = "articles"
-    iterator = 'xml' 
-    itertag = 'item'  # Standard RSS tag for an article
-    namespaces = [
-        ('dc', 'http://purl.org/dc/elements/1.1/'),
-        ('content', 'http://purl.org/rss/1.0/modules/content/'),
-        ('media', 'http://search.yahoo.com/mrss/')
-    ]
+    # 0 = start URL (homepage), 1 = discovered feed, 2 = article
+    # DEPTH_LIMIT must be >= 2 (see settings.py).
 
     def __init__(self, filename='urls.txt', *args, **kwargs):
         super(NewsRSSSpider, self).__init__(*args, **kwargs)
         with open(filename, 'r') as f:
-            # We filter for RSS feeds only here
-            self.start_urls = [line.strip() for line in f if '/rss' in line or '/feed' in line]
+            self.start_urls = [
+                line.strip() for line in f
+                if line.strip() and not line.strip().startswith('#')
+            ]
 
-    def parse_node(self, response, node):
-        """This runs for every <item> found in the RSS XML"""
+    @staticmethod
+    def _looks_like_feed(response) -> bool:
+        """Heuristic: XML body (rss/feed/RDF root) or a feed-looking URL."""
+        ct = (response.headers.get('Content-Type') or b'').decode('latin1', 'ignore').lower()
+        url = response.url.lower()
+        if url.endswith(('.xml', '.rss', '.atom')):
+            return True
+        if 'xml' in ct or 'rss' in ct or 'atom' in ct:
+            return True
+        try:
+            root = response.xpath('local-name(/*)').get()
+            if root and root.lower() in ('rss', 'feed', 'rdf'):
+                return True
+        except Exception:  # noqa: BLE001 — non-XML responses fail XPath root lookup
+            pass
+        return False
+
+    def parse(self, response):
+        """Handle a start URL: parse it directly if it is a feed, otherwise
+        autodiscover its RSS/Atom feed link(s)."""
+        if self._looks_like_feed(response):
+            yield from self.parse_feed(response)
+            return
+
+        # Feed autodiscovery: <link rel="alternate" type="application/rss+xml">
+        feed_links = response.xpath(
+            '//link[@rel="alternate" and '
+            '(contains(@type, "rss") or contains(@type, "atom") or contains(@type, "xml"))]/@href'
+        ).getall()
+        if not feed_links:
+            # Fallback: visible links to /feed or /rss paths on the page.
+            feed_links = response.xpath(
+                '//a[contains(@href, "/feed") or contains(@href, "/rss")]/@href'
+            ).getall()
+        for href in feed_links[:3]:  # cap discovery per homepage
+            yield scrapy.Request(
+                urljoin(response.url, href),
+                callback=self.parse_feed,
+                meta={'source': response.url},
+            )
+
+    def parse_feed(self, response):
+        """Parse an RSS (<item>) or Atom (<entry>) feed into article requests."""
+        for node in response.xpath('//item | //entry'):
             title = node.xpath('title/text()').get()
-        link = node.xpath('link/text()').get()
-        pub_date = node.xpath('pubDate/text()').get()
-        
-        # We now yield a Request to the actual article to get the full text
-        # Since these are RSS links, they are usually 'clean' HTML
+            link = (
+                node.xpath('link/text()').get()
+                or node.xpath('link/@href').get()
+                or node.xpath('guid/text()').get()
+            )
+            pub_date = (
+                node.xpath('pubDate/text()').get()
+                or node.xpath('published/text()').get()
+                or node.xpath('updated/text()').get()
+            )
             if link:
-            yield scrapy.Request(link, callback=self.parse_article, meta={'title': title, 'date': pub_date})
+                yield scrapy.Request(
+                    link,
+                    callback=self.parse_article,
+                    meta={'title': title, 'date': pub_date},
+                )
 
     def parse_article(self, response):
-        
+        """Extract the main article text from the linked article page."""
         title = response.meta.get('title')
 
         # Greedy search for the main text body
-        article_text = response.xpath('//article//p/text() | //main//p/text() | //div[contains(@class, "body")]//p/text()').getall()
+        article_text = response.xpath(
+            '//article//p/text() | //main//p/text() | '
+            '//div[contains(@class, "body")]//p/text() | '
+            '//div[contains(@class, "content")]//p/text()'
+        ).getall()
         pure_text = " ".join(article_text)
         pure_text = re.sub(r'\s+', ' ', pure_text).strip()
 

news/summerizer/summarizer.py

@@ -203,6 +203,40 @@ def build_futures_context() -> str:
 
 # ── DB helpers ─────────────────────────────────────────────────────────────
 
+def ensure_tables() -> None:
+    """Idempotently create the news tables if missing.
+
+    Normally created by alembic 003_news when the app container starts, but
+    this summarizer may boot before the app has run migrations (compose only
+    guarantees `db` is up, not that alembic has run). Mirrors the scraper
+    pipeline's own CREATE TABLE IF NOT EXISTS so either start order is safe.
+    """
+    ddl = """
+        CREATE TABLE IF NOT EXISTS articles (
+            id SERIAL PRIMARY KEY,
+            title TEXT,
+            url TEXT UNIQUE,
+            content TEXT,
+            domain TEXT,
+            timestamp TIMESTAMPTZ
+        );
+        CREATE TABLE IF NOT EXISTS article_summaries (
+            id SERIAL PRIMARY KEY,
+            summary_text TEXT NOT NULL,
+            batch_timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW()
+        );
+    """
+    try:
+        conn = psycopg2.connect(**DB_CONFIG)
+        cur = conn.cursor()
+        cur.execute(ddl)
+        conn.commit()
+        cur.close()
+        conn.close()
+    except Exception as exc:  # noqa: BLE001
+        logger.error("Error ensuring news tables: %s", exc)
+
+
 def get_recent_news() -> list[dict]:
     """Fetch articles from the last SUMMARY_WINDOW_HOURS (content > 100 chars)."""
     query = """
@@ -274,6 +308,7 @@ def build_master_prompt(final_input: str) -> str:
 
 def summarize_news() -> None:
     """Map-reduce summarize recent articles and store the master summary."""
+    ensure_tables()
     articles = get_recent_news()
     if not articles:
         logger.info("No new articles found in the last %sh.", SUMMARY_WINDOW_HOURS)