diff --git a/news/scraper/newsScraper/settings.py b/news/scraper/newsScraper/settings.py
index 498fb2b..9bf79d8 100644
--- a/news/scraper/newsScraper/settings.py
+++ b/news/scraper/newsScraper/settings.py
@@ -60,7 +60,8 @@ DNSCACHE_ENABLED = True
DNSCACHE_SIZE = 20000
DNS_TIMEOUT = 20
DNS_RESOLVER = 'scrapy.resolver.CachingThreadedResolver'
-DEPTH_LIMIT = os.getenv('DEPTH_LIMIT', 1)
+# Depth must allow homepage(0) -> discovered feed(1) -> article(2).
+DEPTH_LIMIT = os.getenv('DEPTH_LIMIT', 3)
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
diff --git a/news/scraper/newsScraper/spiders/news_spider.py b/news/scraper/newsScraper/spiders/news_spider.py
index 1eef1ea..136f1b8 100644
--- a/news/scraper/newsScraper/spiders/news_spider.py
+++ b/news/scraper/newsScraper/spiders/news_spider.py
@@ -1,42 +1,108 @@
import scrapy
-from scrapy.spiders import XMLFeedSpider
-from urllib.parse import urlparse
+from scrapy.spiders import Spider
+from urllib.parse import urljoin, urlparse
import datetime
import re
-class NewsRSSSpider(XMLFeedSpider):
+
+class NewsRSSSpider(Spider):
+ """Crawl the curated news sources in urls.txt and extract articles.
+
+ urls.txt contains 257 news HOMEPAGES (not feed URLs), so this spider
+ implements feed autodiscovery: it fetches each start URL, finds the
+ RSS/Atom feed link (``
+ or a visible /rss|/feed link), follows it, and then follows each feed
+ item's article link to extract the full text.
+
+ Upstream filtered urls.txt for '/rss'/'/feed' lines, which matched ZERO
+ of the 257 homepage URLs — the scraper silently did nothing. This version
+ makes the curated homepage list actually work.
+ """
+
name = "articles"
- iterator = 'xml'
- itertag = 'item' # Standard RSS tag for an article
- namespaces = [
- ('dc', 'http://purl.org/dc/elements/1.1/'),
- ('content', 'http://purl.org/rss/1.0/modules/content/'),
- ('media', 'http://search.yahoo.com/mrss/')
- ]
-
+ # 0 = start URL (homepage), 1 = discovered feed, 2 = article
+ # DEPTH_LIMIT must be >= 2 (see settings.py).
+
def __init__(self, filename='urls.txt', *args, **kwargs):
super(NewsRSSSpider, self).__init__(*args, **kwargs)
with open(filename, 'r') as f:
- # We filter for RSS feeds only here
- self.start_urls = [line.strip() for line in f if '/rss' in line or '/feed' in line]
-
- def parse_node(self, response, node):
- """This runs for every found in the RSS XML"""
- title = node.xpath('title/text()').get()
- link = node.xpath('link/text()').get()
- pub_date = node.xpath('pubDate/text()').get()
-
- # We now yield a Request to the actual article to get the full text
- # Since these are RSS links, they are usually 'clean' HTML
- if link:
- yield scrapy.Request(link, callback=self.parse_article, meta={'title': title, 'date': pub_date})
+ self.start_urls = [
+ line.strip() for line in f
+ if line.strip() and not line.strip().startswith('#')
+ ]
+
+ @staticmethod
+ def _looks_like_feed(response) -> bool:
+ """Heuristic: XML body (rss/feed/RDF root) or a feed-looking URL."""
+ ct = (response.headers.get('Content-Type') or b'').decode('latin1', 'ignore').lower()
+ url = response.url.lower()
+ if url.endswith(('.xml', '.rss', '.atom')):
+ return True
+ if 'xml' in ct or 'rss' in ct or 'atom' in ct:
+ return True
+ try:
+ root = response.xpath('local-name(/*)').get()
+ if root and root.lower() in ('rss', 'feed', 'rdf'):
+ return True
+ except Exception: # noqa: BLE001 — non-XML responses fail XPath root lookup
+ pass
+ return False
+
+ def parse(self, response):
+ """Handle a start URL: parse it directly if it is a feed, otherwise
+ autodiscover its RSS/Atom feed link(s)."""
+ if self._looks_like_feed(response):
+ yield from self.parse_feed(response)
+ return
+
+ # Feed autodiscovery:
+ feed_links = response.xpath(
+ '//link[@rel="alternate" and '
+ '(contains(@type, "rss") or contains(@type, "atom") or contains(@type, "xml"))]/@href'
+ ).getall()
+ if not feed_links:
+ # Fallback: visible links to /feed or /rss paths on the page.
+ feed_links = response.xpath(
+ '//a[contains(@href, "/feed") or contains(@href, "/rss")]/@href'
+ ).getall()
+ for href in feed_links[:3]: # cap discovery per homepage
+ yield scrapy.Request(
+ urljoin(response.url, href),
+ callback=self.parse_feed,
+ meta={'source': response.url},
+ )
+
+ def parse_feed(self, response):
+ """Parse an RSS () or Atom () feed into article requests."""
+ for node in response.xpath('//item | //entry'):
+ title = node.xpath('title/text()').get()
+ link = (
+ node.xpath('link/text()').get()
+ or node.xpath('link/@href').get()
+ or node.xpath('guid/text()').get()
+ )
+ pub_date = (
+ node.xpath('pubDate/text()').get()
+ or node.xpath('published/text()').get()
+ or node.xpath('updated/text()').get()
+ )
+ if link:
+ yield scrapy.Request(
+ link,
+ callback=self.parse_article,
+ meta={'title': title, 'date': pub_date},
+ )
def parse_article(self, response):
-
+ """Extract the main article text from the linked article page."""
title = response.meta.get('title')
-
+
# Greedy search for the main text body
- article_text = response.xpath('//article//p/text() | //main//p/text() | //div[contains(@class, "body")]//p/text()').getall()
+ article_text = response.xpath(
+ '//article//p/text() | //main//p/text() | '
+ '//div[contains(@class, "body")]//p/text() | '
+ '//div[contains(@class, "content")]//p/text()'
+ ).getall()
pure_text = " ".join(article_text)
pure_text = re.sub(r'\s+', ' ', pure_text).strip()
diff --git a/news/summerizer/summarizer.py b/news/summerizer/summarizer.py
index 569a0dc..993f71e 100644
--- a/news/summerizer/summarizer.py
+++ b/news/summerizer/summarizer.py
@@ -203,6 +203,40 @@ def build_futures_context() -> str:
# ── DB helpers ─────────────────────────────────────────────────────────────
+def ensure_tables() -> None:
+ """Idempotently create the news tables if missing.
+
+ Normally created by alembic 003_news when the app container starts, but
+ this summarizer may boot before the app has run migrations (compose only
+ guarantees `db` is up, not that alembic has run). Mirrors the scraper
+ pipeline's own CREATE TABLE IF NOT EXISTS so either start order is safe.
+ """
+ ddl = """
+ CREATE TABLE IF NOT EXISTS articles (
+ id SERIAL PRIMARY KEY,
+ title TEXT,
+ url TEXT UNIQUE,
+ content TEXT,
+ domain TEXT,
+ timestamp TIMESTAMPTZ
+ );
+ CREATE TABLE IF NOT EXISTS article_summaries (
+ id SERIAL PRIMARY KEY,
+ summary_text TEXT NOT NULL,
+ batch_timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW()
+ );
+ """
+ try:
+ conn = psycopg2.connect(**DB_CONFIG)
+ cur = conn.cursor()
+ cur.execute(ddl)
+ conn.commit()
+ cur.close()
+ conn.close()
+ except Exception as exc: # noqa: BLE001
+ logger.error("Error ensuring news tables: %s", exc)
+
+
def get_recent_news() -> list[dict]:
"""Fetch articles from the last SUMMARY_WINDOW_HOURS (content > 100 chars)."""
query = """
@@ -274,6 +308,7 @@ def build_master_prompt(final_input: str) -> str:
def summarize_news() -> None:
"""Map-reduce summarize recent articles and store the master summary."""
+ ensure_tables()
articles = get_recent_news()
if not articles:
logger.info("No new articles found in the last %sh.", SUMMARY_WINDOW_HOURS)