8643153954
Scraper loops with NEWS_SCRAPE_INTERVAL_S (default 10s after each crawl). Summarizer runs every NEWS_SUMMARIZE_INTERVAL_S (default 900) over the last 15 minutes of articles. Feed list replaced from the k8s scrapy-urls configmap (334 sources).
116 lines
4.6 KiB
Python
116 lines
4.6 KiB
Python
import scrapy
|
|
from scrapy.spiders import Spider
|
|
from urllib.parse import urljoin, urlparse
|
|
import datetime
|
|
import re
|
|
|
|
|
|
class NewsRSSSpider(Spider):
|
|
"""Crawl the curated news sources in urls.txt and extract articles.
|
|
|
|
urls.txt contains curated news HOMEPAGES and RSS/Atom feeds, so this spider
|
|
implements feed autodiscovery: it fetches each start URL, finds the
|
|
RSS/Atom feed link (`<link rel="alternate" type="application/rss+xml">`
|
|
or a visible /rss|/feed link), follows it, and then follows each feed
|
|
item's article link to extract the full text.
|
|
|
|
Upstream filtered urls.txt for '/rss'/'/feed' lines, which matched ZERO
|
|
of the 257 homepage URLs — the scraper silently did nothing. This version
|
|
makes the curated homepage list actually work.
|
|
"""
|
|
|
|
name = "articles"
|
|
# 0 = start URL (homepage), 1 = discovered feed, 2 = article
|
|
# DEPTH_LIMIT must be >= 2 (see settings.py).
|
|
|
|
def __init__(self, filename='urls.txt', *args, **kwargs):
|
|
super(NewsRSSSpider, self).__init__(*args, **kwargs)
|
|
with open(filename, 'r') as f:
|
|
self.start_urls = [
|
|
line.strip() for line in f
|
|
if line.strip() and not line.strip().startswith('#')
|
|
]
|
|
|
|
@staticmethod
|
|
def _looks_like_feed(response) -> bool:
|
|
"""Heuristic: XML body (rss/feed/RDF root) or a feed-looking URL."""
|
|
ct = (response.headers.get('Content-Type') or b'').decode('latin1', 'ignore').lower()
|
|
url = response.url.lower()
|
|
if url.endswith(('.xml', '.rss', '.atom')):
|
|
return True
|
|
if 'xml' in ct or 'rss' in ct or 'atom' in ct:
|
|
return True
|
|
try:
|
|
root = response.xpath('local-name(/*)').get()
|
|
if root and root.lower() in ('rss', 'feed', 'rdf'):
|
|
return True
|
|
except Exception: # noqa: BLE001 — non-XML responses fail XPath root lookup
|
|
pass
|
|
return False
|
|
|
|
def parse(self, response):
|
|
"""Handle a start URL: parse it directly if it is a feed, otherwise
|
|
autodiscover its RSS/Atom feed link(s)."""
|
|
if self._looks_like_feed(response):
|
|
yield from self.parse_feed(response)
|
|
return
|
|
|
|
# Feed autodiscovery: <link rel="alternate" type="application/rss+xml">
|
|
feed_links = response.xpath(
|
|
'//link[@rel="alternate" and '
|
|
'(contains(@type, "rss") or contains(@type, "atom") or contains(@type, "xml"))]/@href'
|
|
).getall()
|
|
if not feed_links:
|
|
# Fallback: visible links to /feed or /rss paths on the page.
|
|
feed_links = response.xpath(
|
|
'//a[contains(@href, "/feed") or contains(@href, "/rss")]/@href'
|
|
).getall()
|
|
for href in feed_links[:3]: # cap discovery per homepage
|
|
yield scrapy.Request(
|
|
urljoin(response.url, href),
|
|
callback=self.parse_feed,
|
|
meta={'source': response.url},
|
|
)
|
|
|
|
def parse_feed(self, response):
|
|
"""Parse an RSS (<item>) or Atom (<entry>) feed into article requests."""
|
|
for node in response.xpath('//item | //entry'):
|
|
title = node.xpath('title/text()').get()
|
|
link = (
|
|
node.xpath('link/text()').get()
|
|
or node.xpath('link/@href').get()
|
|
or node.xpath('guid/text()').get()
|
|
)
|
|
pub_date = (
|
|
node.xpath('pubDate/text()').get()
|
|
or node.xpath('published/text()').get()
|
|
or node.xpath('updated/text()').get()
|
|
)
|
|
if link:
|
|
yield scrapy.Request(
|
|
link,
|
|
callback=self.parse_article,
|
|
meta={'title': title, 'date': pub_date},
|
|
)
|
|
|
|
def parse_article(self, response):
|
|
"""Extract the main article text from the linked article page."""
|
|
title = response.meta.get('title')
|
|
|
|
# Greedy search for the main text body
|
|
article_text = response.xpath(
|
|
'//article//p/text() | //main//p/text() | '
|
|
'//div[contains(@class, "body")]//p/text() | '
|
|
'//div[contains(@class, "content")]//p/text()'
|
|
).getall()
|
|
pure_text = " ".join(article_text)
|
|
pure_text = re.sub(r'\s+', ' ', pure_text).strip()
|
|
|
|
if len(pure_text) > 300:
|
|
yield {
|
|
'title': title,
|
|
'url': response.url,
|
|
'text': pure_text,
|
|
'domain': urlparse(response.url).netloc,
|
|
'timestamp': datetime.datetime.now().isoformat()
|
|
}
|