osint-dashboard/news/scraper/newsScraper/spiders/news_spider.py

import scrapy
from scrapy.spiders import Spider
from urllib.parse import urljoin, urlparse
import datetime
import re


class NewsRSSSpider(Spider):
    """Crawl the curated news sources in urls.txt and extract articles.

    urls.txt contains 257 news HOMEPAGES (not feed URLs), so this spider
    implements feed autodiscovery: it fetches each start URL, finds the
    RSS/Atom feed link (`<link rel="alternate" type="application/rss+xml">`
    or a visible /rss|/feed link), follows it, and then follows each feed
    item's article link to extract the full text.

    Upstream filtered urls.txt for '/rss'/'/feed' lines, which matched ZERO
    of the 257 homepage URLs — the scraper silently did nothing. This version
    makes the curated homepage list actually work.
    """

    name = "articles"
    # 0 = start URL (homepage), 1 = discovered feed, 2 = article
    # DEPTH_LIMIT must be >= 2 (see settings.py).

    def __init__(self, filename='urls.txt', *args, **kwargs):
        super(NewsRSSSpider, self).__init__(*args, **kwargs)
        with open(filename, 'r') as f:
            self.start_urls = [
                line.strip() for line in f
                if line.strip() and not line.strip().startswith('#')
            ]

    @staticmethod
    def _looks_like_feed(response) -> bool:
        """Heuristic: XML body (rss/feed/RDF root) or a feed-looking URL."""
        ct = (response.headers.get('Content-Type') or b'').decode('latin1', 'ignore').lower()
        url = response.url.lower()
        if url.endswith(('.xml', '.rss', '.atom')):
            return True
        if 'xml' in ct or 'rss' in ct or 'atom' in ct:
            return True
        try:
            root = response.xpath('local-name(/*)').get()
            if root and root.lower() in ('rss', 'feed', 'rdf'):
                return True
        except Exception:  # noqa: BLE001 — non-XML responses fail XPath root lookup
            pass
        return False

    def parse(self, response):
        """Handle a start URL: parse it directly if it is a feed, otherwise
        autodiscover its RSS/Atom feed link(s)."""
        if self._looks_like_feed(response):
            yield from self.parse_feed(response)
            return

        # Feed autodiscovery: <link rel="alternate" type="application/rss+xml">
        feed_links = response.xpath(
            '//link[@rel="alternate" and '
            '(contains(@type, "rss") or contains(@type, "atom") or contains(@type, "xml"))]/@href'
        ).getall()
        if not feed_links:
            # Fallback: visible links to /feed or /rss paths on the page.
            feed_links = response.xpath(
                '//a[contains(@href, "/feed") or contains(@href, "/rss")]/@href'
            ).getall()
        for href in feed_links[:3]:  # cap discovery per homepage
            yield scrapy.Request(
                urljoin(response.url, href),
                callback=self.parse_feed,
                meta={'source': response.url},
            )

    def parse_feed(self, response):
        """Parse an RSS (<item>) or Atom (<entry>) feed into article requests."""
        for node in response.xpath('//item | //entry'):
            title = node.xpath('title/text()').get()
            link = (
                node.xpath('link/text()').get()
                or node.xpath('link/@href').get()
                or node.xpath('guid/text()').get()
            )
            pub_date = (
                node.xpath('pubDate/text()').get()
                or node.xpath('published/text()').get()
                or node.xpath('updated/text()').get()
            )
            if link:
                yield scrapy.Request(
                    link,
                    callback=self.parse_article,
                    meta={'title': title, 'date': pub_date},
                )

    def parse_article(self, response):
        """Extract the main article text from the linked article page."""
        title = response.meta.get('title')

        # Greedy search for the main text body
        article_text = response.xpath(
            '//article//p/text() | //main//p/text() | '
            '//div[contains(@class, "body")]//p/text() | '
            '//div[contains(@class, "content")]//p/text()'
        ).getall()
        pure_text = " ".join(article_text)
        pure_text = re.sub(r'\s+', ' ', pure_text).strip()

        if len(pure_text) > 300:
            yield {
                'title': title,
                'url': response.url,
                'text': pure_text,
                'domain': urlparse(response.url).netloc,
                'timestamp': datetime.datetime.now().isoformat()
            }