import scrapy
from scrapy.spiders import XMLFeedSpider
from urllib.parse import urlparse
import datetime
import re

class NewsRSSSpider(XMLFeedSpider):
    name = "articles"
    iterator = 'xml' 
    itertag = 'item'  # Standard RSS tag for an article
    namespaces = [
        ('dc', 'http://purl.org/dc/elements/1.1/'),
        ('content', 'http://purl.org/rss/1.0/modules/content/'),
        ('media', 'http://search.yahoo.com/mrss/')
    ]
    
    def __init__(self, filename='urls.txt', *args, **kwargs):
        super(NewsRSSSpider, self).__init__(*args, **kwargs)
        with open(filename, 'r') as f:
            # We filter for RSS feeds only here
            self.start_urls = [line.strip() for line in f if '/rss' in line or '/feed' in line]
        
    def parse_node(self, response, node):
        """This runs for every <item> found in the RSS XML"""
        title = node.xpath('title/text()').get()
        link = node.xpath('link/text()').get()
        pub_date = node.xpath('pubDate/text()').get()
        
        # We now yield a Request to the actual article to get the full text
        # Since these are RSS links, they are usually 'clean' HTML
        if link:
            yield scrapy.Request(link, callback=self.parse_article, meta={'title': title, 'date': pub_date})

    def parse_article(self, response):
        
        title = response.meta.get('title')
        
        # Greedy search for the main text body
        article_text = response.xpath('//article//p/text() | //main//p/text() | //div[contains(@class, "body")]//p/text()').getall()
        pure_text = " ".join(article_text)
        pure_text = re.sub(r'\s+', ' ', pure_text).strip()

        if len(pure_text) > 300:
            yield {
                'title': title,
                'url': response.url,
                'text': pure_text,
                'domain': urlparse(response.url).netloc,
                'timestamp': datetime.datetime.now().isoformat()
            }