51 lines
1.9 KiB
Python
51 lines
1.9 KiB
Python
|
import scrapy
|
||
|
|
from scrapy.spiders import XMLFeedSpider
|
||
|
|
from urllib.parse import urlparse
|
||
|
|
import datetime
|
||
|
|
import re
|
||
|
|
|
||
|
|
class NewsRSSSpider(XMLFeedSpider):
|
||
|
|
name = "articles"
|
||
|
|
iterator = 'xml'
|
||
|
|
itertag = 'item' # Standard RSS tag for an article
|
||
|
|
namespaces = [
|
||
|
|
('dc', 'http://purl.org/dc/elements/1.1/'),
|
||
|
|
('content', 'http://purl.org/rss/1.0/modules/content/'),
|
||
|
|
('media', 'http://search.yahoo.com/mrss/')
|
||
|
|
]
|
||
|
|
|
||
|
|
def __init__(self, filename='urls.txt', *args, **kwargs):
|
||
|
|
super(NewsRSSSpider, self).__init__(*args, **kwargs)
|
||
|
|
with open(filename, 'r') as f:
|
||
|
|
# We filter for RSS feeds only here
|
||
|
|
self.start_urls = [line.strip() for line in f if '/rss' in line or '/feed' in line]
|
||
|
|
|
||
|
|
def parse_node(self, response, node):
|
||
|
|
"""This runs for every <item> found in the RSS XML"""
|
||
|
|
title = node.xpath('title/text()').get()
|
||
|
|
link = node.xpath('link/text()').get()
|
||
|
|
pub_date = node.xpath('pubDate/text()').get()
|
||
|
|
|
||
|
|
# We now yield a Request to the actual article to get the full text
|
||
|
|
# Since these are RSS links, they are usually 'clean' HTML
|
||
|
|
if link:
|
||
|
|
yield scrapy.Request(link, callback=self.parse_article, meta={'title': title, 'date': pub_date})
|
||
|
|
|
||
|
|
def parse_article(self, response):
|
||
|
|
|
||
|
|
title = response.meta.get('title')
|
||
|
|
|
||
|
|
# Greedy search for the main text body
|
||
|
|
article_text = response.xpath('//article//p/text() | //main//p/text() | //div[contains(@class, "body")]//p/text()').getall()
|
||
|
|
pure_text = " ".join(article_text)
|
||
|
|
pure_text = re.sub(r'\s+', ' ', pure_text).strip()
|
||
|
|
|
||
|
|
if len(pure_text) > 300:
|
||
|
|
yield {
|
||
|
|
'title': title,
|
||
|
|
'url': response.url,
|
||
|
|
'text': pure_text,
|
||
|
|
'domain': urlparse(response.url).netloc,
|
||
|
|
'timestamp': datetime.datetime.now().isoformat()
|
||
|
|
}
|