5849d72299
Some checks failed
build-and-deploy / build (push) Failing after 4s
The vendored spider only started on urls.txt lines containing '/rss' or '/feed' — but the curated urls.txt holds 200 homepages, so the crawler matched ZERO feeds and silently did nothing (2ms, 0 items). Rewrite the spider with feed autodiscovery: fetch each homepage, find its <link rel="alternate" type="application/rss+xml"> (or /feed|/rss link), parse the feed, then follow each item to extract the article. Bump DEPTH_LIMIT 1->3 (homepage -> feed -> article). Also make the summarizer resilient to booting before the app container has run alembic (docker-compose only guarantees `db` is up): add idempotent ensure_tables() mirroring the scraper's CREATE TABLE IF NOT EXISTS.
95 lines
2.6 KiB
Python
95 lines
2.6 KiB
Python
import os
|
|
from dotenv import load_dotenv
|
|
load_dotenv()
|
|
|
|
PROXY_USER = os.getenv('PROXY_USER', '').strip()
|
|
PROXY_PASS = os.getenv('PROXY_PASS', '').strip()
|
|
PROXY_ENDPOINT = os.getenv('PROXY_ENDPOINT', '').strip()
|
|
|
|
def get_proxy_url():
|
|
if not PROXY_ENDPOINT:
|
|
return None
|
|
endpoint = PROXY_ENDPOINT.replace('http://', '').replace('https://','')
|
|
|
|
if PROXY_USER and PROXY_PASS:
|
|
return f"http://{PROXY_USER}:{PROXY_PASS}@{endpoint}"
|
|
else:
|
|
return f"http://{endpoint}"
|
|
|
|
PROXY_URL = get_proxy_url()
|
|
|
|
DOWNLOADER_MIDDLEWARES = {
|
|
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 110,
|
|
}
|
|
|
|
BOT_NAME = "newsScraper"
|
|
|
|
SPIDER_MODULES = ["newsScraper.spiders"]
|
|
NEWSPIDER_MODULE = "newsScraper.spiders"
|
|
|
|
ADDONS = {}
|
|
|
|
|
|
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
|
USER_AGENT = os.getenv('USER_AGENT', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
|
DEFAULT_REQUEST_HEADERS = {
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
|
'Accept-Language': 'en-US,en;q=0.5',
|
|
'Accept-Encoding': 'gzip, deflate, br',
|
|
'DNT': '1',
|
|
'Connection': 'keep-alive',
|
|
'Upgrade-Insecure-Requests': '1',
|
|
}
|
|
|
|
# Obey robots.txt rules
|
|
ROBOTSTXT_OBEY = True
|
|
|
|
# Concurrency and throttling settings
|
|
CONCURRENT_REQUESTS = os.getenv('CONCURRENT_REQUESTS', '100').strip()
|
|
CONCURRENT_REQUESTS_PER_DOMAIN = 2
|
|
DOWNLOAD_DELAY = 3
|
|
REACTOR_THREADPOOL_MAXSIZE = 100
|
|
LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
|
|
RETRY_ENABLED = True
|
|
DOWNLOAD_TIMEOUT = 60
|
|
AJAXCRAWL_ENABLED = False
|
|
AUTO_THROTTLE_ENABLED = True
|
|
AUTOTHROTTLE_ENABLED = True
|
|
AUTOTHROTTLE_TARGET_CONCURRENCY = 2.0
|
|
DNSCACHE_ENABLED = True
|
|
DNSCACHE_SIZE = 20000
|
|
DNS_TIMEOUT = 20
|
|
DNS_RESOLVER = 'scrapy.resolver.CachingThreadedResolver'
|
|
# Depth must allow homepage(0) -> discovered feed(1) -> article(2).
|
|
DEPTH_LIMIT = os.getenv('DEPTH_LIMIT', 3)
|
|
|
|
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
|
|
|
|
|
|
FEEDS = {
|
|
'data/hourly_news.jsonl':{
|
|
'format': 'jsonlines',
|
|
'encoding': 'utf8',
|
|
'overwrite': True,
|
|
}
|
|
}
|
|
FEED_EXPORT_ENCODING = "utf-8"
|
|
|
|
ITEM_PIPELINES = {
|
|
'newsScraper.pipelines.PostgresPipeline': 300,
|
|
}
|
|
|
|
# Database Config (These should be in your .env / K8s Secrets)
|
|
|
|
DB_HOST = os.getenv('DB_HOST', 'postgres-service')
|
|
DB_NAME = os.getenv('DB_NAME', 'news_db')
|
|
DB_USER = os.getenv('DB_USER', 'admin')
|
|
DB_PASSWORD = os.getenv('DB_PASSWORD')
|
|
|
|
db_config = {
|
|
'host': DB_HOST,
|
|
'database': DB_NAME,
|
|
'user': DB_USER,
|
|
'password': DB_PASSWORD,
|
|
'port': 5432 # Default postgres port
|
|
}
|