import os 
from dotenv import load_dotenv 
load_dotenv() 

PROXY_USER = os.getenv('PROXY_USER', '').strip()
PROXY_PASS = os.getenv('PROXY_PASS', '').strip()
PROXY_ENDPOINT = os.getenv('PROXY_ENDPOINT', '').strip()

def get_proxy_url():
    if not PROXY_ENDPOINT: 
        return None 
    endpoint = PROXY_ENDPOINT.replace('http://', '').replace('https://','')

    if PROXY_USER and PROXY_PASS:
        return f"http://{PROXY_USER}:{PROXY_PASS}@{endpoint}"
    else:
        return f"http://{endpoint}"

PROXY_URL = get_proxy_url() 

DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 110,
}

BOT_NAME = "newsScraper"

SPIDER_MODULES = ["newsScraper.spiders"]
NEWSPIDER_MODULE = "newsScraper.spiders"

ADDONS = {}


# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = os.getenv('USER_AGENT', "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
DEFAULT_REQUEST_HEADERS = {
   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
   'Accept-Language': 'en-US,en;q=0.5',
   'Accept-Encoding': 'gzip, deflate, br',
   'DNT': '1', 
   'Connection': 'keep-alive',
   'Upgrade-Insecure-Requests': '1',
}

# Obey robots.txt rules
ROBOTSTXT_OBEY = True 

# Concurrency and throttling settings
CONCURRENT_REQUESTS = os.getenv('CONCURRENT_REQUESTS', '100').strip() 
CONCURRENT_REQUESTS_PER_DOMAIN = 2 
DOWNLOAD_DELAY = 3 
REACTOR_THREADPOOL_MAXSIZE = 100
LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
RETRY_ENABLED = True 
DOWNLOAD_TIMEOUT = 60
AJAXCRAWL_ENABLED = False
AUTO_THROTTLE_ENABLED = True   
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_TARGET_CONCURRENCY = 2.0
DNSCACHE_ENABLED = True 
DNSCACHE_SIZE = 20000
DNS_TIMEOUT = 20 
DNS_RESOLVER = 'scrapy.resolver.CachingThreadedResolver'
# Depth must allow homepage(0) -> discovered feed(1) -> article(2).
DEPTH_LIMIT = os.getenv('DEPTH_LIMIT', 3)

TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"


FEEDS = {
    'data/hourly_news.jsonl':{
        'format': 'jsonlines',
        'encoding': 'utf8',
            'overwrite': True, 
    }
}
FEED_EXPORT_ENCODING = "utf-8"

ITEM_PIPELINES = {
    'newsScraper.pipelines.PostgresPipeline': 300,
}

# Database Config (These should be in your .env / K8s Secrets)

DB_HOST = os.getenv('DB_HOST', 'postgres-service')
DB_NAME = os.getenv('DB_NAME', 'news_db')
DB_USER = os.getenv('DB_USER', 'admin')
DB_PASSWORD = os.getenv('DB_PASSWORD')

db_config = {
    'host': DB_HOST,
    'database': DB_NAME,
    'user': DB_USER,
    'password': DB_PASSWORD,
    'port': 5432 # Default postgres port
}