91f436390b
Some checks failed
build-and-deploy / build (push) Failing after 4s
Vendor the newsPipeline scraper + summarizer into the repo and wire them into
docker-compose against the EXISTING osint-db (no second Postgres), replacing
the upstream k8s CronJobs with in-compose wall-clock loops (:00 scrape, :05
summarize).
- news/scraper: vendored Scrapy project (257 RSS feeds) + hourly loop
scheduler (run_news_scraper.py)
- news/summerizer: vendored Gemini map-reduce summarizer, cleaned:
* fix broken google-genai response handling (_extract_text, defensive)
* fix malformed INSERT/GRANT query in save_summary_to_db
* OSINT-neutral default MAP_PROMPT; futures/markets language gated behind
INCLUDE_FUTURES=0 (yfinance lazy-imported)
* env-configurable model, batch size, lookback window
+ hourly loop scheduler (run_news_summarizer.py, :05)
- alembic 003_news: idempotent articles + article_summaries tables
- API: GET /api/news and GET /api/news/summaries (+ models, schemas)
- tests/test_api_news.py: 5 DB-backed contract tests (all pass vs real PG)
- docs/news.md + .env.example updates
Both services run under the `ingest` compose profile (matching the
ingester/camera-scraper pattern) and build arm64 on the Pi via the existing
Forgejo CI workflow. telebot left out of scope (reserved env only).
92 lines
2.7 KiB
Python
92 lines
2.7 KiB
Python
# Define your item pipelines here
|
|
#
|
|
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
|
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
|
|
|
import logging
|
|
import psycopg2
|
|
import os
|
|
from scrapy.exceptions import DropItem
|
|
class PostgresPipeline:
|
|
|
|
def __init__(self, db_config):
|
|
# 1. Store the config
|
|
self.db_config = db_config
|
|
# 2. Initialize the set here so it exists when process_item is called
|
|
self.seen_urls = set()
|
|
|
|
@classmethod
|
|
def from_crawler(cls, crawler):
|
|
db_config = {
|
|
'host': crawler.settings.get('DB_HOST'),
|
|
'database': crawler.settings.get('DB_NAME'),
|
|
'user': crawler.settings.get('DB_USER'),
|
|
'password': crawler.settings.get('DB_PASSWORD'),
|
|
}
|
|
return cls(db_config=db_config)
|
|
|
|
|
|
|
|
|
|
def open_spider(self, spider):
|
|
# Connect using environment variables
|
|
self.connection = psycopg2.connect(
|
|
host=os.getenv('DB_HOST'),
|
|
database=os.getenv('DB_NAME'),
|
|
user=os.getenv('DB_USER'),
|
|
password=os.getenv('DB_PASSWORD'),
|
|
port=os.getenv('DB_PORT', '5432')
|
|
)
|
|
self.cur = self.connection.cursor()
|
|
|
|
# Create table if it doesn't exist
|
|
self.cur.execute("""
|
|
CREATE TABLE IF NOT EXISTS articles (
|
|
id SERIAL PRIMARY KEY,
|
|
title TEXT,
|
|
url TEXT UNIQUE,
|
|
content TEXT,
|
|
domain TEXT,
|
|
timestamp TIMESTAMPTZ
|
|
)
|
|
""")
|
|
self.connection.commit()
|
|
|
|
def process_item(self, item, spider):
|
|
if item ['url'] in self.seen_urls:
|
|
raise DropItem()
|
|
try:
|
|
self.cur.execute("""
|
|
INSERT INTO articles (title, url, content, domain, timestamp)
|
|
VALUES (%s, %s, %s, %s, %s)
|
|
ON CONFLICT (url) DO NOTHING
|
|
""", (
|
|
item['title'],
|
|
item['url'],
|
|
item['text'],
|
|
item['domain'],
|
|
item['timestamp']
|
|
))
|
|
if self.cur.rowcount == 0:
|
|
e = DropItem("Duplicate URL (database conflict)")
|
|
e.log_level = logging.DEBUG
|
|
raise e
|
|
self.connection.commit()
|
|
return item
|
|
except Exception as e:
|
|
spider.logger.error(f"Error saving to Postgres: {e}")
|
|
self.connection.rollback()
|
|
raise
|
|
|
|
def close_spider(self, spider):
|
|
self.cur.close()
|
|
self.connection.close()
|
|
|
|
from itemadapter import ItemAdapter
|
|
|
|
|
|
class NewsscraperPipeline:
|
|
def process_item(self, item, spider):
|
|
return item
|
|
|
|
|