8643153954
Scraper loops with NEWS_SCRAPE_INTERVAL_S (default 10s after each crawl). Summarizer runs every NEWS_SUMMARIZE_INTERVAL_S (default 900) over the last 15 minutes of articles. Feed list replaced from the k8s scrapy-urls configmap (334 sources).
55 lines
1.8 KiB
Python
55 lines
1.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Scheduler loop for the news summarizer — every NEWS_SUMMARIZE_INTERVAL_S.
|
|
|
|
Default 900s (15 minutes). Serial: a slow LLM pass never overlaps the next.
|
|
|
|
Env (all optional, 12-factor):
|
|
NEWS_SUMMARIZE_INTERVAL_S seconds between runs (default 900)
|
|
NEWS_SUMMARIZE_RUN_ON_START "1" to summarize once immediately on boot (default 1)
|
|
NOUS_API_KEY optional in env; Keys UI / api_keys also works
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
logger = logging.getLogger("news.summarizer.scheduler")
|
|
|
|
INTERVAL_S = max(1, int(os.getenv("NEWS_SUMMARIZE_INTERVAL_S", "900")))
|
|
RUN_ON_START = os.getenv("NEWS_SUMMARIZE_RUN_ON_START", "1").lower() in ("1", "true", "yes")
|
|
|
|
|
|
def run_summarize() -> None:
|
|
logger.info("summarize starting at %s", datetime.datetime.now().isoformat(timespec="seconds"))
|
|
try:
|
|
proc = subprocess.run([sys.executable, "summarizer.py"], cwd="/app")
|
|
logger.info("summarize finished rc=%s", proc.returncode)
|
|
except Exception: # noqa: BLE001 — keep the loop alive across failures
|
|
logger.exception("summarize failed")
|
|
|
|
|
|
def main() -> None:
|
|
if not os.getenv("NOUS_API_KEY", "").strip():
|
|
logger.warning(
|
|
"NOUS_API_KEY unset in env — will read api_keys on each run; idle if both empty"
|
|
)
|
|
logger.info(
|
|
"news summarizer loop starting (interval_s=%s, run_on_start=%s)",
|
|
INTERVAL_S, RUN_ON_START,
|
|
)
|
|
if RUN_ON_START:
|
|
run_summarize()
|
|
while True:
|
|
logger.info("next summarize in %ss", INTERVAL_S)
|
|
time.sleep(INTERVAL_S)
|
|
run_summarize()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|