"""OSINT Dashboard — non-secret app settings (keyv-style Postgres table). Model choice lives here so the summarizer container can read it from Postgres. Only whitelisted names are stored — this is not a generic dump. Storage: the table is created lazily with ``CREATE TABLE IF NOT EXISTS`` on first use in each process (same bootstrap pattern as ``api_keys``). """ from __future__ import annotations import asyncio import os import time from datetime import datetime, timezone import httpx from sqlalchemy import Column, DateTime, String, Table, Text, func, select, text from database import async_session, engine, metadata DEFAULT_LLM_URL = "http://127.0.0.1:11434" DEFAULT_SUMMARY_MODEL = "qwen3:30b-a3b" SETTING_SUMMARY_MODEL = "SUMMARY_MODEL" ALLOWED_SETTINGS = frozenset({SETTING_SUMMARY_MODEL}) MODELS_CACHE_TTL_S = 600.0 MODELS_TIMEOUT_S = 8.0 DEFAULT_MODELS_USER_AGENT = "osint-dashboard-news-summarizer" FALLBACK_MODELS = [ "qwen3:30b-a3b", ] app_settings = Table( "app_settings", metadata, Column("name", String(128), primary_key=True), Column("value", Text, nullable=False), Column("updated_at", DateTime(timezone=True), server_default=func.now(), nullable=False), ) _CREATE_TABLE_SQL = text( """ CREATE TABLE IF NOT EXISTS app_settings ( name VARCHAR(128) PRIMARY KEY, value TEXT NOT NULL, updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ) """ ) _ensure_lock = asyncio.Lock() _ensured = False _models_cache: tuple[float, dict] | None = None class SettingsError(ValueError): """Raised when a setting name or value fails validation.""" async def ensure_app_settings_table() -> None: """Create the app_settings table if it doesn't exist (idempotent, per process).""" global _ensured if _ensured: return async with _ensure_lock: if _ensured: return async with engine.begin() as conn: await conn.execute(_CREATE_TABLE_SQL) _ensured = True def llm_url() -> str: """Read-only Ollama origin (env, never writable from the UI).""" raw = (os.getenv("LLM_URL") or "").strip().rstrip("/") for suffix in ("/api/generate", "/api/chat", "/api", "/v1"): if raw.endswith(suffix): raw = raw[: -len(suffix)].rstrip("/") break return raw or DEFAULT_LLM_URL def _validate_summary_model(value: str) -> str: stripped = (value or "").strip() if not stripped or len(stripped) > 128: raise SettingsError("summary_model must be 1–128 chars, not whitespace-only") return stripped async def get_summary_model() -> str: """Stored SUMMARY_MODEL, else env, else qwen3:30b-a3b.""" await ensure_app_settings_table() async with async_session() as session: row = ( await session.execute( select(app_settings).where(app_settings.c.name == SETTING_SUMMARY_MODEL) ) ).mappings().one_or_none() if row and row["value"]: return row["value"] return os.getenv("SUMMARY_MODEL", DEFAULT_SUMMARY_MODEL) async def set_summary_model(value: str) -> dict: """Upsert SUMMARY_MODEL and return the public settings payload.""" value = _validate_summary_model(value) now = datetime.now(timezone.utc) await ensure_app_settings_table() async with async_session() as session: existing = ( await session.execute( select(app_settings).where(app_settings.c.name == SETTING_SUMMARY_MODEL) ) ).mappings().one_or_none() if existing: await session.execute( app_settings.update() .where(app_settings.c.name == SETTING_SUMMARY_MODEL) .values(value=value, updated_at=now) ) else: await session.execute( app_settings.insert().values( name=SETTING_SUMMARY_MODEL, value=value, updated_at=now ) ) await session.commit() return await get_app_settings() async def get_app_settings() -> dict: return { "summary_model": await get_summary_model(), "llm_url": llm_url(), } def _fallback_payload() -> dict: return { "source": "fallback", "models": [{"id": mid} for mid in FALLBACK_MODELS], } def _models_user_agent() -> str: return os.getenv("OSINT_USER_AGENT") or DEFAULT_MODELS_USER_AGENT def _parse_models_payload(body: object) -> list[dict[str, str]]: if isinstance(body, dict): raw = body.get("data", body.get("models", [])) elif isinstance(body, list): raw = body else: raw = [] out: list[dict[str, str]] = [] for item in raw or []: if isinstance(item, str) and item.strip(): out.append({"id": item.strip()}) elif isinstance(item, dict): mid = item.get("id") or item.get("name") or item.get("model") if mid: out.append({"id": str(mid)}) return out async def _http_get(url: str, *, headers: dict[str, str], timeout: float) -> httpx.Response: async with httpx.AsyncClient(timeout=timeout, headers=headers) as client: return await client.get(url) async def list_models() -> dict: """Live ``GET {llm_url}/api/tags``. Never 502 — fallback on failure.""" global _models_cache now = time.monotonic() hit = _models_cache if hit and now - hit[0] < MODELS_CACHE_TTL_S: return hit[1] url = f"{llm_url()}/api/tags" headers = { "User-Agent": _models_user_agent(), "Accept": "application/json", } try: resp = await _http_get(url, headers=headers, timeout=MODELS_TIMEOUT_S) resp.raise_for_status() models = _parse_models_payload(resp.json()) if not models: return _fallback_payload() payload = {"source": "live", "models": models} _models_cache = (now, payload) return payload except Exception: return _fallback_payload()