osint-dashboard/app/settings_store.py

"""OSINT Dashboard — non-secret app settings (keyv-style Postgres table).

Model choice lives here so the summarizer container can read it from Postgres.
Only whitelisted names are stored — this is not a generic dump.

Storage: the table is created lazily with ``CREATE TABLE IF NOT EXISTS`` on
first use in each process (same bootstrap pattern as ``api_keys``).
"""

from __future__ import annotations

import asyncio
import os
import time
from datetime import datetime, timezone

import httpx
from sqlalchemy import Column, DateTime, String, Table, Text, func, select, text

from database import async_session, engine, metadata

DEFAULT_LLM_URL = "http://127.0.0.1:11434"
DEFAULT_SUMMARY_MODEL = "qwen3:30b-a3b"
SETTING_SUMMARY_MODEL = "SUMMARY_MODEL"
ALLOWED_SETTINGS = frozenset({SETTING_SUMMARY_MODEL})
MODELS_CACHE_TTL_S = 600.0
MODELS_TIMEOUT_S = 8.0
DEFAULT_MODELS_USER_AGENT = "osint-dashboard-news-summarizer"

FALLBACK_MODELS = [
    "qwen3:30b-a3b",
]

app_settings = Table(
    "app_settings",
    metadata,
    Column("name", String(128), primary_key=True),
    Column("value", Text, nullable=False),
    Column("updated_at", DateTime(timezone=True), server_default=func.now(), nullable=False),
)

_CREATE_TABLE_SQL = text(
    """
    CREATE TABLE IF NOT EXISTS app_settings (
        name       VARCHAR(128) PRIMARY KEY,
        value      TEXT NOT NULL,
        updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
    )
    """
)

_ensure_lock = asyncio.Lock()
_ensured = False
_models_cache: tuple[float, dict] | None = None


class SettingsError(ValueError):
    """Raised when a setting name or value fails validation."""


async def ensure_app_settings_table() -> None:
    """Create the app_settings table if it doesn't exist (idempotent, per process)."""
    global _ensured
    if _ensured:
        return
    async with _ensure_lock:
        if _ensured:
            return
        async with engine.begin() as conn:
            await conn.execute(_CREATE_TABLE_SQL)
        _ensured = True


def llm_url() -> str:
    """Read-only Ollama origin (env, never writable from the UI)."""
    raw = (os.getenv("LLM_URL") or "").strip().rstrip("/")
    for suffix in ("/api/generate", "/api/chat", "/api", "/v1"):
        if raw.endswith(suffix):
            raw = raw[: -len(suffix)].rstrip("/")
            break
    return raw or DEFAULT_LLM_URL


def _validate_summary_model(value: str) -> str:
    stripped = (value or "").strip()
    if not stripped or len(stripped) > 128:
        raise SettingsError("summary_model must be 1–128 chars, not whitespace-only")
    return stripped


async def get_summary_model() -> str:
    """Stored SUMMARY_MODEL, else env, else qwen3:30b-a3b."""
    await ensure_app_settings_table()
    async with async_session() as session:
        row = (
            await session.execute(
                select(app_settings).where(app_settings.c.name == SETTING_SUMMARY_MODEL)
            )
        ).mappings().one_or_none()
    if row and row["value"]:
        return row["value"]
    return os.getenv("SUMMARY_MODEL", DEFAULT_SUMMARY_MODEL)


async def set_summary_model(value: str) -> dict:
    """Upsert SUMMARY_MODEL and return the public settings payload."""
    value = _validate_summary_model(value)
    now = datetime.now(timezone.utc)

    await ensure_app_settings_table()
    async with async_session() as session:
        existing = (
            await session.execute(
                select(app_settings).where(app_settings.c.name == SETTING_SUMMARY_MODEL)
            )
        ).mappings().one_or_none()
        if existing:
            await session.execute(
                app_settings.update()
                .where(app_settings.c.name == SETTING_SUMMARY_MODEL)
                .values(value=value, updated_at=now)
            )
        else:
            await session.execute(
                app_settings.insert().values(
                    name=SETTING_SUMMARY_MODEL, value=value, updated_at=now
                )
            )
        await session.commit()
    return await get_app_settings()


async def get_app_settings() -> dict:
    return {
        "summary_model": await get_summary_model(),
        "llm_url": llm_url(),
    }


def _fallback_payload() -> dict:
    return {
        "source": "fallback",
        "models": [{"id": mid} for mid in FALLBACK_MODELS],
    }


def _models_user_agent() -> str:
    return os.getenv("OSINT_USER_AGENT") or DEFAULT_MODELS_USER_AGENT


def _parse_models_payload(body: object) -> list[dict[str, str]]:
    if isinstance(body, dict):
        raw = body.get("data", body.get("models", []))
    elif isinstance(body, list):
        raw = body
    else:
        raw = []
    out: list[dict[str, str]] = []
    for item in raw or []:
        if isinstance(item, str) and item.strip():
            out.append({"id": item.strip()})
        elif isinstance(item, dict):
            mid = item.get("id") or item.get("name") or item.get("model")
            if mid:
                out.append({"id": str(mid)})
    return out


async def _http_get(url: str, *, headers: dict[str, str], timeout: float) -> httpx.Response:
    async with httpx.AsyncClient(timeout=timeout, headers=headers) as client:
        return await client.get(url)


async def list_models() -> dict:
    """Live ``GET {llm_url}/api/tags``. Never 502 — fallback on failure."""
    global _models_cache
    now = time.monotonic()
    hit = _models_cache
    if hit and now - hit[0] < MODELS_CACHE_TTL_S:
        return hit[1]

    url = f"{llm_url()}/api/tags"
    headers = {
        "User-Agent": _models_user_agent(),
        "Accept": "application/json",
    }
    try:
        resp = await _http_get(url, headers=headers, timeout=MODELS_TIMEOUT_S)
        resp.raise_for_status()
        models = _parse_models_payload(resp.json())
        if not models:
            return _fallback_payload()
        payload = {"source": "live", "models": models}
        _models_cache = (now, payload)
        return payload
    except Exception:
        return _fallback_payload()