diff --git a/app/main.py b/app/main.py index 52333ab..57c9d33 100644 --- a/app/main.py +++ b/app/main.py @@ -21,6 +21,7 @@ from datetime import datetime, timedelta, timezone from decimal import Decimal from pathlib import Path from typing import NoReturn +from urllib.parse import urlparse from uuid import UUID import httpx @@ -43,7 +44,7 @@ from schemas import ( DashboardSummary, EntityCreate, EntityKind, EntityOut, EventCreate, EventOut, FireOut, NewsArticleOut, NewsMapItemOut, NewsSummaryOut, NewsTickerItemOut, - FeedSourceCreate, FeedSourceOut, + FeedSourceCreate, FeedSourceOut, FeedSourceUpdate, KeyOut, KeyValueIn, NewsModelsOut, SettingsIn, SettingsOut, SearchResult, SentimentSummary, SourceType, @@ -51,7 +52,8 @@ from schemas import ( GeofenceCreate, GeofenceUpdate, ) from ingestor import ingest_event, fetch_and_process -from sources import ingest_rss_feed, ingest_gdelt, ingest_earthquakes, ingest_social_signals +from camera_scraper import is_public_url +from sources import GDELT_API, ingest_rss_feed, ingest_gdelt, ingest_earthquakes, ingest_social_signals from fire_sources import ingest_fires from keystore import KeyFormatError, delete_key, list_keys, set_key from settings_store import SettingsError, get_app_settings, list_models, set_summary_model @@ -366,20 +368,22 @@ async def create_source(payload: FeedSourceCreate): @app.patch("/api/sources/{source_id}") -async def update_source(source_id: UUID, payload: dict): - """Update a feed source (e.g., toggle enabled).""" +async def update_source(source_id: UUID, payload: FeedSourceUpdate): + """Update a feed source (name/url/config/enabled only).""" + values = payload.model_dump(exclude_unset=True) async with async_session() as session: row = (await session.execute( select(feed_sources).where(feed_sources.c.id == source_id) )).mappings().one_or_none() if not row: raise HTTPException(404, "Source not found") - await session.execute( - feed_sources.update() - .where(feed_sources.c.id == source_id) - .values(**payload) - ) - await session.commit() + if values: + await session.execute( + feed_sources.update() + .where(feed_sources.c.id == source_id) + .values(**values) + ) + await session.commit() return {"ok": True} @@ -822,9 +826,15 @@ async def put_settings(payload: SettingsIn): # ── Ingestion Triggers ─────────────────────────────────────────────────── +def _require_public_url(url: str, field: str) -> None: + if not is_public_url(url): + raise HTTPException(400, f"{field} is not a public URL") + + @app.post("/api/ingest/rss") async def trigger_rss_ingest(feed_url: str, source_id: str | None = None): """Trigger RSS feed ingestion.""" + _require_public_url(feed_url, "feed_url") count = await ingest_rss_feed(feed_url, source_id) return {"status": "ok", "items_ingested": count} @@ -832,6 +842,10 @@ async def trigger_rss_ingest(feed_url: str, source_id: str | None = None): @app.post("/api/ingest/gdelt") async def trigger_gdelt_ingest(query: str = "", max_articles: int = 50): """Trigger GDELT data ingestion.""" + _require_public_url(GDELT_API, "GDELT target") + parsed = urlparse(query) + if parsed.scheme in ("http", "https") and parsed.hostname: + _require_public_url(query, "query") count = await ingest_gdelt(query, max_articles) return {"status": "ok", "articles_ingested": count} diff --git a/app/schemas.py b/app/schemas.py index 64b86c5..64676fc 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -7,7 +7,7 @@ from enum import Enum from typing import Literal, Optional from uuid import UUID -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator # ─── Enums ─────────────────────────────────────────────────────────────── @@ -63,6 +63,17 @@ class FeedSourceCreate(BaseModel): config: Optional[dict] = None +class FeedSourceUpdate(BaseModel): + """PATCH /api/sources/{id} — only these keys may be set.""" + + model_config = ConfigDict(extra="forbid") + + name: Optional[str] = None + url: Optional[str] = None + config: Optional[dict] = None + enabled: Optional[bool] = None + + class FeedSourceOut(BaseModel): id: UUID name: str diff --git a/tests/test_ingest_ssrf_and_sources.py b/tests/test_ingest_ssrf_and_sources.py new file mode 100644 index 0000000..21c85b2 --- /dev/null +++ b/tests/test_ingest_ssrf_and_sources.py @@ -0,0 +1,67 @@ +"""SSRF guard on ingest triggers + PATCH /api/sources allowlist.""" + +from __future__ import annotations + +import asyncio + +import httpx +import pytest +from pydantic import ValidationError + +from main import app + +BASE = "http://test" +LINK_LOCAL_META = "http://169.254.169.254/latest/meta-data/" +LOOPBACK = "http://127.0.0.1/secret" + + +async def _req(method: str, path: str, **kw) -> httpx.Response: + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url=BASE) as client: + return await client.request(method, path, **kw) + + +def test_rss_ingest_rejects_link_local_metadata_url(monkeypatch): + called = {"n": 0} + + async def _boom(*_a, **_k): + called["n"] += 1 + raise AssertionError("ingest_rss_feed must not run for a private URL") + + monkeypatch.setattr("main.ingest_rss_feed", _boom) + resp = asyncio.run(_req("POST", "/api/ingest/rss", params={"feed_url": LINK_LOCAL_META})) + assert resp.status_code == 400 + assert called["n"] == 0 + + +def test_gdelt_ingest_rejects_private_query_url(monkeypatch): + called = {"n": 0} + + async def _boom(*_a, **_k): + called["n"] += 1 + raise AssertionError("ingest_gdelt must not run for a private URL query") + + monkeypatch.setattr("main.ingest_gdelt", _boom) + resp = asyncio.run(_req("POST", "/api/ingest/gdelt", params={"query": LOOPBACK})) + assert resp.status_code == 400 + assert called["n"] == 0 + + +def test_update_source_rejects_unknown_fields(): + sid = "00000000-0000-0000-0000-000000000001" + resp = asyncio.run(_req("PATCH", f"/api/sources/{sid}", json={"enabled": True, "source_type": "rss"})) + assert resp.status_code == 422 + + +def test_feed_source_update_allowlist_only(): + from schemas import FeedSourceUpdate + + payload = FeedSourceUpdate(name="n", url="https://example.com/rss", config={"k": 1}, enabled=False) + assert payload.model_dump(exclude_unset=True) == { + "name": "n", + "url": "https://example.com/rss", + "config": {"k": 1}, + "enabled": False, + } + with pytest.raises(ValidationError): + FeedSourceUpdate.model_validate({"enabled": True, "id": "00000000-0000-0000-0000-000000000001"})