fix(api): SSRF guard on ingest; whitelist PATCH /api/sources

Reject private/loopback/link-local hosts on POST /api/ingest/rss and
URL-shaped GDELT queries via camera_scraper.is_public_url (HTTP 400).
PATCH /api/sources/{id} only accepts name, url, config, enabled (422 else).
This commit is contained in:
Sirius DevOps 2026-09-01 00:47:15 -04:00
parent ef27e13e50
commit 59974be696
3 changed files with 103 additions and 11 deletions

View file

@ -21,6 +21,7 @@ from datetime import datetime, timedelta, timezone
from decimal import Decimal from decimal import Decimal
from pathlib import Path from pathlib import Path
from typing import NoReturn from typing import NoReturn
from urllib.parse import urlparse
from uuid import UUID from uuid import UUID
import httpx import httpx
@ -43,7 +44,7 @@ from schemas import (
DashboardSummary, EntityCreate, EntityKind, EntityOut, DashboardSummary, EntityCreate, EntityKind, EntityOut,
EventCreate, EventOut, FireOut, NewsArticleOut, NewsMapItemOut, EventCreate, EventOut, FireOut, NewsArticleOut, NewsMapItemOut,
NewsSummaryOut, NewsTickerItemOut, NewsSummaryOut, NewsTickerItemOut,
FeedSourceCreate, FeedSourceOut, FeedSourceCreate, FeedSourceOut, FeedSourceUpdate,
KeyOut, KeyValueIn, KeyOut, KeyValueIn,
NewsModelsOut, SettingsIn, SettingsOut, NewsModelsOut, SettingsIn, SettingsOut,
SearchResult, SentimentSummary, SourceType, SearchResult, SentimentSummary, SourceType,
@ -51,7 +52,8 @@ from schemas import (
GeofenceCreate, GeofenceUpdate, GeofenceCreate, GeofenceUpdate,
) )
from ingestor import ingest_event, fetch_and_process from ingestor import ingest_event, fetch_and_process
from sources import ingest_rss_feed, ingest_gdelt, ingest_earthquakes, ingest_social_signals from camera_scraper import is_public_url
from sources import GDELT_API, ingest_rss_feed, ingest_gdelt, ingest_earthquakes, ingest_social_signals
from fire_sources import ingest_fires from fire_sources import ingest_fires
from keystore import KeyFormatError, delete_key, list_keys, set_key from keystore import KeyFormatError, delete_key, list_keys, set_key
from settings_store import SettingsError, get_app_settings, list_models, set_summary_model from settings_store import SettingsError, get_app_settings, list_models, set_summary_model
@ -366,20 +368,22 @@ async def create_source(payload: FeedSourceCreate):
@app.patch("/api/sources/{source_id}") @app.patch("/api/sources/{source_id}")
async def update_source(source_id: UUID, payload: dict): async def update_source(source_id: UUID, payload: FeedSourceUpdate):
"""Update a feed source (e.g., toggle enabled).""" """Update a feed source (name/url/config/enabled only)."""
values = payload.model_dump(exclude_unset=True)
async with async_session() as session: async with async_session() as session:
row = (await session.execute( row = (await session.execute(
select(feed_sources).where(feed_sources.c.id == source_id) select(feed_sources).where(feed_sources.c.id == source_id)
)).mappings().one_or_none() )).mappings().one_or_none()
if not row: if not row:
raise HTTPException(404, "Source not found") raise HTTPException(404, "Source not found")
await session.execute( if values:
feed_sources.update() await session.execute(
.where(feed_sources.c.id == source_id) feed_sources.update()
.values(**payload) .where(feed_sources.c.id == source_id)
) .values(**values)
await session.commit() )
await session.commit()
return {"ok": True} return {"ok": True}
@ -822,9 +826,15 @@ async def put_settings(payload: SettingsIn):
# ── Ingestion Triggers ─────────────────────────────────────────────────── # ── Ingestion Triggers ───────────────────────────────────────────────────
def _require_public_url(url: str, field: str) -> None:
if not is_public_url(url):
raise HTTPException(400, f"{field} is not a public URL")
@app.post("/api/ingest/rss") @app.post("/api/ingest/rss")
async def trigger_rss_ingest(feed_url: str, source_id: str | None = None): async def trigger_rss_ingest(feed_url: str, source_id: str | None = None):
"""Trigger RSS feed ingestion.""" """Trigger RSS feed ingestion."""
_require_public_url(feed_url, "feed_url")
count = await ingest_rss_feed(feed_url, source_id) count = await ingest_rss_feed(feed_url, source_id)
return {"status": "ok", "items_ingested": count} return {"status": "ok", "items_ingested": count}
@ -832,6 +842,10 @@ async def trigger_rss_ingest(feed_url: str, source_id: str | None = None):
@app.post("/api/ingest/gdelt") @app.post("/api/ingest/gdelt")
async def trigger_gdelt_ingest(query: str = "", max_articles: int = 50): async def trigger_gdelt_ingest(query: str = "", max_articles: int = 50):
"""Trigger GDELT data ingestion.""" """Trigger GDELT data ingestion."""
_require_public_url(GDELT_API, "GDELT target")
parsed = urlparse(query)
if parsed.scheme in ("http", "https") and parsed.hostname:
_require_public_url(query, "query")
count = await ingest_gdelt(query, max_articles) count = await ingest_gdelt(query, max_articles)
return {"status": "ok", "articles_ingested": count} return {"status": "ok", "articles_ingested": count}

View file

@ -7,7 +7,7 @@ from enum import Enum
from typing import Literal, Optional from typing import Literal, Optional
from uuid import UUID from uuid import UUID
from pydantic import BaseModel, Field, field_validator from pydantic import BaseModel, ConfigDict, Field, field_validator
# ─── Enums ─────────────────────────────────────────────────────────────── # ─── Enums ───────────────────────────────────────────────────────────────
@ -63,6 +63,17 @@ class FeedSourceCreate(BaseModel):
config: Optional[dict] = None config: Optional[dict] = None
class FeedSourceUpdate(BaseModel):
"""PATCH /api/sources/{id} — only these keys may be set."""
model_config = ConfigDict(extra="forbid")
name: Optional[str] = None
url: Optional[str] = None
config: Optional[dict] = None
enabled: Optional[bool] = None
class FeedSourceOut(BaseModel): class FeedSourceOut(BaseModel):
id: UUID id: UUID
name: str name: str

View file

@ -0,0 +1,67 @@
"""SSRF guard on ingest triggers + PATCH /api/sources allowlist."""
from __future__ import annotations
import asyncio
import httpx
import pytest
from pydantic import ValidationError
from main import app
BASE = "http://test"
LINK_LOCAL_META = "http://169.254.169.254/latest/meta-data/"
LOOPBACK = "http://127.0.0.1/secret"
async def _req(method: str, path: str, **kw) -> httpx.Response:
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
return await client.request(method, path, **kw)
def test_rss_ingest_rejects_link_local_metadata_url(monkeypatch):
called = {"n": 0}
async def _boom(*_a, **_k):
called["n"] += 1
raise AssertionError("ingest_rss_feed must not run for a private URL")
monkeypatch.setattr("main.ingest_rss_feed", _boom)
resp = asyncio.run(_req("POST", "/api/ingest/rss", params={"feed_url": LINK_LOCAL_META}))
assert resp.status_code == 400
assert called["n"] == 0
def test_gdelt_ingest_rejects_private_query_url(monkeypatch):
called = {"n": 0}
async def _boom(*_a, **_k):
called["n"] += 1
raise AssertionError("ingest_gdelt must not run for a private URL query")
monkeypatch.setattr("main.ingest_gdelt", _boom)
resp = asyncio.run(_req("POST", "/api/ingest/gdelt", params={"query": LOOPBACK}))
assert resp.status_code == 400
assert called["n"] == 0
def test_update_source_rejects_unknown_fields():
sid = "00000000-0000-0000-0000-000000000001"
resp = asyncio.run(_req("PATCH", f"/api/sources/{sid}", json={"enabled": True, "source_type": "rss"}))
assert resp.status_code == 422
def test_feed_source_update_allowlist_only():
from schemas import FeedSourceUpdate
payload = FeedSourceUpdate(name="n", url="https://example.com/rss", config={"k": 1}, enabled=False)
assert payload.model_dump(exclude_unset=True) == {
"name": "n",
"url": "https://example.com/rss",
"config": {"k": 1},
"enabled": False,
}
with pytest.raises(ValidationError):
FeedSourceUpdate.model_validate({"enabled": True, "id": "00000000-0000-0000-0000-000000000001"})