fix(api): SSRF guard on ingest; whitelist PATCH /api/sources #46

Merged
sirius merged 2 commits from feat/ssrf-ingest-source-whitelist into master 2026-09-01 00:54:14 -04:00
3 changed files with 103 additions and 11 deletions
Showing only changes of commit 59974be696 - Show all commits

View file

@ -21,6 +21,7 @@ from datetime import datetime, timedelta, timezone
from decimal import Decimal
from pathlib import Path
from typing import NoReturn
from urllib.parse import urlparse
from uuid import UUID
import httpx
@ -43,7 +44,7 @@ from schemas import (
DashboardSummary, EntityCreate, EntityKind, EntityOut,
EventCreate, EventOut, FireOut, NewsArticleOut, NewsMapItemOut,
NewsSummaryOut, NewsTickerItemOut,
FeedSourceCreate, FeedSourceOut,
FeedSourceCreate, FeedSourceOut, FeedSourceUpdate,
KeyOut, KeyValueIn,
NewsModelsOut, SettingsIn, SettingsOut,
SearchResult, SentimentSummary, SourceType,
@ -51,7 +52,8 @@ from schemas import (
GeofenceCreate, GeofenceUpdate,
)
from ingestor import ingest_event, fetch_and_process
from sources import ingest_rss_feed, ingest_gdelt, ingest_earthquakes, ingest_social_signals
from camera_scraper import is_public_url
from sources import GDELT_API, ingest_rss_feed, ingest_gdelt, ingest_earthquakes, ingest_social_signals
from fire_sources import ingest_fires
from keystore import KeyFormatError, delete_key, list_keys, set_key
from settings_store import SettingsError, get_app_settings, list_models, set_summary_model
@ -366,20 +368,22 @@ async def create_source(payload: FeedSourceCreate):
@app.patch("/api/sources/{source_id}")
async def update_source(source_id: UUID, payload: dict):
"""Update a feed source (e.g., toggle enabled)."""
async def update_source(source_id: UUID, payload: FeedSourceUpdate):
"""Update a feed source (name/url/config/enabled only)."""
values = payload.model_dump(exclude_unset=True)
async with async_session() as session:
row = (await session.execute(
select(feed_sources).where(feed_sources.c.id == source_id)
)).mappings().one_or_none()
if not row:
raise HTTPException(404, "Source not found")
await session.execute(
feed_sources.update()
.where(feed_sources.c.id == source_id)
.values(**payload)
)
await session.commit()
if values:
await session.execute(
feed_sources.update()
.where(feed_sources.c.id == source_id)
.values(**values)
)
await session.commit()
return {"ok": True}
@ -822,9 +826,15 @@ async def put_settings(payload: SettingsIn):
# ── Ingestion Triggers ───────────────────────────────────────────────────
def _require_public_url(url: str, field: str) -> None:
if not is_public_url(url):
raise HTTPException(400, f"{field} is not a public URL")
@app.post("/api/ingest/rss")
async def trigger_rss_ingest(feed_url: str, source_id: str | None = None):
"""Trigger RSS feed ingestion."""
_require_public_url(feed_url, "feed_url")
count = await ingest_rss_feed(feed_url, source_id)
return {"status": "ok", "items_ingested": count}
@ -832,6 +842,10 @@ async def trigger_rss_ingest(feed_url: str, source_id: str | None = None):
@app.post("/api/ingest/gdelt")
async def trigger_gdelt_ingest(query: str = "", max_articles: int = 50):
"""Trigger GDELT data ingestion."""
_require_public_url(GDELT_API, "GDELT target")
parsed = urlparse(query)
if parsed.scheme in ("http", "https") and parsed.hostname:
_require_public_url(query, "query")
count = await ingest_gdelt(query, max_articles)
return {"status": "ok", "articles_ingested": count}

View file

@ -7,7 +7,7 @@ from enum import Enum
from typing import Literal, Optional
from uuid import UUID
from pydantic import BaseModel, Field, field_validator
from pydantic import BaseModel, ConfigDict, Field, field_validator
# ─── Enums ───────────────────────────────────────────────────────────────
@ -63,6 +63,17 @@ class FeedSourceCreate(BaseModel):
config: Optional[dict] = None
class FeedSourceUpdate(BaseModel):
"""PATCH /api/sources/{id} — only these keys may be set."""
model_config = ConfigDict(extra="forbid")
name: Optional[str] = None
url: Optional[str] = None
config: Optional[dict] = None
enabled: Optional[bool] = None
class FeedSourceOut(BaseModel):
id: UUID
name: str

View file

@ -0,0 +1,67 @@
"""SSRF guard on ingest triggers + PATCH /api/sources allowlist."""
from __future__ import annotations
import asyncio
import httpx
import pytest
from pydantic import ValidationError
from main import app
BASE = "http://test"
LINK_LOCAL_META = "http://169.254.169.254/latest/meta-data/"
LOOPBACK = "http://127.0.0.1/secret"
async def _req(method: str, path: str, **kw) -> httpx.Response:
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url=BASE) as client:
return await client.request(method, path, **kw)
def test_rss_ingest_rejects_link_local_metadata_url(monkeypatch):
called = {"n": 0}
async def _boom(*_a, **_k):
called["n"] += 1
raise AssertionError("ingest_rss_feed must not run for a private URL")
monkeypatch.setattr("main.ingest_rss_feed", _boom)
resp = asyncio.run(_req("POST", "/api/ingest/rss", params={"feed_url": LINK_LOCAL_META}))
assert resp.status_code == 400
assert called["n"] == 0
def test_gdelt_ingest_rejects_private_query_url(monkeypatch):
called = {"n": 0}
async def _boom(*_a, **_k):
called["n"] += 1
raise AssertionError("ingest_gdelt must not run for a private URL query")
monkeypatch.setattr("main.ingest_gdelt", _boom)
resp = asyncio.run(_req("POST", "/api/ingest/gdelt", params={"query": LOOPBACK}))
assert resp.status_code == 400
assert called["n"] == 0
def test_update_source_rejects_unknown_fields():
sid = "00000000-0000-0000-0000-000000000001"
resp = asyncio.run(_req("PATCH", f"/api/sources/{sid}", json={"enabled": True, "source_type": "rss"}))
assert resp.status_code == 422
def test_feed_source_update_allowlist_only():
from schemas import FeedSourceUpdate
payload = FeedSourceUpdate(name="n", url="https://example.com/rss", config={"k": 1}, enabled=False)
assert payload.model_dump(exclude_unset=True) == {
"name": "n",
"url": "https://example.com/rss",
"config": {"k": 1},
"enabled": False,
}
with pytest.raises(ValidationError):
FeedSourceUpdate.model_validate({"enabled": True, "id": "00000000-0000-0000-0000-000000000001"})