news/summerizer/intel.py

@@ -0,0 +1,90 @@
+"""Pure parser for the news-summarizer reduce JSON / geo / importance contract."""
+
+from __future__ import annotations
+
+import json
+import re
+
+_EMPTY = {"summary_en": "", "ticker": [], "map_items": []}
+_KEEP = frozenset({"critical", "high"})
+_THINK_RE = re.compile(r"<think>.*?</think>", re.DOTALL)
+_FENCE_RE = re.compile(r"```(?:json)?", re.IGNORECASE)
+
+TICKER_HEADLINE_MAX = 140
+MAP_HEADLINE_MAX = 160
+TICKER_CAP = 12
+
+
+def parse_reduce_json(raw: str) -> dict:
+    try:
+        text = _THINK_RE.sub("", raw or "")
+        text = _FENCE_RE.sub("", text)
+        start = text.find("{")
+        end = text.rfind("}")
+        if start == -1 or end == -1 or end < start:
+            return dict(_EMPTY)
+        data = json.loads(text[start : end + 1])
+        if not isinstance(data, dict):
+            return dict(_EMPTY)
+        summary = data.get("summary_en", "")
+        ticker = data.get("ticker", [])
+        map_items = data.get("map_items", [])
+        return {
+            "summary_en": summary if isinstance(summary, str) else "",
+            "ticker": ticker if isinstance(ticker, list) else [],
+            "map_items": map_items if isinstance(map_items, list) else [],
+        }
+    except Exception:
+        return dict(_EMPTY)
+
+
+def clamp_coords(lat, lon) -> tuple[float, float] | None:
+    try:
+        lat_f = float(lat)
+        lon_f = float(lon)
+    except (TypeError, ValueError):
+        return None
+    if not (-90 <= lat_f <= 90 and -180 <= lon_f <= 180):
+        return None
+    return (lat_f, lon_f)
+
+
+def _trimmed_headline(row: dict, limit: int) -> str:
+    headline = row.get("headline") or ""
+    if not isinstance(headline, str):
+        headline = str(headline)
+    return headline.strip()[:limit]
+
+
+def select_ticker(rows: list) -> list:
+    out = []
+    for row in rows:
+        if row.get("importance") not in _KEEP:
+            continue
+        headline = _trimmed_headline(row, TICKER_HEADLINE_MAX)
+        if not headline:
+            continue
+        item = dict(row)
+        item["headline"] = headline
+        out.append(item)
+        if len(out) >= TICKER_CAP:
+            break
+    return out
+
+
+def select_map(items: list) -> list:
+    out = []
+    for row in items:
+        if row.get("importance") not in _KEEP:
+            continue
+        headline = _trimmed_headline(row, MAP_HEADLINE_MAX)
+        if not headline:
+            continue
+        coords = clamp_coords(row.get("lat"), row.get("lon"))
+        if coords is None:
+            continue
+        item = dict(row)
+        item["headline"] = headline
+        item["lat"], item["lon"] = coords
+        out.append(item)
+    return out

news/summerizer/tests/test_intel.py

@@ -0,0 +1,42 @@
+from intel import parse_reduce_json, clamp_coords, select_ticker, select_map
+
+FENCED = """```json
+{"summary_en": "Brief.", "ticker": [
+  {"headline": "Blast in Kyiv", "importance": "critical", "url": "https://ex", "location_name": "Kyiv"}
+], "map_items": [
+  {"headline": "Blast in Kyiv", "importance": "critical", "location_name": "Kyiv, Ukraine",
+   "lat": 50.45, "lon": 30.52, "location_confidence": "city", "category": "military/conflict", "url": "https://ex"}
+]}
+```"""
+
+def test_parse_strips_fence_and_think_tags():
+    raw = "<think>nope</think>\n" + FENCED
+    out = parse_reduce_json(raw)
+    assert out["summary_en"] == "Brief."
+    assert len(out["ticker"]) == 1
+
+def test_parse_empty_and_garbage_returns_empty_struct():
+    assert parse_reduce_json("")["summary_en"] == ""
+    assert parse_reduce_json("not json")["ticker"] == []
+
+def test_clamp_coords_drops_out_of_range_and_unknown():
+    assert clamp_coords(50.45, 30.52) == (50.45, 30.52)
+    assert clamp_coords(95.0, 10.0) is None
+    assert clamp_coords(None, 10.0) is None
+    assert clamp_coords("50.45", "30.52") == (50.45, 30.52)
+
+def test_select_ticker_keeps_critical_high_caps_12():
+    rows = [{"headline": f"h{i}", "importance": "critical"} for i in range(15)]
+    rows.append({"headline": "skip", "importance": "low"})
+    out = select_ticker(rows)
+    assert len(out) == 12
+    assert all(r["importance"] in ("critical", "high") for r in out)
+
+def test_select_map_requires_valid_coords_and_flag():
+    items = [
+        {"headline": "A", "importance": "critical", "lat": 50.45, "lon": 30.52, "location_name": "Kyiv"},
+        {"headline": "B", "importance": "critical", "lat": None, "lon": None, "location_name": "Unknown"},
+        {"headline": "C", "importance": "low", "lat": 1.0, "lon": 2.0, "location_name": "x"},
+    ]
+    out = select_map(items)
+    assert [r["headline"] for r in out] == ["A"]