feat: load k8s news prompts from env/files, not Python
Bundled MAP_PROMPT/SUMMARY_PROMPT from the customer1 deepseek configmap. Env wins over prompt files; blank compose injection is treated as unset. Parser maps market_overview JSON onto the dashboard ticker/map contract.
This commit is contained in:
Sirius DevOps
parent
c10b617f1f
commit
1c47ecbc5a
11 changed files with 444 additions and 73 deletions
|
|
@ -85,6 +85,11 @@ SUMMARY_WINDOW_HOURS=1
|
||||||
# Futures/markets coupling from the upstream pipeline is OFF by default
|
# Futures/markets coupling from the upstream pipeline is OFF by default
|
||||||
# (irrelevant to OSINT). Set INCLUDE_FUTURES=1 + install yfinance to enable.
|
# (irrelevant to OSINT). Set INCLUDE_FUTURES=1 + install yfinance to enable.
|
||||||
INCLUDE_FUTURES=0
|
INCLUDE_FUTURES=0
|
||||||
|
# Prompts: bundled from k8s deepseek-configmap (prompt_files/).
|
||||||
|
# Set MAP_PROMPT / SUMMARY_PROMPT in this file to override without rebuilding.
|
||||||
|
# Blank is treated as unset (compose injects ${MAP_PROMPT:-}).
|
||||||
|
# MAP_PROMPT=
|
||||||
|
# SUMMARY_PROMPT=
|
||||||
# Wall-clock scheduling (k8s CronJob replacement): scrape minute, summarize minute
|
# Wall-clock scheduling (k8s CronJob replacement): scrape minute, summarize minute
|
||||||
NEWS_SCRAPE_MINUTE=0
|
NEWS_SCRAPE_MINUTE=0
|
||||||
NEWS_SUMMARIZE_MINUTE=5
|
NEWS_SUMMARIZE_MINUTE=5
|
||||||
|
|
|
||||||
|
|
@ -234,6 +234,10 @@ services:
|
||||||
NEWS_SUMMARIZE_MINUTE: ${NEWS_SUMMARIZE_MINUTE:-5}
|
NEWS_SUMMARIZE_MINUTE: ${NEWS_SUMMARIZE_MINUTE:-5}
|
||||||
NEWS_SUMMARIZE_RUN_ON_START: ${NEWS_SUMMARIZE_RUN_ON_START:-1}
|
NEWS_SUMMARIZE_RUN_ON_START: ${NEWS_SUMMARIZE_RUN_ON_START:-1}
|
||||||
NEWS_SUMMARIZE_FORCE: ${NEWS_SUMMARIZE_FORCE:-0}
|
NEWS_SUMMARIZE_FORCE: ${NEWS_SUMMARIZE_FORCE:-0}
|
||||||
|
MAP_PROMPT: ${MAP_PROMPT:-}
|
||||||
|
SUMMARY_PROMPT: ${SUMMARY_PROMPT:-}
|
||||||
|
MAP_PROMPT_FILE: ${MAP_PROMPT_FILE:-/app/prompt_files/map.txt}
|
||||||
|
SUMMARY_PROMPT_FILE: ${SUMMARY_PROMPT_FILE:-/app/prompt_files/summary.txt}
|
||||||
command: ["python", "run_news_summarizer.py"]
|
command: ["python", "run_news_summarizer.py"]
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
|
|
|
||||||
19
docs/news.md
19
docs/news.md
|
|
@ -258,11 +258,20 @@ No Hermes-4 reasoning system prompt. Reduce uses `json_mode=True`.
|
||||||
|
|
||||||
## Prompts
|
## Prompts
|
||||||
|
|
||||||
Both prompts are env-overridable — the default `MAP_PROMPT` is OSINT-neutral
|
Map/reduce prompts are the k8s `deepseek-configmap` text in
|
||||||
(facts, locations, entities, category, OSINT signal per article; English) and
|
`news/summerizer/prompt_files/{map,summary}.txt` — **not** hardcoded in
|
||||||
the default `SUMMARY_PROMPT` demands the reduce JSON above (with a
|
Python. Resolution order: `MAP_PROMPT` / `SUMMARY_PROMPT` env (non-blank
|
||||||
"no qualifying events" escape hatch). Upstream's futures/markets prompt
|
wins) → `MAP_PROMPT_FILE` / `SUMMARY_PROMPT_FILE` → bundled files.
|
||||||
language is gated behind `INCLUDE_FUTURES=1`.
|
|
||||||
|
Compose passes `${MAP_PROMPT:-}` so a host `.env` swap takes effect on
|
||||||
|
container recreate (no image rebuild). Blank env is treated as unset.
|
||||||
|
|
||||||
|
The reduce JSON is the market-intel schema (`market_overview` +
|
||||||
|
`geopolitical_osint`). `intel.parse_reduce_json` maps it onto the dashboard
|
||||||
|
`summary_en` / ticker / map_items contract. Additive `lat`/`lon` on
|
||||||
|
`critical_events` and `active_conflicts` feed Critical News pins.
|
||||||
|
|
||||||
|
Upstream's futures/markets tape is gated behind `INCLUDE_FUTURES=1`.
|
||||||
|
|
||||||
## Tests
|
## Tests
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
COPY summarizer.py run_news_summarizer.py ./
|
COPY intel.py nous_client.py prompts.py summarizer.py run_news_summarizer.py ./
|
||||||
|
COPY prompt_files ./prompt_files
|
||||||
|
|
||||||
# Security: run as a non-privileged user.
|
# Security: run as a non-privileged user.
|
||||||
RUN useradd -m summarizer_user
|
RUN useradd -m summarizer_user
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,115 @@ TICKER_CAP = 12
|
||||||
MAP_CAP = 20
|
MAP_CAP = 20
|
||||||
|
|
||||||
|
|
||||||
|
def _impact_to_importance(level) -> str | None:
|
||||||
|
lv = str(level or "").strip().lower()
|
||||||
|
if lv in {"high", "critical"}:
|
||||||
|
return "critical" if lv == "critical" else "high"
|
||||||
|
if lv == "medium":
|
||||||
|
return "high"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _from_market_intel(data: dict) -> dict:
|
||||||
|
"""Map the k8s deepseek reduce JSON onto summary_en / ticker / map_items."""
|
||||||
|
raw_ov = data.get("market_overview")
|
||||||
|
ov: dict = raw_ov if isinstance(raw_ov, dict) else {}
|
||||||
|
raw_geo = data.get("geopolitical_osint")
|
||||||
|
geo: dict = raw_geo if isinstance(raw_geo, dict) else {}
|
||||||
|
raw_events = ov.get("critical_events")
|
||||||
|
events: list = raw_events if isinstance(raw_events, list) else []
|
||||||
|
raw_themes = ov.get("key_themes")
|
||||||
|
themes: list = raw_themes if isinstance(raw_themes, list) else []
|
||||||
|
raw_conflicts = geo.get("active_conflicts")
|
||||||
|
conflicts: list = raw_conflicts if isinstance(raw_conflicts, list) else []
|
||||||
|
|
||||||
|
parts: list[str] = []
|
||||||
|
sentiment = ov.get("overall_sentiment")
|
||||||
|
if sentiment:
|
||||||
|
score = ov.get("sentiment_score", "")
|
||||||
|
parts.append(f"**Sentiment:** {sentiment}" + (f" ({score})" if score != "" else ""))
|
||||||
|
if themes:
|
||||||
|
parts.append("**Themes:** " + ", ".join(str(t) for t in themes[:5] if t))
|
||||||
|
for ev in events[:8]:
|
||||||
|
if not isinstance(ev, dict):
|
||||||
|
continue
|
||||||
|
headline = (ev.get("headline") or "").strip()
|
||||||
|
desc = (ev.get("description") or "").strip()
|
||||||
|
if headline and desc:
|
||||||
|
parts.append(f"### {headline}\n{desc}")
|
||||||
|
elif headline:
|
||||||
|
parts.append(f"### {headline}")
|
||||||
|
elif desc:
|
||||||
|
parts.append(desc)
|
||||||
|
for c in conflicts[:4]:
|
||||||
|
if not isinstance(c, dict):
|
||||||
|
continue
|
||||||
|
intel = (c.get("intelligence") or "").strip()
|
||||||
|
region = (c.get("region") or "").strip()
|
||||||
|
if intel:
|
||||||
|
parts.append(f"**{region or 'Conflict'}:** {intel}")
|
||||||
|
summary_en = "\n\n".join(parts)
|
||||||
|
|
||||||
|
ticker: list[dict] = []
|
||||||
|
for ev in events:
|
||||||
|
if not isinstance(ev, dict):
|
||||||
|
continue
|
||||||
|
importance = _impact_to_importance(ev.get("impact_level"))
|
||||||
|
if not importance:
|
||||||
|
continue
|
||||||
|
headline = (ev.get("headline") or "").strip()
|
||||||
|
if not headline:
|
||||||
|
continue
|
||||||
|
ticker.append({
|
||||||
|
"headline": headline,
|
||||||
|
"importance": importance,
|
||||||
|
"url": ev.get("url") or "",
|
||||||
|
"location_name": ev.get("location_name") or ev.get("region") or "",
|
||||||
|
})
|
||||||
|
|
||||||
|
map_items: list[dict] = []
|
||||||
|
for c in conflicts:
|
||||||
|
if not isinstance(c, dict):
|
||||||
|
continue
|
||||||
|
status = str(c.get("status") or "").lower()
|
||||||
|
importance = "critical" if status == "escalating" else "high"
|
||||||
|
headline = (c.get("intelligence") or c.get("region") or "").strip()
|
||||||
|
if not headline:
|
||||||
|
continue
|
||||||
|
map_items.append({
|
||||||
|
"headline": headline,
|
||||||
|
"importance": importance,
|
||||||
|
"location_name": c.get("location_name") or c.get("region") or "",
|
||||||
|
"lat": c.get("lat"),
|
||||||
|
"lon": c.get("lon"),
|
||||||
|
"location_confidence": "region",
|
||||||
|
"category": "military/conflict",
|
||||||
|
"url": "",
|
||||||
|
})
|
||||||
|
for ev in events:
|
||||||
|
if not isinstance(ev, dict):
|
||||||
|
continue
|
||||||
|
importance = _impact_to_importance(ev.get("impact_level"))
|
||||||
|
if not importance:
|
||||||
|
continue
|
||||||
|
if clamp_coords(ev.get("lat"), ev.get("lon")) is None:
|
||||||
|
continue
|
||||||
|
headline = (ev.get("headline") or "").strip()
|
||||||
|
if not headline:
|
||||||
|
continue
|
||||||
|
map_items.append({
|
||||||
|
"headline": headline,
|
||||||
|
"importance": importance,
|
||||||
|
"location_name": ev.get("location_name") or "",
|
||||||
|
"lat": ev.get("lat"),
|
||||||
|
"lon": ev.get("lon"),
|
||||||
|
"location_confidence": "city",
|
||||||
|
"category": ev.get("category") or "other",
|
||||||
|
"url": ev.get("url") or "",
|
||||||
|
})
|
||||||
|
return {"summary_en": summary_en, "ticker": ticker, "map_items": map_items}
|
||||||
|
|
||||||
|
|
||||||
def parse_reduce_json(raw: str) -> dict:
|
def parse_reduce_json(raw: str) -> dict:
|
||||||
try:
|
try:
|
||||||
text = _THINK_RE.sub("", raw or "")
|
text = _THINK_RE.sub("", raw or "")
|
||||||
|
|
@ -27,14 +136,18 @@ def parse_reduce_json(raw: str) -> dict:
|
||||||
data = json.loads(text[start : end + 1])
|
data = json.loads(text[start : end + 1])
|
||||||
if not isinstance(data, dict):
|
if not isinstance(data, dict):
|
||||||
return dict(_EMPTY)
|
return dict(_EMPTY)
|
||||||
summary = data.get("summary_en", "")
|
if isinstance(data.get("summary_en"), str) or isinstance(data.get("ticker"), list):
|
||||||
ticker = data.get("ticker", [])
|
summary = data.get("summary_en", "")
|
||||||
map_items = data.get("map_items", [])
|
ticker = data.get("ticker", [])
|
||||||
return {
|
map_items = data.get("map_items", [])
|
||||||
"summary_en": summary if isinstance(summary, str) else "",
|
return {
|
||||||
"ticker": ticker if isinstance(ticker, list) else [],
|
"summary_en": summary if isinstance(summary, str) else "",
|
||||||
"map_items": map_items if isinstance(map_items, list) else [],
|
"ticker": ticker if isinstance(ticker, list) else [],
|
||||||
}
|
"map_items": map_items if isinstance(map_items, list) else [],
|
||||||
|
}
|
||||||
|
if "market_overview" in data or "geopolitical_osint" in data:
|
||||||
|
return _from_market_intel(data)
|
||||||
|
return dict(_EMPTY)
|
||||||
except Exception:
|
except Exception:
|
||||||
return dict(_EMPTY)
|
return dict(_EMPTY)
|
||||||
|
|
||||||
|
|
|
||||||
50
news/summerizer/prompt_files/map.txt
Normal file
50
news/summerizer/prompt_files/map.txt
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
# ROLE
|
||||||
|
Market Intelligence Extractor — Map Phase.
|
||||||
|
You receive a batch of full-text news articles. Extract structured, machine-readable intelligence from each.
|
||||||
|
|
||||||
|
# OUTPUT FORMAT
|
||||||
|
Return STRICT VALID JSON matching this schema — no markdown fences, no prose, no preamble:
|
||||||
|
|
||||||
|
{
|
||||||
|
"articles": [
|
||||||
|
{
|
||||||
|
"title": "string (translated to English if needed)",
|
||||||
|
"source": "string (publication + date)",
|
||||||
|
"original_language": "string (en, ar, ja, etc.)",
|
||||||
|
"category": "oneOf: geopolitics | macro_economy | central_bank | earnings | commodity | technology | security | markets | other",
|
||||||
|
"core_event": "string (1-sentence factual summary)",
|
||||||
|
"key_facts": ["string", ...],
|
||||||
|
"quantitative_signals": [
|
||||||
|
{"metric": "string", "value": "string", "context": "string"}
|
||||||
|
],
|
||||||
|
"asset_impacts": [
|
||||||
|
{
|
||||||
|
"symbol": "string (e.g. /GC, /CL, /ES)",
|
||||||
|
"direction": "oneOf: bullish | bearish | neutral | uncertain",
|
||||||
|
"impact_level": "oneOf: high | medium | low",
|
||||||
|
"reasoning": "string"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"credibility_score": "number (1-5, 5=highest)",
|
||||||
|
"source_bias": "string (e.g. 'state media', 'financial press', 'neutral wire')",
|
||||||
|
"osint_tags": ["string", ...]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"batch_metadata": {
|
||||||
|
"total_articles": "number",
|
||||||
|
"dominant_themes": ["string", ...],
|
||||||
|
"contradictions": ["string (article A vs article B conflict)", ...]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# EXTRACTION RULES
|
||||||
|
- Translate ALL non-English content to English. Preserve original_language field.
|
||||||
|
- Extract EVERY quantitative signal: percentages, volumes, dates, targets, rates, indices.
|
||||||
|
- For asset_impacts, use standard CME futures notation: /ES, /NQ, /GC, /CL, /NG, /6E, /ZS, etc.
|
||||||
|
- osint_tags: domain-agnostic labels useful for non-trading consumers (e.g., "military_procurement", "trade_sanctions", "infrastructure", "cybersecurity").
|
||||||
|
- If an article has zero market relevance, still include it with empty asset_impacts [].
|
||||||
|
- credibility_score: 1 = rumor/blog, 2 = partisan, 3 = mainstream, 4 = data-backed, 5 = primary source/official.
|
||||||
|
- Output ONLY the JSON object. No backticks, no ```json, no explanation.
|
||||||
|
|
||||||
|
# DATA TO PROCESS
|
||||||
|
{batch_text}
|
||||||
119
news/summerizer/prompt_files/summary.txt
Normal file
119
news/summerizer/prompt_files/summary.txt
Normal file
|
|
@ -0,0 +1,119 @@
|
||||||
|
# ROLE
|
||||||
|
Lead Market Intelligence Analyst — Reduce Phase.
|
||||||
|
Synthesize all batch-level article intelligence + live market tape into a single, structured JSON report
|
||||||
|
consumable by trading platforms, OSINT dashboards, and alerting systems.
|
||||||
|
|
||||||
|
# OUTPUT FORMAT
|
||||||
|
Return STRICT VALID JSON — no markdown fences, no prose wrapper, no preamble:
|
||||||
|
|
||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"generated_at": "ISO-8601 timestamp",
|
||||||
|
"batch_id": "YYYY-MM-DDTHH (hour of analysis)",
|
||||||
|
"articles_processed": "number",
|
||||||
|
"data_sources": ["string", ...]
|
||||||
|
},
|
||||||
|
"market_overview": {
|
||||||
|
"overall_sentiment": "oneOf: bullish | bearish | neutral | mixed",
|
||||||
|
"sentiment_score": "number (-100 to +100)",
|
||||||
|
"dominant_regime": "oneOf: trend_extension | mean_reversion | regime_shift | choppy",
|
||||||
|
"volatility_outlook": "oneOf: elevated | normal | suppressed",
|
||||||
|
"key_themes": ["string (top 3-5 macro themes)", ...],
|
||||||
|
"critical_events": [
|
||||||
|
{
|
||||||
|
"headline": "string",
|
||||||
|
"category": "string",
|
||||||
|
"impact_level": "oneOf: high | medium | low",
|
||||||
|
"description": "string (2-3 sentences)",
|
||||||
|
"source_credibility": "number (1-5)",
|
||||||
|
"location_name": "string (city/region/country if the event is geolocated; else empty)",
|
||||||
|
"lat": "number or null (estimated WGS84; omit/null if unknown)",
|
||||||
|
"lon": "number or null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"trading_signals": [
|
||||||
|
{
|
||||||
|
"ticker": "string (e.g. /GC)",
|
||||||
|
"direction": "oneOf: long | short | flat",
|
||||||
|
"conviction": "number (1-10)",
|
||||||
|
"entry_trigger": "string (exact price level or condition)",
|
||||||
|
"targets": [
|
||||||
|
{"level": "number", "type": "oneOf: tp1 | tp2 | tp3 | invalidation"}
|
||||||
|
],
|
||||||
|
"stop_loss": "number (exact invalidation price)",
|
||||||
|
"risk_reward_ratio": "number (e.g. 2.5)",
|
||||||
|
"expected_move_pct": "number",
|
||||||
|
"catalyst": "string (what drives this setup)",
|
||||||
|
"rationale": "string (news synthesis + technical anchor)",
|
||||||
|
"time_horizon": "oneOf: intraday | swing_1d | swing_3d | swing_5d",
|
||||||
|
"invalidation_event": "string (what kills the thesis)",
|
||||||
|
"news_sources_count": "number (how many articles support this)"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"geopolitical_osint": {
|
||||||
|
"risk_score": "number (1-10, 10=max disruption)",
|
||||||
|
"active_conflicts": [
|
||||||
|
{
|
||||||
|
"region": "string",
|
||||||
|
"status": "oneOf: escalating | stable | de-escalating",
|
||||||
|
"markets_at_risk": ["string (futures symbols)", ...],
|
||||||
|
"intelligence": "string (what changed and why it matters)",
|
||||||
|
"location_name": "string (city/region/country)",
|
||||||
|
"lat": "number or null (estimated WGS84; omit/null if unknown)",
|
||||||
|
"lon": "number or null"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"policy_shifts": [
|
||||||
|
{
|
||||||
|
"jurisdiction": "string",
|
||||||
|
"policy": "string",
|
||||||
|
"market_impact": "string"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"technology_intelligence": [
|
||||||
|
{
|
||||||
|
"sector": "string",
|
||||||
|
"development": "string",
|
||||||
|
"relevance": "string (why this matters)"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"watchlist": [
|
||||||
|
{
|
||||||
|
"ticker": "string",
|
||||||
|
"reason": "string (1-line monitoring note)",
|
||||||
|
"key_level": "number (price to watch)"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"regime_summary": {
|
||||||
|
"trend_bias": "oneOf: bullish | bearish | neutral",
|
||||||
|
"breadth": "string (e.g. 'broad-based rally' or 'narrow leadership')",
|
||||||
|
"liquidity": "oneOf: abundant | normal | draining",
|
||||||
|
"key_resistance": "string (macro resistance zone or event)",
|
||||||
|
"key_support": "string (macro support zone or event)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# REASONING PROTOCOL (INTERNAL — do NOT include in output)
|
||||||
|
Execute these steps internally before producing JSON:
|
||||||
|
|
||||||
|
1. ARTICLE SYNTHESIS — Merge all article summaries. Identify reinforcing themes and contradictions.
|
||||||
|
2. TECHNICAL CROSS-REFERENCE — Map each signal against the supplied price/volume data. Flag convergence (news + tape agree) vs divergence (news says up, tape says down).
|
||||||
|
3. IMPACT SCORING — Rate each setup by (conviction × magnitude × R:R). Rank top 5-7.
|
||||||
|
4. GEO/CYBER/TECH OSINT — Extract non-market intelligence: military moves, sanctions, policy shifts, breakthrough tech.
|
||||||
|
5. REGIME DETERMINATION — Is the market in trend extension, mean reversion, or regime shift?
|
||||||
|
6. SELF-CRITIQUE — Challenge conviction scores. Downgrade if evidence is thin or sources conflict.
|
||||||
|
7. LEVEL VALIDATION — Every price level must trace to supplied tape data. Never fabricate.
|
||||||
|
|
||||||
|
# CONSTRAINTS
|
||||||
|
- Output MUST be valid JSON parseable by json.loads(). No trailing commas, no comments.
|
||||||
|
- No ```json fences — raw JSON only.
|
||||||
|
- All prices as numbers, not strings.
|
||||||
|
- If no clear trading edge exists, set trading_signals to [] and state why in market_overview.key_themes.
|
||||||
|
- sentiment_score: -100 = max bearish, 0 = neutral, +100 = max bullish.
|
||||||
|
- Use standard CME futures notation everywhere: /CL, /GC, /ES, /NQ, /NG, /6E, /ZS, etc.
|
||||||
|
- Keep descriptions concise. This JSON is consumed programmatically AND rendered for humans.
|
||||||
|
|
||||||
|
# INPUT DATA
|
||||||
|
{final_input}
|
||||||
36
news/summerizer/prompts.py
Normal file
36
news/summerizer/prompts.py
Normal file
|
|
@ -0,0 +1,36 @@
|
||||||
|
"""Load map/reduce prompts: env wins, else file, else bundled k8s prompts.
|
||||||
|
|
||||||
|
Blank ``MAP_PROMPT`` / ``SUMMARY_PROMPT`` (compose ``${VAR:-}``) is treated as
|
||||||
|
unset so a host .env can swap text without rebuilding the image.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_BUNDLED = Path(__file__).resolve().parent / "prompt_files"
|
||||||
|
_MAP_FILE = "map.txt"
|
||||||
|
_SUMMARY_FILE = "summary.txt"
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_prompt(env_name: str, file_env: str, bundled_name: str) -> str:
|
||||||
|
raw = (os.getenv(env_name) or "").strip()
|
||||||
|
if raw:
|
||||||
|
return raw
|
||||||
|
path = (os.getenv(file_env) or "").strip() or str(_BUNDLED / bundled_name)
|
||||||
|
try:
|
||||||
|
text = Path(path).read_text(encoding="utf-8")
|
||||||
|
except OSError as exc:
|
||||||
|
raise RuntimeError(f"{env_name} unset and prompt file unreadable: {path}") from exc
|
||||||
|
if not text.strip():
|
||||||
|
raise RuntimeError(f"{env_name} unset and prompt file empty: {path}")
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def map_prompt_template() -> str:
|
||||||
|
return resolve_prompt("MAP_PROMPT", "MAP_PROMPT_FILE", _MAP_FILE)
|
||||||
|
|
||||||
|
|
||||||
|
def summary_prompt_template() -> str:
|
||||||
|
return resolve_prompt("SUMMARY_PROMPT", "SUMMARY_PROMPT_FILE", _SUMMARY_FILE)
|
||||||
|
|
@ -19,6 +19,8 @@ of each summarize_news() — env wins, else api_keys / app_settings:
|
||||||
OSINT_USER_AGENT default osint-dashboard-news-summarizer
|
OSINT_USER_AGENT default osint-dashboard-news-summarizer
|
||||||
MAP_PROMPT override map-phase prompt (uses {batch_text})
|
MAP_PROMPT override map-phase prompt (uses {batch_text})
|
||||||
SUMMARY_PROMPT override reduce-phase prompt (uses {final_input})
|
SUMMARY_PROMPT override reduce-phase prompt (uses {final_input})
|
||||||
|
MAP_PROMPT_FILE path to map prompt (default /app/prompt_files/map.txt)
|
||||||
|
SUMMARY_PROMPT_FILE path to reduce prompt
|
||||||
NEWS_SUMMARIZE_FORCE "1" to ignore the current-UTC-hour idempotency skip
|
NEWS_SUMMARIZE_FORCE "1" to ignore the current-UTC-hour idempotency skip
|
||||||
INCLUDE_FUTURES "1" to prepend live futures prices (default 0)
|
INCLUDE_FUTURES "1" to prepend live futures prices (default 0)
|
||||||
|
|
||||||
|
|
@ -38,6 +40,7 @@ import psycopg2
|
||||||
|
|
||||||
from intel import parse_reduce_json, select_map, select_ticker
|
from intel import parse_reduce_json, select_map, select_ticker
|
||||||
from nous_client import chat
|
from nous_client import chat
|
||||||
|
from prompts import map_prompt_template, summary_prompt_template
|
||||||
|
|
||||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||||
logger = logging.getLogger("news.summarizer")
|
logger = logging.getLogger("news.summarizer")
|
||||||
|
|
@ -66,63 +69,8 @@ FUTURES_TICKERS = {
|
||||||
"Currencies": ["6E=F", "6J=F", "6B=F"],
|
"Currencies": ["6E=F", "6J=F", "6B=F"],
|
||||||
}
|
}
|
||||||
|
|
||||||
# ── OSINT-neutral default prompts (env-overridable via MAP_PROMPT/SUMMARY_PROMPT) ──
|
# Prompts live in prompt_files/ (k8s deepseek-configmap). Override at runtime
|
||||||
|
# with MAP_PROMPT / SUMMARY_PROMPT (env wins) or MAP_PROMPT_FILE / SUMMARY_PROMPT_FILE.
|
||||||
MAP_PROMPT_DEFAULT = """\
|
|
||||||
You are a precise, factual OSINT news processor. Your ONLY source of information is the articles provided below. Do NOT add external knowledge, assumptions, training data, or invented facts.
|
|
||||||
|
|
||||||
Write every field in English. Translate if the article is not English.
|
|
||||||
|
|
||||||
For EACH article in the batch:
|
|
||||||
1. Extract 2-4 key factual bullet points (who, what, when, where, numbers, quotes — stay very close to the text).
|
|
||||||
2. Location: country/city/region or Unknown. If you can estimate coordinates, emit them as numbers; otherwise omit.
|
|
||||||
3. Entities: list the key people, organizations, or governments mentioned (comma-separated, only names present in the text), else "None".
|
|
||||||
4. Category: pick one — politics, military/conflict, economy, technology, environment/disaster, health, crime, society, sport, other.
|
|
||||||
5. OSINT signal: if the article describes an event with geopolitical, security, military, economic, or disaster significance, say so in one short sentence. Otherwise write: "No notable OSINT signal."
|
|
||||||
6. Importance: critical (breaking geopolitical/military/disaster with immediate impact), high, medium, low, none.
|
|
||||||
|
|
||||||
If several articles cover the same story, add one short batch-level note at the end: "Batch theme: [one sentence]".
|
|
||||||
|
|
||||||
Output format — strictly one block per article:
|
|
||||||
|
|
||||||
Article 1:
|
|
||||||
- Fact bullet 1
|
|
||||||
- Fact bullet 2
|
|
||||||
- Location: ...
|
|
||||||
- Entities: ...
|
|
||||||
- Category: ...
|
|
||||||
- OSINT signal: ...
|
|
||||||
- Importance: ...
|
|
||||||
- Lat: ...
|
|
||||||
- Lon: ...
|
|
||||||
|
|
||||||
Article 2:
|
|
||||||
...
|
|
||||||
|
|
||||||
Articles in this batch:
|
|
||||||
{batch_text}
|
|
||||||
"""
|
|
||||||
|
|
||||||
SUMMARY_PROMPT_DEFAULT = """\
|
|
||||||
CRITICAL INSTRUCTION - REPEAT 3 TIMES: YOU MUST USE ONLY THE DATA PROVIDED BELOW. DO NOT INVENT, RECALL, OR ADD ANY EVENTS, NAMES, DATES, IMPLICATIONS, PROJECTS, OR DETAILS NOT EXPLICITLY PRESENT IN THE DATA. IF THE DATA HAS NO MAJOR GEOPOLITICAL/TECH/MILITARY/ECONOMIC/IMPACTFUL EVENTS OR UNUSUAL STORIES, set summary_en to exactly: "No qualifying impactful or unusual events in the recent hourly news data." and use empty ticker and map_items arrays. AND STOP. NO EXTERNAL KNOWLEDGE FROM TRAINING.
|
|
||||||
|
|
||||||
All text in English.
|
|
||||||
|
|
||||||
Demand a single JSON object (no markdown fences) with this exact shape:
|
|
||||||
|
|
||||||
{
|
|
||||||
"summary_en": "English markdown brief or the no-qualifying-events sentence",
|
|
||||||
"ticker": [{"headline": "", "importance": "critical", "url": "", "location_name": ""}],
|
|
||||||
"map_items": [{"headline": "", "importance": "critical", "location_name": "", "lat": 0, "lon": 0, "location_confidence": "city", "category": "military/conflict", "url": ""}]
|
|
||||||
}
|
|
||||||
|
|
||||||
ticker: only critical and high, max 12, ≤140 chars, no markdown.
|
|
||||||
map_items: only critical and high where a real-world location is explicit in the data. Estimate lat/lon. If location is Unknown or not in the data, omit the item. Never invent a place. Max 20.
|
|
||||||
summary_en: English markdown brief for an operator HUD.
|
|
||||||
|
|
||||||
DATA:
|
|
||||||
{final_input}
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
# ── LLM helpers ────────────────────────────────────────────────────────────
|
# ── LLM helpers ────────────────────────────────────────────────────────────
|
||||||
|
|
@ -399,7 +347,7 @@ def build_map_prompt(batch: list[dict]) -> str:
|
||||||
f"Title: {a['title']}\nSource: {a['domain']}\nURL: {a['url']}\nContent: {a['content'][:1500]}"
|
f"Title: {a['title']}\nSource: {a['domain']}\nURL: {a['url']}\nContent: {a['content'][:1500]}"
|
||||||
for a in batch
|
for a in batch
|
||||||
)
|
)
|
||||||
template = os.getenv("MAP_PROMPT", MAP_PROMPT_DEFAULT)
|
template = map_prompt_template()
|
||||||
prefix = build_futures_context() + "\n" if INCLUDE_FUTURES else ""
|
prefix = build_futures_context() + "\n" if INCLUDE_FUTURES else ""
|
||||||
try:
|
try:
|
||||||
return prefix + template.format(batch_text=batch_text)
|
return prefix + template.format(batch_text=batch_text)
|
||||||
|
|
@ -408,7 +356,7 @@ def build_map_prompt(batch: list[dict]) -> str:
|
||||||
|
|
||||||
|
|
||||||
def build_master_prompt(final_input: str) -> str:
|
def build_master_prompt(final_input: str) -> str:
|
||||||
template = os.getenv("SUMMARY_PROMPT", SUMMARY_PROMPT_DEFAULT)
|
template = summary_prompt_template()
|
||||||
prefix = build_futures_context() + "\n" if INCLUDE_FUTURES else ""
|
prefix = build_futures_context() + "\n" if INCLUDE_FUTURES else ""
|
||||||
try:
|
try:
|
||||||
return prefix + template.format(final_input=final_input)
|
return prefix + template.format(final_input=final_input)
|
||||||
|
|
|
||||||
|
|
@ -49,3 +49,49 @@ def test_select_map_caps_20():
|
||||||
out = select_map(items)
|
out = select_map(items)
|
||||||
assert len(out) == 20
|
assert len(out) == 20
|
||||||
assert all(r["importance"] in ("critical", "high") for r in out)
|
assert all(r["importance"] in ("critical", "high") for r in out)
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_k8s_market_intel_reduce_json():
|
||||||
|
raw = """{
|
||||||
|
"market_overview": {
|
||||||
|
"overall_sentiment": "bearish",
|
||||||
|
"sentiment_score": -40,
|
||||||
|
"key_themes": ["oil supply", "rates"],
|
||||||
|
"critical_events": [
|
||||||
|
{
|
||||||
|
"headline": "Strike on Kharkiv",
|
||||||
|
"impact_level": "high",
|
||||||
|
"description": "Infrastructure hit.",
|
||||||
|
"location_name": "Kharkiv",
|
||||||
|
"lat": 49.99,
|
||||||
|
"lon": 36.23
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"headline": "Mild CPI print",
|
||||||
|
"impact_level": "low",
|
||||||
|
"description": "No surprise."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"geopolitical_osint": {
|
||||||
|
"active_conflicts": [
|
||||||
|
{
|
||||||
|
"region": "Ukraine",
|
||||||
|
"status": "escalating",
|
||||||
|
"intelligence": "Front-line push near Kharkiv",
|
||||||
|
"lat": 50.0,
|
||||||
|
"lon": 36.2
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}"""
|
||||||
|
out = parse_reduce_json(raw)
|
||||||
|
assert "bearish" in out["summary_en"]
|
||||||
|
assert "Strike on Kharkiv" in out["summary_en"]
|
||||||
|
assert [t["headline"] for t in out["ticker"]] == ["Strike on Kharkiv"]
|
||||||
|
assert out["ticker"][0]["importance"] == "high"
|
||||||
|
mapped = select_map(out["map_items"])
|
||||||
|
headlines = [r["headline"] for r in mapped]
|
||||||
|
assert "Front-line push near Kharkiv" in headlines
|
||||||
|
assert "Strike on Kharkiv" in headlines
|
||||||
|
|
||||||
|
|
|
||||||
40
news/summerizer/tests/test_prompts.py
Normal file
40
news/summerizer/tests/test_prompts.py
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from prompts import map_prompt_template, resolve_prompt, summary_prompt_template
|
||||||
|
|
||||||
|
|
||||||
|
def test_bundled_prompts_contain_placeholders():
|
||||||
|
assert "{batch_text}" in map_prompt_template()
|
||||||
|
assert "{final_input}" in summary_prompt_template()
|
||||||
|
assert "Market Intelligence Extractor" in map_prompt_template()
|
||||||
|
assert "Lead Market Intelligence Analyst" in summary_prompt_template()
|
||||||
|
|
||||||
|
|
||||||
|
def test_env_prompt_wins_over_file(monkeypatch, tmp_path):
|
||||||
|
monkeypatch.setenv("MAP_PROMPT", "ENV {batch_text}")
|
||||||
|
monkeypatch.setenv("MAP_PROMPT_FILE", str(tmp_path / "missing.txt"))
|
||||||
|
assert resolve_prompt("MAP_PROMPT", "MAP_PROMPT_FILE", "map.txt") == "ENV {batch_text}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_blank_env_falls_through_to_file(monkeypatch, tmp_path):
|
||||||
|
f = tmp_path / "custom.txt"
|
||||||
|
f.write_text("FILE {batch_text}\n")
|
||||||
|
monkeypatch.setenv("MAP_PROMPT", " ")
|
||||||
|
monkeypatch.setenv("MAP_PROMPT_FILE", str(f))
|
||||||
|
assert resolve_prompt("MAP_PROMPT", "MAP_PROMPT_FILE", "map.txt") == "FILE {batch_text}\n"
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_file_raises(monkeypatch, tmp_path):
|
||||||
|
monkeypatch.delenv("MAP_PROMPT", raising=False)
|
||||||
|
monkeypatch.setenv("MAP_PROMPT_FILE", str(tmp_path / "nope.txt"))
|
||||||
|
try:
|
||||||
|
resolve_prompt("MAP_PROMPT", "MAP_PROMPT_FILE", "map.txt")
|
||||||
|
assert False, "expected RuntimeError"
|
||||||
|
except RuntimeError as exc:
|
||||||
|
assert "unreadable" in str(exc)
|
||||||
|
|
||||||
|
|
||||||
|
def test_bundled_files_exist():
|
||||||
|
root = Path(__file__).resolve().parents[1] / "prompt_files"
|
||||||
|
assert (root / "map.txt").is_file()
|
||||||
|
assert (root / "summary.txt").is_file()
|
||||||
Loading…
Add table
Reference in a new issue