add more sources for news scraper

apps/base/customer1/news_bot/scrapy-urls-configmap.yaml

@@ -627,3 +627,41 @@ data:
     https://www.thejakartapost.com/rss/business 
 
     https://www.nationthailand.com/rss/business
+    https://www.aramco.com/api/v1/com/rss/news?sc_lang=en
+    https://www.worldoil.com/rss?feed=topic:saudi+arabia
+    https://www.worldoil.com/rss?feed=topic:iraq
+    https://www.worldoil.com/rss?feed=topic:uae
+    https://www.worldoil.com/rss?feed=topic:russia
+    https://www.worldoil.com/rss?feed=topic:canada
+    https://www.worldoil.com/rss?feed=topic:oil+sands
+    https://www.rigzone.com/news/europe_russia/production/rss/
+    https://www.argusmedia.com/en/news-and-insights/latest-market-news/rss
+    https://www.eia.gov/rss/
+    https://www.opec.org/opec_web/en/pressreleases.rss (or check https://www.opec.org for updated press releases feed)
+    https://www.rosneft.com/press/news/rss/
+    https://feeds.content.dowjones.io/public/rss/RSSMarketsMain
+    https://feeds.content.dowjones.io/public/rss/socialeconomyfeed
+    https://feeds.content.dowjones.io/public/rss/WSJcomUSBusiness
+    https://feeds.content.dowjones.io/public/rss/RSSWorldNews
+    http://feeds.feedburner.com/EconomicEventsAgriculture
+    http://feeds.feedburner.com/EconomicEventsEnergy
+    http://feeds.feedburner.com/EconomicEventsInterestRates
+    http://feeds.feedburner.com/mediaroom/CMsF
+    http://feeds.feedburner.com/CMEClearPortNoticesRss
+    http://feeds.feedburner.com/GlobexAdvisories
+    https://feeds.content.dowjones.io/public/rss/mw_topstories
+    https://feeds.content.dowjones.io/public/rss/mw_realtimeheadlines
+    http://feeds.marketwatch.com/marketwatch/bulletins
+    https://feeds.content.dowjones.io/public/rss/mw_marketpulse
+    https://www.nasdaqtrader.com/rss.aspx?feed=currentheadlines&categorylist=51,52,53
+    https://www.nasdaqtrader.com/rss.aspx?feed=currentheadlines&categorylist=11,12,13
+    https://www.investing.com/rss/stock_Futures.rss
+    https://www.investing.com/rss/stock_Options.rss
+    https://www.investing.com/rss/news_11.rss
+    https://www.investing.com/rss/news_25.rss
+    https://www.nasdaq.com/feed/rssoutbound?category=Markets
+    https://www.nasdaq.com/feed/rssoutbound?category=Commodities
+    https://www.barchart.com/news/rss/financials/options-news
+    https://www.barchart.com/news/rss/commodities/futures-news
+    https://www.spglobal.com/spdji/en/rss
+    https://www.litefinance.org/rss/analytics/

grafana-secrets.yaml

@@ -5,5 +5,5 @@ data:
 kind: Secret
 metadata:
   creationTimestamp: null
-  name: grafana_secrets
+  name: grafana-secrets
   namespace: customer1

grafanaSecrets1.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+data:
+  grafana_admin_password: UGV0dWsxazE=
+  grafana_admin_user: c2lyaXVz
+kind: Secret
+metadata:
+  creationTimestamp: null
+  name: grafana_secrets
+  namespace: customer1

monitoring/controllers/base/kube-prometheus-stack/release.yaml

@@ -0,0 +1,33 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: kube-prometheus-stack
+  namespace: monitoring
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: kube-prometheus-stack
+      version: "66.x"
+      
+
+      sourceRef:
+        kind: HelmRepository
+        name: kube-prometheus-stack
+        namespace: monitoring
+      interval: 12h
+  install:
+    crds: Create
+  upgrade:
+    crds: CreateReplace
+  driftDetection:
+    mode: enabled
+    ignore:
+      # Ignore "validated" annotation which is not inserted during install
+      - paths: ["/metadata/annotations/prometheus-operator-validated"]
+        target:
+          kind: PrometheusRule
+  
+  valuesFrom:
+    - kind: Secret  
+      name: grafana-secrets  

monitoring/controllers/staging/kustomization.yaml

@@ -0,0 +1,28 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - repository.yaml
+  - release.yaml
+  - secrets.yaml
+
+patches:
+  - target:
+      kind: SecretProviderClass
+      name: grafana-secrets
+    patch: |-
+      - op: replace
+        path: /spec/parameters/userAssignedIdentityID
+        value: "10ebab78-f6c8-4d5e-ac64-c0d85fb031d0"
+      - op: replace
+        path: /spec/parameters/tenantId
+        value: "6c0d1498-a808-4d41-a865-fdebeb412738"
+
+configMapGenerator:
+  - name: kube-prometheus-stack-values
+    namespace: flux-system
+    files:
+      - values.yaml=values.yaml
+
+configurations:
+  - kustomizeconfig.yaml

monitoring/controllers/staging/kustomizeconfig.yaml

@@ -0,0 +1,6 @@
+nameReference:
+  - kind: ConfigMap
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease

monitoring/controllers/staging/release.yaml

@@ -0,0 +1,33 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: kube-prometheus-stack
+  namespace: flux-system
+spec:
+  targetNamespace: monitoring
+  interval: 30m
+  chart:
+    spec:
+      chart: kube-prometheus-stack
+      version: "80.2.0"
+      sourceRef:
+        kind: HelmRepository
+        name: prometheus-community
+        namespace: flux-system
+      interval: 12h
+
+  valuesFrom:
+    - kind: ConfigMap
+      name: kube-prometheus-stack-values
+
+  install:
+    createNamespace: true
+    crds: Create
+  upgrade:
+    crds: CreateReplace
+  driftDetection:
+    mode: enabled
+    ignore:
+      - paths: ["/metadata/annotations/prometheus-operator-validated"]
+        target:
+          kind: PrometheusRule

monitoring/controllers/staging/repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: prometheus-community
+  namespace: flux-system
+spec:
+  interval: 24h
+  url: https://prometheus-community.github.io/helm-charts

monitoring/controllers/staging/values.yaml

@@ -0,0 +1,73 @@
+# Alertmanager disabled - using Grafana-native alerting
+alertmanager:
+  enabled: false
+
+# Prometheus configuration
+prometheus:
+  prometheusSpec:
+    retention: 7d
+    # Select PodMonitors from all namespaces (needed for CNPG database monitoring)
+    podMonitorNamespaceSelector: {}
+    podMonitorSelectorNilUsesHelmValues: false
+    probeSelectorNilUsesHelmValues: false
+    ruleSelectorNilUsesHelmValues: false
+    serviceMonitorSelectorNilUsesHelmValues: false
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          accessModes: ["ReadWriteOnce"]
+          resources:
+            requests:
+              storage: 10Gi
+
+# Grafana configuration
+grafana:
+  # Admin credentials - username set directly, password from Key Vault
+  adminUser: admin
+  admin:
+    existingSecret: grafana-admin-credentials
+    userKey: admin-user
+    passwordKey: admin-password
+
+  # Inject Telegram credentials from Key Vault
+  envFromSecret: grafana-env
+
+  # Ingress configuration
+  ingress:
+    enabled: true
+    ingressClassName: traefik
+    annotations:
+      cert-manager.io/cluster-issuer: letsencrypt-prod
+    hosts:
+      - grafana.mercury-staging.kubecraftlabs.com
+    tls:
+      - secretName: grafana-tls
+        hosts:
+          - grafana.mercury-staging.kubecraftlabs.com
+
+  # Mount CSI secrets volume to trigger secret sync
+  extraSecretMounts:
+    - name: secrets-store
+      mountPath: /mnt/secrets-store
+      csi:
+        driver: secrets-store.csi.k8s.io
+        readOnly: true
+        volumeAttributes:
+          secretProviderClass: grafana-secrets
+
+  sidecar:
+    dashboards:
+      enabled: true
+      label: grafana_dashboard
+      labelValue: "1"
+      searchNamespace: monitoring
+      folderAnnotation: grafana_folder
+      provider:
+        allowUiUpdates: true
+        folder: Kubernetes
+    alerts:
+      enabled: true
+      label: grafana_alert
+      labelValue: "1"
+      searchNamespace: monitoring
+      initAlerts: true