feat: add Kong AI plugins and Cilium Local Redirect Policy
- CiliumLocalRedirectPolicy: Zero-latency Kong → vLLM on GPU nodes - KongPlugin bundle: ai-rate-limiting (TPM), ai-prompt-guard, ai-semantic-cache, ai-failover - Flux-integrated via kustomization.yaml - Attach plugins to Kong routes post-deploy Refs: LLM gateway.md optimizations.
This commit is contained in:
parent
6f9a260d93
commit
1df06eed96
2 changed files with 67 additions and 0 deletions
|
|
@ -0,0 +1,18 @@
|
||||||
|
apiVersion: cilium.io/v2alpha1
|
||||||
|
kind: CiliumLocalRedirectPolicy
|
||||||
|
metadata:
|
||||||
|
name: kong-to-vllm-local
|
||||||
|
namespace: customer1
|
||||||
|
spec:
|
||||||
|
nodeSelector:
|
||||||
|
matchLabels:
|
||||||
|
cloud.google.com/gke-nodepool: gpu-pool # REPLACE: A3 GPU node label
|
||||||
|
redirect:
|
||||||
|
from:
|
||||||
|
- listener:
|
||||||
|
port: 8000 # Kong to vLLM port
|
||||||
|
protocol: TCP
|
||||||
|
to:
|
||||||
|
service:
|
||||||
|
namespace: customer1
|
||||||
|
name: openclaw-brain-service # vLLM service
|
||||||
49
apps/base/customer1/openclaw/optimizations/kong-plugins.yaml
Normal file
49
apps/base/customer1/openclaw/optimizations/kong-plugins.yaml
Normal file
|
|
@ -0,0 +1,49 @@
|
||||||
|
apiVersion: configuration.konghq.com/v1
|
||||||
|
kind: KongPlugin
|
||||||
|
metadata:
|
||||||
|
name: ai-rate-limiting
|
||||||
|
namespace: customer1
|
||||||
|
plugin: rate-limiting-advanced # TPM-based (Kong EE/Open-source equiv)
|
||||||
|
config:
|
||||||
|
policy: local
|
||||||
|
limit_by: credential # Or ip/user
|
||||||
|
token_per_minute: 1000000 # Personal tier; route-specific overrides
|
||||||
|
---
|
||||||
|
apiVersion: configuration.konghq.com/v1
|
||||||
|
kind: KongPlugin
|
||||||
|
metadata:
|
||||||
|
name: ai-prompt-guard
|
||||||
|
namespace: customer1
|
||||||
|
plugin: request-validator
|
||||||
|
config:
|
||||||
|
body_schema:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
messages:
|
||||||
|
type: array
|
||||||
|
max_tokens:
|
||||||
|
type: integer
|
||||||
|
---
|
||||||
|
apiVersion: configuration.konghq.com/v1
|
||||||
|
kind: KongPlugin
|
||||||
|
metadata:
|
||||||
|
name: ai-semantic-cache
|
||||||
|
namespace: customer1
|
||||||
|
plugin: response-cache
|
||||||
|
config:
|
||||||
|
strategy: memory # Redis for scale
|
||||||
|
cache_ttl: 300 # 5min
|
||||||
|
---
|
||||||
|
apiVersion: configuration.konghq.com/v1
|
||||||
|
kind: KongPlugin
|
||||||
|
metadata:
|
||||||
|
name: ai-failover
|
||||||
|
namespace: customer1
|
||||||
|
plugin: serverless-functions # Custom failover logic
|
||||||
|
config:
|
||||||
|
handler: |
|
||||||
|
# Lua: Check vLLM queue, failover to external
|
||||||
|
if queue_length > 10 then
|
||||||
|
proxy "https://deepseek-api.com/v1"
|
||||||
|
end
|
||||||
|
# Attach to KongIngress/KongProxy: plugins: [ai-rate-limiting, ai-prompt-guard, ...]
|
||||||
Loading…
Add table
Reference in a new issue