Multi Tenant LLM Hosting

# vllm-deployment.yaml - Multi-model serving with vLLM
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-gpt4o-equivalent
  namespace: llm-serving
  labels:
    app: vllm
    model-tier: premium
spec:
  replicas: 3
  selector:
    matchLabels:
      app: vllm
      model-tier: premium
  template:
    metadata:
      labels:
        app: vllm
        model-tier: premium
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8080"
    spec:
      containers:
        - name: vllm
          image: vllm/vllm-openai:v0.4.1
          args:
            - "--model=/models/llama-3.1-70b"
            - "--tensor-parallel-size=2"
            - "--max-model-len=8192"
            - "--gpu-memory-utilization=0.90"
            - "--max-num-seqs=128"
            - "--enable-prefix-caching"
          ports:
            - containerPort: 8000
              name: inference
            - containerPort: 8080
              name: metrics
          resources:
            requests:
              nvidia.com/gpu: 2
              cpu: "8"
              memory: "64Gi"
            limits:
              nvidia.com/gpu: 2
              cpu: "16"
              memory: "128Gi"
          volumeMounts:
            - name: model-weights
              mountPath: /models
              readOnly: true
      volumes:
        - name: model-weights
          persistentVolumeClaim:
            claimName: premium-model-weights
      tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
      nodeSelector:
        gpu-type: a100
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-economy
  namespace: llm-serving
  labels:
    app: vllm
    model-tier: economy
spec:
  replicas: 2
  selector:
    matchLabels:
      app: vllm
      model-tier: economy
  template:
    metadata:
      labels:
        app: vllm
        model-tier: economy
    spec:
      containers:
        - name: vllm
          image: vllm/vllm-openai:v0.4.1
          args:
            - "--model=/models/llama-3.1-8b"
            - "--max-model-len=4096"
            - "--gpu-memory-utilization=0.85"
            - "--max-num-seqs=256"
            - "--enable-prefix-caching"
          ports:
            - containerPort: 8000
              name: inference
            - containerPort: 8080
              name: metrics
          resources:
            requests:
              nvidia.com/gpu: 1
              cpu: "4"
              memory: "32Gi"
            limits:
              nvidia.com/gpu: 1
              cpu: "8"
              memory: "64Gi"
          volumeMounts:
            - name: model-weights
              mountPath: /models
              readOnly: true
      volumes:
        - name: model-weights
          persistentVolumeClaim:
            claimName: economy-model-weights
      tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule

# tenant-quotas-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: tenant-quotas
  namespace: llm-serving
data:
  quotas.yaml: |
    tenants:
      acme-corp:
        tier: enterprise
        models_allowed:
          - llama-3.1-70b
          - llama-3.1-8b
          - nomic-embed-text
        rate_limits:
          requests_per_minute: 300
          tokens_per_minute: 500000
          concurrent_requests: 50
        budget:
          daily_limit_usd: 500.00
          monthly_limit_usd: 10000.00
          alert_threshold_percent: 80
        priority: high

      startup-xyz:
        tier: standard
        models_allowed:
          - llama-3.1-8b
          - nomic-embed-text
        rate_limits:
          requests_per_minute: 60
          tokens_per_minute: 100000
          concurrent_requests: 10
        budget:
          daily_limit_usd: 50.00
          monthly_limit_usd: 1000.00
          alert_threshold_percent: 80
        priority: medium

      internal-dev:
        tier: free
        models_allowed:
          - llama-3.1-8b
        rate_limits:
          requests_per_minute: 20
          tokens_per_minute: 50000
          concurrent_requests: 5
        budget:
          daily_limit_usd: 10.00
          monthly_limit_usd: 200.00
          alert_threshold_percent: 90
        priority: low

# tenant-namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: tenant-acme-corp
  labels:
    tenant: acme-corp
    isolation: strict
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: tenant-isolation
  namespace: tenant-acme-corp
spec:
  podSelector: {}
  policyTypes:
    - Ingress
    - Egress
  ingress:
    - from:
        - namespaceSelector:
            matchLabels:
              name: llm-gateway
  egress:
    - to:
        - namespaceSelector:
            matchLabels:
              name: llm-serving
      ports:
        - port: 8000
          protocol: TCP
    - to:
        - namespaceSelector:
            matchLabels:
              name: kube-dns
      ports:
        - port: 53
          protocol: UDP
---
apiVersion: v1
kind: ResourceQuota
metadata:
  name: tenant-quota
  namespace: tenant-acme-corp
spec:
  hard:
    requests.cpu: "16"
    requests.memory: "64Gi"
    limits.cpu: "32"
    limits.memory: "128Gi"
    requests.nvidia.com/gpu: "4"
    pods: "20"

# gateway_router.py
"""Multi-tenant request router with rate limiting and model routing."""
import time
import json
import redis
from fastapi import FastAPI, HTTPException, Header, Request
from typing import Optional
import httpx
import yaml

app = FastAPI()
redis_client = redis.Redis(host="redis", port=6379, decode_responses=True)

# Load tenant config
with open("/etc/config/quotas.yaml") as f:
    TENANT_CONFIG = yaml.safe_load(f)["tenants"]

MODEL_ENDPOINTS = {
    "llama-3.1-70b": "http://vllm-gpt4o-equivalent:8000",
    "llama-3.1-8b": "http://vllm-economy:8000",
    "nomic-embed-text": "http://embedding-service:8000",
}

def check_rate_limit(tenant_id: str, config: dict) -> bool:
    """Check and update rate limit for a tenant."""
    key = f"ratelimit:{tenant_id}:{int(time.time() // 60)}"
    current = redis_client.incr(key)
    if current == 1:
        redis_client.expire(key, 120)
    return current <= config["rate_limits"]["requests_per_minute"]

def check_concurrent(tenant_id: str, config: dict) -> bool:
    """Check concurrent request limit."""
    key = f"concurrent:{tenant_id}"
    current = int(redis_client.get(key) or 0)
    return current < config["rate_limits"]["concurrent_requests"]

def check_budget(tenant_id: str, config: dict) -> bool:
    """Check if tenant is within daily budget."""
    key = f"spend:{tenant_id}:{time.strftime('%Y-%m-%d')}"
    current_spend = float(redis_client.get(key) or 0)
    return current_spend < config["budget"]["daily_limit_usd"]

def record_usage(tenant_id: str, model: str, prompt_tokens: int, completion_tokens: int):
    """Record token usage and cost for billing."""
    # Cost rates per 1K tokens
    rates = {
        "llama-3.1-70b": {"prompt": 0.004, "completion": 0.012},
        "llama-3.1-8b": {"prompt": 0.0005, "completion": 0.0015},
        "nomic-embed-text": {"prompt": 0.0001, "completion": 0.0},
    }
    rate = rates.get(model, {"prompt": 0.001, "completion": 0.003})
    cost = (prompt_tokens * rate["prompt"] + completion_tokens * rate["completion"]) / 1000

    # Update daily spend
    spend_key = f"spend:{tenant_id}:{time.strftime('%Y-%m-%d')}"
    redis_client.incrbyfloat(spend_key, cost)
    redis_client.expire(spend_key, 172800)

    # Record for billing export
    billing_key = f"billing:{tenant_id}:{time.strftime('%Y-%m')}"
    redis_client.rpush(billing_key, json.dumps({
        "timestamp": time.time(),
        "model": model,
        "prompt_tokens": prompt_tokens,
        "completion_tokens": completion_tokens,
        "cost_usd": cost,
    }))

@app.post("/v1/chat/completions")
async def chat_completions(
    request: Request,
    x_tenant_id: str = Header(...),
    x_api_key: str = Header(...),
):
    """Route chat completion request with tenant controls."""
    if x_tenant_id not in TENANT_CONFIG:
        raise HTTPException(status_code=403, detail="Unknown tenant")

    config = TENANT_CONFIG[x_tenant_id]
    body = await request.json()
    model = body.get("model", "llama-3.1-8b")

    # Check model access
    if model not in config["models_allowed"]:
        raise HTTPException(status_code=403, detail=f"Model {model} not allowed for tenant")

    # Check rate limit
    if not check_rate_limit(x_tenant_id, config):
        raise HTTPException(status_code=429, detail="Rate limit exceeded")

    # Check concurrent requests
    if not check_concurrent(x_tenant_id, config):
        raise HTTPException(status_code=429, detail="Concurrent request limit exceeded")

    # Check budget
    if not check_budget(x_tenant_id, config):
        raise HTTPException(status_code=402, detail="Daily budget exceeded")

    # Route to model endpoint
    endpoint = MODEL_ENDPOINTS.get(model)
    if not endpoint:
        raise HTTPException(status_code=404, detail=f"Model {model} not available")

    # Track concurrent requests
    concurrent_key = f"concurrent:{x_tenant_id}"
    redis_client.incr(concurrent_key)

    try:
        async with httpx.AsyncClient(timeout=120.0) as client:
            response = await client.post(
                f"{endpoint}/v1/chat/completions",
                json=body,
                headers={"Content-Type": "application/json"},
            )
            result = response.json()

            # Record usage
            usage = result.get("usage", {})
            record_usage(
                x_tenant_id, model,
                usage.get("prompt_tokens", 0),
                usage.get("completion_tokens", 0),
            )

            return result
    finally:
        redis_client.decr(concurrent_key)