Federator.ai GPU Booster Plus - Complete Implementation Guide

┌─────────────────┐     ┌──────────────────┐     ┌─────────────────────┐
│  User Requests  │────▶│  Load Balancer   │────▶│  vLLM Pods (1-3)   │
└─────────────────┘     │     (NGINX)      │     │  on H100 nodes     │
                        └──────────────────┘     └─────────────────────┘
                                                           │
                        ┌──────────────────┐               │
                        │     Grafana      │◀──────────────┤
                        └──────────────────┘               │
                                ▲                          │
                        ┌──────────────────┐               │
                        │   Prometheus     │◀──────────────┤
                        └──────────────────┘               │
                                ▲                          │
                        ┌──────────────────┐               │
                        │  DCGM Exporter   │◀──────────────┘
                        └──────────────────┘
                                ▲
                        ┌──────────────────┐
                        │  GPU Booster     │
                        │     Plugin       │
                        └──────────────────┘

# namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: gpu-booster
  labels:
    name: gpu-booster
    monitoring: prometheus
---
# GPU node labeling
apiVersion: v1
kind: ConfigMap
metadata:
  name: node-labeler
  namespace: gpu-booster
data:
  label-nodes.sh: |
    #!/bin/bash
    # Label GPU nodes for scheduling
    for node in $(kubectl get nodes -l nvidia.com/gpu.present=true -o name); do
      kubectl label $node gpu-type=h100 --overwrite
      kubectl label $node gpu-count=8 --overwrite
      kubectl label $node workload=inference --overwrite
    done

# dcgm-exporter.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: dcgm-exporter
  namespace: gpu-booster
spec:
  selector:
    matchLabels:
      app: dcgm-exporter
  template:
    metadata:
      labels:
        app: dcgm-exporter
    spec:
      serviceAccountName: dcgm-exporter
      nodeSelector:
        nvidia.com/gpu.present: "true"
      tolerations:
      - key: nvidia.com/gpu
        operator: Exists
        effect: NoSchedule
      containers:
      - name: dcgm-exporter
        image: nvcr.io/nvidia/k8s/dcgm-exporter:3.1.0-3.1.0-ubuntu20.04
        args:
        - -f
        - /etc/dcgm-exporter/dcp-metrics-included.csv
        - -c
        - "15000"  # 15 second collection interval
        env:
        - name: DCGM_EXPORTER_KUBERNETES_GPU_ID_TYPE
          value: "device-name"
        - name: DCGM_EXPORTER_LISTEN
          value: ":9400"
        ports:
        - name: metrics
          containerPort: 9400
        resources:
          requests:
            memory: "128Mi"
            cpu: "100m"
          limits:
            memory: "512Mi"
            cpu: "500m"
        volumeMounts:
        - name: gpu-metrics-config
          mountPath: /etc/dcgm-exporter
        securityContext:
          privileged: true
      volumes:
      - name: gpu-metrics-config
        configMap:
          name: dcgm-metrics-config
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: dcgm-metrics-config
  namespace: gpu-booster
data:
  dcp-metrics-included.csv: |
    # Format: dcgm_field_name,Prometheus_metric_name,help_text,metric_type
    DCGM_FI_DEV_GPU_UTIL,dcgm_gpu_utilization,GPU utilization percentage,gauge
    DCGM_FI_DEV_MEM_COPY_UTIL,dcgm_gpu_mem_copy_utilization,GPU memory bandwidth utilization,gauge
    DCGM_FI_DEV_FB_FREE,dcgm_fb_free,Framebuffer memory free in MB,gauge
    DCGM_FI_DEV_FB_USED,dcgm_fb_used,Framebuffer memory used in MB,gauge
    DCGM_FI_DEV_POWER_USAGE,dcgm_power_usage,Power usage in watts,gauge
    DCGM_FI_DEV_GPU_TEMP,dcgm_gpu_temp,GPU temperature in Celsius,gauge
    DCGM_FI_DEV_SM_CLOCK,dcgm_sm_clock,SM clock frequency in MHz,gauge
    DCGM_FI_DEV_MEM_CLOCK,dcgm_memory_clock,Memory clock frequency in MHz,gauge
    DCGM_FI_PROF_DRAM_ACTIVE,dcgm_dram_active,DRAM active cycles,gauge
    DCGM_FI_PROF_PIPE_TENSOR_ACTIVE,dcgm_tensor_active,Tensor core active cycles,gauge
---
apiVersion: v1
kind: Service
metadata:
  name: dcgm-exporter-service
  namespace: gpu-booster
  labels:
    app: dcgm-exporter
spec:
  selector:
    app: dcgm-exporter
  ports:
  - name: metrics
    port: 9400
    targetPort: 9400

# prometheus-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: gpu-booster
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      evaluation_interval: 15s
    
    scrape_configs:
    - job_name: 'dcgm-exporter'
      kubernetes_sd_configs:
      - role: pod
        namespaces:
          names:
          - gpu-booster
      relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_app]
        action: keep
        regex: dcgm-exporter
      - source_labels: [__meta_kubernetes_pod_node_name]
        target_label: node
      - source_labels: [__meta_kubernetes_pod_name]
        target_label: pod
    
    - job_name: 'vllm-metrics'
      kubernetes_sd_configs:
      - role: pod
        namespaces:
          names:
          - gpu-booster
      relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_app]
        action: keep
        regex: vllm-deepseek
      - source_labels: [__meta_kubernetes_pod_name]
        target_label: instance
      - source_labels: [__meta_kubernetes_pod_node_name]
        target_label: node
    
    - job_name: 'kubernetes-pods'
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus
  namespace: gpu-booster
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus
  template:
    metadata:
      labels:
        app: prometheus
    spec:
      serviceAccountName: prometheus
      containers:
      - name: prometheus
        image: prom/prometheus:v2.45.0
        args:
        - '--config.file=/etc/prometheus/prometheus.yml'
        - '--storage.tsdb.path=/prometheus/'
        - '--web.console.libraries=/etc/prometheus/console_libraries'
        - '--web.console.templates=/etc/prometheus/consoles'
        - '--storage.tsdb.retention.time=30d'
        - '--web.enable-lifecycle'
        ports:
        - containerPort: 9090
        resources:
          requests:
            memory: "2Gi"
            cpu: "1000m"
          limits:
            memory: "4Gi"
            cpu: "2000m"
        volumeMounts:
        - name: prometheus-config
          mountPath: /etc/prometheus/
        - name: prometheus-storage
          mountPath: /prometheus/
      volumes:
      - name: prometheus-config
        configMap:
          name: prometheus-config
      - name: prometheus-storage
        persistentVolumeClaim:
          claimName: prometheus-pvc

# vllm-deepseek.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: vllm-config
  namespace: gpu-booster
data:
  startup.sh: |
    #!/bin/bash
    set -e
    
    # Set CUDA device visibility for 8-way tensor parallelism
    export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
    
    # Configure NCCL for optimal multi-GPU communication
    export NCCL_IB_DISABLE=0
    export NCCL_IB_GID_INDEX=3
    export NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3
    export NCCL_NET_GDR_LEVEL=2
    export NCCL_P2P_LEVEL=NVL
    
    # Start vLLM server with optimizations
    python -m vllm.entrypoints.openai.api_server \
      --model="/models/deepseek-r1" \
      --tensor-parallel-size=8 \
      --dtype=float8_e4m3fn \
      --max-model-len=32768 \
      --max-num-batched-tokens=8192 \
      --max-num-seqs=256 \
      --enable-prefix-caching \
      --enable-chunked-prefill \
      --max-num-batched-tokens=8192 \
      --gpu-memory-utilization=0.95 \
      --swap-space=0 \
      --disable-log-requests \
      --port=8000 \
      --host=0.0.0.0 \
      --quantization="fp8" \
      --kv-cache-dtype="fp8_e5m2" \
      --disable-custom-all-reduce
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-deepseek
  namespace: gpu-booster
spec:
  replicas: 1
  selector:
    matchLabels:
      app: vllm-deepseek
  template:
    metadata:
      labels:
        app: vllm-deepseek
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8000"
        prometheus.io/path: "/metrics"
    spec:
      serviceAccountName: vllm-service
      nodeSelector:
        gpu-type: h100
        gpu-count: "8"
      tolerations:
      - key: nvidia.com/gpu
        operator: Exists
        effect: NoSchedule
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - vllm-deepseek
              topologyKey: kubernetes.io/hostname
      containers:
      - name: vllm
        image: vllm/vllm-openai:v0.4.0
        command: ["/bin/bash"]
        args: ["/config/startup.sh"]
        ports:
        - containerPort: 8000
          name: http
        env:
        - name: PYTHONUNBUFFERED
          value: "1"
        - name: CUDA_DEVICE_ORDER
          value: "PCI_BUS_ID"
        - name: NVIDIA_VISIBLE_DEVICES
          value: "all"
        - name: VLLM_ATTENTION_BACKEND
          value: "FLASHINFER"
        resources:
          requests:
            nvidia.com/gpu: 8
            memory: "900Gi"
            cpu: "64"
          limits:
            nvidia.com/gpu: 8
            memory: "1000Gi"
            cpu: "96"
        volumeMounts:
        - name: vllm-config
          mountPath: /config
        - name: model-storage
          mountPath: /models
        - name: shm
          mountPath: /dev/shm
        livenessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 600
          periodSeconds: 30
          timeoutSeconds: 10
          failureThreshold: 3
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 3
      volumes:
      - name: vllm-config
        configMap:
          name: vllm-config
          defaultMode: 0755
      - name: model-storage
        persistentVolumeClaim:
          claimName: deepseek-model-pvc
      - name: shm
        emptyDir:
          medium: Memory
          sizeLimit: 128Gi

# custom-metrics-adapter.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: adapter-config
  namespace: gpu-booster
data:
  config.yaml: |
    rules:
    - seriesQuery: 'dcgm_gpu_utilization{namespace="gpu-booster",pod!=""}'
      resources:
        overrides:
          namespace: {resource: "namespace"}
          pod: {resource: "pod"}
      name:
        matches: "^dcgm_gpu_utilization"
        as: "gpu_utilization_percentage"
      metricsQuery: 'avg(dcgm_gpu_utilization{<<.LabelMatchers>>})'
    
    - seriesQuery: 'dcgm_gpu_temp{namespace="gpu-booster",pod!=""}'
      resources:
        overrides:
          namespace: {resource: "namespace"}
          pod: {resource: "pod"}
      name:
        matches: "^dcgm_gpu_temp"
        as: "gpu_temperature_celsius"
      metricsQuery: 'max(dcgm_gpu_temp{<<.LabelMatchers>>})'
    
    - seriesQuery: 'dcgm_power_usage{namespace="gpu-booster",pod!=""}'
      resources:
        overrides:
          namespace: {resource: "namespace"}
          pod: {resource: "pod"}
      name:
        matches: "^dcgm_power_usage"
        as: "gpu_power_watts"
      metricsQuery: 'sum(dcgm_power_usage{<<.LabelMatchers>>})'
    
    - seriesQuery: 'vllm_request_queue_length{namespace="gpu-booster",pod!=""}'
      resources:
        overrides:
          namespace: {resource: "namespace"}
          pod: {resource: "pod"}
      name:
        matches: "^vllm_request_queue_length"
        as: "request_queue_length"
      metricsQuery: 'vllm_request_queue_length{<<.LabelMatchers>>}'
    
    - seriesQuery: 'vllm_avg_tokens_per_second{namespace="gpu-booster",pod!=""}'
      resources:
        overrides:
          namespace: {resource: "namespace"}
          pod: {resource: "pod"}
      name:
        matches: "^vllm_avg_tokens_per_second"
        as: "tokens_per_second"
      metricsQuery: 'vllm_avg_tokens_per_second{<<.LabelMatchers>>}'
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: custom-metrics-adapter
  namespace: gpu-booster
spec:
  replicas: 1
  selector:
    matchLabels:
      app: custom-metrics-adapter
  template:
    metadata:
      labels:
        app: custom-metrics-adapter
    spec:
      serviceAccountName: custom-metrics-adapter
      containers:
      - name: custom-metrics-adapter
        image: k8s.gcr.io/prometheus-adapter/prometheus-adapter:v0.11.0
        args:
        - --cert-dir=/tmp/cert
        - --config=/config/config.yaml
        - --logtostderr=true
        - --prometheus-url=http://prometheus-service.gpu-booster.svc:9090/
        - --metrics-relist-interval=30s
        - --secure-port=6443
        ports:
        - containerPort: 6443
          name: https
        resources:
          requests:
            memory: "256Mi"
            cpu: "250m"
          limits:
            memory: "512Mi"
            cpu: "500m"
        volumeMounts:
        - name: config
          mountPath: /config
          readOnly: true
        - name: tmp
          mountPath: /tmp
      volumes:
      - name: config
        configMap:
          name: adapter-config
      - name: tmp
        emptyDir: {}

# hpa-config.yaml
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: vllm-deepseek-hpa
  namespace: gpu-booster
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: vllm-deepseek
  minReplicas: 1
  maxReplicas: 3
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 300  # 5 minute stabilization
      policies:
      - type: Percent
        value: 50  # Scale down by 50% max
        periodSeconds: 60
      selectPolicy: Min
    scaleUp:
      stabilizationWindowSeconds: 60
      policies:
      - type: Percent
        value: 100  # Double the replicas
        periodSeconds: 60
      - type: Pods
        value: 1
        periodSeconds: 60
      selectPolicy: Max
  metrics:
  - type: Pods
    pods:
      metric:
        name: gpu_utilization_percentage
      target:
        type: AverageValue
        averageValue: "85"  # Target 85% GPU utilization
  - type: Pods
    pods:
      metric:
        name: request_queue_length
      target:
        type: AverageValue
        averageValue: "25"  # Scale up if queue > 25 requests
  - type: Pods
    pods:
      metric:
        name: gpu_temperature_celsius
      target:
        type: AverageValue
        averageValue: "80"  # Thermal threshold

# gpu-booster-plugin.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: gpu-booster-plugin
  namespace: gpu-booster
data:
  gpu-booster.py: |
    #!/usr/bin/env python3
    """
    Federator.ai GPU Booster Plugin
    Implements thermal-aware placement and optimization
    """
    import json
    import logging
    import os
    import time
    from dataclasses import dataclass
    from typing import Dict, List, Optional
    
    import numpy as np
    from kubernetes import client, config
    from prometheus_api_client import PrometheusConnect
    
    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger(__name__)
    
    @dataclass
    class GPUNode:
        name: str
        gpu_count: int
        current_temp: float
        current_util: float
        power_usage: float
        cooling_factor: float = 0.8
    
    class GPUBoosterOptimizer:
        def __init__(self, prometheus_url: str):
            self.prom = PrometheusConnect(url=prometheus_url)
            config.load_incluster_config()
            self.v1 = client.CoreV1Api()
            self.apps_v1 = client.AppsV1Api()
            
        def get_gpu_nodes(self) -> List[GPUNode]:
            """Fetch GPU node information with current metrics"""
            nodes = []
            
            # Get all GPU nodes
            node_list = self.v1.list_node(
                label_selector="gpu-type=h100"
            )
            
            for node in node_list.items:
                node_name = node.metadata.name
                
                # Get current GPU metrics
                temp_query = f'max(dcgm_gpu_temp{{node="{node_name}"}})'
                util_query = f'avg(dcgm_gpu_utilization{{node="{node_name}"}})'
                power_query = f'sum(dcgm_power_usage{{node="{node_name}"}})'
                
                try:
                    temp = float(self.prom.custom_query(temp_query)[0]['value'][1])
                    util = float(self.prom.custom_query(util_query)[0]['value'][1])
                    power = float(self.prom.custom_query(power_query)[0]['value'][1])
                except (IndexError, KeyError):
                    temp = 0.0
                    util = 0.0
                    power = 0.0
                
                nodes.append(GPUNode(
                    name=node_name,
                    gpu_count=8,
                    current_temp=temp,
                    current_util=util,
                    power_usage=power
                ))
            
            return nodes
        
        def calculate_placement_score(self, node: GPUNode) -> float:
            """Calculate placement score based on thermal and utilization factors"""
            # Thermal penalty (exponential as we approach 80°C)
            thermal_score = 1.0 - np.exp((node.current_temp - 80) / 10) if node.current_temp < 80 else 0.1
            
            # Utilization score (prefer nodes with headroom)
            util_score = 1.0 - (node.current_util / 100) ** 2
            
            # Power efficiency score
            power_score = 1.0 - (node.power_usage / 2400)  # H100 max ~300W * 8
            
            # Combined score with weights
            return (thermal_score * 0.4 + util_score * 0.4 + power_score * 0.2) * node.cooling_factor
        
        def optimize_placement(self, deployment_name: str = "vllm-deepseek"):
            """Apply thermal-aware placement constraints"""
            nodes = self.get_gpu_nodes()
            
            # Sort nodes by placement score
            sorted_nodes = sorted(nodes, key=self.calculate_placement_score, reverse=True)
            
            # Update deployment with node affinity
            deployment = self.apps_v1.read_namespaced_deployment(
                name=deployment_name,
                namespace="gpu-booster"
            )
            
            # Create preferred node affinity
            preferred_nodes = [node.name for node in sorted_nodes[:2]]  # Top 2 nodes
            
            node_affinity = client.V1NodeAffinity(
                preferred_during_scheduling_ignored_during_execution=[
                    client.V1PreferredSchedulingTerm(
                        weight=100,
                        preference=client.V1NodeSelectorTerm(
                            match_expressions=[
                                client.V1NodeSelectorRequirement(
                                    key="kubernetes.io/hostname",
                                    operator="In",
                                    values=preferred_nodes
                                )
                            ]
                        )
                    )
                ]
            )
            
            # Update deployment
            deployment.spec.template.spec.affinity = client.V1Affinity(
                node_affinity=node_affinity
            )
            
            self.apps_v1.patch_namespaced_deployment(
                name=deployment_name,
                namespace="gpu-booster",
                body=deployment
            )
            
            logger.info(f"Updated placement preferences: {preferred_nodes}")
        
        def monitor_and_optimize(self, interval: int = 60):
            """Continuous monitoring and optimization loop"""
            while True:
                try:
                    self.optimize_placement()
                    
                    # Log current state
                    nodes = self.get_gpu_nodes()
                    for node in nodes:
                        logger.info(
                            f"Node {node.name}: Temp={node.current_temp}°C, "
                            f"Util={node.current_util}%, Power={node.power_usage}W"
                        )
                    
                except Exception as e:
                    logger.error(f"Optimization error: {e}")
                
                time.sleep(interval)
    
    if __name__ == "__main__":
        prometheus_url = os.getenv("PROMETHEUS_URL", "http://prometheus-service:9090")
        optimizer = GPUBoosterOptimizer(prometheus_url)
        optimizer.monitor_and_optimize()
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: gpu-booster-plugin
  namespace: gpu-booster
spec:
  replicas: 1
  selector:
    matchLabels:
      app: gpu-booster-plugin
  template:
    metadata:
      labels:
        app: gpu-booster-plugin
    spec:
      serviceAccountName: gpu-booster
      containers:
      - name: gpu-booster
        image: python:3.11-slim
        command: ["python", "/app/gpu-booster.py"]
        env:
        - name: PROMETHEUS_URL
          value: "http://prometheus-service:9090"
        resources:
          requests:
            memory: "256Mi"
            cpu: "250m"
          limits:
            memory: "512Mi"
            cpu: "500m"
        volumeMounts:
        - name: plugin-code
          mountPath: /app
      volumes:
      - name: plugin-code
        configMap:
          name: gpu-booster-plugin

# nginx-lb.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-config
  namespace: gpu-booster
data:
  nginx.conf: |
    user nginx;
    worker_processes auto;
    worker_cpu_affinity auto;
    error_log /var/log/nginx/error.log notice;
    pid /var/run/nginx.pid;
    
    events {
        worker_connections 4096;
        use epoll;
        multi_accept on;
    }
    
    http {
        include /etc/nginx/mime.types;
        default_type application/octet-stream;
        
        sendfile on;
        tcp_nopush on;
        tcp_nodelay on;
        keepalive_timeout 65;
        keepalive_requests 100;
        
        # Upstream configuration with least connections
        upstream vllm_backend {
            least_conn;
            keepalive 32;
            
            server vllm-service:8000 max_fails=3 fail_timeout=30s;
        }
        
        server {
            listen 80;
            server_name _;
            
            # Increase buffer sizes for large requests
            client_body_buffer_size 128k;
            client_max_body_size 10m;
            client_header_buffer_size 1k;
            large_client_header_buffers 4 16k;
            
            # Request timeout settings
            proxy_connect_timeout 600s;
            proxy_send_timeout 600s;
            proxy_read_timeout 600s;
            
            location / {
                proxy_pass http://vllm_backend;
                proxy_http_version 1.1;
                
                # Headers
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Forwarded-Proto $scheme;
                
                # Keep-alive
                proxy_set_header Connection "";
                
                # Buffering settings
                proxy_buffering off;
                proxy_request_buffering off;
            }
            
            # Health check endpoint
            location /health {
                access_log off;
                return 200 "healthy\n";
            }
            
            # Metrics endpoint
            location /nginx-metrics {
                stub_status on;
                access_log off;
                allow 10.0.0.0/8;
                deny all;
            }
        }
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-lb
  namespace: gpu-booster
spec:
  replicas: 2
  selector:
    matchLabels:
      app: nginx-lb
  template:
    metadata:
      labels:
        app: nginx-lb
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - nginx-lb
            topologyKey: kubernetes.io/hostname
      containers:
      - name: nginx
        image: nginx:1.25-alpine
        ports:
        - containerPort: 80
          name: http
        resources:
          requests:
            memory: "256Mi"
            cpu: "500m"
          limits:
            memory: "512Mi"
            cpu: "1000m"
        volumeMounts:
        - name: nginx-config
          mountPath: /etc/nginx/nginx.conf
          subPath: nginx.conf
        livenessProbe:
          httpGet:
            path: /health
            port: 80
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /health
            port: 80
          periodSeconds: 5
      volumes:
      - name: nginx-config
        configMap:
          name: nginx-config
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-lb-service
  namespace: gpu-booster
spec:
  type: LoadBalancer
  selector:
    app: nginx-lb
  ports:
  - port: 80
    targetPort: 80
    protocol: TCP

# grafana-dashboard.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-dashboard
  namespace: gpu-booster
data:
  gpu-booster-dashboard.json: |
    {
      "dashboard": {
        "title": "GPU Booster Plus - DeepSeek R1 Performance",
        "panels": [
          {
            "title": "GPU Utilization",
            "targets": [
              {
                "expr": "avg(dcgm_gpu_utilization{namespace=\"gpu-booster\"})",
                "legendFormat": "Average GPU Utilization"
              }
            ],
            "type": "graph",
            "gridPos": {"x": 0, "y": 0, "w": 8, "h": 8}
          },
          {
            "title": "Tokens per Second",
            "targets": [
              {
                "expr": "sum(rate(vllm_generation_tokens_total[5m]))",
                "legendFormat": "Tokens/s"
              }
            ],
            "type": "graph",
            "gridPos": {"x": 8, "y": 0, "w": 8, "h": 8}
          },
          {
            "title": "Request Latency P99",
            "targets": [
              {
                "expr": "histogram_quantile(0.99, rate(vllm_request_duration_seconds_bucket[5m]))",
                "legendFormat": "P99 Latency"
              }
            ],
            "type": "graph",
            "gridPos": {"x": 16, "y": 0, "w": 8, "h": 8}
          },
          {
            "title": "GPU Temperature",
            "targets": [
              {
                "expr": "max(dcgm_gpu_temp{namespace=\"gpu-booster\"}) by (node)",
                "legendFormat": "{{node}}"
              }
            ],
            "type": "heatmap",
            "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8}
          },
          {
            "title": "Power Consumption",
            "targets": [
              {
                "expr": "sum(dcgm_power_usage{namespace=\"gpu-booster\"}) by (pod)",
                "legendFormat": "{{pod}}"
              }
            ],
            "type": "graph",
            "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8}
          },
          {
            "title": "Active Replicas",
            "targets": [
              {
                "expr": "kube_deployment_status_replicas{deployment=\"vllm-deepseek\",namespace=\"gpu-booster\"}",
                "legendFormat": "Replicas"
              }
            ],
            "type": "stat",
            "gridPos": {"x": 0, "y": 16, "w": 6, "h": 4}
          },
          {
            "title": "Tokens per Dollar",
            "targets": [
              {
                "expr": "sum(rate(vllm_generation_tokens_total[5m])) / (sum(dcgm_power_usage) * 0.0001)",
                "legendFormat": "Tokens/$"
              }
            ],
            "type": "stat",
            "gridPos": {"x": 6, "y": 16, "w": 6, "h": 4}
          },
          {
            "title": "Queue Length",
            "targets": [
              {
                "expr": "sum(vllm_request_queue_length{namespace=\"gpu-booster\"})",
                "legendFormat": "Queue Size"
              }
            ],
            "type": "gauge",
            "gridPos": {"x": 12, "y": 16, "w": 6, "h": 4}
          }
        ]
      }
    }

#!/bin/bash
# deploy-gpu-booster.sh

set -e

echo "🚀 Deploying Federator.ai GPU Booster Plus"

# Color codes for output
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
NC='\033[0m'

# Function to check if resource exists
check_resource() {
    kubectl get $1 $2 -n $3 &> /dev/null
}

# Function to wait for deployment
wait_for_deployment() {
    echo -e "${YELLOW}Waiting for $1 to be ready...${NC}"
    kubectl rollout status deployment/$1 -n gpu-booster --timeout=600s
}

# Create namespace
echo -e "${GREEN}Creating namespace...${NC}"
kubectl apply -f namespace.yaml

# Label GPU nodes
echo -e "${GREEN}Labeling GPU nodes...${NC}"
kubectl apply -f namespace.yaml
kubectl create job node-labeler --from=cronjob/node-labeler -n gpu-booster || true

# Deploy DCGM Exporter
echo -e "${GREEN}Deploying DCGM Exporter...${NC}"
kubectl apply -f dcgm-exporter.yaml

# Deploy Prometheus
echo -e "${GREEN}Deploying Prometheus...${NC}"
kubectl apply -f prometheus-config.yaml
wait_for_deployment prometheus

# Deploy Custom Metrics Adapter
echo -e "${GREEN}Deploying Custom Metrics Adapter...${NC}"
kubectl apply -f custom-metrics-adapter.yaml
wait_for_deployment custom-metrics-adapter

# Deploy vLLM Service
echo -e "${GREEN}Deploying vLLM DeepSeek R1 Service...${NC}"
kubectl apply -f vllm-deepseek.yaml
wait_for_deployment vllm-deepseek

# Deploy HPA
echo -e "${GREEN}Configuring Horizontal Pod Autoscaler...${NC}"
kubectl apply -f hpa-config.yaml

# Deploy GPU Booster Plugin
echo -e "${GREEN}Deploying GPU Booster Plugin...${NC}"
kubectl apply -f gpu-booster-plugin.yaml
wait_for_deployment gpu-booster-plugin

# Deploy Load Balancer
echo -e "${GREEN}Deploying NGINX Load Balancer...${NC}"
kubectl apply -f nginx-lb.yaml
wait_for_deployment nginx-lb

# Configure Grafana Dashboard
echo -e "${GREEN}Configuring Grafana Dashboard...${NC}"
kubectl apply -f grafana-dashboard.yaml

# Run validation tests
echo -e "${GREEN}Running validation tests...${NC}"
./validate-deployment.sh

echo -e "${GREEN}✅ Deployment complete!${NC}"
echo -e "${GREEN}Access the service at: $(kubectl get svc nginx-lb-service -n gpu-booster -o jsonpath='{.status.loadBalancer.ingress[0].ip}')${NC}"

#!/bin/bash
# validate-deployment.sh

set -e

echo "🔍 Validating GPU Booster Deployment"

# Test GPU metrics collection
echo "Testing DCGM metrics..."
METRICS=$(kubectl exec -n gpu-booster deployment/prometheus -- wget -qO- http://localhost:9090/api/v1/query?query=dcgm_gpu_utilization | jq '.data.result | length')
if [ "$METRICS" -gt 0 ]; then
    echo "✅ GPU metrics collection working"
else
    echo "❌ GPU metrics not found"
    exit 1
fi

# Test vLLM health
echo "Testing vLLM service..."
POD=$(kubectl get pod -n gpu-booster -l app=vllm-deepseek -o jsonpath='{.items[0].metadata.name}')
if kubectl exec -n gpu-booster $POD -- curl -s http://localhost:8000/health | grep -q "ok"; then
    echo "✅ vLLM service healthy"
else
    echo "❌ vLLM service unhealthy"
    exit 1
fi

# Test HPA configuration
echo "Testing HPA..."
if kubectl get hpa vllm-deepseek-hpa -n gpu-booster -o jsonpath='{.status.currentMetrics}' | grep -q "gpu_utilization"; then
    echo "✅ HPA configured correctly"
else
    echo "❌ HPA metrics not available"
    exit 1
fi

# Performance benchmark
echo "Running performance benchmark..."
./performance-test.sh

echo "✅ All validations passed!"

#!/usr/bin/env python3
# performance-test.py

import asyncio
import time
import statistics
from typing import List
import aiohttp
import json

class PerformanceTest:
    def __init__(self, endpoint: str):
        self.endpoint = endpoint
        self.results = {
            "latencies": [],
            "tokens_per_second": [],
            "errors": 0,
            "total_requests": 0
        }
    
    async def send_request(self, session: aiohttp.ClientSession, prompt: str):
        """Send a single inference request"""
        payload = {
            "model": "deepseek-r1",
            "prompt": prompt,
            "max_tokens": 256,
            "temperature": 0.7,
            "stream": False
        }
        
        start_time = time.time()
        try:
            async with session.post(
                f"{self.endpoint}/v1/completions",
                json=payload,
                timeout=aiohttp.ClientTimeout(total=60)
            ) as response:
                if response.status == 200:
                    data = await response.json()
                    latency = time.time() - start_time
                    tokens = len(data['choices'][0]['text'].split())
                    
                    self.results["latencies"].append(latency)
                    self.results["tokens_per_second"].append(tokens / latency)
                else:
                    self.results["errors"] += 1
        except Exception as e:
            print(f"Request error: {e}")
            self.results["errors"] += 1
        finally:
            self.results["total_requests"] += 1
    
    async def run_test(self, num_requests: int = 100, concurrency: int = 10):
        """Run performance test with specified concurrency"""
        test_prompts = [
            "Explain the concept of quantum computing in simple terms.",
            "Write a Python function to calculate fibonacci numbers.",
            "What are the main differences between TCP and UDP?",
            "Describe the process of photosynthesis.",
            "How does a neural network learn?",
        ]
        
        async with aiohttp.ClientSession() as session:
            tasks = []
            for i in range(num_requests):
                prompt = test_prompts[i % len(test_prompts)]
                task = self.send_request(session, prompt)
                tasks.append(task)
                
                # Control concurrency
                if len(tasks) >= concurrency:
                    await asyncio.gather(*tasks)
                    tasks = []
            
            # Process remaining tasks
            if tasks:
                await asyncio.gather(*tasks)
    
    def print_results(self):
        """Print test results"""
        if not self.results["latencies"]:
            print("No successful requests")
            return
        
        avg_latency = statistics.mean(self.results["latencies"])
        p99_latency = sorted(self.results["latencies"])[int(len(self.results["latencies"]) * 0.99)]
        avg_tps = statistics.mean(self.results["tokens_per_second"])
        
        print("\n=== Performance Test Results ===")
        print(f"Total Requests: {self.results['total_requests']}")
        print(f"Successful: {len(self.results['latencies'])}")
        print(f"Errors: {self.results['errors']}")
        print(f"Average Latency: {avg_latency:.2f}s")
        print(f"P99 Latency: {p99_latency:.2f}s")
        print(f"Average Tokens/s: {avg_tps:.2f}")
        print(f"Success Rate: {(len(self.results['latencies']) / self.results['total_requests'] * 100):.1f}%")
        
        # Check against targets
        baseline_latency = 2.0  # Example baseline
        if p99_latency <= baseline_latency * 1.1:
            print("✅ Latency target met")
        else:
            print("❌ Latency target not met")

if __name__ == "__main__":
    import sys
    
    endpoint = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:80"
    
    test = PerformanceTest(endpoint)
    asyncio.run(test.run_test(num_requests=100, concurrency=10))
    test.print_results()

   chmod +x deploy-gpu-booster.sh
   ./deploy-gpu-booster.sh

   kubectl get pods -n gpu-booster -w

   kubectl port-forward -n gpu-booster svc/grafana 3000:3000

   python3 performance-test.py http://$(kubectl get svc nginx-lb-service -n gpu-booster -o jsonpath='{.status.loadBalancer.ingress[0].ip}')

# Check GPU metrics
kubectl exec -n gpu-booster deployment/prometheus -- promtool query instant http://localhost:9090 'dcgm_gpu_utilization'

# View HPA status
kubectl describe hpa vllm-deepseek-hpa -n gpu-booster

# Check GPU Booster logs
kubectl logs -n gpu-booster deployment/gpu-booster-plugin