Part 5 of 5: Deploy vLLM

2026-07-22 · 2 min read

Deploy vLLM

resource "kubernetes_namespace_v1" "vllm" {
  metadata {
    name = "vllm"
  }
}

resource "kubernetes_persistent_volume_claim_v1" "vllm_data" {
  metadata {
    name      = "vllm-data-pvc"
    namespace = kubernetes_namespace_v1.vllm.metadata[0].name
  }

  spec {
    storage_class_name = "ceph-hdd-ec"
    access_modes       = ["ReadWriteMany"]

    resources {
      requests = {
        storage = "1Ti"
      }
    }
  }
}


resource "kubernetes_deployment_v1" "inference_llm" {
  metadata {
    name      = "inference-llm"
    namespace = kubernetes_namespace_v1.vllm.metadata[0].name
  }

  spec {
    replicas = 1

    strategy {
      type = "Recreate"
    }

    selector {
      match_labels = {
        app = "inference-llm"
      }
    }

    template {
      metadata {
        labels = {
          app = "inference-llm"
        }
      }

      spec {
        enable_service_links = false

        volume {
          name = "vllm-data"
          persistent_volume_claim {
            claim_name = kubernetes_persistent_volume_claim_v1.vllm_data.metadata[0].name
          }
        }

        container {
          name  = "vllm"
          image = "vllm-xpu:v0.26.0"
          tty = true

          command = [
            "bash", "-c"
          ]
          
          args = [
            <<EOT
              source /opt/intel/oneapi/setvars.sh --force
              source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force
              vllm serve \
                Qwen/Qwen3.6-27B \
                --safetensors-load-strategy prefetch \
                --gpu-memory-util 0.95 \
                --dtype bfloat16 \
                --trust-remote-code \
                --disable-sliding-window \
                --block-size 64 \
                --tensor-parallel-size 2 \
                --kv-cache-dtype fp8 \
                --max-num-seqs 6 \
                --enable-chunked-prefill \
                --max-num-batched-tokens 832 \
                --language-model-only \
                --enable-auto-tool-choice \
                --tool-call-parser qwen3_xml \
                --reasoning-parser deepseek_r1 \
                --max-model-len 32768 \
                --served-model-name Qwen3.6-27B \
                --speculative-config '{"method":"mtp","num_speculative_tokens":2}'
            EOT
          ]

          volume_mount {
            name       = "vllm-data"
            mount_path = "/root/.cache"
          }

          port {
            container_port = 8000
            protocol       = "TCP"
          }

          resources {
            requests = {
              "gpu.intel.com/xe" = 2
            }
            limits = {
              "gpu.intel.com/xe" = 2
            }
          }
        }
      }
    }
  }
}