Multi-node serving on Dynamo

# EKS g6.8xlarge, one NVIDIA L4 per node, with EFA. A single L4 can't hold the
# 14B model, so the gang spans two nodes; EFA gives their cross-node traffic and
# ModelExpress's peer-to-peer weight transfer an RDMA fabric instead of plain
# TCP. Its 128 GiB of memory holds the weights as they load, and the 100 GB disk
# holds the vLLM image.
#
# Both the GPU and the EFA fabric are claim: DRA devices. A gang's nodeSelector
# requests both, so DRA binds one L4 and one EFA interface per pod. The EFA
# device is installed by the EFA DRA driver the Dynamo stack runs.
apiVersion: modelplane.ai/v1alpha1
kind: InferenceClass
metadata:
  name: l4-1x-g6-efa
spec:
  description: "EKS g6.8xlarge, 1x NVIDIA L4, EFA"
  provisioning:
    provider: EKS
    eks:
      instanceType: g6.8xlarge
      diskSizeGb: 100
      accelerator:
        type: nvidia-l4
        count: 1
  devices:
  - name: gpu
    claim: DRA
    driver: gpu.nvidia.com
    deviceClassName: gpu.nvidia.com
    count: 1
    attributes:
      architecture: { string: Ada Lovelace }
    capacity:
      memory: { value: "23034Mi" }   # L4's real reported VRAM (not the nominal 24GB)
  - name: efa
    claim: DRA
    driver: dra.net
    deviceClassName: efa.networking.k8s.aws
    count: 1

# An EKS cluster running the Dynamo serving stack, with a two-node L4 pool so a
# gang can span both nodes. spec.stack: Dynamo installs Grove and the KAI
# Scheduler, which gang-schedule the leader and worker together and compose them
# as a Grove PodCliqueSet.
#
# fabric.type: EFA turns on Elastic Fabric Adapter for the pool, so the gang's
# cross-node traffic and ModelExpress's peer-to-peer weight transfer run over an
# RDMA fabric. Without it multi-node NCCL falls back to TCP, which is slow and
# unstable.
apiVersion: modelplane.ai/v1alpha1
kind: InferenceCluster
metadata:
  name: eks-us-east
  labels:
    modelplane.ai/region: us-east
spec:
  stack: Dynamo
  cluster:
    source: EKS
    eks:
      region: us-east-1
  nodePools:
  - name: gpu-l4
    className: l4-1x-g6-efa
    nodeCount: 2
    minNodeCount: 2
    maxNodeCount: 2
    zones:
    - us-east-1b
    fabric:
      type: EFA

# The shared read-write-many cache the gang serves from, hydrated once from
# Hugging Face. Both gang pods mount it and read weights from it over EFS,
# instead of each pulling its own copy. Qwen2.5-14B is open, so it needs no
# token. Its FP16 weights are about 29 GB, so sizeGiB leaves headroom.
apiVersion: modelplane.ai/v1alpha1
kind: ModelCache
metadata:
  name: qwen2-5-14b
  namespace: ml-team
spec:
  source: HuggingFace
  huggingFace:
    repo: Qwen/Qwen2.5-14B-Instruct
    sizeGiB: 40

# Qwen2.5-14B served across two L4 nodes as a gang. The FP16 weights (~29 GB)
# don't fit one L4's 23 GB, so the engine is a Leader + Worker gang,
# pipeline-parallel across two g6.8xlarge nodes with one L4 each. Both pods mount
# the shared ModelCache and claim an EFA interface for the fabric.
#
# The cluster runs the Dynamo stack, so Grove and the KAI Scheduler gang-schedule
# the two pods, and Modelplane composes them as a Grove PodCliqueSet.
# $(MODELPLANE_LEADER_ADDRESS) resolves to the leader on Dynamo, but
# $(MODELPLANE_RANK) isn't injected there yet (modelplaneai/modelplane#418), so
# each command sets its own --node-rank: 0 on the leader, and
# $$((GROVE_PCLQ_POD_INDEX + 1)) on the worker ($$ escapes past Kubernetes,
# leaving $((...)) for the shell to evaluate).
#
# Notes on the engine flags:
#   --pipeline-parallel-size=2 splits the model across the two nodes;
#     --tensor-parallel-size=1 keeps one GPU per node. Pipeline parallelism sends
#     only activations between nodes, so it stays light on the fabric.
#   --distributed-executor-backend=mp is vLLM's native multiprocessing multi-node
#     path; vllm/vllm-openai:v0.23.0 no longer ships Ray.
#   --load-format modelexpress loads weights through the ModelExpress server the
#     Dynamo stack runs: the leader seeds from the cache and publishes itself, and
#     the worker pulls peer-to-peer over EFA rather than reading the cache again.
#     The vLLM image doesn't ship the loader, so pip install it first.
#     --load-format=runai_streamer is the alternative that reads the cache
#     directly, on any stack.
#   --max-model-len=8192 caps context so the KV cache fits alongside the weights.
# FI_PROVIDER=efa points libfabric at the EFA interface; NCCL_DEBUG=INFO logs the
# transport NCCL picks, so you can confirm it's EFA and not TCP.
apiVersion: modelplane.ai/v1alpha1
kind: ModelDeployment
metadata:
  name: qwen2-5-14b
  namespace: ml-team
spec:
  replicas: 1
  template:
    spec:
      modelCacheRef:
        name: qwen2-5-14b
      engines:
      - name: qwen
        members:
        - role: Leader
          nodeSelector:
            devices:
            - name: gpu
              count: 1
              selectors:
              - cel: |
                  device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("20Gi")) >= 0
            - name: efa
              count: 1
              selectors:
              - cel: |
                  device.driver == "dra.net"
          template:
            spec:
              containers:
              - name: engine
                image: vllm/vllm-openai:v0.23.0
                env:
                - name: FI_PROVIDER
                  value: "efa"
                - name: NCCL_DEBUG
                  value: "INFO"
                command:
                - /bin/sh
                - -c
                - >-
                  pip install --index-url https://pypi.nvidia.com modelexpress &&
                  exec vllm serve Qwen/Qwen2.5-14B-Instruct
                  --served-model-name=qwen2.5-14b
                  --tensor-parallel-size=1
                  --pipeline-parallel-size=2
                  --distributed-executor-backend=mp
                  --nnodes=2 --node-rank=0
                  --master-addr=$(MODELPLANE_LEADER_ADDRESS)
                  --load-format modelexpress
                  --max-model-len=8192
                  --gpu-memory-utilization=0.90
                  --port=8000
        - role: Worker
          worker:
            nodes: 1
          nodeSelector:
            devices:
            - name: gpu
              count: 1
              selectors:
              - cel: |
                  device.capacity["gpu.nvidia.com"].memory.compareTo(quantity("20Gi")) >= 0
            - name: efa
              count: 1
              selectors:
              - cel: |
                  device.driver == "dra.net"
          template:
            spec:
              containers:
              - name: engine
                image: vllm/vllm-openai:v0.23.0
                env:
                - name: FI_PROVIDER
                  value: "efa"
                - name: NCCL_DEBUG
                  value: "INFO"
                command:
                - /bin/sh
                - -c
                - >-
                  pip install --index-url https://pypi.nvidia.com modelexpress &&
                  exec vllm serve Qwen/Qwen2.5-14B-Instruct
                  --served-model-name=qwen2.5-14b
                  --tensor-parallel-size=1
                  --pipeline-parallel-size=2
                  --distributed-executor-backend=mp
                  --nnodes=2 --node-rank=$$((GROVE_PCLQ_POD_INDEX + 1))
                  --master-addr=$(MODELPLANE_LEADER_ADDRESS)
                  --headless
                  --load-format modelexpress
                  --max-model-len=8192
                  --gpu-memory-utilization=0.90
                  --port=8000

# Exposes the gang as one OpenAI-compatible URL. Modelplane composes one
# ModelEndpoint per replica, labeled modelplane.ai/deployment: qwen2-5-14b, so
# this selector reaches it. Read the public address from status.address:
#   kubectl get ms qwen2-5-14b -n ml-team -o jsonpath='{.status.address}'
apiVersion: modelplane.ai/v1alpha1
kind: ModelService
metadata:
  name: qwen2-5-14b
  namespace: ml-team
spec:
  endpoints:
  - selector:
      matchLabels:
        modelplane.ai/deployment: qwen2-5-14b