Interconnecting GPUs in Managed Service for Kubernetes® clusters using InfiniBand™

export INFINIBAND_FABRIC=<fabric>

export GPU_CLUSTER_ID=$(nebius compute gpu-cluster create \
  --name gpu-cluster-name \
  --infiniband-fabric $INFINIBAND_FABRIC \
  --format jsonpath='{.metadata.id}')

nebius mk8s node-group create \
  --template-resources-platform gpu-h100-sxm \
  --template-resources-preset 8gpu-128vcpu-1600gb \
  --template-gpu-cluster-id $GPU_CLUSTER_ID \
  --template-gpu-settings-drivers-preset cuda13.0 \
  ...

infinibandFabric := "<fabric>"

gpuClusterOperation, err := sdk.Services().Compute().V1().
    GpuCluster().Create(
        ctx,
        &compute.CreateGpuClusterRequest{
            Metadata: &common.ResourceMetadata{
                Name: "gpu-cluster-name",
            },
            Spec: &compute.GpuClusterSpec{
                InfinibandFabric: infinibandFabric,
            },
        },
    )
if err != nil {
    return err
}
if _, err = gpuClusterOperation.Wait(ctx); err != nil {
    return err
}
gpuClusterID := gpuClusterOperation.ResourceID()
if gpuClusterID == "" {
    return fmt.Errorf("GPU cluster ID is missing")
}

nodeGroupOperation, err := sdk.Services().MK8S().V1().
    NodeGroup().Create(
        ctx,
        &mk8s.CreateNodeGroupRequest{
            Metadata: &common.ResourceMetadata{
                ParentId: mk8sClusterID,
                Name:     "node-group-example",
            },
            Spec: &mk8s.NodeGroupSpec{
                Version: "1.36",
                Size: &mk8s.NodeGroupSpec_FixedNodeCount{
                    FixedNodeCount: 1,
                },
                Template: &mk8s.NodeTemplate{
                    Resources: &mk8s.ResourcesSpec{
                        Platform: "gpu-h100-sxm",
                        Size: &mk8s.ResourcesSpec_Preset{
                            Preset: "8gpu-128vcpu-1600gb",
                        },
                    },
                    BootDisk: &mk8s.DiskSpec{
                        Size: &mk8s.DiskSpec_SizeGibibytes{
                            SizeGibibytes: 100,
                        },
                        Type: mk8s.DiskSpec_NETWORK_SSD,
                    },
                    GpuCluster: &mk8s.GpuClusterSpec{
                        Id: gpuClusterID,
                    },
                    GpuSettings: &mk8s.GpuSettings{
                        DriversPreset: "cuda13.0",
                    },
                },
            },
        },
    )
if err != nil {
    return err
}
if _, err = nodeGroupOperation.Wait(ctx); err != nil {
    return err
}

infiniband_fabric = "<fabric>"

gpu_cluster_service = GpuClusterServiceClient(sdk)
gpu_cluster_operation = await gpu_cluster_service.create(
    CreateGpuClusterRequest(
        metadata=ResourceMetadata(name="gpu-cluster-name"),
        spec=GpuClusterSpec(
            infiniband_fabric=infiniband_fabric,
        ),
    ),
)
await gpu_cluster_operation.wait()
gpu_cluster_id = gpu_cluster_operation.resource_id
if not gpu_cluster_id:
    raise ValueError("GPU cluster ID is missing")

node_group_service = NodeGroupServiceClient(sdk)
node_group_operation = await node_group_service.create(
    CreateNodeGroupRequest(
        metadata=ResourceMetadata(
            parent_id=mk8s_cluster_id,
            name="node-group-example",
        ),
        spec=NodeGroupSpec(
            version="1.36",
            fixed_node_count=1,
            template=NodeTemplate(
                resources=ResourcesSpec(
                    platform="gpu-h100-sxm",
                    preset="8gpu-128vcpu-1600gb",
                ),
                boot_disk=DiskSpec(
                    size_gibibytes=100,
                    type=DiskSpec__DiskType.NETWORK_SSD,
                ),
                gpu_cluster=GpuClusterSpec(id=gpu_cluster_id),
                gpu_settings=GpuSettings(drivers_preset="cuda13.0"),
            ),
        ),
    ),
)
await node_group_operation.wait()

const infinibandFabric = "<fabric>";

const gpuClusterService = new GpuClusterService(sdk);
const gpuClusterOperation = await gpuClusterService.create(
  CreateGpuClusterRequest.create({
    metadata: ResourceMetadata.create({
      name: "gpu-cluster-name",
    }),
    spec: GpuClusterSpec.create({
      infinibandFabric,
    }),
  }),
).result;
await gpuClusterOperation.wait();
const gpuClusterId = gpuClusterOperation.resourceId();
if (!gpuClusterId) {
  throw new Error("GPU cluster ID is missing");
}

const nodeGroupService = new NodeGroupService(sdk);
const nodeGroupOperation = await nodeGroupService.create(
  CreateNodeGroupRequest.create({
    metadata: ResourceMetadata.create({
      parentId: mk8sClusterId,
      name: "node-group-example",
    }),
    spec: NodeGroupSpec.create({
      version: "1.36",
      size: {
        $case: "fixedNodeCount",
        fixedNodeCount: 1,
      },
      template: NodeTemplate.create({
        resources: ResourcesSpec.create({
          platform: "gpu-h100-sxm",
          size: {
            $case: "preset",
            preset: "8gpu-128vcpu-1600gb",
          },
        }),
        bootDisk: DiskSpec.create({
          size: {
            $case: "sizeGibibytes",
            sizeGibibytes: 100,
          },
          type: DiskSpec_DiskType.NETWORK_SSD,
        }),
        gpuCluster: GpuClusterSpec.create({
          id: gpuClusterId,
        }),
        gpuSettings: GpuSettings.create({
          driversPreset: "cuda13.0",
        }),
      }),
    }),
  }),
).result;
await nodeGroupOperation.wait();