InfiniBand™ networking for Compute virtual machines with GPUs

cat ~/.nebius/config.yaml

nebius profile update --parent-id <project_ID>

export INFINIBAND_FABRIC=<fabric>

export GPU_CLUSTER_ID=$(nebius compute gpu-cluster create \
  --name <gpu_cluster_name> \
  --infiniband-fabric $INFINIBAND_FABRIC \
  --format json \
  | jq -r ".metadata.id")

export BOOT_DISK_ID=$(nebius compute disk create \
  --name <boot_disk_name> \
  --size-gibibytes 200 \
  --type network_ssd \
  --source-image-family-image-family ubuntu24.04-cuda13.0 \
  --block-size-bytes 4096 \
  --format json \
  | jq -r ".metadata.id")

nebius compute instance create \
  --resources-platform gpu-h100-sxm \
  --resources-preset 8gpu-128vcpu-1600gb \
  --gpu-cluster-id $GPU_CLUSTER_ID \
  --boot-disk-existing-disk-id $BOOT_DISK_ID \
  ...

infinibandFabric := "<fabric>"

gpuClusterOperation, err := sdk.Services().Compute().V1().
    GpuCluster().Create(
        ctx,
        &compute.CreateGpuClusterRequest{
            Metadata: &common.ResourceMetadata{
                Name: "<gpu_cluster_name>",
            },
            Spec: &compute.GpuClusterSpec{
                InfinibandFabric: infinibandFabric,
            },
        },
    )
if err != nil {
    return err
}
if _, err = gpuClusterOperation.Wait(ctx); err != nil {
    return err
}
gpuClusterID := gpuClusterOperation.ResourceID()

diskOperation, err := sdk.Services().Compute().V1().
    Disk().Create(
        ctx,
        &compute.CreateDiskRequest{
            Metadata: &common.ResourceMetadata{
                Name: "<boot_disk_name>",
            },
            Spec: &compute.DiskSpec{
                Size: &compute.DiskSpec_SizeGibibytes{
                    SizeGibibytes: 200,
                },
                BlockSizeBytes: 4096,
                Type:           compute.DiskSpec_NETWORK_SSD,
                Source: &compute.DiskSpec_SourceImageFamily{
                    SourceImageFamily: &compute.SourceImageFamily{
                        ImageFamily: "ubuntu24.04-cuda13.0",
                    },
                },
            },
        },
    )
if err != nil {
    return err
}
if _, err = diskOperation.Wait(ctx); err != nil {
    return err
}
bootDiskID := diskOperation.ResourceID()

attachMode := compute.AttachedDiskSpec_READ_WRITE
instanceOperation, err := sdk.Services().Compute().V1().
    Instance().Create(
        ctx,
        &compute.CreateInstanceRequest{
            Metadata: &common.ResourceMetadata{
                Name: "vm",
            },
            Spec: &compute.InstanceSpec{
                Stopped: true,
                Resources: &compute.ResourcesSpec{
                    Platform: "gpu-h100-sxm",
                    Size: &compute.ResourcesSpec_Preset{
                        Preset: "8gpu-128vcpu-1600gb",
                    },
                },
                GpuCluster: &compute.InstanceGpuClusterSpec{
                    Id: gpuClusterID,
                },
                BootDisk: &compute.AttachedDiskSpec{
                    AttachMode: attachMode,
                    Type: &compute.AttachedDiskSpec_ExistingDisk{
                        ExistingDisk: &compute.ExistingDisk{
                            Id: bootDiskID,
                        },
                    },
                },
                NetworkInterfaces: []*compute.NetworkInterfaceSpec{
                    {
                        Name:      "ni",
                        SubnetId:  subnetID,
                        IpAddress: &compute.IPAddress{},
                        PublicIpAddress: &compute.PublicIPAddress{},
                    },
                },
            },
        },
    )
if err != nil {
    return err
}
if _, err = instanceOperation.Wait(ctx); err != nil {
    return err
}

infiniband_fabric = "<fabric>"

gpu_cluster_service = GpuClusterServiceClient(sdk)
create_cluster_operation = await gpu_cluster_service.create(
    CreateGpuClusterRequest(
        metadata=ResourceMetadata(
            name="<gpu_cluster_name>",
        ),
        spec=GpuClusterSpec(
            infiniband_fabric=infiniband_fabric,
        ),
    ),
)
await create_cluster_operation.wait()
gpu_cluster_id = create_cluster_operation.resource_id

disk_service = DiskServiceClient(sdk)
create_disk_operation = await disk_service.create(
    CreateDiskRequest(
        metadata=ResourceMetadata(
            name="<boot_disk_name>",
        ),
        spec=DiskSpec(
            block_size_bytes=4096,
            type=DiskSpec.DiskType.NETWORK_SSD,
            source_image_family=SourceImageFamily(
                image_family="ubuntu24.04-cuda13.0",
            ),
            size_gibibytes=200,
        ),
    ),
)
await create_disk_operation.wait()
boot_disk_id = create_disk_operation.resource_id

instance_service = InstanceServiceClient(sdk)
create_instance_operation = await instance_service.create(
    CreateInstanceRequest(
        metadata=ResourceMetadata(
            name="vm",
        ),
        spec=InstanceSpec(
            stopped=True,
            resources=ResourcesSpec(
                platform="gpu-h100-sxm",
                preset="8gpu-128vcpu-1600gb",
            ),
            gpu_cluster=InstanceGpuClusterSpec(
                id=gpu_cluster_id,
            ),
            boot_disk=AttachedDiskSpec(
                attach_mode=(
                    AttachedDiskSpec.AttachMode.READ_WRITE
                ),
                existing_disk=ExistingDisk(id=boot_disk_id),
            ),
            network_interfaces=[
                NetworkInterfaceSpec(
                    name="ni",
                    subnet_id=subnet_id,
                    ip_address=IPAddress(),
                    public_ip_address=PublicIPAddress(),
                ),
            ],
        ),
    ),
)
await create_instance_operation.wait()

const infinibandFabric = "<fabric>";

const gpuClusterService = new GpuClusterService(sdk);
const createGpuClusterOperation = await gpuClusterService.create(
  CreateGpuClusterRequest.create({
    metadata: ResourceMetadata.create({
      name: "<gpu_cluster_name>",
    }),
    spec: GpuClusterSpec.create({
      infinibandFabric,
    }),
  }),
).result;
await createGpuClusterOperation.wait();
const gpuClusterId = createGpuClusterOperation.resourceId();

const diskService = new DiskService(sdk);
const createDiskOperation = await diskService.create(
  CreateDiskRequest.create({
    metadata: ResourceMetadata.create({
      name: "<boot_disk_name>",
    }),
    spec: DiskSpec.create({
      blockSizeBytes: 4096,
      type: DiskSpec_DiskType.NETWORK_SSD,
      source: {
        $case: "sourceImageFamily",
        sourceImageFamily: SourceImageFamily.create({
          imageFamily: "ubuntu24.04-cuda13.0",
        }),
      },
      size: {
        $case: "sizeGibibytes",
        sizeGibibytes: 200,
      },
    }),
  }),
).result;
await createDiskOperation.wait();
const bootDiskId = createDiskOperation.resourceId();

const instanceService = new InstanceService(sdk);
const createInstanceOperation = await instanceService.create(
  CreateInstanceRequest.create({
    metadata: ResourceMetadata.create({
      name: "vm",
    }),
    spec: InstanceSpec.create({
      stopped: true,
      resources: ResourcesSpec.create({
        platform: "gpu-h100-sxm",
        size: {
          $case: "preset",
          preset: "8gpu-128vcpu-1600gb",
        },
      }),
      gpuCluster: InstanceGpuClusterSpec.create({
        id: gpuClusterId,
      }),
      bootDisk: AttachedDiskSpec.create({
        attachMode: AttachedDiskSpec_AttachMode.READ_WRITE,
        type: {
          $case: "existingDisk",
          existingDisk: ExistingDisk.create({
            id: bootDiskId,
          }),
        },
      }),
      networkInterfaces: [
        NetworkInterfaceSpec.create({
          name: "ni",
          subnetId,
          ipAddress: IPAddress.create({}),
          publicIpAddress: PublicIPAddress.create({}),
        }),
      ],
    }),
  }),
).result;
await createInstanceOperation.wait();

nebius compute gpu-cluster list

nebius compute gpu-cluster delete <GPU_cluster_ID>

gpuClusters, err := sdk.Services().Compute().V1().
    GpuCluster().List(
        ctx,
        &compute.ListGpuClustersRequest{
            ParentId: nbProject,
        },
    )
if err != nil {
    return err
}
fmt.Println(gpuClusters)

delClusterOp, err := sdk.Services().Compute().V1().
    GpuCluster().Delete(
        ctx,
        &compute.DeleteGpuClusterRequest{
            Id: "<GPU_cluster_ID>",
        },
    )
if err != nil {
    return err
}
if _, err = delClusterOp.Wait(ctx); err != nil {
    return err
}

gpu_cluster_service = GpuClusterServiceClient(sdk)
gpu_clusters = await gpu_cluster_service.list(
    ListGpuClustersRequest(parent_id=nb_project),
)
print(gpu_clusters)

gpu_cluster_service = GpuClusterServiceClient(sdk)
delete_cluster_operation = await gpu_cluster_service.delete(
    DeleteGpuClusterRequest(id="<GPU_cluster_ID>"),
)
await delete_cluster_operation.wait()

const listGpuClusterService = new GpuClusterService(sdk);
const gpuClusters = await listGpuClusterService.list(
  ListGpuClustersRequest.create({
    parentId: nbProject,
  }),
);
console.log(gpuClusters);

const deleteGpuClusterService = new GpuClusterService(sdk);
const deleteClusterOperation = await deleteGpuClusterService.delete(
  DeleteGpuClusterRequest.create({
    id: "<GPU_cluster_ID>",
  }),
).result;
await deleteClusterOperation.wait();