// resource requirements
var skuNumGPUs int
// initially respect the user setting by deploying the model on the same number of nodes as the user requested
numNodes := *workspaceObj.Resource.Count
gpuConfig, err := utils.GetGPUConfigBySKU(workspaceObj.Resource.InstanceType)
if err != nil {
	gpuConfig, err = utils.TryGetGPUConfigFromNode(ctx, kubeClient, workspaceObj.Status.WorkerNodes)
	if err != nil {
		defaultNumGPU := resource.MustParse(model.GetInferenceParameters().GPUCountRequirement)
		skuNumGPUs = int(defaultNumGPU.Value())
	}
}
if gpuConfig != nil {
	skuNumGPUs = gpuConfig.GPUCount
	// Calculate the minimum number of nodes required to satisfy the model's total GPU memory requirement.
	// The goal is to maximize GPU utilization and not spread the model across too many nodes.
	totalGPUMemoryRequired := resource.MustParse(model.GetInferenceParameters().TotalGPUMemoryRequirement)
	totalGPUMemoryPerNode := resource.NewQuantity(int64(gpuConfig.GPUMemGB)*consts.GiBToBytes, resource.BinarySI)

	minimumNodes := 0
	for ; totalGPUMemoryRequired.Sign() > 0; totalGPUMemoryRequired.Sub(*totalGPUMemoryPerNode) {
		minimumNodes++
	}
	if minimumNodes < numNodes {
		numNodes = minimumNodes
	}
}

gctx := &generator.WorkspaceGeneratorContext{
	Ctx:        ctx,
	KubeClient: kubeClient,
	Workspace:  workspaceObj,
	Model:      model,
}
podOpts := []generator.TypedManifestModifier[generator.WorkspaceGeneratorContext, corev1.PodSpec]{
	GenerateInferencePodSpec(gpuConfig, skuNumGPUs, numNodes),
	SetModelDownloadInfo,
	SetAdapterPuller,
}

var depObj client.Object
// For multi-node distributed inference with vLLM, we need to use a StatefulSet instead of a Deployment
// to ensure pods are created with individual identities (their ordinal indexes) -
// https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-identity
runtimeName := v1beta1.GetWorkspaceRuntimeName(workspaceObj)
if model.SupportDistributedInference() && runtimeName == pkgmodel.RuntimeNameVLLM && numNodes > 1 {
	podOpts = append(podOpts, SetDistributedInferenceProbe)

	var ssOpts []generator.TypedManifestModifier[generator.WorkspaceGeneratorContext, appsv1.StatefulSet]
	if checkIfNVMeAvailable(ctx, gpuConfig, kubeClient) {
		ssOpts = append(ssOpts, manifests.AddStatefulSetVolumeClaimTemplates(GenerateModelWeightsCacheVolume(ctx, workspaceObj, model)))
	} else {
		podOpts = append(podOpts, SetDefaultModelWeightsVolume)
	}

	podSpec, err := generator.GenerateManifest(gctx, podOpts...)
	if err != nil {
		return nil, err
	}

	ssOpts = append(ssOpts,
		manifests.GenerateStatefulSetManifest(revisionNum, numNodes),
		manifests.SetStatefulSetPodSpec(podSpec),
	)

	depObj, err = generator.GenerateManifest(gctx, ssOpts...)
} else {
	podOpts = append(podOpts, SetDefaultModelWeightsVolume)

	podSpec, err := generator.GenerateManifest(gctx, podOpts...)
	if err != nil {
		return nil, err
	}

	depObj, err = generator.GenerateManifest(gctx,
		manifests.GenerateDeploymentManifest(revisionNum, numNodes),
		manifests.SetDeploymentPodSpec(podSpec),
	)
}

func GenerateStatefulSetManifest(revisionNum string, replicas int) func(*generator.WorkspaceGeneratorContext, *appsv1.StatefulSet) error {
	return func(ctx *generator.WorkspaceGeneratorContext, ss *appsv1.StatefulSet) error {
		selector := map[string]string{
			kaitov1beta1.LabelWorkspaceName: ctx.Workspace.Name,
		}
		labelselector := &v1.LabelSelector{
			MatchLabels: selector,
		}

		ss.ObjectMeta = v1.ObjectMeta{
			Name:      ctx.Workspace.Name,
			Namespace: ctx.Workspace.Namespace,
			Annotations: map[string]string{
				kaitov1beta1.WorkspaceRevisionAnnotation: revisionNum,
			},
			OwnerReferences: []v1.OwnerReference{
				*v1.NewControllerRef(ctx.Workspace, kaitov1beta1.GroupVersion.WithKind("Workspace")),
			},
		}
		ss.Spec = appsv1.StatefulSetSpec{
			Replicas:            lo.ToPtr(int32(replicas)),
			PodManagementPolicy: appsv1.ParallelPodManagement,
			PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{
				WhenScaled:  appsv1.RetainPersistentVolumeClaimRetentionPolicyType,
				WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType,
			},
			Selector: labelselector,
			Template: corev1.PodTemplateSpec{
				ObjectMeta: v1.ObjectMeta{
					Labels: selector,
				},
			},
		}

		ss.Spec.ServiceName = fmt.Sprintf("%s-headless", ctx.Workspace.Name)
		return nil
	}

spec.Volumes = append(spec.Volumes, adapterVolume)
for i := range spec.Containers { // FIXME: assume only one container in the pod
	spec.Containers[i].VolumeMounts = append(spec.Containers[i].VolumeMounts, adapterVolumeMount)