From 52b457421a9bcccc9e8d7f4e63c755e1fd635581 Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Thu, 27 Mar 2025 18:22:50 +0000 Subject: [PATCH 1/6] Pod level hugepage cgroup when unset in container --- pkg/apis/core/types.go | 2 +- .../kuberuntime_container_linux.go | 51 ++++++++++++------- .../kuberuntime_container_linux_test.go | 4 +- staging/src/k8s.io/api/core/v1/types.go | 2 +- 4 files changed, 38 insertions(+), 21 deletions(-) diff --git a/pkg/apis/core/types.go b/pkg/apis/core/types.go index 32707e245d2..b544f3a867d 100644 --- a/pkg/apis/core/types.go +++ b/pkg/apis/core/types.go @@ -3787,7 +3787,7 @@ type PodSpec struct { ResourceClaims []PodResourceClaim // Resources is the total amount of CPU and Memory resources required by all // containers in the pod. It supports specifying Requests and Limits for - // "cpu" and "memory" resource names only. ResourceClaims are not supported. + // "cpu", "memory" and "hugepages-" resource names only. ResourceClaims are not supported. // // This field enables fine-grained control over resource allocation for the // entire pod, allowing resource sharing among containers in a pod. diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go b/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go index 1a62f6ce780..c3cb5fcd231 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go @@ -145,7 +145,7 @@ func (m *kubeGenericRuntimeManager) generateLinuxContainerResources(ctx context. lcr.OomScoreAdj = int64(qos.GetContainerOOMScoreAdjust(pod, container, int64(m.machineInfo.MemoryCapacity))) - lcr.HugepageLimits = GetHugepageLimitsFromResources(ctx, container.Resources) + lcr.HugepageLimits = GetHugepageLimitsFromResources(ctx, pod, container.Resources) // Configure swap for the container m.configureContainerSwapResources(ctx, lcr, pod, container) @@ -323,8 +323,7 @@ func (m *kubeGenericRuntimeManager) calculateLinuxResources(cpuRequest, cpuLimit } // GetHugepageLimitsFromResources returns limits of each hugepages from resources. -func GetHugepageLimitsFromResources(ctx context.Context, resources v1.ResourceRequirements) []*runtimeapi.HugepageLimit { - logger := klog.FromContext(ctx) +func GetHugepageLimitsFromResources(ctx context.Context, pod *v1.Pod, containerResources v1.ResourceRequirements) []*runtimeapi.HugepageLimit { var hugepageLimits []*runtimeapi.HugepageLimit // For each page size, limit to 0. @@ -336,23 +335,20 @@ func GetHugepageLimitsFromResources(ctx context.Context, resources v1.ResourceRe } requiredHugepageLimits := map[string]uint64{} - for resourceObj, amountObj := range resources.Limits { - if !v1helper.IsHugePageResourceName(resourceObj) { - continue - } - pageSize, err := v1helper.HugePageSizeFromResourceName(resourceObj) - if err != nil { - logger.Info("Failed to get hugepage size from resource", "object", resourceObj, "err", err) - continue + // When hugepage limits are specified at pod level and no hugepage limits are + // specified at container level, the container's cgroup will reflect the pod level limit. + if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodLevelResources) && resourcehelper.IsPodLevelResourcesSet(pod) { + for limitName, limitAmount := range pod.Spec.Resources.Limits { + readAndDefineRequiredHugepageLimit(ctx, requiredHugepageLimits, limitName, limitAmount) } + } - sizeString, err := v1helper.HugePageUnitSizeFromByteSize(pageSize.Value()) - if err != nil { - logger.Info("Size is invalid", "object", resourceObj, "err", err) - continue - } - requiredHugepageLimits[sizeString] = uint64(amountObj.Value()) + // If both the pod level and the container level specify hugepages, the container + // level will have precedence, so the container's hugepages limit will be reflected + // in the container's cgroup values. + for resourceObj, amountObj := range containerResources.Limits { + readAndDefineRequiredHugepageLimit(ctx, requiredHugepageLimits, resourceObj, amountObj) } for _, hugepageLimit := range hugepageLimits { @@ -364,6 +360,27 @@ func GetHugepageLimitsFromResources(ctx context.Context, resources v1.ResourceRe return hugepageLimits } +func readAndDefineRequiredHugepageLimit(ctx context.Context, requiredHugepageLimits map[string]uint64, resourceObj v1.ResourceName, amountObj resource.Quantity) { + logger := klog.FromContext(ctx) + + if !v1helper.IsHugePageResourceName(resourceObj) { + return + } + + pageSize, err := v1helper.HugePageSizeFromResourceName(resourceObj) + if err != nil { + logger.Info("Failed to get hugepage size from resource", "object", resourceObj, "err", err) + return + } + + sizeString, err := v1helper.HugePageUnitSizeFromByteSize(pageSize.Value()) + if err != nil { + logger.Info("Size is invalid", "object", resourceObj, "err", err) + return + } + requiredHugepageLimits[sizeString] = uint64(amountObj.Value()) +} + func toKubeContainerResources(statusResources *runtimeapi.ContainerResources) *kubecontainer.ContainerResources { var cStatusResources *kubecontainer.ContainerResources runtimeStatusResources := statusResources.GetLinux() diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go index 41378547592..9a73204f8f9 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go @@ -50,7 +50,7 @@ import ( "k8s.io/utils/ptr" ) -func makeExpectedConfig(t *testing.T, tCtx context.Context, m *kubeGenericRuntimeManager, pod *v1.Pod, containerIndex int, enforceMemoryQoS bool) *runtimeapi.ContainerConfig { +func makeExpectedConfig(_ *testing.T, tCtx context.Context, m *kubeGenericRuntimeManager, pod *v1.Pod, containerIndex int, enforceMemoryQoS bool) *runtimeapi.ContainerConfig { container := &pod.Spec.Containers[containerIndex] podIP := "" restartCount := 0 @@ -676,7 +676,7 @@ func TestGetHugepageLimitsFromResources(t *testing.T) { } } - results := GetHugepageLimitsFromResources(tCtx, test.resources) + results := GetHugepageLimitsFromResources(tCtx, &v1.Pod{}, test.resources) if !reflect.DeepEqual(expectedHugepages, results) { t.Errorf("%s test failed. Expected %v but got %v", test.name, expectedHugepages, results) } diff --git a/staging/src/k8s.io/api/core/v1/types.go b/staging/src/k8s.io/api/core/v1/types.go index 5b4602e1dbb..924dc31d32a 100644 --- a/staging/src/k8s.io/api/core/v1/types.go +++ b/staging/src/k8s.io/api/core/v1/types.go @@ -4338,7 +4338,7 @@ type PodSpec struct { ResourceClaims []PodResourceClaim `json:"resourceClaims,omitempty" patchStrategy:"merge,retainKeys" patchMergeKey:"name" protobuf:"bytes,39,rep,name=resourceClaims"` // Resources is the total amount of CPU and Memory resources required by all // containers in the pod. It supports specifying Requests and Limits for - // "cpu" and "memory" resource names only. ResourceClaims are not supported. + // "cpu", "memory" and "hugepages-" resource names only. ResourceClaims are not supported. // // This field enables fine-grained control over resource allocation for the // entire pod, allowing resource sharing among containers in a pod. From 8e3f93c879654c4f3a9c745a7c739540730e88c9 Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Tue, 24 Jun 2025 21:31:33 +0000 Subject: [PATCH 2/6] Unit test propagate pod level hugepages to containers --- .../kuberuntime_container_linux_test.go | 151 ++++++++++++++++-- 1 file changed, 141 insertions(+), 10 deletions(-) diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go index 9a73204f8f9..8162dd82163 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go @@ -542,9 +542,11 @@ func TestGetHugepageLimitsFromResources(t *testing.T) { } tests := []struct { - name string - resources v1.ResourceRequirements - expected []*runtimeapi.HugepageLimit + name string + podResources v1.ResourceRequirements + resources v1.ResourceRequirements + expected []*runtimeapi.HugepageLimit + podLevelResourcesEnabled bool }{ { name: "Success2MB", @@ -606,9 +608,8 @@ func TestGetHugepageLimitsFromResources(t *testing.T) { name: "Success2MBand1GB", resources: v1.ResourceRequirements{ Limits: v1.ResourceList{ - v1.ResourceName(v1.ResourceCPU): resource.MustParse("0"), - "hugepages-2Mi": resource.MustParse("2Mi"), - "hugepages-1Gi": resource.MustParse("2Gi"), + "hugepages-2Mi": resource.MustParse("2Mi"), + "hugepages-1Gi": resource.MustParse("2Gi"), }, }, expected: []*runtimeapi.HugepageLimit{ @@ -626,9 +627,8 @@ func TestGetHugepageLimitsFromResources(t *testing.T) { name: "Skip2MBand1GB", resources: v1.ResourceRequirements{ Limits: v1.ResourceList{ - v1.ResourceName(v1.ResourceCPU): resource.MustParse("0"), - "hugepages-2MB": resource.MustParse("2Mi"), - "hugepages-1GB": resource.MustParse("2Gi"), + "hugepages-2MB": resource.MustParse("2Mi"), + "hugepages-1GB": resource.MustParse("2Gi"), }, }, expected: []*runtimeapi.HugepageLimit{ @@ -642,6 +642,131 @@ func TestGetHugepageLimitsFromResources(t *testing.T) { }, }, }, + // Pod level hugepage set 2MB, Container level hugepage unset + { + name: "PodLevel2MB", + podResources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "hugepages-2Mi": resource.MustParse("2Mi"), + }, + }, + resources: v1.ResourceRequirements{}, + expected: []*runtimeapi.HugepageLimit{ + { + PageSize: "2MB", + Limit: 2097152, + }, + }, + podLevelResourcesEnabled: true, + }, + // Pod level hugepage set 1GB, Container level hugepage unset + { + name: "PodLevel1GB", + podResources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "hugepages-1Gi": resource.MustParse("2Gi"), + }, + }, + resources: v1.ResourceRequirements{}, + expected: []*runtimeapi.HugepageLimit{ + { + PageSize: "1GB", + Limit: 2147483648, + }, + }, + podLevelResourcesEnabled: true, + }, + // Pod level hugepage set 2MB 1GB, Container level hugepage unset + { + name: "PodLevel2MBand1GB", + podResources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "hugepages-2Mi": resource.MustParse("2Mi"), + "hugepages-1Gi": resource.MustParse("2Gi"), + }, + }, + resources: v1.ResourceRequirements{}, + expected: []*runtimeapi.HugepageLimit{ + { + PageSize: "2MB", + Limit: 2097152, + }, + { + PageSize: "1GB", + Limit: 2147483648, + }, + }, + podLevelResourcesEnabled: true, + }, + // Pod level hugepage set 2MB, Container level hugepage set + { + name: "PodLevel2MBContainerLevel2MB", + podResources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "hugepages-2Mi": resource.MustParse("4Mi"), + }, + }, + resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "hugepages-2Mi": resource.MustParse("2Mi"), + }, + }, + expected: []*runtimeapi.HugepageLimit{ + { + PageSize: "2MB", + Limit: 2097152, + }, + }, + podLevelResourcesEnabled: true, + }, + // Pod level hugepage set 1GB, Container level hugepage set + { + name: "PodLevel1GBContainerLevel1GB", + podResources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "hugepages-1Gi": resource.MustParse("4Gi"), + }, + }, + resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "hugepages-1Gi": resource.MustParse("2Gi"), + }, + }, + expected: []*runtimeapi.HugepageLimit{ + { + PageSize: "1GB", + Limit: 2147483648, + }, + }, + podLevelResourcesEnabled: true, + }, + // Pod level hugepage set 2MB 1GB, Container level hugepage set + { + name: "PodLevel2MBand1GBContainerLevel2MBand1GB", + podResources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "hugepages-2Mi": resource.MustParse("4Mi"), + "hugepages-1Gi": resource.MustParse("4Gi"), + }, + }, + resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "hugepages-2Mi": resource.MustParse("2Mi"), + "hugepages-1Gi": resource.MustParse("2Gi"), + }, + }, + expected: []*runtimeapi.HugepageLimit{ + { + PageSize: "2MB", + Limit: 2097152, + }, + { + PageSize: "1GB", + Limit: 2147483648, + }, + }, + podLevelResourcesEnabled: true, + }, } for _, test := range tests { @@ -676,7 +801,13 @@ func TestGetHugepageLimitsFromResources(t *testing.T) { } } - results := GetHugepageLimitsFromResources(tCtx, &v1.Pod{}, test.resources) + testPod := &v1.Pod{} + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLevelResources, test.podLevelResourcesEnabled) + if test.podLevelResourcesEnabled { + testPod.Spec.Resources = &test.podResources + } + + results := GetHugepageLimitsFromResources(tCtx, testPod, test.resources) if !reflect.DeepEqual(expectedHugepages, results) { t.Errorf("%s test failed. Expected %v but got %v", test.name, expectedHugepages, results) } From 845e94d37075d199cd3cc23ac05f05a95e2c15b8 Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Wed, 25 Jun 2025 17:40:01 +0000 Subject: [PATCH 3/6] Validation logic and Defaulting update for pod level hugepages The hugepage aggregated container limits cannot be greater than pod-level limits. This was already enforced with the defaulted requests from the specfied limits, however it did not make it clear about both hugepage requests and limits. --- pkg/apis/core/v1/defaults.go | 30 ++++++++++++++++---------- pkg/apis/core/validation/validation.go | 23 ++++++++++++++++++++ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/pkg/apis/core/v1/defaults.go b/pkg/apis/core/v1/defaults.go index fc6a6ad25ee..74dbc832d4a 100644 --- a/pkg/apis/core/v1/defaults.go +++ b/pkg/apis/core/v1/defaults.go @@ -455,9 +455,8 @@ func defaultPodRequests(obj *v1.Pod) { // PodLevelResources feature) and pod-level requests are not set, the pod-level requests // default to the effective requests of all the containers for that resource. for key, aggrCtrLim := range aggrCtrReqs { - // Defaulting for pod level hugepages requests takes them directly from the pod limit, - // hugepages cannot be overcommited and must have the limit, so we skip them here. - if _, exists := podReqs[key]; !exists && resourcehelper.IsSupportedPodLevelResource(key) && !corev1helper.IsHugePageResourceName(key) { + // Default pod level requests for overcommittable resources from aggregated container requests. + if _, exists := podReqs[key]; !exists && resourcehelper.IsSupportedPodLevelResource(key) && corev1helper.IsOvercommitAllowed(key) { podReqs[key] = aggrCtrLim.DeepCopy() } } @@ -487,29 +486,38 @@ func defaultPodRequests(obj *v1.Pod) { // limits set: // The pod-level limit becomes equal to the aggregated hugepages limit of all // the containers in the pod. -func defaultHugePagePodLimits(obj *v1.Pod) { - // We only populate defaults when the pod-level resources are partly specified already. - if obj.Spec.Resources == nil { +func defaultHugePagePodLimits(pod *v1.Pod) { + // We only populate hugepage limit defaults when the pod-level resources are partly specified. + if pod.Spec.Resources == nil { return } - if len(obj.Spec.Resources.Limits) == 0 && len(obj.Spec.Resources.Requests) == 0 { + if len(pod.Spec.Resources.Limits) == 0 && len(pod.Spec.Resources.Requests) == 0 { return } var podLims v1.ResourceList - podLims = obj.Spec.Resources.Limits + podLims = pod.Spec.Resources.Limits if podLims == nil { podLims = make(v1.ResourceList) } - aggrCtrLims := resourcehelper.AggregateContainerLimits(obj, resourcehelper.PodResourcesOptions{}) + aggrCtrLims := resourcehelper.AggregateContainerLimits(pod, resourcehelper.PodResourcesOptions{}) // When containers specify limits for hugepages and pod-level limits are not // set for that resource, the pod-level limit will default to the aggregated // hugepages limit of all the containers. for key, aggrCtrLim := range aggrCtrLims { - if _, exists := podLims[key]; !exists && resourcehelper.IsSupportedPodLevelResource(key) && corev1helper.IsHugePageResourceName(key) { + if !resourcehelper.IsSupportedPodLevelResource(key) || !corev1helper.IsHugePageResourceName(key) { + continue + } + + // We do not default pod-level hugepage limits if there is a hugepage request. + if _, exists := pod.Spec.Resources.Requests[key]; exists { + continue + } + + if _, exists := podLims[key]; !exists { podLims[key] = aggrCtrLim.DeepCopy() } } @@ -517,6 +525,6 @@ func defaultHugePagePodLimits(obj *v1.Pod) { // Only set pod-level resource limits in the PodSpec if the requirements map // contains entries after collecting container-level limits and pod-level limits for hugepages. if len(podLims) > 0 { - obj.Spec.Resources.Limits = podLims + pod.Spec.Resources.Limits = podLims } } diff --git a/pkg/apis/core/validation/validation.go b/pkg/apis/core/validation/validation.go index f294ff3d072..d7a41d7d684 100644 --- a/pkg/apis/core/validation/validation.go +++ b/pkg/apis/core/validation/validation.go @@ -4596,6 +4596,29 @@ func validatePodResourceConsistency(spec *core.PodSpec, fldPath *field.Path) fie } } + // Pod level hugepage limits must be always equal or greater than the aggregated + // container level hugepage limits, this is due to the hugepage resources being + // treated as a non overcommitable resource (request and limit must be equal) + // for the current container level hugepage behavior. + // This is also why hugepages overcommitment is not allowed in pod level resources, + // the pod cgroup values must reflect the request/limit set at pod level, and the + // container level cgroup values must be within that limit. + aggrContainerLims := resourcehelper.AggregateContainerLimits(&v1.Pod{Spec: *v1PodSpec}, resourcehelper.PodResourcesOptions{}) + for resourceName, ctrLims := range aggrContainerLims { + if !helper.IsHugePageResourceName(core.ResourceName(resourceName)) { + continue + } + + podSpecLimits, hasLimit := spec.Resources.Limits[core.ResourceName(resourceName)] + if !hasLimit { + continue + } + + if ctrLims.Cmp(podSpecLimits) > 0 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("limits").Key(string(resourceName)), podSpecLimits.String(), fmt.Sprintf("must be greater than or equal to aggregate container limits of %s", ctrLims.String()))) + } + } + // Individual Container limits must be <= Pod-level limits. for i, ctr := range spec.Containers { for resourceName, ctrLimit := range ctr.Resources.Limits { From 9f5b09eb7bb95c61b32680bbf433faa8824c9e99 Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Wed, 25 Jun 2025 17:49:50 +0000 Subject: [PATCH 4/6] Unit test pod level hugepage Default and Validation logic --- .../validate_pod_level_defaults_test.go | 366 ++++++++++++++++++ pkg/apis/core/v1/defaults_test.go | 64 ++- pkg/apis/core/validation/validation_test.go | 42 +- 3 files changed, 469 insertions(+), 3 deletions(-) create mode 100644 pkg/api/testing/validate_pod_level_defaults_test.go diff --git a/pkg/api/testing/validate_pod_level_defaults_test.go b/pkg/api/testing/validate_pod_level_defaults_test.go new file mode 100644 index 00000000000..44b8c43c0fd --- /dev/null +++ b/pkg/api/testing/validate_pod_level_defaults_test.go @@ -0,0 +1,366 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testing + +import ( + "fmt" + "testing" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation/field" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" + "k8s.io/kubernetes/pkg/api/legacyscheme" + apisapps1 "k8s.io/kubernetes/pkg/apis/apps/v1" + "k8s.io/kubernetes/pkg/apis/core" + corev1 "k8s.io/kubernetes/pkg/apis/core/v1" + "k8s.io/kubernetes/pkg/apis/core/validation" + "k8s.io/kubernetes/pkg/features" +) + +func TestPodLevelResourcesDefaults(t *testing.T) { + resCPU := v1.ResourceName(v1.ResourceCPU) + valCPU1 := resource.MustParse("1") + resMemory := v1.ResourceName(v1.ResourceMemory) + valMem100Mi := resource.MustParse("100Mi") + resHugepage2Mi := v1.ResourceName("hugepages-2Mi") + valhugepage2Mi := resource.MustParse("2Mi") + valhugepage10Mi := resource.MustParse("10Mi") + resHugepage1Gi := v1.ResourceName("hugepages-1Gi") + valHugepage1Gi := resource.MustParse("1Gi") + + testCases := []struct { + name string + podResources *v1.ResourceRequirements + containerResources []v1.ResourceRequirements + expectErr bool + }{ + { + name: "no pod-level resources, container hugepages-2Mi:R", + containerResources: []v1.ResourceRequirements{ + { + Limits: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + }, + Requests: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage2Mi, + }, + }, + }, + expectErr: true, + }, + { + name: "no pod-level resources, container hugepages-2Mi:L", + containerResources: []v1.ResourceRequirements{ + { + Limits: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage2Mi, + }, + Requests: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + }, + }, + }, + expectErr: false, + }, + { + name: "no pod-level resources, container hugepages-2Mi:R/L", + containerResources: []v1.ResourceRequirements{ + { + Limits: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage2Mi, + }, + Requests: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage2Mi, + }, + }, + }, + expectErr: false, + }, + { + name: "no pod-level resources, container hugepages-2Mi:R/L hugepages-1Gi:R/L", + containerResources: []v1.ResourceRequirements{ + { + Limits: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage2Mi, + resHugepage1Gi: valHugepage1Gi, + }, + Requests: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage2Mi, + resHugepage1Gi: valHugepage1Gi, + }, + }, + }, + expectErr: false, + }, + { + name: "pod-level resources hugepages-2Mi:R, container hugepages-2Mi:none", + podResources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + }, + Requests: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage10Mi, + }, + }, + containerResources: []v1.ResourceRequirements{ + { + Limits: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + }, + Requests: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + }, + }, + }, + expectErr: true, + }, + { + name: "pod-level resources hugepages-2Mi:L, container hugepages-2Mi:L", + podResources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage10Mi, + }, + Requests: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + }, + }, + containerResources: []v1.ResourceRequirements{ + { + Limits: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage2Mi, + }, + Requests: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + }, + }, + }, + expectErr: false, + }, + { + name: "pod-level resources hugepages-2Mi:R/L, container hugepages-2Mi:R/L", + podResources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage10Mi, + }, + Requests: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage10Mi, + }, + }, + containerResources: []v1.ResourceRequirements{ + { + Limits: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage2Mi, + }, + Requests: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage2Mi, + }, + }, + }, + expectErr: false, + }, + { + name: "pod-level resources hugepages-2Mi:R/L hugepages-1Gi:R/L, container hugepages-2Mi:R/L hugepages-1Gi:R/L", + podResources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage10Mi, + resHugepage1Gi: valHugepage1Gi, + }, + Requests: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage10Mi, + resHugepage1Gi: valHugepage1Gi, + }, + }, + containerResources: []v1.ResourceRequirements{ + { + Limits: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage2Mi, + resHugepage1Gi: valHugepage1Gi, + }, + Requests: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage2Mi, + resHugepage1Gi: valHugepage1Gi, + }, + }, + }, + expectErr: false, + }, + { + name: "pod-level resources hugepages-2Mi:R, container hugepages-2Mi:L", + podResources: &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + }, + Requests: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage10Mi, + }, + }, + containerResources: []v1.ResourceRequirements{ + { + Limits: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + resHugepage2Mi: valhugepage2Mi, + }, + Requests: v1.ResourceList{ + resCPU: valCPU1, + resMemory: valMem100Mi, + }, + }, + }, + expectErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLevelResources, true) + opts := validation.PodValidationOptions{PodLevelResourcesEnabled: true} + + // Step 1: Pod defaulting and validation + podForPodValidation1 := makePod(tc.podResources, tc.containerResources) + + corev1.SetDefaults_Pod(podForPodValidation1) + corev1.SetDefaults_PodSpec(&podForPodValidation1.Spec) + + internalPod := &core.Pod{} + if err := legacyscheme.Scheme.Convert(podForPodValidation1, internalPod, nil); err != nil { + t.Fatalf("Step 1: Failed to convert v1.Pod to core.Pod: %v", err) + } + + podErrs := validation.ValidatePodSpec(&internalPod.Spec, &internalPod.ObjectMeta, field.NewPath("spec"), opts) + if len(podErrs) > 0 != tc.expectErr { + t.Errorf("Step 1: Pod validation failed. expectErr=%v, got errs: %v", tc.expectErr, podErrs.ToAggregate()) + } + + // Step 2: Deployment defaulting and validation + podForPodValidation2 := makePod(tc.podResources, tc.containerResources) + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "test-deploy", Namespace: "test-ns"}, + Spec: appsv1.DeploymentSpec{ + Template: v1.PodTemplateSpec{ + Spec: podForPodValidation2.Spec, + }, + }, + } + + apisapps1.SetDefaults_Deployment(deployment) + corev1.SetDefaults_PodSpec(&deployment.Spec.Template.Spec) + internalPodTemplateSpec := &core.PodTemplateSpec{} + if err := legacyscheme.Scheme.Convert(&deployment.Spec.Template, internalPodTemplateSpec, nil); err != nil { + t.Fatalf("Step 2: Failed to convert v1.PodTemplateSpec to core.PodTemplateSpec: %v", err) + } + + podErrs = validation.ValidatePodTemplateSpec(internalPodTemplateSpec, field.NewPath("template"), opts) + if len(podErrs) > 0 != tc.expectErr { + t.Errorf("Step 2: Pod Template validation failed. expectErr=%v, got errs: %v", tc.expectErr, podErrs.ToAggregate()) + } + + // Step 3: Validate the defaulted pod spec from the deployment + podForPodValidation3 := &v1.Pod{ + Spec: deployment.Spec.Template.Spec, + } + corev1.SetDefaults_Pod(podForPodValidation3) + corev1.SetDefaults_PodSpec(&podForPodValidation3.Spec) + internalPod = &core.Pod{} + if err := legacyscheme.Scheme.Convert(podForPodValidation3, internalPod, nil); err != nil { + t.Fatalf("Step 3: Failed to convert v1.Pod to core.Pod: %v", err) + } + + podErrs = validation.ValidatePodSpec(&internalPod.Spec, &internalPod.ObjectMeta, field.NewPath("spec"), opts) + if len(podErrs) > 0 != tc.expectErr { + t.Errorf("Step 3: Pod validation failed. expectErr=%v, got errs: %v", tc.expectErr, podErrs.ToAggregate()) + } + }) + } +} + +func makePod(podResources *v1.ResourceRequirements, containersResources []v1.ResourceRequirements) *v1.Pod { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test-pod", Namespace: "test-ns"}, + Spec: v1.PodSpec{ + Containers: makeContainers(containersResources), + }, + } + + if podResources != nil { + pod.Spec.Resources = podResources + } + + return pod +} + +func makeContainers(resourceRequirements []v1.ResourceRequirements) []v1.Container { + containers := []v1.Container{} + for idx, containerResources := range resourceRequirements { + container := v1.Container{ + Name: fmt.Sprintf("container-%d", idx), + Image: "img", ImagePullPolicy: "Never", TerminationMessagePolicy: "File", + Resources: containerResources, + } + containers = append(containers, container) + } + + return containers +} diff --git a/pkg/apis/core/v1/defaults_test.go b/pkg/apis/core/v1/defaults_test.go index 3af21ca06ed..9f070a49650 100644 --- a/pkg/apis/core/v1/defaults_test.go +++ b/pkg/apis/core/v1/defaults_test.go @@ -1229,7 +1229,7 @@ func TestPodResourcesDefaults(t *testing.T) { }, }, }, { - name: "pod hugepages requests=unset limits=unset, container hugepages requests=unset limits=set", + name: "pod has cpu limit with hugepages requests=unset limits=unset, container hugepages requests=unset limits=set", podLevelResourcesEnabled: true, podResources: &v1.ResourceRequirements{ Limits: v1.ResourceList{ @@ -1290,6 +1290,68 @@ func TestPodResourcesDefaults(t *testing.T) { }, }, }, + }, { + name: "pod has cpu request with hugepages requests=unset limits=unset, container hugepages requests=unset limits=set", + podLevelResourcesEnabled: true, + podResources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + }, + }, + containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("4Mi"), + }, + }, + }, { + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + }, + }, + }, + expectedPodSpec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("6Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("5m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("6Mi"), + }, + }, + Containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("4Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("2m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("4Mi"), + }, + }, + }, { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + Limits: v1.ResourceList{ + "cpu": resource.MustParse("1m"), + v1.ResourceHugePagesPrefix + "2Mi": resource.MustParse("2Mi"), + }, + }, + }, + }, + }, }, { name: "pod hugepages requests=unset limits=set, container hugepages requests=unset limits=set", podLevelResourcesEnabled: true, diff --git a/pkg/apis/core/validation/validation_test.go b/pkg/apis/core/validation/validation_test.go index 0f1560641f7..ba34eff5d8f 100644 --- a/pkg/apis/core/validation/validation_test.go +++ b/pkg/apis/core/validation/validation_test.go @@ -20173,12 +20173,12 @@ func TestValidatePodResourceConsistency(t *testing.T) { Resources: core.ResourceRequirements{ Requests: core.ResourceList{ core.ResourceCPU: resource.MustParse("8"), - core.ResourceMemory: resource.MustParse("3Mi"), + core.ResourceMemory: resource.MustParse("7Mi"), }, }, }, }, - expectedErrors: []string{"must be greater than or equal to aggregate container requests"}, + expectedErrors: []string{"must be greater than or equal to aggregate container requests", "must be greater than or equal to aggregate container requests"}, }, { name: "container requests with resources unsupported at pod-level", podResources: core.ResourceRequirements{ @@ -20212,6 +20212,7 @@ func TestValidatePodResourceConsistency(t *testing.T) { Limits: core.ResourceList{ core.ResourceCPU: resource.MustParse("10"), core.ResourceMemory: resource.MustParse("10Mi"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("10Mi"), }, }, containers: []core.Container{ @@ -20220,6 +20221,7 @@ func TestValidatePodResourceConsistency(t *testing.T) { Limits: core.ResourceList{ core.ResourceCPU: resource.MustParse("5"), core.ResourceMemory: resource.MustParse("5Mi"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("4Mi"), }, }, }, { @@ -20227,6 +20229,7 @@ func TestValidatePodResourceConsistency(t *testing.T) { Limits: core.ResourceList{ core.ResourceCPU: resource.MustParse("4"), core.ResourceMemory: resource.MustParse("3Mi"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("4Mi"), }, }, }, @@ -20237,6 +20240,7 @@ func TestValidatePodResourceConsistency(t *testing.T) { Limits: core.ResourceList{ core.ResourceCPU: resource.MustParse("10"), core.ResourceMemory: resource.MustParse("10Mi"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("8Mi"), }, }, containers: []core.Container{ @@ -20245,6 +20249,7 @@ func TestValidatePodResourceConsistency(t *testing.T) { Limits: core.ResourceList{ core.ResourceCPU: resource.MustParse("5"), core.ResourceMemory: resource.MustParse("5Mi"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("4Mi"), }, }, }, { @@ -20252,6 +20257,7 @@ func TestValidatePodResourceConsistency(t *testing.T) { Limits: core.ResourceList{ core.ResourceCPU: resource.MustParse("5"), core.ResourceMemory: resource.MustParse("5Mi"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("4Mi"), }, }, }, @@ -20281,12 +20287,41 @@ func TestValidatePodResourceConsistency(t *testing.T) { }, }, }, + }, { + name: "hugepage aggregate container limits greater than pod limits", + podResources: core.ResourceRequirements{ + Limits: core.ResourceList{ + core.ResourceCPU: resource.MustParse("10"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("4Mi"), + }, + }, + containers: []core.Container{ + { + Resources: core.ResourceRequirements{ + Limits: core.ResourceList{ + core.ResourceCPU: resource.MustParse("6"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("4Mi"), + }, + }, + }, { + Resources: core.ResourceRequirements{ + Limits: core.ResourceList{ + core.ResourceCPU: resource.MustParse("6"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("4Mi"), + }, + }, + }, + }, + expectedErrors: []string{ + "must be greater than or equal to aggregate container limits", + }, }, { name: "individual container limits greater than pod limits", podResources: core.ResourceRequirements{ Limits: core.ResourceList{ core.ResourceCPU: resource.MustParse("10"), core.ResourceMemory: resource.MustParse("10Mi"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("2Mi"), }, }, containers: []core.Container{ @@ -20295,11 +20330,14 @@ func TestValidatePodResourceConsistency(t *testing.T) { Limits: core.ResourceList{ core.ResourceCPU: resource.MustParse("11"), core.ResourceMemory: resource.MustParse("12Mi"), + core.ResourceName(core.ResourceHugePagesPrefix + "2Mi"): resource.MustParse("4Mi"), }, }, }, }, expectedErrors: []string{ + "must be greater than or equal to aggregate container limits", + "must be less than or equal to pod limits", "must be less than or equal to pod limits", "must be less than or equal to pod limits", }, From f925e555483b10991f516cfc4dec843a16d855a2 Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Wed, 25 Jun 2025 23:32:30 +0000 Subject: [PATCH 5/6] E2E tests for container hugepage resources immutability Pod level hugepage resources are not propagated to the containers, only pod level cgroup values are propagated to the containers when they do not specify hugepage resources. --- test/e2e_node/hugepages_test.go | 231 +++++++++++++++++++++++++++----- 1 file changed, 195 insertions(+), 36 deletions(-) diff --git a/test/e2e_node/hugepages_test.go b/test/e2e_node/hugepages_test.go index 12900faeed2..7094833cd32 100644 --- a/test/e2e_node/hugepages_test.go +++ b/test/e2e_node/hugepages_test.go @@ -33,6 +33,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" + corev1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/test/e2e/feature" @@ -178,7 +179,7 @@ func isHugePageAvailable(hugepagesSize int) bool { return true } -func getHugepagesTestPod(f *framework.Framework, podLimits v1.ResourceList, containerLimits v1.ResourceList, mounts []v1.VolumeMount, volumes []v1.Volume) *v1.Pod { +func getHugepagesTestPod(f *framework.Framework, podResources *v1.ResourceRequirements, containerLimits v1.ResourceList, mounts []v1.VolumeMount, volumes []v1.Volume) *v1.Pod { pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "hugepages-", @@ -200,10 +201,8 @@ func getHugepagesTestPod(f *framework.Framework, podLimits v1.ResourceList, cont }, } - if podLimits != nil { - pod.Spec.Resources = &v1.ResourceRequirements{ - Limits: podLimits, - } + if podResources.Requests != nil || podResources.Limits != nil { + pod.Spec.Resources = podResources } return pod @@ -519,7 +518,7 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func }) // Serial because the test updates kubelet configuration. -var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), feature.PodLevelResources, func() { +var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), feature.PodLevelResources, framework.WithFeatureGate(features.PodLevelResources), func() { f := framework.NewDefaultFramework("pod-level-hugepages-resources") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -527,7 +526,7 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea var ( testpod *v1.Pod expectedHugepageLimits v1.ResourceList - podLimits v1.ResourceList + podResources *v1.ResourceRequirements containerLimits v1.ResourceList mounts []v1.VolumeMount volumes []v1.Volume @@ -545,12 +544,46 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea waitForHugepages(f, ctx, hugepages) - pod := getHugepagesTestPod(f, podLimits, containerLimits, mounts, volumes) + pod := getHugepagesTestPod(f, podResources, containerLimits, mounts, volumes) + + // Capture the initial containers spec resources before deployment + initialContainerResources := make(map[string]*v1.ResourceRequirements) + for _, container := range pod.Spec.Containers { + initialContainerResources[container.Name] = container.Resources.DeepCopy() + } ginkgo.By("by running a test pod that requests hugepages") testpod = e2epod.NewPodClient(f).CreateSync(ctx, pod) + // Verify that the testpod container spec resources were not modified after being deployed + // This has the objective to explicitly show that the the pod level hugepage + // resources are not propagated to the containers, only pod level cgroup values + // are propagated to the containers when they do not specify hugepage resources. + ginkgo.By("Verifying that the testpod spec resources were not modified after being deployed") + retrievedPod, err := e2epod.NewPodClient(f).Get(ctx, testpod.Name, metav1.GetOptions{}) + gomega.Expect(err).To(gomega.Succeed(), "Failed to get the deployed pod") + + // Iterate over the containers and check that the resources are equal to the initial spec before deploying + for _, container := range retrievedPod.Spec.Containers { + initialResources, ok := initialContainerResources[container.Name] + gomega.Expect(ok).To(gomega.BeTrueBecause("Container %s not found in initialContainerResources", container.Name)) + + // The container limits must be maintained equally to the initial spec + for resourceName, resourceValue := range initialResources.Limits { + if corev1helper.IsHugePageResourceName(resourceName) { + gomega.Expect(container.Resources.Limits[resourceName]).To(gomega.Equal(resourceValue), fmt.Sprintf("Pod.Spec.Containers.Resources.Limits.%s should remain unchanged after deployment", resourceName)) + } + } + + // Since container requests were not specified, they are defaulted to the limits + for resourceName, resourceValue := range initialResources.Limits { + if corev1helper.IsHugePageResourceName(resourceName) { + gomega.Expect(container.Resources.Requests[resourceName]).To(gomega.Equal(resourceValue), fmt.Sprintf("Pod.Spec.Containers.Resources.Requests.%s should default to limits after deployment", resourceName)) + } + } + } + framework.Logf("Test pod name: %s", testpod.Name) }) @@ -575,10 +608,12 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea expectedHugepageLimits = v1.ResourceList{ hugepagesResourceName2Mi: resource.MustParse("6Mi"), } - podLimits = v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - hugepagesResourceName2Mi: resource.MustParse("6Mi"), + podResources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + }, } containerLimits = v1.ResourceList{} mounts = []v1.VolumeMount{ @@ -618,10 +653,12 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea expectedHugepageLimits = v1.ResourceList{ hugepagesResourceName2Mi: resource.MustParse("6Mi"), } - podLimits = v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - hugepagesResourceName2Mi: resource.MustParse("6Mi"), + podResources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + }, } containerLimits = v1.ResourceList{ v1.ResourceCPU: resource.MustParse("10m"), @@ -665,9 +702,59 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea expectedHugepageLimits = v1.ResourceList{ hugepagesResourceName2Mi: resource.MustParse("4Mi"), } - podLimits = v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("100Mi"), + podResources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + containerLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("4Mi"), + } + mounts = []v1.VolumeMount{ + { + Name: "hugepages-2mi", + MountPath: "/hugepages-2Mi", + }, + } + volumes = []v1.Volume{ + { + Name: "hugepages-2mi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages2Mi, + }, + }, + }, + } + }) + + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { + runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages) + }) + + ginkgo.JustAfterEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 0, + } + }) + }) + + ginkgo.Context("only pod level requests no pod hugepages, container hugepages, single page size", func() { + ginkgo.BeforeEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 5, + } + expectedHugepageLimits = v1.ResourceList{ + hugepagesResourceName2Mi: resource.MustParse("4Mi"), + } + podResources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, } containerLimits = v1.ResourceList{ v1.ResourceCPU: resource.MustParse("10m"), @@ -713,11 +800,13 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea hugepagesResourceName2Mi: resource.MustParse("6Mi"), hugepagesResourceName1Gi: resource.MustParse("1Gi"), } - podLimits = v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - hugepagesResourceName2Mi: resource.MustParse("6Mi"), - hugepagesResourceName1Gi: resource.MustParse("1Gi"), + podResources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + }, } containerLimits = v1.ResourceList{} mounts = []v1.VolumeMount{ @@ -772,11 +861,13 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea hugepagesResourceName2Mi: resource.MustParse("6Mi"), hugepagesResourceName1Gi: resource.MustParse("1Gi"), } - podLimits = v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - hugepagesResourceName2Mi: resource.MustParse("6Mi"), - hugepagesResourceName1Gi: resource.MustParse("1Gi"), + podResources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + }, } containerLimits = v1.ResourceList{ v1.ResourceCPU: resource.MustParse("10m"), @@ -836,9 +927,75 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea hugepagesResourceName2Mi: resource.MustParse("4Mi"), hugepagesResourceName1Gi: resource.MustParse("1Gi"), } - podLimits = v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("100Mi"), + podResources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + containerLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("4Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + mounts = []v1.VolumeMount{ + { + Name: "hugepages-2mi", + MountPath: "/hugepages-2Mi", + }, + { + Name: "hugepages-1gi", + MountPath: "/hugepages-1Gi", + }, + } + volumes = []v1.Volume{ + { + Name: "hugepages-2mi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages2Mi, + }, + }, + }, + { + Name: "hugepages-1gi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages1Gi, + }, + }, + }, + } + }) + + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { + runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages) + }) + + ginkgo.JustAfterEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 0, + hugepagesResourceName1Gi: 0, + } + }) + }) + + ginkgo.Context("only pod level requests no pod hugepages, container hugepages, multiple page size", func() { + ginkgo.BeforeEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 5, + hugepagesResourceName1Gi: 1, + } + expectedHugepageLimits = v1.ResourceList{ + hugepagesResourceName2Mi: resource.MustParse("4Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + podResources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, } containerLimits = v1.ResourceList{ v1.ResourceCPU: resource.MustParse("10m"), @@ -898,10 +1055,12 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea hugepagesResourceName2Mi: resource.MustParse("6Mi"), hugepagesResourceName1Gi: resource.MustParse("1Gi"), } - podLimits = v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - hugepagesResourceName2Mi: resource.MustParse("6Mi"), + podResources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + }, } containerLimits = v1.ResourceList{ v1.ResourceCPU: resource.MustParse("10m"), From 1bc995c195ec6fc2e97e32976805606c03d099b4 Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Tue, 1 Jul 2025 21:42:45 +0000 Subject: [PATCH 6/6] Generated files --- api/openapi-spec/swagger.json | 2 +- api/openapi-spec/v3/api__v1_openapi.json | 2 +- api/openapi-spec/v3/apis__apps__v1_openapi.json | 2 +- api/openapi-spec/v3/apis__batch__v1_openapi.json | 2 +- pkg/generated/openapi/zz_generated.openapi.go | 2 +- staging/src/k8s.io/api/core/v1/generated.proto | 2 +- staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 3f135fd3149..05cb409cce7 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -10097,7 +10097,7 @@ }, "resources": { "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements", - "description": "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\" and \"memory\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate." + "description": "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\", \"memory\" and \"hugepages-\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate." }, "restartPolicy": { "description": "Restart policy for all containers within the pod. One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. Default to Always. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy", diff --git a/api/openapi-spec/v3/api__v1_openapi.json b/api/openapi-spec/v3/api__v1_openapi.json index 43415059e37..cb360f0b5eb 100644 --- a/api/openapi-spec/v3/api__v1_openapi.json +++ b/api/openapi-spec/v3/api__v1_openapi.json @@ -5799,7 +5799,7 @@ "$ref": "#/components/schemas/io.k8s.api.core.v1.ResourceRequirements" } ], - "description": "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\" and \"memory\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate." + "description": "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\", \"memory\" and \"hugepages-\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate." }, "restartPolicy": { "description": "Restart policy for all containers within the pod. One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. Default to Always. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy", diff --git a/api/openapi-spec/v3/apis__apps__v1_openapi.json b/api/openapi-spec/v3/apis__apps__v1_openapi.json index 98c90805f1f..93457373623 100644 --- a/api/openapi-spec/v3/apis__apps__v1_openapi.json +++ b/api/openapi-spec/v3/apis__apps__v1_openapi.json @@ -4027,7 +4027,7 @@ "$ref": "#/components/schemas/io.k8s.api.core.v1.ResourceRequirements" } ], - "description": "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\" and \"memory\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate." + "description": "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\", \"memory\" and \"hugepages-\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate." }, "restartPolicy": { "description": "Restart policy for all containers within the pod. One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. Default to Always. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy", diff --git a/api/openapi-spec/v3/apis__batch__v1_openapi.json b/api/openapi-spec/v3/apis__batch__v1_openapi.json index 95e0848d129..8cb3e3e9f32 100644 --- a/api/openapi-spec/v3/apis__batch__v1_openapi.json +++ b/api/openapi-spec/v3/apis__batch__v1_openapi.json @@ -3219,7 +3219,7 @@ "$ref": "#/components/schemas/io.k8s.api.core.v1.ResourceRequirements" } ], - "description": "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\" and \"memory\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate." + "description": "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\", \"memory\" and \"hugepages-\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate." }, "restartPolicy": { "description": "Restart policy for all containers within the pod. One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. Default to Always. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy", diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 21889c1cf62..6c80c0b5dad 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -29834,7 +29834,7 @@ func schema_k8sio_api_core_v1_PodSpec(ref common.ReferenceCallback) common.OpenA }, "resources": { SchemaProps: spec.SchemaProps{ - Description: "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\" and \"memory\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate.", + Description: "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\", \"memory\" and \"hugepages-\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate.", Ref: ref("k8s.io/api/core/v1.ResourceRequirements"), }, }, diff --git a/staging/src/k8s.io/api/core/v1/generated.proto b/staging/src/k8s.io/api/core/v1/generated.proto index 0570afdb0b3..af5140d09b7 100644 --- a/staging/src/k8s.io/api/core/v1/generated.proto +++ b/staging/src/k8s.io/api/core/v1/generated.proto @@ -4617,7 +4617,7 @@ message PodSpec { // Resources is the total amount of CPU and Memory resources required by all // containers in the pod. It supports specifying Requests and Limits for - // "cpu" and "memory" resource names only. ResourceClaims are not supported. + // "cpu", "memory" and "hugepages-" resource names only. ResourceClaims are not supported. // // This field enables fine-grained control over resource allocation for the // entire pod, allowing resource sharing among containers in a pod. diff --git a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go index 1e2526f3612..9d88d91590b 100644 --- a/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/core/v1/types_swagger_doc_generated.go @@ -1877,7 +1877,7 @@ var map_PodSpec = map[string]string{ "hostUsers": "Use the host's user namespace. Optional: Default to true. If set to true or not present, the pod will be run in the host user namespace, useful for when the pod needs a feature only available to the host user namespace, such as loading a kernel module with CAP_SYS_MODULE. When set to false, a new userns is created for the pod. Setting false is useful for mitigating container breakout vulnerabilities even allowing users to run their containers as root without actually having root privileges on the host. This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature.", "schedulingGates": "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the scheduler will not attempt to schedule the pod.\n\nSchedulingGates can only be set at pod creation time, and be removed only afterwards.", "resourceClaims": "ResourceClaims defines which ResourceClaims must be allocated and reserved before the Pod is allowed to start. The resources will be made available to those containers which consume them by name.\n\nThis is an alpha field and requires enabling the DynamicResourceAllocation feature gate.\n\nThis field is immutable.", - "resources": "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\" and \"memory\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate.", + "resources": "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\", \"memory\" and \"hugepages-\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate.", "hostnameOverride": "HostnameOverride specifies an explicit override for the pod's hostname as perceived by the pod. This field only specifies the pod's hostname and does not affect its DNS records. When this field is set to a non-empty string: - It takes precedence over the values set in `hostname` and `subdomain`. - The Pod's hostname will be set to this value. - `setHostnameAsFQDN` must be nil or set to false. - `hostNetwork` must be set to false.\n\nThis field must be a valid DNS subdomain as defined in RFC 1123 and contain at most 64 characters. Requires the HostnameOverride feature gate to be enabled.", }