From f925e555483b10991f516cfc4dec843a16d855a2 Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Wed, 25 Jun 2025 23:32:30 +0000 Subject: [PATCH] E2E tests for container hugepage resources immutability Pod level hugepage resources are not propagated to the containers, only pod level cgroup values are propagated to the containers when they do not specify hugepage resources. --- test/e2e_node/hugepages_test.go | 231 +++++++++++++++++++++++++++----- 1 file changed, 195 insertions(+), 36 deletions(-) diff --git a/test/e2e_node/hugepages_test.go b/test/e2e_node/hugepages_test.go index 12900faeed2..7094833cd32 100644 --- a/test/e2e_node/hugepages_test.go +++ b/test/e2e_node/hugepages_test.go @@ -33,6 +33,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" + corev1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/test/e2e/feature" @@ -178,7 +179,7 @@ func isHugePageAvailable(hugepagesSize int) bool { return true } -func getHugepagesTestPod(f *framework.Framework, podLimits v1.ResourceList, containerLimits v1.ResourceList, mounts []v1.VolumeMount, volumes []v1.Volume) *v1.Pod { +func getHugepagesTestPod(f *framework.Framework, podResources *v1.ResourceRequirements, containerLimits v1.ResourceList, mounts []v1.VolumeMount, volumes []v1.Volume) *v1.Pod { pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "hugepages-", @@ -200,10 +201,8 @@ func getHugepagesTestPod(f *framework.Framework, podLimits v1.ResourceList, cont }, } - if podLimits != nil { - pod.Spec.Resources = &v1.ResourceRequirements{ - Limits: podLimits, - } + if podResources.Requests != nil || podResources.Limits != nil { + pod.Spec.Resources = podResources } return pod @@ -519,7 +518,7 @@ var _ = SIGDescribe("HugePages", framework.WithSerial(), feature.HugePages, func }) // Serial because the test updates kubelet configuration. -var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), feature.PodLevelResources, func() { +var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), feature.PodLevelResources, framework.WithFeatureGate(features.PodLevelResources), func() { f := framework.NewDefaultFramework("pod-level-hugepages-resources") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged @@ -527,7 +526,7 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea var ( testpod *v1.Pod expectedHugepageLimits v1.ResourceList - podLimits v1.ResourceList + podResources *v1.ResourceRequirements containerLimits v1.ResourceList mounts []v1.VolumeMount volumes []v1.Volume @@ -545,12 +544,46 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea waitForHugepages(f, ctx, hugepages) - pod := getHugepagesTestPod(f, podLimits, containerLimits, mounts, volumes) + pod := getHugepagesTestPod(f, podResources, containerLimits, mounts, volumes) + + // Capture the initial containers spec resources before deployment + initialContainerResources := make(map[string]*v1.ResourceRequirements) + for _, container := range pod.Spec.Containers { + initialContainerResources[container.Name] = container.Resources.DeepCopy() + } ginkgo.By("by running a test pod that requests hugepages") testpod = e2epod.NewPodClient(f).CreateSync(ctx, pod) + // Verify that the testpod container spec resources were not modified after being deployed + // This has the objective to explicitly show that the the pod level hugepage + // resources are not propagated to the containers, only pod level cgroup values + // are propagated to the containers when they do not specify hugepage resources. + ginkgo.By("Verifying that the testpod spec resources were not modified after being deployed") + retrievedPod, err := e2epod.NewPodClient(f).Get(ctx, testpod.Name, metav1.GetOptions{}) + gomega.Expect(err).To(gomega.Succeed(), "Failed to get the deployed pod") + + // Iterate over the containers and check that the resources are equal to the initial spec before deploying + for _, container := range retrievedPod.Spec.Containers { + initialResources, ok := initialContainerResources[container.Name] + gomega.Expect(ok).To(gomega.BeTrueBecause("Container %s not found in initialContainerResources", container.Name)) + + // The container limits must be maintained equally to the initial spec + for resourceName, resourceValue := range initialResources.Limits { + if corev1helper.IsHugePageResourceName(resourceName) { + gomega.Expect(container.Resources.Limits[resourceName]).To(gomega.Equal(resourceValue), fmt.Sprintf("Pod.Spec.Containers.Resources.Limits.%s should remain unchanged after deployment", resourceName)) + } + } + + // Since container requests were not specified, they are defaulted to the limits + for resourceName, resourceValue := range initialResources.Limits { + if corev1helper.IsHugePageResourceName(resourceName) { + gomega.Expect(container.Resources.Requests[resourceName]).To(gomega.Equal(resourceValue), fmt.Sprintf("Pod.Spec.Containers.Resources.Requests.%s should default to limits after deployment", resourceName)) + } + } + } + framework.Logf("Test pod name: %s", testpod.Name) }) @@ -575,10 +608,12 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea expectedHugepageLimits = v1.ResourceList{ hugepagesResourceName2Mi: resource.MustParse("6Mi"), } - podLimits = v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - hugepagesResourceName2Mi: resource.MustParse("6Mi"), + podResources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + }, } containerLimits = v1.ResourceList{} mounts = []v1.VolumeMount{ @@ -618,10 +653,12 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea expectedHugepageLimits = v1.ResourceList{ hugepagesResourceName2Mi: resource.MustParse("6Mi"), } - podLimits = v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - hugepagesResourceName2Mi: resource.MustParse("6Mi"), + podResources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + }, } containerLimits = v1.ResourceList{ v1.ResourceCPU: resource.MustParse("10m"), @@ -665,9 +702,59 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea expectedHugepageLimits = v1.ResourceList{ hugepagesResourceName2Mi: resource.MustParse("4Mi"), } - podLimits = v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("100Mi"), + podResources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + containerLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("4Mi"), + } + mounts = []v1.VolumeMount{ + { + Name: "hugepages-2mi", + MountPath: "/hugepages-2Mi", + }, + } + volumes = []v1.Volume{ + { + Name: "hugepages-2mi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages2Mi, + }, + }, + }, + } + }) + + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { + runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages) + }) + + ginkgo.JustAfterEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 0, + } + }) + }) + + ginkgo.Context("only pod level requests no pod hugepages, container hugepages, single page size", func() { + ginkgo.BeforeEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 5, + } + expectedHugepageLimits = v1.ResourceList{ + hugepagesResourceName2Mi: resource.MustParse("4Mi"), + } + podResources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, } containerLimits = v1.ResourceList{ v1.ResourceCPU: resource.MustParse("10m"), @@ -713,11 +800,13 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea hugepagesResourceName2Mi: resource.MustParse("6Mi"), hugepagesResourceName1Gi: resource.MustParse("1Gi"), } - podLimits = v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - hugepagesResourceName2Mi: resource.MustParse("6Mi"), - hugepagesResourceName1Gi: resource.MustParse("1Gi"), + podResources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + }, } containerLimits = v1.ResourceList{} mounts = []v1.VolumeMount{ @@ -772,11 +861,13 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea hugepagesResourceName2Mi: resource.MustParse("6Mi"), hugepagesResourceName1Gi: resource.MustParse("1Gi"), } - podLimits = v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - hugepagesResourceName2Mi: resource.MustParse("6Mi"), - hugepagesResourceName1Gi: resource.MustParse("1Gi"), + podResources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + }, } containerLimits = v1.ResourceList{ v1.ResourceCPU: resource.MustParse("10m"), @@ -836,9 +927,75 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea hugepagesResourceName2Mi: resource.MustParse("4Mi"), hugepagesResourceName1Gi: resource.MustParse("1Gi"), } - podLimits = v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("100Mi"), + podResources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + containerLimits = v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("4Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + mounts = []v1.VolumeMount{ + { + Name: "hugepages-2mi", + MountPath: "/hugepages-2Mi", + }, + { + Name: "hugepages-1gi", + MountPath: "/hugepages-1Gi", + }, + } + volumes = []v1.Volume{ + { + Name: "hugepages-2mi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages2Mi, + }, + }, + }, + { + Name: "hugepages-1gi", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: mediumHugepages1Gi, + }, + }, + }, + } + }) + + ginkgo.It("should set correct hugetlb mount and limit under the container cgroup", func(ctx context.Context) { + runHugePagesTests(f, ctx, testpod, expectedHugepageLimits, mounts, hugepages) + }) + + ginkgo.JustAfterEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 0, + hugepagesResourceName1Gi: 0, + } + }) + }) + + ginkgo.Context("only pod level requests no pod hugepages, container hugepages, multiple page size", func() { + ginkgo.BeforeEach(func() { + hugepages = map[string]int{ + hugepagesResourceName2Mi: 5, + hugepagesResourceName1Gi: 1, + } + expectedHugepageLimits = v1.ResourceList{ + hugepagesResourceName2Mi: resource.MustParse("4Mi"), + hugepagesResourceName1Gi: resource.MustParse("1Gi"), + } + podResources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, } containerLimits = v1.ResourceList{ v1.ResourceCPU: resource.MustParse("10m"), @@ -898,10 +1055,12 @@ var _ = SIGDescribe("Pod Level HugePages Resources", framework.WithSerial(), fea hugepagesResourceName2Mi: resource.MustParse("6Mi"), hugepagesResourceName1Gi: resource.MustParse("1Gi"), } - podLimits = v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("10m"), - v1.ResourceMemory: resource.MustParse("100Mi"), - hugepagesResourceName2Mi: resource.MustParse("6Mi"), + podResources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("10m"), + v1.ResourceMemory: resource.MustParse("100Mi"), + hugepagesResourceName2Mi: resource.MustParse("6Mi"), + }, } containerLimits = v1.ResourceList{ v1.ResourceCPU: resource.MustParse("10m"),