From 7fbf63a23fcf393d4ba310412155dfbbaa36bb9a Mon Sep 17 00:00:00 2001 From: Luiz Oliveira Date: Tue, 29 Jul 2025 12:02:26 -0400 Subject: [PATCH] HPA support for pod-level resource specifications (#132430) * HPA support for pod-level resource specifications * Add e2e tests for HPA support for pod-level resource specifications --- .../podautoscaler/replica_calculator.go | 84 +++++++++--- .../podautoscaler/replica_calculator_test.go | 127 +++++++++++++++--- test/e2e/autoscaling/autoscaling_timer.go | 2 +- .../autoscaling/horizontal_pod_autoscaling.go | 114 +++++++++++++++- .../horizontal_pod_autoscaling_behavior.go | 22 +-- .../autoscaling/autoscaling_utils.go | 17 ++- .../autoscaling/horizontal_pod_autoscalers.go | 4 +- test/utils/runners.go | 35 +++-- 8 files changed, 339 insertions(+), 66 deletions(-) diff --git a/pkg/controller/podautoscaler/replica_calculator.go b/pkg/controller/podautoscaler/replica_calculator.go index bbaf7acc800..656cf3ca51c 100644 --- a/pkg/controller/podautoscaler/replica_calculator.go +++ b/pkg/controller/podautoscaler/replica_calculator.go @@ -27,9 +27,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/util/feature" corelisters "k8s.io/client-go/listers/core/v1" + resourcehelpers "k8s.io/component-helpers/resource" podutil "k8s.io/kubernetes/pkg/api/v1/pod" metricsclient "k8s.io/kubernetes/pkg/controller/podautoscaler/metrics" + "k8s.io/kubernetes/pkg/features" ) const ( @@ -94,7 +97,7 @@ func (c *ReplicaCalculator) GetResourceReplicas(ctx context.Context, currentRepl return 0, 0, 0, time.Time{}, fmt.Errorf("did not receive metrics for targeted pods (pods might be unready)") } - requests, err := calculatePodRequests(podList, container, resource) + requests, err := calculateRequests(podList, container, resource) if err != nil { return 0, 0, 0, time.Time{}, err } @@ -449,31 +452,76 @@ func groupPods(pods []*v1.Pod, metrics metricsclient.PodMetricsInfo, resource v1 return } -func calculatePodRequests(pods []*v1.Pod, container string, resource v1.ResourceName) (map[string]int64, error) { +// calculateRequests computes the request value for each pod for the specified +// resource. +// If container is non-empty, it uses the request of that specific container. +// If container is empty, it uses pod-level requests if pod-level requests are +// set on the pod. Otherwise, it sums the requests of all containers in the pod +// (including restartable init containers). +// It returns a map of pod names to their calculated request values. +func calculateRequests(pods []*v1.Pod, container string, resource v1.ResourceName) (map[string]int64, error) { + podLevelResourcesEnabled := feature.DefaultFeatureGate.Enabled(features.PodLevelResources) requests := make(map[string]int64, len(pods)) for _, pod := range pods { - podSum := int64(0) - // Calculate all regular containers and restartable init containers requests. - containers := append([]v1.Container{}, pod.Spec.Containers...) - for _, c := range pod.Spec.InitContainers { - if c.RestartPolicy != nil && *c.RestartPolicy == v1.ContainerRestartPolicyAlways { - containers = append(containers, c) - } + var request int64 + var err error + // Determine if we should use pod-level requests: see KEP-2837 + // https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/2837-pod-level-resource-spec/README.md + usePodLevelRequests := podLevelResourcesEnabled && + resourcehelpers.IsPodLevelRequestsSet(pod) && + // If a container name is specified in the HPA, it takes precedence over + // the pod-level requests. + container == "" + + if usePodLevelRequests { + request, err = calculatePodLevelRequests(pod, resource) + } else { + request, err = calculatePodRequestsFromContainers(pod, container, resource) } - for _, c := range containers { - if container == "" || container == c.Name { - if containerRequest, ok := c.Resources.Requests[resource]; ok { - podSum += containerRequest.MilliValue() - } else { - return nil, fmt.Errorf("missing request for %s in container %s of Pod %s", resource, c.Name, pod.ObjectMeta.Name) - } - } + if err != nil { + return nil, err } - requests[pod.Name] = podSum + requests[pod.Name] = request } return requests, nil } +// calculatePodLevelRequests computes the requests for the specific resource at +// the pod level. +func calculatePodLevelRequests(pod *v1.Pod, resource v1.ResourceName) (int64, error) { + podLevelRequests := resourcehelpers.PodRequests(pod, resourcehelpers.PodResourcesOptions{}) + podRequest, ok := podLevelRequests[resource] + if !ok { + return 0, fmt.Errorf("missing pod-level request for %s in Pod %s", resource, pod.Name) + } + return podRequest.MilliValue(), nil +} + +// calculatePodRequestsFromContainers computes the requests for the specified +// resource by summing requests from all containers in the pod. +// If a container name is specified, it uses only that container. +func calculatePodRequestsFromContainers(pod *v1.Pod, container string, resource v1.ResourceName) (int64, error) { + // Calculate all regular containers and restartable init containers requests. + containers := append([]v1.Container{}, pod.Spec.Containers...) + for _, c := range pod.Spec.InitContainers { + if c.RestartPolicy != nil && *c.RestartPolicy == v1.ContainerRestartPolicyAlways { + containers = append(containers, c) + } + } + + request := int64(0) + for _, c := range containers { + if container == "" || container == c.Name { + containerRequest, ok := c.Resources.Requests[resource] + if !ok { + return 0, fmt.Errorf("missing request for %s in container %s of Pod %s", resource, c.Name, pod.Name) + } + request += containerRequest.MilliValue() + } + } + return request, nil +} + func removeMetricsForPods(metrics metricsclient.PodMetricsInfo, pods sets.Set[string]) { for _, pod := range pods.UnsortedList() { delete(metrics, pod) diff --git a/pkg/controller/podautoscaler/replica_calculator_test.go b/pkg/controller/podautoscaler/replica_calculator_test.go index 3522c0f1d9a..fc164f7c6ab 100644 --- a/pkg/controller/podautoscaler/replica_calculator_test.go +++ b/pkg/controller/podautoscaler/replica_calculator_test.go @@ -31,13 +31,16 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/controller" metricsclient "k8s.io/kubernetes/pkg/controller/podautoscaler/metrics" + "k8s.io/kubernetes/pkg/features" cmapi "k8s.io/metrics/pkg/apis/custom_metrics/v1beta2" emapi "k8s.io/metrics/pkg/apis/external_metrics/v1beta1" metricsapi "k8s.io/metrics/pkg/apis/metrics/v1beta1" @@ -2225,17 +2228,18 @@ func TestGroupPods(t *testing.T) { } } -func TestCalculatePodRequests(t *testing.T) { +func TestCalculateRequests(t *testing.T) { containerRestartPolicyAlways := v1.ContainerRestartPolicyAlways testPod := "test-pod" tests := []struct { - name string - pods []*v1.Pod - container string - resource v1.ResourceName - expectedRequests map[string]int64 - expectedError error + name string + pods []*v1.Pod + container string + resource v1.ResourceName + enablePodLevelResources bool + expectedRequests map[string]int64 + expectedError error }{ { name: "void", @@ -2246,7 +2250,7 @@ func TestCalculatePodRequests(t *testing.T) { expectedError: nil, }, { - name: "pod with regular containers", + name: "Sum container requests if pod-level feature is disabled", pods: []*v1.Pod{{ ObjectMeta: metav1.ObjectMeta{ Name: testPod, @@ -2265,7 +2269,93 @@ func TestCalculatePodRequests(t *testing.T) { expectedError: nil, }, { - name: "calculate requests with special container", + name: "Pod-level resources are enabled, but not set: fallback to sum container requests", + enablePodLevelResources: true, + pods: []*v1.Pod{{ + ObjectMeta: metav1.ObjectMeta{ + Name: testPod, + Namespace: testNamespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + {Name: "container1", Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI)}}}, + {Name: "container2", Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(50, resource.DecimalSI)}}}, + }, + }, + }}, + container: "", + resource: v1.ResourceCPU, + expectedRequests: map[string]int64{testPod: 150}, + expectedError: nil, + }, + { + name: "Pod-level resources override container requests when feature enabled and pod resources specified", + enablePodLevelResources: true, + pods: []*v1.Pod{{ + + ObjectMeta: metav1.ObjectMeta{ + Name: testPod, + Namespace: testNamespace, + }, + Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(800, resource.DecimalSI)}, + }, + Containers: []v1.Container{ + {Name: "container1", Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI)}}}, + {Name: "container2", Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(50, resource.DecimalSI)}}}, + }, + }, + }}, + container: "", + resource: v1.ResourceCPU, + expectedRequests: map[string]int64{testPod: 800}, + expectedError: nil, + }, + { + name: "Fail if at least one of the containers is missing requests and pod-level feature/requests are not set", + pods: []*v1.Pod{{ + ObjectMeta: metav1.ObjectMeta{ + Name: testPod, + Namespace: testNamespace, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + {Name: "container1"}, + {Name: "container2", Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(50, resource.DecimalSI)}}}, + }, + }, + }}, + container: "", + resource: v1.ResourceCPU, + expectedRequests: nil, + expectedError: fmt.Errorf("missing request for %s in container %s of Pod %s", v1.ResourceCPU, "container1", testPod), + }, + { + name: "Pod-level resources override missing container requests when feature enabled and pod resources specified", + enablePodLevelResources: true, + pods: []*v1.Pod{{ + ObjectMeta: metav1.ObjectMeta{ + Name: testPod, + Namespace: testNamespace, + }, + Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(800, resource.DecimalSI)}, + }, + Containers: []v1.Container{ + {Name: "container1"}, + {Name: "container2", Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(50, resource.DecimalSI)}}}, + }, + }, + }}, + container: "", + resource: v1.ResourceCPU, + expectedRequests: map[string]int64{testPod: 800}, + expectedError: nil, + }, + { + name: "Container: if a container name is specified, calculate requests only for that container", pods: []*v1.Pod{{ ObjectMeta: metav1.ObjectMeta{ Name: testPod, @@ -2284,22 +2374,27 @@ func TestCalculatePodRequests(t *testing.T) { expectedError: nil, }, { - name: "container missing requests", + name: "Container: if a container name is specified, calculate requests only for that container and ignore pod-level requests", + enablePodLevelResources: true, pods: []*v1.Pod{{ ObjectMeta: metav1.ObjectMeta{ Name: testPod, Namespace: testNamespace, }, Spec: v1.PodSpec{ + Resources: &v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(800, resource.DecimalSI)}, + }, Containers: []v1.Container{ - {Name: "container1"}, + {Name: "container1", Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(100, resource.DecimalSI)}}}, + {Name: "container2", Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: *resource.NewMilliQuantity(50, resource.DecimalSI)}}}, }, }, }}, - container: "", + container: "container1", resource: v1.ResourceCPU, - expectedRequests: nil, - expectedError: fmt.Errorf("missing request for %s in container %s of Pod %s", v1.ResourceCPU, "container1", testPod), + expectedRequests: map[string]int64{testPod: 100}, + expectedError: nil, }, { name: "pod with restartable init containers", @@ -2327,7 +2422,9 @@ func TestCalculatePodRequests(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - requests, err := calculatePodRequests(tc.pods, tc.container, tc.resource) + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLevelResources, tc.enablePodLevelResources) + + requests, err := calculateRequests(tc.pods, tc.container, tc.resource) assert.Equal(t, tc.expectedRequests, requests, "requests should be as expected") assert.Equal(t, tc.expectedError, err, "error should be as expected") }) diff --git a/test/e2e/autoscaling/autoscaling_timer.go b/test/e2e/autoscaling/autoscaling_timer.go index 49d4370e861..b74e7b83bb1 100644 --- a/test/e2e/autoscaling/autoscaling_timer.go +++ b/test/e2e/autoscaling/autoscaling_timer.go @@ -86,7 +86,7 @@ var _ = SIGDescribe(feature.ClusterSizeAutoscalingScaleUp, framework.WithSlow(), nodeMemoryMB := (&nodeMemoryBytes).Value() / 1024 / 1024 memRequestMB := nodeMemoryMB / 10 // Ensure each pod takes not more than 10% of node's allocatable memory. replicas := 1 - resourceConsumer := e2eautoscaling.NewDynamicResourceConsumer(ctx, "resource-consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, replicas, 0, 0, 0, cpuRequestMillis, memRequestMB, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle) + resourceConsumer := e2eautoscaling.NewDynamicResourceConsumer(ctx, "resource-consumer", f.Namespace.Name, e2eautoscaling.KindDeployment, replicas, 0, 0, 0, cpuRequestMillis, memRequestMB, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, nil) ginkgo.DeferCleanup(resourceConsumer.CleanUp) resourceConsumer.WaitForReplicas(ctx, replicas, 1*time.Minute) // Should finish ~immediately, so 1 minute is more than enough. diff --git a/test/e2e/autoscaling/horizontal_pod_autoscaling.go b/test/e2e/autoscaling/horizontal_pod_autoscaling.go index 41038c919a2..79b8e65f80e 100644 --- a/test/e2e/autoscaling/horizontal_pod_autoscaling.go +++ b/test/e2e/autoscaling/horizontal_pod_autoscaling.go @@ -25,7 +25,9 @@ import ( autoscalingv2 "k8s.io/api/autoscaling/v2" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" e2eautoscaling "k8s.io/kubernetes/test/e2e/framework/autoscaling" @@ -67,6 +69,18 @@ var _ = SIGDescribe(feature.HPA, "Horizontal pod autoscaling (scale resource: CP }) }) + f.Describe("Deployment (Pod-level Resources Resource Metric)", framework.WithFeatureGate(features.PodLevelResources), func() { + f.It(titleUp+titleAverageUtilization, func(ctx context.Context) { + scaleUpPodLevelResources(ctx, "test-deployment-pod-level", e2eautoscaling.KindDeployment, autoscalingv2.ResourceMetricSourceType, f) + }) + }) + + f.Describe("Deployment (Pod-level Resources ContainerResource Metric)", framework.WithFeatureGate(features.PodLevelResources), func() { + f.It(titleUp+titleAverageUtilization, func(ctx context.Context) { + scaleUpPodLevelResources(ctx, "test-deployment-pod-level", e2eautoscaling.KindDeployment, autoscalingv2.ContainerResourceMetricSourceType, f) + }) + }) + f.Describe("ReplicaSet", func() { ginkgo.It(titleUp, func(ctx context.Context) { scaleUp(ctx, "rs", e2eautoscaling.KindReplicaSet, cpuResource, utilizationMetricType, false, f) @@ -202,7 +216,7 @@ func (st *HPAScaleTest) run(ctx context.Context, name string, kind schema.GroupV } else if st.resourceType == memResource { initMemTotal = st.initMemTotal } - rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, name, f.Namespace.Name, kind, st.initPods, initCPUTotal, initMemTotal, 0, st.perPodCPURequest, st.perPodMemRequest, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle) + rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, name, f.Namespace.Name, kind, st.initPods, initCPUTotal, initMemTotal, 0, st.perPodCPURequest, st.perPodMemRequest, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, nil) ginkgo.DeferCleanup(rc.CleanUp) hpa := e2eautoscaling.CreateResourceHorizontalPodAutoscaler(ctx, rc, st.resourceType, st.metricTargetType, st.targetValue, st.minPods, st.maxPods) ginkgo.DeferCleanup(e2eautoscaling.DeleteHorizontalPodAutoscaler, rc, hpa.Name) @@ -310,7 +324,7 @@ func (st *HPAContainerResourceScaleTest) run(ctx context.Context, name string, k } else if st.resourceType == memResource { initMemTotal = st.initMemTotal } - rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, name, f.Namespace.Name, kind, st.initPods, initCPUTotal, initMemTotal, 0, st.perContainerCPURequest, st.perContainerMemRequest, f.ClientSet, f.ScalesGetter, st.sidecarStatus, st.sidecarType) + rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, name, f.Namespace.Name, kind, st.initPods, initCPUTotal, initMemTotal, 0, st.perContainerCPURequest, st.perContainerMemRequest, f.ClientSet, f.ScalesGetter, st.sidecarStatus, st.sidecarType, nil) ginkgo.DeferCleanup(rc.CleanUp) hpa := e2eautoscaling.CreateContainerResourceHorizontalPodAutoscaler(ctx, rc, st.resourceType, st.metricTargetType, st.targetValue, st.minPods, st.maxPods) ginkgo.DeferCleanup(e2eautoscaling.DeleteContainerResourceHPA, rc, hpa.Name) @@ -411,9 +425,105 @@ func doNotScaleOnBusySidecar(ctx context.Context, name string, kind schema.Group st.run(ctx, name, kind, f) } +// HPAPodResourceScaleTest is a struct that defines the parameters for +// a pod-level resource scaling test. +type HPAPodResourceScaleTest struct { + initPods int + initCPUTotal int + metricSourceType autoscalingv2.MetricSourceType + perPodRequests *v1.ResourceRequirements + perContainerCPURequest int64 + perContainerMemRequest int64 + targetValue int32 + minPods int32 + maxPods int32 + firstScale int + cpuBurst int + secondScale int32 +} + +// run executes the HPA pod-level resource scaling test. +// It creates a resource consumer and an HPA, then verifies that the number of +// replicas scales up to the expected number of pods based on the initial CPU +// consumption. +// It also optionally verifies a second scaling event based on a CPU burst. +func (st *HPAPodResourceScaleTest) run(ctx context.Context, name string, kind schema.GroupVersionKind, f *framework.Framework) { + const timeToWait = 15 * time.Minute + resourceType := cpuResource + rc := e2eautoscaling.NewDynamicResourceConsumer(ctx, name, f.Namespace.Name, + kind, st.initPods, st.initCPUTotal, 0, 0, st.perContainerCPURequest, + st.perContainerMemRequest, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, + e2eautoscaling.Idle, st.perPodRequests) + ginkgo.DeferCleanup(rc.CleanUp) + + createHPAFn := e2eautoscaling.CreateResourceHorizontalPodAutoscaler + if st.metricSourceType == autoscalingv2.ContainerResourceMetricSourceType { + createHPAFn = e2eautoscaling.CreateContainerResourceHorizontalPodAutoscaler + } + hpa := createHPAFn(ctx, rc, resourceType, utilizationMetricType, + st.targetValue, st.minPods, st.maxPods) + ginkgo.DeferCleanup(e2eautoscaling.DeleteHorizontalPodAutoscaler, rc, hpa.Name) + + rc.WaitForReplicas(ctx, st.firstScale, timeToWait) + + if st.cpuBurst > 0 && st.secondScale > 0 { + rc.ConsumeCPU(st.cpuBurst) + rc.WaitForReplicas(ctx, int(st.secondScale), timeToWait) + } +} + +// scaleUpPodLevelResources configures and runs a test that scales up a workload +// that has pod-level resources set based on a Utilization metric type HPA. +// It also handles the case where the metric source is ContainerResource, +// adjusting the parameters accordingly. +func scaleUpPodLevelResources(ctx context.Context, name string, kind schema.GroupVersionKind, metricSourceType autoscalingv2.MetricSourceType, f *framework.Framework) { + st := &HPAPodResourceScaleTest{ + metricSourceType: metricSourceType, + perPodRequests: resourceRequirements(500, 500), + perContainerCPURequest: 0, + perContainerMemRequest: 0, + initCPUTotal: 250, + cpuBurst: 700, + targetValue: 20, + minPods: 1, + maxPods: 5, + initPods: 1, + firstScale: 3, + secondScale: 5, + } + + if metricSourceType == autoscalingv2.ContainerResourceMetricSourceType { + // When pod-level resources are set and the HPA is configured on + // ContainerResource metric type, HPA considers the target container + // resource requests, instead of the pod-level resources during + // calculations. + // The values below make sure that HPA is autoscaling based on + // perContainerCPURequest instead of perPodRequests (HPA would not scale + // up if it was considering perPodRequests). + st.perContainerCPURequest = 250 + st.perContainerMemRequest = 250 + st.initCPUTotal = 125 + st.cpuBurst = 350 + } + st.run(ctx, name, kind, f) +} + func getTargetValueByType(averageValueTarget, averageUtilizationTarget int, targetType autoscalingv2.MetricTargetType) int32 { if targetType == utilizationMetricType { return int32(averageUtilizationTarget) } return int32(averageValueTarget) } + +func resourceRequirements(cpuMillis, memMb int64) *v1.ResourceRequirements { + return &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(cpuMillis, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(memMb*1024*1024, resource.BinarySI), // ResourceMemory is in bytes + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(cpuMillis, resource.DecimalSI), + v1.ResourceMemory: *resource.NewQuantity(memMb*1024*1024, resource.BinarySI), // ResourceMemory is in bytes + }, + } +} diff --git a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go index 0ffb0353564..5d41c0bde17 100644 --- a/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go +++ b/test/e2e/autoscaling/horizontal_pod_autoscaling_behavior.go @@ -69,7 +69,7 @@ var _ = SIGDescribe(feature.HPA, "Horizontal pod autoscaling (non-default behavi hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, - ) + nil) ginkgo.DeferCleanup(rc.CleanUp) hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, @@ -110,7 +110,7 @@ var _ = SIGDescribe(feature.HPA, "Horizontal pod autoscaling (non-default behavi hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, - ) + nil) ginkgo.DeferCleanup(rc.CleanUp) hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, @@ -149,7 +149,7 @@ var _ = SIGDescribe(feature.HPA, "Horizontal pod autoscaling (non-default behavi hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, - ) + nil) ginkgo.DeferCleanup(rc.CleanUp) hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, @@ -185,7 +185,7 @@ var _ = SIGDescribe(feature.HPA, "Horizontal pod autoscaling (non-default behavi hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, - ) + nil) ginkgo.DeferCleanup(rc.CleanUp) hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, @@ -227,7 +227,7 @@ var _ = SIGDescribe(feature.HPA, "Horizontal pod autoscaling (non-default behavi hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, - ) + nil) ginkgo.DeferCleanup(rc.CleanUp) hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, @@ -269,7 +269,7 @@ var _ = SIGDescribe(feature.HPA, "Horizontal pod autoscaling (non-default behavi hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, - ) + nil) ginkgo.DeferCleanup(rc.CleanUp) hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, @@ -313,7 +313,7 @@ var _ = SIGDescribe(feature.HPA, "Horizontal pod autoscaling (non-default behavi hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, - ) + nil) ginkgo.DeferCleanup(rc.CleanUp) hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, @@ -356,7 +356,7 @@ var _ = SIGDescribe(feature.HPA, "Horizontal pod autoscaling (non-default behavi hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, - ) + nil) ginkgo.DeferCleanup(rc.CleanUp) hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, @@ -403,7 +403,7 @@ var _ = SIGDescribe(feature.HPA, "Horizontal pod autoscaling (non-default behavi hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, - ) + nil) ginkgo.DeferCleanup(rc.CleanUp) hpa := e2eautoscaling.CreateCPUHorizontalPodAutoscalerWithBehavior(ctx, @@ -455,7 +455,7 @@ var _ = SIGDescribe(feature.HPA, "Horizontal pod autoscaling (non-default behavi hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, - ) + nil) ginkgo.DeferCleanup(rc.CleanUp) scaleUpRule := e2eautoscaling.HPAScalingRuleWithScalingPolicy(autoscalingv2.PodsScalingPolicy, int32(podsLimitPerMinute), int32(limitWindowLength.Seconds())) @@ -516,7 +516,7 @@ var _ = SIGDescribe(feature.HPAConfigurableTolerance, framework.WithFeatureGate( hpaName, f.Namespace.Name, e2eautoscaling.KindDeployment, initPods, initCPUUsageTotal, 0, 0, int64(podCPURequest), 200, f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, e2eautoscaling.Idle, - ) + nil) ginkgo.DeferCleanup(rc.CleanUp) scaleRule := e2eautoscaling.HPAScalingRuleWithToleranceMilli(10000) diff --git a/test/e2e/framework/autoscaling/autoscaling_utils.go b/test/e2e/framework/autoscaling/autoscaling_utils.go index 8fa90de5f4a..799cd56e965 100644 --- a/test/e2e/framework/autoscaling/autoscaling_utils.go +++ b/test/e2e/framework/autoscaling/autoscaling_utils.go @@ -131,9 +131,9 @@ type ResourceConsumer struct { } // NewDynamicResourceConsumer is a wrapper to create a new dynamic ResourceConsumer -func NewDynamicResourceConsumer(ctx context.Context, name, nsName string, kind schema.GroupVersionKind, replicas, initCPUTotal, initMemoryTotal, initCustomMetric int, cpuLimit, memLimit int64, clientset clientset.Interface, scaleClient scaleclient.ScalesGetter, enableSidecar SidecarStatusType, sidecarType SidecarWorkloadType) *ResourceConsumer { +func NewDynamicResourceConsumer(ctx context.Context, name, nsName string, kind schema.GroupVersionKind, replicas, initCPUTotal, initMemoryTotal, initCustomMetric int, cpuLimit, memLimit int64, clientset clientset.Interface, scaleClient scaleclient.ScalesGetter, enableSidecar SidecarStatusType, sidecarType SidecarWorkloadType, podResources *v1.ResourceRequirements) *ResourceConsumer { return newResourceConsumer(ctx, name, nsName, kind, replicas, initCPUTotal, initMemoryTotal, initCustomMetric, dynamicConsumptionTimeInSeconds, - dynamicRequestSizeInMillicores, dynamicRequestSizeInMegabytes, dynamicRequestSizeCustomMetric, cpuLimit, memLimit, clientset, scaleClient, nil, nil, enableSidecar, sidecarType) + dynamicRequestSizeInMillicores, dynamicRequestSizeInMegabytes, dynamicRequestSizeCustomMetric, cpuLimit, memLimit, clientset, scaleClient, nil, nil, enableSidecar, sidecarType, podResources) } // getSidecarContainer returns sidecar container @@ -171,7 +171,7 @@ memLimit argument is in megabytes, memLimit is a maximum amount of memory that c cpuLimit argument is in millicores, cpuLimit is a maximum amount of cpu that can be consumed by a single pod */ func newResourceConsumer(ctx context.Context, name, nsName string, kind schema.GroupVersionKind, replicas, initCPUTotal, initMemoryTotal, initCustomMetric, consumptionTimeInSeconds, requestSizeInMillicores, - requestSizeInMegabytes int, requestSizeCustomMetric int, cpuLimit, memLimit int64, clientset clientset.Interface, scaleClient scaleclient.ScalesGetter, podAnnotations, serviceAnnotations map[string]string, sidecarStatus SidecarStatusType, sidecarType SidecarWorkloadType) *ResourceConsumer { + requestSizeInMegabytes int, requestSizeCustomMetric int, cpuLimit, memLimit int64, clientset clientset.Interface, scaleClient scaleclient.ScalesGetter, podAnnotations, serviceAnnotations map[string]string, sidecarStatus SidecarStatusType, sidecarType SidecarWorkloadType, podResources *v1.ResourceRequirements) *ResourceConsumer { if podAnnotations == nil { podAnnotations = make(map[string]string) } @@ -194,7 +194,7 @@ func newResourceConsumer(ctx context.Context, name, nsName string, kind schema.G framework.ExpectNoError(err) resourceClient := dynamicClient.Resource(schema.GroupVersionResource{Group: crdGroup, Version: crdVersion, Resource: crdNamePlural}).Namespace(nsName) - runServiceAndWorkloadForResourceConsumer(ctx, clientset, resourceClient, apiExtensionClient, nsName, name, kind, replicas, cpuLimit, memLimit, podAnnotations, serviceAnnotations, additionalContainers) + runServiceAndWorkloadForResourceConsumer(ctx, clientset, resourceClient, apiExtensionClient, nsName, name, kind, replicas, cpuLimit, memLimit, podAnnotations, serviceAnnotations, additionalContainers, podResources) controllerName := name + "-ctrl" // If sidecar is enabled and busy, run service and consumer for sidecar if sidecarStatus == Enable && sidecarType == Busy { @@ -617,7 +617,7 @@ func runServiceAndSidecarForResourceConsumer(ctx context.Context, c clientset.In ctx, c, ns, controllerName, 1, startServiceInterval, startServiceTimeout)) } -func runServiceAndWorkloadForResourceConsumer(ctx context.Context, c clientset.Interface, resourceClient dynamic.ResourceInterface, apiExtensionClient crdclientset.Interface, ns, name string, kind schema.GroupVersionKind, replicas int, cpuLimitMillis, memLimitMb int64, podAnnotations, serviceAnnotations map[string]string, additionalContainers []v1.Container) { +func runServiceAndWorkloadForResourceConsumer(ctx context.Context, c clientset.Interface, resourceClient dynamic.ResourceInterface, apiExtensionClient crdclientset.Interface, ns, name string, kind schema.GroupVersionKind, replicas int, cpuLimitMillis, memLimitMb int64, podAnnotations, serviceAnnotations map[string]string, additionalContainers []v1.Container, podResources *v1.ResourceRequirements) { ginkgo.By(fmt.Sprintf("Running consuming RC %s via %s with %v replicas", name, kind, replicas)) _, err := createService(ctx, c, name, ns, serviceAnnotations, map[string]string{"name": name}, port, targetPort) framework.ExpectNoError(err) @@ -629,13 +629,16 @@ func runServiceAndWorkloadForResourceConsumer(ctx context.Context, c clientset.I Namespace: ns, Timeout: timeoutRC, Replicas: replicas, - CpuRequest: cpuLimitMillis, - CpuLimit: cpuLimitMillis, + CPURequest: cpuLimitMillis, + CPULimit: cpuLimitMillis, MemRequest: memLimitMb * 1024 * 1024, // MemLimit is in bytes MemLimit: memLimitMb * 1024 * 1024, Annotations: podAnnotations, AdditionalContainers: additionalContainers, } + if podResources != nil { + rcConfig.PodResources = podResources.DeepCopy() + } dpConfig := testutils.DeploymentConfig{ RCConfig: rcConfig, diff --git a/test/e2e/upgrades/autoscaling/horizontal_pod_autoscalers.go b/test/e2e/upgrades/autoscaling/horizontal_pod_autoscalers.go index ddac480002a..89fe2e04c69 100644 --- a/test/e2e/upgrades/autoscaling/horizontal_pod_autoscalers.go +++ b/test/e2e/upgrades/autoscaling/horizontal_pod_autoscalers.go @@ -53,7 +53,9 @@ func (t *HPAUpgradeTest) Setup(ctx context.Context, f *framework.Framework) { f.ClientSet, f.ScalesGetter, e2eautoscaling.Disable, - e2eautoscaling.Idle) + e2eautoscaling.Idle, + nil, + ) t.hpa = e2eautoscaling.CreateCPUResourceHorizontalPodAutoscaler(ctx, t.rc, 20, /* targetCPUUtilizationPercent */ diff --git a/test/utils/runners.go b/test/utils/runners.go index b04af70fda6..c8b8bc94759 100644 --- a/test/utils/runners.go +++ b/test/utils/runners.go @@ -119,11 +119,12 @@ type RCConfig struct { Timeout time.Duration PodStatusFile *os.File Replicas int - CpuRequest int64 // millicores - CpuLimit int64 // millicores - MemRequest int64 // bytes - MemLimit int64 // bytes - GpuLimit int64 // count + CPURequest int64 // millicores + CPULimit int64 // millicores + MemRequest int64 // bytes + MemLimit int64 // bytes + GpuLimit int64 // count + PodResources *v1.ResourceRequirements // Pod-level resources ReadinessProbe *v1.Probe DNSPolicy *v1.DNSPolicy PriorityClassName string @@ -331,6 +332,10 @@ func (config *DeploymentConfig) create() error { }, } + if config.PodResources != nil { + deployment.Spec.Template.Spec.Resources = config.PodResources.DeepCopy() + } + if len(config.AdditionalContainers) > 0 { deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, config.AdditionalContainers...) } @@ -402,6 +407,10 @@ func (config *ReplicaSetConfig) create() error { }, } + if config.PodResources != nil { + rs.Spec.Template.Spec.Resources = config.PodResources.DeepCopy() + } + if len(config.AdditionalContainers) > 0 { rs.Spec.Template.Spec.Containers = append(rs.Spec.Template.Spec.Containers, config.AdditionalContainers...) } @@ -478,6 +487,10 @@ func (config *RCConfig) create() error { }, } + if config.PodResources != nil { + rc.Spec.Template.Spec.Resources = config.PodResources.DeepCopy() + } + if len(config.AdditionalContainers) > 0 { rc.Spec.Template.Spec.Containers = append(rc.Spec.Template.Spec.Containers, config.AdditionalContainers...) } @@ -521,20 +534,20 @@ func (config *RCConfig) applyTo(template *v1.PodTemplateSpec) { c := &template.Spec.Containers[0] c.Ports = append(c.Ports, v1.ContainerPort{Name: k, ContainerPort: int32(v), HostPort: int32(v)}) } - if config.CpuLimit > 0 || config.MemLimit > 0 || config.GpuLimit > 0 { + if config.CPULimit > 0 || config.MemLimit > 0 || config.GpuLimit > 0 { template.Spec.Containers[0].Resources.Limits = v1.ResourceList{} } - if config.CpuLimit > 0 { - template.Spec.Containers[0].Resources.Limits[v1.ResourceCPU] = *resource.NewMilliQuantity(config.CpuLimit, resource.DecimalSI) + if config.CPULimit > 0 { + template.Spec.Containers[0].Resources.Limits[v1.ResourceCPU] = *resource.NewMilliQuantity(config.CPULimit, resource.DecimalSI) } if config.MemLimit > 0 { template.Spec.Containers[0].Resources.Limits[v1.ResourceMemory] = *resource.NewQuantity(config.MemLimit, resource.DecimalSI) } - if config.CpuRequest > 0 || config.MemRequest > 0 { + if config.CPURequest > 0 || config.MemRequest > 0 { template.Spec.Containers[0].Resources.Requests = v1.ResourceList{} } - if config.CpuRequest > 0 { - template.Spec.Containers[0].Resources.Requests[v1.ResourceCPU] = *resource.NewMilliQuantity(config.CpuRequest, resource.DecimalSI) + if config.CPURequest > 0 { + template.Spec.Containers[0].Resources.Requests[v1.ResourceCPU] = *resource.NewMilliQuantity(config.CPURequest, resource.DecimalSI) } if config.MemRequest > 0 { template.Spec.Containers[0].Resources.Requests[v1.ResourceMemory] = *resource.NewQuantity(config.MemRequest, resource.DecimalSI)