Add scheduler_perf test case for AssignedPodDelete event handling

test/integration/scheduler_perf/config/event_handling/poddelete-pod-blocker-affinity.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-blocker-affinity-
+  labels:
+    color: green
+    type: blocker
+spec:
+  containers:
+  - image: registry.k8s.io/pause:3.10
+    name: pause
+  terminationGracePeriodSeconds: 0

test/integration/scheduler_perf/config/event_handling/poddelete-pod-blocker-topology-ports-resources.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-blocker-topology-
+  labels:
+    topology: blue
+    type: blocker
+spec:
+  topologySpreadConstraints:
+    - maxSkew: 10
+      minDomains: 10000
+      topologyKey: kubernetes.io/hostname
+      whenUnsatisfiable: DoNotSchedule
+      labelSelector:
+        matchLabels:
+          topology: blue
+  containers:
+  - image: registry.k8s.io/pause:3.10
+    name: pause
+    ports:
+    - hostPort: 8{{ mod .Index 12 }}
+      containerPort: 8{{ mod .Index 12 }}
+    resources:
+      requests:
+        cpu: 0.35
+        memory: 3Gi
+  terminationGracePeriodSeconds: 0

test/integration/scheduler_perf/config/event_handling/poddelete-pod-interpodaffinity.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-unsched-
+  labels:
+    color: green
+    type: unsched
+spec:
+  affinity:
+    podAntiAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+      - labelSelector:
+          matchLabels:
+            color: green
+        topologyKey: kubernetes.io/hostname
+        namespaces: ["blockeraffinity"]
+  containers:
+  - image: registry.k8s.io/pause:3.10
+    name: pause

test/integration/scheduler_perf/config/event_handling/poddelete-pod-nodeports.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-unsched-
+  labels:
+    type: unsched
+spec:
+  containers:
+  - image: registry.k8s.io/pause:3.10
+    name: pause
+    ports:
+    - hostPort: 8{{ mod .Index 12 }}
+      containerPort: 8{{ mod .Index 12 }}

test/integration/scheduler_perf/config/event_handling/poddelete-pod-noderesources.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-unsched-
+  labels:
+    type: unsched
+spec:
+  containers:
+  - image: registry.k8s.io/pause:3.10
+    name: pause
+    resources:
+      requests:
+        cpu: 0.2
+        memory: 1Gi

test/integration/scheduler_perf/config/event_handling/poddelete-pod-nodevolumelimits.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-unsched-
+  labels:
+    type: unsched
+spec:
+  containers:
+  - image: registry.k8s.io/pause:3.10
+    name: pause

test/integration/scheduler_perf/config/event_handling/poddelete-pod-podtopologyspread.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-unsched-
+  labels:
+    topology: blue
+    type: unsched
+spec:
+  topologySpreadConstraints:
+    - maxSkew: 11
+      minDomains: 10000
+      topologyKey: kubernetes.io/hostname
+      whenUnsatisfiable: DoNotSchedule
+      labelSelector:
+        matchLabels:
+          topology: blue
+  containers:
+  - image: registry.k8s.io/pause:3.10
+    name: pause

test/integration/scheduler_perf/config/event_handling/poddelete-pod-volumerestrictions.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: pod-unsched-
+  labels:
+    type: unsched
+spec:
+  containers:
+  - image: registry.k8s.io/pause:3.10
+    name: pause
+  volumes:
+  - name: vol
+    persistentVolumeClaim:
+      claimName: pvc-{{ .Index }}

test/integration/scheduler_perf/config/performance-config.yaml

@@ -1548,3 +1548,118 @@
       initNodes: 1000
       deletingPods: 1000
       measurePods: 1000
+
+# This test case is used to measure the performance of queuing hints when handling the AssignedPodDelete events.
+# First, two groups of blocker pods are created, which will prevents other pods from being scheduled.
+# Then multiple types of pods are created, and each group is filtered by different plugin.
+# Next, blocker pods are gradually deleted and previously unscheduled pods can be scheduled.
+# Plugins covered: InterPodAffinity, NodePorts, NodeResources, NodeVolumeLimits, PodTopologySpread and VolumeRestrictions.
+- name: EventHandlingPodDelete
+  featureGates:
+    SchedulerQueueingHints: true
+  workloadTemplate:
+  - opcode: createNodes
+    countParam: $initNodes
+    nodeTemplatePath: config/templates/node-default.yaml
+    # Allow max 20 volumes per node.
+    nodeAllocatableStrategy:
+      nodeAllocatable:
+        attachable-volumes-csi-ebs.csi.aws.com: "20"
+      csiNodeAllocatable:
+        ebs.csi.aws.com:
+          count: 20
+  # Create pods that will block other pods from being scheduled.
+  # They'll block using NodePorts, NodeResources, NodeVolumeLimits and PodTopologySpread plugins.
+  - opcode: createPods
+    countParam: $blockerPods
+    podTemplatePath: config/event_handling/poddelete-pod-blocker-topology-ports-resources.yaml
+    persistentVolumeTemplatePath: config/templates/pv-csi.yaml
+    persistentVolumeClaimTemplatePath: config/templates/pvc.yaml
+    namespace: blockertopologyportsresources
+  # Create second group of pods that will block another pods from being scheduled.
+  # They'll block using InterPodAffinity and VolumeRestrictions plugins.
+  - opcode: createPods
+    countParam: $blockerPods
+    podTemplatePath: config/event_handling/poddelete-pod-blocker-affinity.yaml
+    persistentVolumeTemplatePath: config/templates/pv-csi.yaml
+    persistentVolumeClaimTemplatePath: config/templates/pvc-once-pod.yaml
+    namespace: blockeraffinity
+  # Collect metrics from all createPods ops below.
+  - opcode: startCollectingMetrics
+    name: unschedPods
+    namespaces: [blockertopologyportsresources, blockeraffinity, nodeports, noderesources, nodevolumelimits, interpodaffinity]
+    labelSelector:
+      type: unsched
+  # Create pods blocked using PodTopologySpread plugin.
+  # Note: for this plugin, namespace has to match the blocker's namespace,
+  # so has to be "blockertopologyportsresources".
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/event_handling/poddelete-pod-podtopologyspread.yaml
+    skipWaitToCompletion: true
+    namespace: blockertopologyportsresources
+  # Create pods blocked using VolumeRestrictions plugin.
+  # Note: these pods uses PVCs and PVs created for second blocker pods,
+  # so the count needs to be equal to $blockerPods
+  # and namespace has to be "blockeraffinity".
+  - opcode: createPods
+    countParam: $blockerPods
+    podTemplatePath: config/event_handling/poddelete-pod-volumerestrictions.yaml
+    skipWaitToCompletion: true
+    namespace: blockeraffinity
+  # Create pods blocked using NodePorts plugin.
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/event_handling/poddelete-pod-nodeports.yaml
+    skipWaitToCompletion: true
+    namespace: nodeports
+  # Create pods blocked using NodeResources plugin.
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/event_handling/poddelete-pod-noderesources.yaml
+    skipWaitToCompletion: true
+    namespace: noderesources
+  # Create pods blocked using NodeVolumeLimits plugin.
+  - opcode: createPods
+    countParam: $blockerPods
+    podTemplatePath: config/event_handling/poddelete-pod-nodevolumelimits.yaml
+    persistentVolumeTemplatePath: config/templates/pv-csi.yaml
+    persistentVolumeClaimTemplatePath: config/templates/pvc.yaml
+    skipWaitToCompletion: true
+    namespace: nodevolumelimits
+  # Create pods blocked using InterPodAffinity plugin.
+  - opcode: createPods
+    countParam: $measurePods
+    podTemplatePath: config/event_handling/poddelete-pod-interpodaffinity.yaml
+    skipWaitToCompletion: true
+    namespace: interpodaffinity
+  # Wait for unschedulable pods to be processed by the scheduler.
+  - opcode: barrier
+    stageRequirement: Attempted
+    labelSelector:
+      type: unsched
+  # Start deleting blocker pods.
+  - opcode: deletePods
+    deletePodsPerSecond: 100
+    namespace: blockertopologyportsresources
+    labelSelector:
+      type: blocker
+    skipWaitToCompletion: true
+  - opcode: deletePods
+    deletePodsPerSecond: 100
+    namespace: blockeraffinity
+    labelSelector:
+      type: blocker
+    skipWaitToCompletion: true
+  # Wait for previously unschedulable pods to be scheduled.
+  - opcode: barrier
+    labelSelector:
+      type: unsched
+  - opcode: stopCollectingMetrics
+  workloads:
+  - name: 50Nodes_500Pods
+    labels: [performance, short]
+    params:
+      initNodes: 50
+      blockerPods: 480 # Must be slightly below initNodes * 10 to be stable
+      measurePods: 500 # Must be initNodes * 10

test/integration/scheduler_perf/config/templates/pvc-once-pod.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  annotations:
+    pv.kubernetes.io/bind-completed: "true"
+spec:
+  accessModes:
+  - ReadWriteOncePod
+  resources:
+    requests:
+      storage: 1Gi

test/integration/scheduler_perf/create.go

@@ -155,7 +155,10 @@ func getSpecFromTextTemplateFile(path string, env map[string]any, spec interface
 	fm := template.FuncMap{"div": func(a, b int) int {
 		return a / b
 	}}
-	tmpl, err := template.New("object").Funcs(fm).Parse(string(content))
+	modFn := template.FuncMap{"mod": func(a, b int) int {
+		return a % b
+	}}
+	tmpl, err := template.New("object").Funcs(fm).Funcs(modFn).Parse(string(content))
 	if err != nil {
 		return err
 	}

test/utils/runners.go

@@ -1194,6 +1194,10 @@ func CreatePodWithPersistentVolume(ctx context.Context, client clientset.Interfa
 		// PVs are cluster-wide resources.
 		// Prepend a namespace to make the name globally unique.
 		pv.Name = fmt.Sprintf("%s-%s", namespace, pv.Name)
+		pvs := pv.Spec.PersistentVolumeSource
+		if pvs.CSI != nil {
+			pvs.CSI.VolumeHandle = pv.Name
+		}
 		if bindVolume {
 			// bind pv to "pvc-$i"
 			pv.Spec.ClaimRef = &v1.ObjectReference{