Make scheduler_perf stable

test/integration/scheduler_perf/config/performance-config.yaml

@@ -27,15 +27,18 @@
     countParam: $initNodes
     uniqueNodeLabelStrategy:
       labelKey: kubernetes.io/hostname
+  - opcode: createNamespaces
+    prefix: sched
+    count: 2
   - opcode: createPods
     countParam: $initPods
     podTemplatePath: config/pod-with-pod-anti-affinity.yaml
-    namespace: sched-setup
+    namespace: sched-0
   - opcode: createPods
     countParam: $measurePods
     podTemplatePath: config/pod-with-pod-anti-affinity.yaml
     collectMetrics: true
-    namespace: sched-test
+    namespace: sched-1
   workloads:
   - name: 500Nodes
     params:
@@ -173,14 +176,17 @@
     labelNodePrepareStrategy:
       labelKey: "topology.kubernetes.io/zone"
       labelValues: ["zone1"]
+  - opcode: createNamespaces
+    prefix: sched
+    count: 2
   - opcode: createPods
     countParam: $initPods
     podTemplatePath: config/pod-with-pod-affinity.yaml
-    namespace: sched-setup
+    namespace: sched-0
   - opcode: createPods
     countParam: $measurePods
     podTemplatePath: config/pod-with-pod-affinity.yaml
-    namespace: sched-test
+    namespace: sched-1
     collectMetrics: true
   workloads:
   - name: 500Nodes
@@ -200,14 +206,17 @@
     countParam: $initNodes
     uniqueNodeLabelStrategy:
       labelKey: kubernetes.io/hostname
+  - opcode: createNamespaces
+    prefix: sched
+    count: 2
   - opcode: createPods
     countParam: $initPods
     podTemplatePath: config/pod-with-preferred-pod-affinity.yaml
-    namespace: sched-setup
+    namespace: sched-0
   - opcode: createPods
     countParam: $measurePods
     podTemplatePath: config/pod-with-preferred-pod-affinity.yaml
-    namespace: sched-test
+    namespace: sched-1
     collectMetrics: true
   workloads:
   - name: 500Nodes
@@ -227,14 +236,17 @@
     countParam: $initNodes
     uniqueNodeLabelStrategy:
       labelKey: kubernetes.io/hostname
+  - opcode: createNamespaces
+    prefix: sched
+    count: 2
   - opcode: createPods
     countParam: $initPods
     podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml
-    namespace: sched-setup
+    namespace: sched-0
   - opcode: createPods
     countParam: $measurePods
     podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml
-    namespace: sched-test
+    namespace: sched-1
     collectMetrics: true
   workloads:
   - name: 500Nodes
@@ -337,26 +349,29 @@
     labelNodePrepareStrategy:
       labelKey: "topology.kubernetes.io/zone"
       labelValues: ["zone1"]
+  - opcode: createNamespaces
+    prefix: sched
+    count: 1
   - opcode: createPods
     countParam: $initPods
     podTemplatePath: config/pod-default.yaml
-    namespace: sched-setup
+    namespace: sched-0
   - opcode: createPods
     countParam: $initPods
     podTemplatePath: config/pod-with-pod-affinity.yaml
-    namespace: sched-setup
+    namespace: sched-0
   - opcode: createPods
     countParam: $initPods
     podTemplatePath: config/pod-with-pod-anti-affinity.yaml
-    namespace: sched-setup
+    namespace: sched-0
   - opcode: createPods
     countParam: $initPods
     podTemplatePath: config/pod-with-preferred-pod-affinity.yaml
-    namespace: sched-setup
+    namespace: sched-0
   - opcode: createPods
     countParam: $initPods
     podTemplatePath: config/pod-with-preferred-pod-anti-affinity.yaml
-    namespace: sched-setup
+    namespace: sched-0
   - opcode: createPods
     countParam: $measurePods
     podTemplatePath: config/pod-default.yaml
@@ -478,7 +493,7 @@
 
 - name: SchedulingRequiredPodAntiAffinityWithNSSelector
   featureGates:
-    PodAffinityNamespaceSelector: true  
+    PodAffinityNamespaceSelector: true
   workloadTemplate:
   - opcode: createNodes
     countParam: $initNodes
@@ -511,10 +526,10 @@
       initPodsPerNamespace: 40
       initNamespaces: 100
       measurePods: 1000
-      
+
 - name: SchedulingPreferredAntiAffinityWithNSSelector
   featureGates:
-    PodAffinityNamespaceSelector: true  
+    PodAffinityNamespaceSelector: true
   workloadTemplate:
   - opcode: createNodes
     countParam: $initNodes
@@ -550,7 +565,7 @@
 
 - name: SchedulingRequiredPodAffinityWithNSSelector
   featureGates:
-    PodAffinityNamespaceSelector: true  
+    PodAffinityNamespaceSelector: true
   workloadTemplate:
   - opcode: createNodes
     countParam: $initNodes
@@ -584,10 +599,10 @@
       initPodsPerNamespace: 50
       initNamespaces: 100
       measurePods: 1000
-      
+
 - name: SchedulingPreferredAffinityWithNSSelector
   featureGates:
-    PodAffinityNamespaceSelector: true  
+    PodAffinityNamespaceSelector: true
   workloadTemplate:
   - opcode: createNodes
     countParam: $initNodes
@@ -620,4 +635,4 @@
       initPodsPerNamespace: 50
       initNamespaces: 100
       measurePods: 1000
-    
+

test/integration/scheduler_perf/config/pod-with-pod-affinity.yaml

@@ -12,7 +12,7 @@ spec:
           matchLabels:
             color: blue
         topologyKey: topology.kubernetes.io/zone
-        namespaces: ["sched-test", "sched-setup"]
+        namespaces: ["sched-1", "sched-0"]
   containers:
   - image: k8s.gcr.io/pause:3.6
     name: pause

test/integration/scheduler_perf/config/pod-with-pod-anti-affinity.yaml

@@ -12,7 +12,7 @@ spec:
           matchLabels:
             color: green
         topologyKey: kubernetes.io/hostname
-        namespaces: ["sched-test", "sched-setup"]
+        namespaces: ["sched-1", "sched-0"]
   containers:
   - image: k8s.gcr.io/pause:3.6
     name: pause

test/integration/scheduler_perf/config/pod-with-preferred-pod-affinity.yaml

@@ -13,7 +13,7 @@ spec:
               matchLabels:
                 color: red
             topologyKey: kubernetes.io/hostname
-            namespaces: ["sched-test", "sched-setup"]
+            namespaces: ["sched-1", "sched-0"]
           weight: 1
   containers:
     - image: k8s.gcr.io/pause:3.6

test/integration/scheduler_perf/config/pod-with-preferred-pod-anti-affinity.yaml

@@ -13,7 +13,7 @@ spec:
               matchLabels:
                 color: yellow
             topologyKey: kubernetes.io/hostname
-            namespaces: ["sched-test", "sched-setup"]
+            namespaces: ["sched-1", "sched-0"]
           weight: 1
   containers:
     - image: k8s.gcr.io/pause:3.6

test/integration/scheduler_perf/scheduler_perf_test.go

@@ -600,8 +600,17 @@ func runWorkload(b *testing.B, tc *testCase, w *workload) []DataItem {
 
 	var mu sync.Mutex
 	var dataItems []DataItem
-	numPodsScheduledPerNamespace := make(map[string]int)
 	nextNodeIndex := 0
+	// numPodsScheduledPerNamespace has all namespaces created in workload and the number of pods they (will) have.
+	// All namespaces listed in numPodsScheduledPerNamespace will be cleaned up.
+	numPodsScheduledPerNamespace := make(map[string]int)
+	b.Cleanup(func() {
+		for namespace, _ := range numPodsScheduledPerNamespace {
+			if err := client.CoreV1().Namespaces().Delete(context.Background(), namespace, metav1.DeleteOptions{}); err != nil {
+				b.Errorf("Deleting Namespace in numPodsScheduledPerNamespace: %v", err)
+			}
+		}
+	})
 
 	for opIndex, op := range unrollWorkloadTemplate(b, tc.WorkloadTemplate, w) {
 		realOp, err := op.realOp.patchParams(w)
@@ -636,27 +645,29 @@ func runWorkload(b *testing.B, tc *testCase, w *workload) []DataItem {
 				nsPreparer.cleanup()
 				b.Fatalf("op %d: %v", opIndex, err)
 			}
-			b.Cleanup(func() {
-				nsPreparer.cleanup()
-			})
+			for _, n := range nsPreparer.namespaces() {
+				if _, ok := numPodsScheduledPerNamespace[n]; ok {
+					// this namespace has been already created.
+					continue
+				}
+				numPodsScheduledPerNamespace[n] = 0
+			}
 
 		case *createPodsOp:
 			var namespace string
+			// define Pod's namespace automatically, and create that namespace.
+			namespace = fmt.Sprintf("namespace-%d", opIndex)
 			if concreteOp.Namespace != nil {
 				namespace = *concreteOp.Namespace
-			} else {
-				// define Pod's namespace automatically, and create that namespace.
-				namespace = fmt.Sprintf("namespace-%d", opIndex)
+			}
+			if _, ok := numPodsScheduledPerNamespace[namespace]; !ok {
+				// The namespace has not created yet.
+				// So, creat that and register it to numPodsScheduledPerNamespace.
 				_, err := client.CoreV1().Namespaces().Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, metav1.CreateOptions{})
-				if err != nil && !apierrors.IsAlreadyExists(err) {
+				if err != nil {
 					b.Fatalf("failed to create namespace for Pod: %v", namespace)
 				}
-				b.Cleanup(func() {
-					err := client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{})
-					if err != nil {
-						b.Errorf("failed to delete namespace %v", namespace)
-					}
-				})
+				numPodsScheduledPerNamespace[namespace] = 0
 			}
 			var collectors []testDataCollector
 			var collectorCtx context.Context
@@ -1104,6 +1115,15 @@ func newNamespacePreparer(cno *createNamespacesOp, clientset clientset.Interface
 	}, nil
 }
 
+// namespaces returns namespace names have been (or will be) created by this namespacePreparer
+func (p *namespacePreparer) namespaces() []string {
+	namespaces := make([]string, p.count)
+	for i := 0; i < p.count; i++ {
+		namespaces[i] = fmt.Sprintf("%s-%d", p.prefix, i)
+	}
+	return namespaces
+}
+
 // prepare creates the namespaces.
 func (p *namespacePreparer) prepare() error {
 	base := &v1.Namespace{}
@@ -1115,7 +1135,7 @@ func (p *namespacePreparer) prepare() error {
 		n := base.DeepCopy()
 		n.Name = fmt.Sprintf("%s-%d", p.prefix, i)
 		if err := testutils.RetryWithExponentialBackOff(func() (bool, error) {
-			_, err := p.client.CoreV1().Namespaces().Create(context.TODO(), n, metav1.CreateOptions{})
+			_, err := p.client.CoreV1().Namespaces().Create(context.Background(), n, metav1.CreateOptions{})
 			return err == nil || apierrors.IsAlreadyExists(err), nil
 		}); err != nil {
 			return err
@@ -1129,7 +1149,7 @@ func (p *namespacePreparer) cleanup() error {
 	var errRet error
 	for i := 0; i < p.count; i++ {
 		n := fmt.Sprintf("%s-%d", p.prefix, i)
-		if err := p.client.CoreV1().Namespaces().Delete(context.TODO(), n, metav1.DeleteOptions{}); err != nil {
+		if err := p.client.CoreV1().Namespaces().Delete(context.Background(), n, metav1.DeleteOptions{}); err != nil {
 			klog.Errorf("Deleting Namespace: %v", err)
 			errRet = err
 		}

test/integration/util/util.go

@@ -120,6 +120,7 @@ func StartFakePVController(clientSet clientset.Interface) ShutdownFunc {
 	pvInformer := informerFactory.Core().V1().PersistentVolumes()
 
 	syncPV := func(obj *v1.PersistentVolume) {
+		ctx := context.Background()
 		if obj.Spec.ClaimRef != nil {
 			claimRef := obj.Spec.ClaimRef
 			pvc, err := clientSet.CoreV1().PersistentVolumeClaims(claimRef.Namespace).Get(ctx, claimRef.Name, metav1.GetOptions{})

test/utils/create_resources.go

@@ -59,6 +59,9 @@ func CreatePodWithRetries(c clientset.Interface, namespace string, obj *v1.Pod)
 	}
 	createFunc := func() (bool, error) {
 		_, err := c.CoreV1().Pods(namespace).Create(context.TODO(), obj, metav1.CreateOptions{})
+		if isGenerateNameConflict(obj.ObjectMeta, err) {
+			return false, nil
+		}
 		if err == nil || apierrors.IsAlreadyExists(err) {
 			return true, nil
 		}
@@ -73,6 +76,9 @@ func CreateRCWithRetries(c clientset.Interface, namespace string, obj *v1.Replic
 	}
 	createFunc := func() (bool, error) {
 		_, err := c.CoreV1().ReplicationControllers(namespace).Create(context.TODO(), obj, metav1.CreateOptions{})
+		if isGenerateNameConflict(obj.ObjectMeta, err) {
+			return false, nil
+		}
 		if err == nil || apierrors.IsAlreadyExists(err) {
 			return true, nil
 		}
@@ -87,6 +93,9 @@ func CreateReplicaSetWithRetries(c clientset.Interface, namespace string, obj *a
 	}
 	createFunc := func() (bool, error) {
 		_, err := c.AppsV1().ReplicaSets(namespace).Create(context.TODO(), obj, metav1.CreateOptions{})
+		if isGenerateNameConflict(obj.ObjectMeta, err) {
+			return false, nil
+		}
 		if err == nil || apierrors.IsAlreadyExists(err) {
 			return true, nil
 		}
@@ -101,6 +110,9 @@ func CreateDeploymentWithRetries(c clientset.Interface, namespace string, obj *a
 	}
 	createFunc := func() (bool, error) {
 		_, err := c.AppsV1().Deployments(namespace).Create(context.TODO(), obj, metav1.CreateOptions{})
+		if isGenerateNameConflict(obj.ObjectMeta, err) {
+			return false, nil
+		}
 		if err == nil || apierrors.IsAlreadyExists(err) {
 			return true, nil
 		}
@@ -115,6 +127,9 @@ func CreateDaemonSetWithRetries(c clientset.Interface, namespace string, obj *ap
 	}
 	createFunc := func() (bool, error) {
 		_, err := c.AppsV1().DaemonSets(namespace).Create(context.TODO(), obj, metav1.CreateOptions{})
+		if isGenerateNameConflict(obj.ObjectMeta, err) {
+			return false, nil
+		}
 		if err == nil || apierrors.IsAlreadyExists(err) {
 			return true, nil
 		}
@@ -129,6 +144,9 @@ func CreateJobWithRetries(c clientset.Interface, namespace string, obj *batch.Jo
 	}
 	createFunc := func() (bool, error) {
 		_, err := c.BatchV1().Jobs(namespace).Create(context.TODO(), obj, metav1.CreateOptions{})
+		if isGenerateNameConflict(obj.ObjectMeta, err) {
+			return false, nil
+		}
 		if err == nil || apierrors.IsAlreadyExists(err) {
 			return true, nil
 		}
@@ -143,6 +161,9 @@ func CreateSecretWithRetries(c clientset.Interface, namespace string, obj *v1.Se
 	}
 	createFunc := func() (bool, error) {
 		_, err := c.CoreV1().Secrets(namespace).Create(context.TODO(), obj, metav1.CreateOptions{})
+		if isGenerateNameConflict(obj.ObjectMeta, err) {
+			return false, nil
+		}
 		if err == nil || apierrors.IsAlreadyExists(err) {
 			return true, nil
 		}
@@ -157,6 +178,9 @@ func CreateConfigMapWithRetries(c clientset.Interface, namespace string, obj *v1
 	}
 	createFunc := func() (bool, error) {
 		_, err := c.CoreV1().ConfigMaps(namespace).Create(context.TODO(), obj, metav1.CreateOptions{})
+		if isGenerateNameConflict(obj.ObjectMeta, err) {
+			return false, nil
+		}
 		if err == nil || apierrors.IsAlreadyExists(err) {
 			return true, nil
 		}
@@ -171,6 +195,9 @@ func CreateServiceWithRetries(c clientset.Interface, namespace string, obj *v1.S
 	}
 	createFunc := func() (bool, error) {
 		_, err := c.CoreV1().Services(namespace).Create(context.TODO(), obj, metav1.CreateOptions{})
+		if isGenerateNameConflict(obj.ObjectMeta, err) {
+			return false, nil
+		}
 		if err == nil || apierrors.IsAlreadyExists(err) {
 			return true, nil
 		}
@@ -185,6 +212,9 @@ func CreateStorageClassWithRetries(c clientset.Interface, obj *storage.StorageCl
 	}
 	createFunc := func() (bool, error) {
 		_, err := c.StorageV1().StorageClasses().Create(context.TODO(), obj, metav1.CreateOptions{})
+		if isGenerateNameConflict(obj.ObjectMeta, err) {
+			return false, nil
+		}
 		if err == nil || apierrors.IsAlreadyExists(err) {
 			return true, nil
 		}
@@ -199,6 +229,9 @@ func CreateResourceQuotaWithRetries(c clientset.Interface, namespace string, obj
 	}
 	createFunc := func() (bool, error) {
 		_, err := c.CoreV1().ResourceQuotas(namespace).Create(context.TODO(), obj, metav1.CreateOptions{})
+		if isGenerateNameConflict(obj.ObjectMeta, err) {
+			return false, nil
+		}
 		if err == nil || apierrors.IsAlreadyExists(err) {
 			return true, nil
 		}
@@ -213,6 +246,9 @@ func CreatePersistentVolumeWithRetries(c clientset.Interface, obj *v1.Persistent
 	}
 	createFunc := func() (bool, error) {
 		_, err := c.CoreV1().PersistentVolumes().Create(context.TODO(), obj, metav1.CreateOptions{})
+		if isGenerateNameConflict(obj.ObjectMeta, err) {
+			return false, nil
+		}
 		if err == nil || apierrors.IsAlreadyExists(err) {
 			return true, nil
 		}
@@ -227,6 +263,9 @@ func CreatePersistentVolumeClaimWithRetries(c clientset.Interface, namespace str
 	}
 	createFunc := func() (bool, error) {
 		_, err := c.CoreV1().PersistentVolumeClaims(namespace).Create(context.TODO(), obj, metav1.CreateOptions{})
+		if isGenerateNameConflict(obj.ObjectMeta, err) {
+			return false, nil
+		}
 		if err == nil || apierrors.IsAlreadyExists(err) {
 			return true, nil
 		}
@@ -234,3 +273,11 @@ func CreatePersistentVolumeClaimWithRetries(c clientset.Interface, namespace str
 	}
 	return RetryWithExponentialBackOff(createFunc)
 }
+
+// isGenerateNameConflict returns whether the error is generateName conflict or not.
+func isGenerateNameConflict(meta metav1.ObjectMeta, err error) bool {
+	if apierrors.IsAlreadyExists(err) && meta.Name == "" {
+		return true
+	}
+	return false
+}