Fix multinode storage e2e tests for multizone clusters

test/e2e/framework/pod/node_selection.go

@@ -48,6 +48,28 @@ func setNodeAffinityRequirement(nodeSelection *NodeSelection, operator v1.NodeSe
 		})
 }
 
+// SetNodeAffinityTopologyRequirement sets node affinity to a specified topology
+func SetNodeAffinityTopologyRequirement(nodeSelection *NodeSelection, topology map[string]string) {
+	if nodeSelection.Affinity == nil {
+		nodeSelection.Affinity = &v1.Affinity{}
+	}
+	if nodeSelection.Affinity.NodeAffinity == nil {
+		nodeSelection.Affinity.NodeAffinity = &v1.NodeAffinity{}
+	}
+	if nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
+		nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{}
+	}
+	for k, v := range topology {
+		nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms,
+			v1.NodeSelectorTerm{
+				MatchExpressions: []v1.NodeSelectorRequirement{
+					{Key: k, Operator: v1.NodeSelectorOpIn, Values: []string{v}},
+				},
+			})
+
+	}
+}
+
 // SetAffinity sets affinity to nodeName to nodeSelection
 func SetAffinity(nodeSelection *NodeSelection, nodeName string) {
 	setNodeAffinityRequirement(nodeSelection, v1.NodeSelectorOpIn, nodeName)

test/e2e/storage/drivers/in_tree.go

@@ -370,12 +370,14 @@ func InitISCSIDriver() testsuites.TestDriver {
 				//"ext3",
 				"ext4",
 			),
+			TopologyKeys: []string{v1.LabelHostname},
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence: true,
 				testsuites.CapFsGroup:     true,
 				testsuites.CapBlock:       true,
 				testsuites.CapExec:        true,
 				testsuites.CapMultiPODs:   true,
+				testsuites.CapTopology:    true,
 			},
 		},
 	}
@@ -772,10 +774,12 @@ func InitHostPathDriver() testsuites.TestDriver {
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 			),
+			TopologyKeys: []string{v1.LabelHostname},
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence:      true,
 				testsuites.CapMultiPODs:        true,
 				testsuites.CapSingleNodeVolume: true,
+				testsuites.CapTopology:         true,
 			},
 		},
 	}
@@ -845,10 +849,12 @@ func InitHostPathSymlinkDriver() testsuites.TestDriver {
 			SupportedFsType: sets.NewString(
 				"", // Default fsType
 			),
+			TopologyKeys: []string{v1.LabelHostname},
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence:      true,
 				testsuites.CapMultiPODs:        true,
 				testsuites.CapSingleNodeVolume: true,
+				testsuites.CapTopology:         true,
 			},
 		},
 	}
@@ -1059,6 +1065,7 @@ func InitCinderDriver() testsuites.TestDriver {
 				"", // Default fsType
 				"ext3",
 			),
+			TopologyKeys: []string{v1.LabelZoneFailureDomain},
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence: true,
 				testsuites.CapFsGroup:     true,
@@ -1067,6 +1074,7 @@ func InitCinderDriver() testsuites.TestDriver {
 				// Cinder supports volume limits, but the test creates large
 				// number of volumes and times out test suites.
 				testsuites.CapVolumeLimits: false,
+				testsuites.CapTopology:     true,
 			},
 		},
 	}
@@ -1366,11 +1374,13 @@ func InitVSphereDriver() testsuites.TestDriver {
 				"", // Default fsType
 				"ext4",
 			),
+			TopologyKeys: []string{v1.LabelZoneFailureDomain},
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence: true,
 				testsuites.CapFsGroup:     true,
 				testsuites.CapExec:        true,
 				testsuites.CapMultiPODs:   true,
+				testsuites.CapTopology:    true,
 			},
 		},
 	}
@@ -1490,6 +1500,7 @@ func InitAzureDiskDriver() testsuites.TestDriver {
 				"ext4",
 				"xfs",
 			),
+			TopologyKeys: []string{v1.LabelZoneFailureDomain},
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence: true,
 				testsuites.CapFsGroup:     true,
@@ -1499,6 +1510,7 @@ func InitAzureDiskDriver() testsuites.TestDriver {
 				// Azure supports volume limits, but the test creates large
 				// number of volumes and times out test suites.
 				testsuites.CapVolumeLimits: false,
+				testsuites.CapTopology:     true,
 			},
 		},
 	}
@@ -1621,6 +1633,7 @@ func InitAwsDriver() testsuites.TestDriver {
 				"ntfs",
 			),
 			SupportedMountOption: sets.NewString("debug", "nouid32"),
+			TopologyKeys:         []string{v1.LabelZoneFailureDomain},
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence:         true,
 				testsuites.CapFsGroup:             true,
@@ -1632,6 +1645,7 @@ func InitAwsDriver() testsuites.TestDriver {
 				// AWS supports volume limits, but the test creates large
 				// number of volumes and times out test suites.
 				testsuites.CapVolumeLimits: false,
+				testsuites.CapTopology:     true,
 			},
 		},
 	}

test/e2e/storage/testsuites/multivolume.go

@@ -180,6 +180,14 @@ func (t *multiVolumeTestSuite) DefineTests(driver TestDriver, pattern testpatter
 		if l.config.ClientNodeName != "" {
 			e2eskipper.Skipf("Driver %q requires to deploy on a specific node - skipping", l.driver.GetDriverInfo().Name)
 		}
+		// For multi-node tests there must be enough nodes with the same toopology to schedule the pods
+		nodeSelection := e2epod.NodeSelection{Name: l.config.ClientNodeName}
+		topologyKeys := dInfo.TopologyKeys
+		if len(topologyKeys) != 0 {
+			if err = ensureTopologyRequirements(&nodeSelection, nodes, l.cs, topologyKeys, 2); err != nil {
+				framework.Failf("Error setting topology requirements: %v", err)
+			}
+		}
 
 		var pvcs []*v1.PersistentVolumeClaim
 		numVols := 2
@@ -192,7 +200,7 @@ func (t *multiVolumeTestSuite) DefineTests(driver TestDriver, pattern testpatter
 		}
 
 		TestAccessMultipleVolumesAcrossPodRecreation(l.config.Framework, l.cs, l.ns.Name,
-			e2epod.NodeSelection{Name: l.config.ClientNodeName}, pvcs, false /* sameNode */)
+			nodeSelection, pvcs, false /* sameNode */)
 	})
 
 	// This tests below configuration (only <block, filesystem> pattern is tested):
@@ -266,6 +274,14 @@ func (t *multiVolumeTestSuite) DefineTests(driver TestDriver, pattern testpatter
 		if l.config.ClientNodeName != "" {
 			e2eskipper.Skipf("Driver %q requires to deploy on a specific node - skipping", l.driver.GetDriverInfo().Name)
 		}
+		// For multi-node tests there must be enough nodes with the same toopology to schedule the pods
+		nodeSelection := e2epod.NodeSelection{Name: l.config.ClientNodeName}
+		topologyKeys := dInfo.TopologyKeys
+		if len(topologyKeys) != 0 {
+			if err = ensureTopologyRequirements(&nodeSelection, nodes, l.cs, topologyKeys, 2); err != nil {
+				framework.Failf("Error setting topology requirements: %v", err)
+			}
+		}
 
 		var pvcs []*v1.PersistentVolumeClaim
 		numVols := 2
@@ -283,7 +299,7 @@ func (t *multiVolumeTestSuite) DefineTests(driver TestDriver, pattern testpatter
 		}
 
 		TestAccessMultipleVolumesAcrossPodRecreation(l.config.Framework, l.cs, l.ns.Name,
-			e2epod.NodeSelection{Name: l.config.ClientNodeName}, pvcs, false /* sameNode */)
+			nodeSelection, pvcs, false /* sameNode */)
 	})
 
 	// This tests below configuration:
@@ -335,6 +351,14 @@ func (t *multiVolumeTestSuite) DefineTests(driver TestDriver, pattern testpatter
 		if l.config.ClientNodeName != "" {
 			e2eskipper.Skipf("Driver %q requires to deploy on a specific node - skipping", l.driver.GetDriverInfo().Name)
 		}
+		// For multi-node tests there must be enough nodes with the same toopology to schedule the pods
+		nodeSelection := e2epod.NodeSelection{Name: l.config.ClientNodeName}
+		topologyKeys := dInfo.TopologyKeys
+		if len(topologyKeys) != 0 {
+			if err = ensureTopologyRequirements(&nodeSelection, nodes, l.cs, topologyKeys, 2); err != nil {
+				framework.Failf("Error setting topology requirements: %v", err)
+			}
+		}
 
 		// Create volume
 		testVolumeSizeRange := t.GetTestSuiteInfo().SupportedSizeRange
@@ -343,7 +367,7 @@ func (t *multiVolumeTestSuite) DefineTests(driver TestDriver, pattern testpatter
 
 		// Test access to the volume from pods on different node
 		TestConcurrentAccessToSingleVolume(l.config.Framework, l.cs, l.ns.Name,
-			e2epod.NodeSelection{Name: l.config.ClientNodeName}, resource.Pvc, numPods, false /* sameNode */)
+			nodeSelection, resource.Pvc, numPods, false /* sameNode */)
 	})
 }
 
@@ -504,3 +528,56 @@ func TestConcurrentAccessToSingleVolume(f *framework.Framework, cs clientset.Int
 		utils.CheckReadFromPath(f, pod, *pvc.Spec.VolumeMode, path, byteLen, seed)
 	}
 }
+
+// getCurrentTopologies() goes through all Nodes and returns unique driver topologies and count of Nodes per topology
+func getCurrentTopologiesNumber(cs clientset.Interface, nodes *v1.NodeList, keys []string) ([]topology, []int, error) {
+	topos := []topology{}
+	topoCount := []int{}
+
+	// TODO: scale?
+	for _, n := range nodes.Items {
+		topo := map[string]string{}
+		for _, k := range keys {
+			v, ok := n.Labels[k]
+			if ok {
+				topo[k] = v
+			}
+		}
+
+		found := false
+		for i, existingTopo := range topos {
+			if topologyEqual(existingTopo, topo) {
+				found = true
+				topoCount[i]++
+				break
+			}
+		}
+		if !found {
+			framework.Logf("found topology %v", topo)
+			topos = append(topos, topo)
+			topoCount = append(topoCount, 1)
+		}
+	}
+	return topos, topoCount, nil
+}
+
+// ensureTopologyRequirements sets nodeSelection affinity according to given topology keys for drivers that provide them
+func ensureTopologyRequirements(nodeSelection *e2epod.NodeSelection, nodes *v1.NodeList, cs clientset.Interface, topologyKeys []string, minCount int) error {
+	topologyList, topologyCount, err := getCurrentTopologiesNumber(cs, nodes, topologyKeys)
+	if err != nil {
+		return err
+	}
+	suitableTopologies := []topology{}
+	for i, topo := range topologyList {
+		if topologyCount[i] >= minCount {
+			suitableTopologies = append(suitableTopologies, topo)
+		}
+	}
+	if len(suitableTopologies) == 0 {
+		framework.Skipf("No topology with at least %d nodes found - skipping", minCount)
+	}
+	// Take the first suitable topology
+	e2epod.SetNodeAffinityTopologyRequirement(nodeSelection, suitableTopologies[0])
+
+	return nil
+}