Merge pull request #40919 from shashidharatd/fed-e2e-1

Automatic merge from submit-queue (batch tested with PRs 40175, 41107, 41111, 40893, 40919)

[Federation][e2e] Move Cluster Registration to federation-up.sh

**What this PR does / why we need it**:
Remove cluster register/unregister calls from test case BeforeEach/AfterEach blocks.
Register clusters once in federation-up.sh


**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #40768

**Special notes for your reviewer**:

**Release note**: `NONE`

cc: @madhusudancs @kubernetes/sig-federation-pr-reviews

federation/cluster/federation-up.sh

@@ -85,23 +85,26 @@ function init() {
       --image="${kube_registry}/hyperkube-amd64:${kube_version}"
 }
 
-# create_cluster_secrets creates the secrets containing the kubeconfigs
-# of the participating clusters in the host cluster. The kubeconfigs itself
-# are created while deploying clusters, i.e. when kube-up is run.
-function create_cluster_secrets() {
-  local -r kubeconfig_dir="$(dirname ${DEFAULT_KUBECONFIG})"
-  local -r base_dir="${kubeconfig_dir}/federation/kubernetes-apiserver"
+# join_cluster_to_federation joins the clusters in the local kubeconfig to federation. The clusters
+# and their kubeconfig entries in the local kubeconfig are created while deploying clusters, i.e. when kube-up is run.
+function join_cluster_to_federation() {
+  for cluster in $("${KUBE_ROOT}/cluster/kubectl.sh" config get-clusters |sed -n '1!p'); do
+    # Skip federation context
+    if [[ "${cluster}" == "${FEDERATION_NAME}" ]]; then
+      continue
+    fi
+    # Skip contexts not beginning with "federation"
+    if [[ "${cluster}" != federation* ]]; then
+      continue
+    fi
 
-  # Create secrets with all the kubernetes-apiserver's kubeconfigs.
-  for dir in $(ls "${base_dir}"); do
-    # We create a secret with the same name as the directory name (which is
-    # same as cluster name in kubeconfig).
-    # Massage the name so that it is valid (should not contain "_" and max 253
-    # chars)
-    name=$(echo "${dir}" | sed -e "s/_/-/g")  # Replace "_" by "-"
-    name=${name:0:252}
-    kube::log::status "Creating secret with name: ${name} in namespace ${FEDERATION_NAMESPACE}"
-    "${KUBE_ROOT}/cluster/kubectl.sh" create secret generic ${name} --from-file="${base_dir}/${dir}/kubeconfig" --namespace="${FEDERATION_NAMESPACE}"
+  kube::log::status "Joining cluster with name '${cluster}' to federation with name '${FEDERATION_NAME}'"
+
+  "${KUBE_ROOT}/federation/develop/kubefed.sh" join \
+      "${cluster}" \
+      --host-cluster-context="${HOST_CLUSTER_CONTEXT}" \
+      --context="${FEDERATION_NAME}" \
+      --secret-name="${cluster//_/-}"    # Replace "_" by "-"
   done
 }
 
@@ -109,11 +112,8 @@ USE_KUBEFED="${USE_KUBEFED:-}"
 
 if [[ "${USE_KUBEFED}" == "true" ]]; then
   init
-  # TODO(madhusudancs): Call to create_cluster_secrets and the function
-  # itself must be removed after implementing cluster join with kubefed
-  # here. This call is now required for the cluster joins in the
-  # BeforeEach blocks of each e2e test to work.
-  create_cluster_secrets
+
+  join_cluster_to_federation
 else
   export FEDERATION_IMAGE_TAG="$(get_version)"
   create-federation-api-objects

test/e2e_federation/federated-daemonset.go

@@ -51,8 +51,7 @@ var _ = framework.KubeDescribe("Federation daemonsets [Feature:Federation]", fun
 
 		BeforeEach(func() {
 			fedframework.SkipUnlessFederated(f.ClientSet)
-			clusters = map[string]*cluster{}
-			registerClusters(clusters, UserAgentName, "", f)
+			clusters, _ = getRegisteredClusters(UserAgentName, f)
 		})
 
 		AfterEach(func() {
@@ -60,7 +59,6 @@ var _ = framework.KubeDescribe("Federation daemonsets [Feature:Federation]", fun
 			// Delete all daemonsets.
 			nsName := f.FederationNamespace.Name
 			deleteAllDaemonSetsOrFail(f.FederationClientset, nsName)
-			unregisterClusters(clusters, f)
 		})
 
 		It("should be created and deleted successfully", func() {

test/e2e_federation/federated-deployment.go

@@ -18,7 +18,6 @@ package e2e_federation
 
 import (
 	"fmt"
-	"os"
 	"strings"
 	"time"
 
@@ -71,22 +70,16 @@ var _ = framework.KubeDescribe("Federation deployments [Feature:Federation]", fu
 	// e2e cases for federated deployment controller
 	Describe("Federated Deployment", func() {
 		var (
-			clusters       map[string]*cluster
-			federationName string
+			clusters map[string]*cluster
 		)
 		BeforeEach(func() {
 			fedframework.SkipUnlessFederated(f.ClientSet)
-			if federationName = os.Getenv("FEDERATION_NAME"); federationName == "" {
-				federationName = DefaultFederationName
-			}
-			clusters = map[string]*cluster{}
-			registerClusters(clusters, UserAgentName, federationName, f)
+			clusters, _ = getRegisteredClusters(UserAgentName, f)
 		})
 
 		AfterEach(func() {
 			nsName := f.FederationNamespace.Name
 			deleteAllDeploymentsOrFail(f.FederationClientset, nsName)
-			unregisterClusters(clusters, f)
 		})
 
 		It("should create and update matching deployments in underlying clusters", func() {

test/e2e_federation/federated-ingress.go

@@ -94,8 +94,7 @@ var _ = framework.KubeDescribe("Federated ingresses [Feature:Federation]", func(
 				federationName = DefaultFederationName
 			}
 			jig = newFederationTestJig(f.FederationClientset)
-			clusters = map[string]*cluster{}
-			primaryClusterName = registerClusters(clusters, UserAgentName, federationName, f)
+			clusters, primaryClusterName = getRegisteredClusters(UserAgentName, f)
 			ns = f.FederationNamespace.Name
 		})
 
@@ -103,7 +102,6 @@ var _ = framework.KubeDescribe("Federated ingresses [Feature:Federation]", func(
 			// Delete all ingresses.
 			nsName := f.FederationNamespace.Name
 			deleteAllIngressesOrFail(f.FederationClientset, nsName)
-			unregisterClusters(clusters, f)
 		})
 
 		It("should create and update matching ingresses in underlying clusters", func() {

test/e2e_federation/federated-namespace.go

@@ -18,7 +18,6 @@ package e2e_federation
 
 import (
 	"fmt"
-	"os"
 	"strings"
 	"time"
 
@@ -44,19 +43,11 @@ var _ = framework.KubeDescribe("Federation namespace [Feature:Federation]", func
 	f := fedframework.NewDefaultFederatedFramework("federation-namespace")
 
 	Describe("Namespace objects", func() {
-		var federationName string
 		var clusters map[string]*cluster // All clusters, keyed by cluster name
 
 		BeforeEach(func() {
 			fedframework.SkipUnlessFederated(f.ClientSet)
-
-			// TODO: Federation API server should be able to answer this.
-			if federationName = os.Getenv("FEDERATION_NAME"); federationName == "" {
-				federationName = DefaultFederationName
-			}
-
-			clusters = map[string]*cluster{}
-			registerClusters(clusters, UserAgentName, federationName, f)
+			clusters, _ = getRegisteredClusters(UserAgentName, f)
 		})
 
 		AfterEach(func() {
@@ -69,7 +60,6 @@ var _ = framework.KubeDescribe("Federation namespace [Feature:Federation]", func
 					cluster.Core().Namespaces().List,
 					cluster.Core().Namespaces().Delete)
 			}
-			unregisterClusters(clusters, f)
 		})
 
 		It("should be created and deleted successfully", func() {

test/e2e_federation/federated-replicaset.go

@@ -18,7 +18,6 @@ package e2e_federation
 
 import (
 	"fmt"
-	"os"
 	"strings"
 	"time"
 
@@ -73,23 +72,17 @@ var _ = framework.KubeDescribe("Federation replicasets [Feature:Federation]", fu
 	// e2e cases for federated replicaset controller
 	Describe("Federated ReplicaSet", func() {
 		var (
-			clusters       map[string]*cluster
-			federationName string
+			clusters map[string]*cluster
 		)
 		BeforeEach(func() {
 			fedframework.SkipUnlessFederated(f.ClientSet)
-			if federationName = os.Getenv("FEDERATION_NAME"); federationName == "" {
-				federationName = DefaultFederationName
-			}
-			clusters = map[string]*cluster{}
-			registerClusters(clusters, UserAgentName, federationName, f)
+			clusters, _ = getRegisteredClusters(UserAgentName, f)
 		})
 
 		AfterEach(func() {
 			// Delete all replicasets.
 			nsName := f.FederationNamespace.Name
 			deleteAllReplicaSetsOrFail(f.FederationClientset, nsName)
-			unregisterClusters(clusters, f)
 		})
 
 		It("should create and update matching replicasets in underling clusters", func() {

test/e2e_federation/federated-secret.go

@@ -50,8 +50,7 @@ var _ = framework.KubeDescribe("Federation secrets [Feature:Federation]", func()
 
 		BeforeEach(func() {
 			fedframework.SkipUnlessFederated(f.ClientSet)
-			clusters = map[string]*cluster{}
-			registerClusters(clusters, UserAgentName, "", f)
+			clusters, _ = getRegisteredClusters(UserAgentName, f)
 		})
 
 		AfterEach(func() {
@@ -59,8 +58,6 @@ var _ = framework.KubeDescribe("Federation secrets [Feature:Federation]", func()
 			// Delete all secrets.
 			nsName := f.FederationNamespace.Name
 			deleteAllSecretsOrFail(f.FederationClientset, nsName)
-			unregisterClusters(clusters, f)
-
 		})
 
 		It("should be created and deleted successfully", func() {

test/e2e_federation/federated-service.go

@@ -89,12 +89,7 @@ var _ = framework.KubeDescribe("Federated Services [Feature:Federation]", func()
 				federationName = DefaultFederationName
 			}
 
-			clusters = map[string]*cluster{}
-			primaryClusterName = registerClusters(clusters, UserAgentName, federationName, f)
-		})
-
-		AfterEach(func() {
-			unregisterClusters(clusters, f)
+			clusters, primaryClusterName = getRegisteredClusters(UserAgentName, f)
 		})
 
 		Describe("service creation", func() {

test/e2e_federation/federation-util.go

@@ -100,9 +100,9 @@ func clusterIsReadyOrFail(f *fedframework.Framework, context *fedframework.E2ECo
 	framework.Logf("Cluster %s is Ready", context.Name)
 }
 
-// waitForAllClustersReady wait for all clusters defined in e2e context to be created
+// waitForAllRegisteredClusters waits for all clusters defined in e2e context to be created
 // return ClusterList until the listed cluster items equals clusterCount
-func waitForAllClustersReady(f *fedframework.Framework, clusterCount int) *federationapi.ClusterList {
+func waitForAllRegisteredClusters(f *fedframework.Framework, clusterCount int) *federationapi.ClusterList {
 	var clusterList *federationapi.ClusterList
 	if err := wait.PollImmediate(framework.Poll, FederatedServiceTimeout, func() (bool, error) {
 		var err error
@@ -187,15 +187,12 @@ func unregisterClusters(clusters map[string]*cluster, f *fedframework.Framework)
 }
 
 // can not be moved to util, as By and Expect must be put in Ginkgo test unit
-func registerClusters(clusters map[string]*cluster, userAgentName, federationName string, f *fedframework.Framework) string {
+func getRegisteredClusters(userAgentName string, f *fedframework.Framework) (map[string]*cluster, string) {
+	clusters := make(map[string]*cluster)
 	contexts := f.GetUnderlyingFederatedContexts()
 
-	for _, context := range contexts {
-		createClusterObjectOrFail(f, &context)
-	}
-
 	By("Obtaining a list of all the clusters")
-	clusterList := waitForAllClustersReady(f, len(contexts))
+	clusterList := waitForAllRegisteredClusters(f, len(contexts))
 
 	framework.Logf("Checking that %d clusters are Ready", len(contexts))
 	for _, context := range contexts {
@@ -211,7 +208,7 @@ func registerClusters(clusters map[string]*cluster, userAgentName, federationNam
 		clusters[c.Name] = &cluster{c.Name, createClientsetForCluster(c, i, userAgentName), false, nil}
 	}
 	createNamespaceInClusters(clusters, f)
-	return primaryClusterName
+	return clusters, primaryClusterName
 }
 
 /*

test/e2e_federation/framework/framework.go

@@ -140,9 +140,6 @@ func (f *Framework) FederationAfterEach() {
 			framework.Logf("Warning: framework is marked federated, but has no federation 1.5 clientset")
 			return
 		}
-		if err := f.FederationClientset.Federation().Clusters().DeleteCollection(nil, metav1.ListOptions{}); err != nil {
-			framework.Logf("Error: failed to delete Clusters: %+v", err)
-		}
 	}()
 
 	// Print events if the test failed.