Merge pull request #26325 from quinton-hoole/2016-05-25-federation-cluster-status-zone-update

Automatic merge from submit-queue

Add federation cluster status zone update

Hi Nikhil

Here's the cluster zone update stuff we discussed today.

What's remaining:

1. Rebase against #26308 (as soon as it merges) to expose the zones and region fields of cluster status.
2. Get hold of a cluster client via which to query nodes in the cluster. Currently we only have a DiscoveryClient, which doesn't know how to list nodes.

Any advice on the latter much appreciated.  Specifically how do I get a cluster client from a discovery client?

Q

federation/pkg/federation-controller/cluster/cluster_client.go

@@ -22,15 +22,19 @@ import (
 	"os"
 	"strings"
 
+	"github.com/golang/glog"
 	federation_v1alpha1 "k8s.io/kubernetes/federation/apis/federation/v1alpha1"
+	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/api/unversioned"
 	"k8s.io/kubernetes/pkg/api/v1"
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	"k8s.io/kubernetes/pkg/client/restclient"
 	"k8s.io/kubernetes/pkg/client/typed/discovery"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
 	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
 	clientcmdapi "k8s.io/kubernetes/pkg/client/unversioned/clientcmd/api"
 	utilnet "k8s.io/kubernetes/pkg/util/net"
+	"k8s.io/kubernetes/pkg/util/sets"
 )
 
 const (
@@ -68,6 +72,7 @@ var KubeconfigGetterForCluster = func(c *federation_v1alpha1.Cluster) clientcmd.
 
 type ClusterClient struct {
 	discoveryClient *discovery.DiscoveryClient
+	kubeClient      *clientset.Clientset
 }
 
 func NewClusterClientSet(c *federation_v1alpha1.Cluster) (*ClusterClient, error) {
@@ -101,6 +106,10 @@ func NewClusterClientSet(c *federation_v1alpha1.Cluster) (*ClusterClient, error)
 		if clusterClientSet.discoveryClient == nil {
 			return nil, nil
 		}
+		clusterClientSet.kubeClient = clientset.NewForConfigOrDie((restclient.AddUserAgent(clusterConfig, UserAgentName)))
+		if clusterClientSet.kubeClient == nil {
+			return nil, nil
+		}
 	}
 	return &clusterClientSet, err
 }
@@ -153,3 +162,59 @@ func (self *ClusterClient) GetClusterHealthStatus() *federation_v1alpha1.Cluster
 	}
 	return &clusterStatus
 }
+
+// GetClusterZones gets the kubernetes cluster zones and region by inspecting labels on nodes in the cluster.
+func (self *ClusterClient) GetClusterZones() (zones []string, region string, err error) {
+	return getZoneNames(self.kubeClient)
+}
+
+// Find the name of the zone in which a Node is running
+func getZoneNameForNode(node api.Node) (string, error) {
+	for key, value := range node.Labels {
+		if key == unversioned.LabelZoneFailureDomain {
+			return value, nil
+		}
+	}
+	return "", fmt.Errorf("Zone name for node %s not found. No label with key %s",
+		node.Name, unversioned.LabelZoneFailureDomain)
+}
+
+// Find the name of the region in which a Node is running
+func getRegionNameForNode(node api.Node) (string, error) {
+	for key, value := range node.Labels {
+		if key == unversioned.LabelZoneRegion {
+			return value, nil
+		}
+	}
+	return "", fmt.Errorf("Region name for node %s not found. No label with key %s",
+		node.Name, unversioned.LabelZoneRegion)
+}
+
+// Find the names of all zones and the region in which we have nodes in this cluster.
+func getZoneNames(client *clientset.Clientset) (zones []string, region string, err error) {
+	zoneNames := sets.NewString()
+	nodes, err := client.Core().Nodes().List(api.ListOptions{})
+	if err != nil {
+		glog.Errorf("Failed to list nodes while getting zone names: %v", err)
+		return nil, "", err
+	}
+	for i, node := range nodes.Items {
+		// TODO: quinton-hoole make this more efficient.
+		//       For non-multi-zone clusters the zone will
+		//       be identical for all nodes, so we only need to look at one node
+		//       For multi-zone clusters we know at build time
+		//       which zones are included.  Rather get this info from there, because it's cheaper.
+		zoneName, err := getZoneNameForNode(node)
+		if err != nil {
+			return nil, "", err
+		}
+		zoneNames.Insert(zoneName)
+		if i == 0 {
+			region, err = getRegionNameForNode(node)
+			if err != nil {
+				return nil, "", err
+			}
+		}
+	}
+	return zoneNames.List(), region, nil
+}

federation/pkg/federation-controller/cluster/clustercontroller.go

@@ -184,11 +184,26 @@ func (cc *ClusterController) UpdateClusterStatus() error {
 				}
 			}
 		}
+		clusterClient, found := cc.clusterKubeClientMap[cluster.Name]
+		if !found {
+			glog.Warningf("Failed to client for cluster %s", cluster.Name)
+			continue
+		}
+
+		zones, region, err := clusterClient.GetClusterZones()
+		if err != nil {
+			glog.Warningf("Failed to get zones and region for cluster %s: %v", cluster.Name, err)
+			// Don't return err here, as we want the rest of the status update to proceed.
+		} else {
+			clusterStatusNew.Zones = zones
+			clusterStatusNew.Region = region
+		}
 		cc.clusterClusterStatusMap[cluster.Name] = *clusterStatusNew
 		cluster.Status = *clusterStatusNew
 		cluster, err := cc.federationClient.Federation().Clusters().UpdateStatus(&cluster)
 		if err != nil {
-			glog.Infof("Failed to update the status of cluster: %v ,error is : %v", cluster.Name, err)
+			glog.Warningf("Failed to update the status of cluster: %v ,error is : %v", cluster.Name, err)
+			// Don't return err here, as we want to continue processing remaining clusters.
 			continue
 		}
 	}