Merge pull request #37835 from luxas/kubeadm_cloudprovider_and_dns

Automatic merge from submit-queue (batch tested with PRs 38294, 37009, 36778, 38130, 37835)

Add a cloudprovider validator flag to kubeadm and update the DNS spec

Broken out from: https://github.com/kubernetes/kubernetes/pull/37568

This PR creates a flag for `cloud-provider` that validates the value before `RunInit()` is run, which makes it now act as a "real" flag
Then it removes the `k8s.io/kubernetes/pkg/cloudprovider` dependency, which makes the binary ~40MB smaller! That's _really_ worth it!

In the second commit, the DNS addon is updated to the latest version: https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/skydns-rc.yaml.base

@kubernetes/sig-cluster-lifecycle

cmd/kubeadm/app/cmd/BUILD

@@ -24,13 +24,12 @@ go_library(
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//cmd/kubeadm/app/apis/kubeadm/v1alpha1:go_default_library",
+        "//cmd/kubeadm/app/cmd/flags:go_default_library",
         "//cmd/kubeadm/app/master:go_default_library",
         "//cmd/kubeadm/app/node:go_default_library",
         "//cmd/kubeadm/app/preflight:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
         "//pkg/api:go_default_library",
-        "//pkg/cloudprovider:go_default_library",
-        "//pkg/cloudprovider/providers:go_default_library",
         "//pkg/kubectl/cmd/util:go_default_library",
         "//pkg/runtime:go_default_library",
         "//pkg/util/flag:go_default_library",

cmd/kubeadm/app/cmd/flags/BUILD

@@ -0,0 +1,18 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+    "go_test",
+    "cgo_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["cloudprovider.go"],
+    tags = ["automanaged"],
+    deps = ["//vendor:github.com/spf13/pflag"],
+)

cmd/kubeadm/app/cmd/flags/cloudprovider.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flags
+
+import (
+	"fmt"
+
+	"github.com/spf13/pflag"
+)
+
+var cloudproviders = []string{
+	"aws",
+	"azure",
+	"cloudstack",
+	"gce",
+	"mesos",
+	"openstack",
+	"ovirt",
+	"photon",
+	"rackspace",
+	"vsphere",
+}
+
+func NewCloudProviderFlag(provider *string) pflag.Value {
+	return &cloudProviderValue{provider: provider}
+}
+
+type cloudProviderValue struct {
+	provider *string
+}
+
+func (c *cloudProviderValue) String() string {
+	return *c.provider
+}
+
+func (c *cloudProviderValue) Set(s string) error {
+	if ValidateCloudProvider(s) {
+		*c.provider = s
+		return nil
+	}
+
+	return fmt.Errorf("cloud provider %q is not supported, you can use any of %v", s, cloudproviders)
+}
+
+func (c *cloudProviderValue) Type() string {
+	return "cloudprovider"
+}
+
+func ValidateCloudProvider(provider string) bool {
+	for _, supported := range cloudproviders {
+		if provider == supported {
+			return true
+		}
+	}
+	return false
+}

cmd/kubeadm/app/cmd/init.go

@@ -28,12 +28,11 @@ import (
 
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/flags"
 	kubemaster "k8s.io/kubernetes/cmd/kubeadm/app/master"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/pkg/api"
-	"k8s.io/kubernetes/pkg/cloudprovider"
-	_ "k8s.io/kubernetes/pkg/cloudprovider/providers"
 	"k8s.io/kubernetes/pkg/runtime"
 	netutil "k8s.io/kubernetes/pkg/util/net"
 )
@@ -103,9 +102,9 @@ func NewCmdInit(out io.Writer) *cobra.Command {
 		&cfg.Networking.DNSDomain, "service-dns-domain", cfg.Networking.DNSDomain,
 		`Use alternative domain for services, e.g. "myorg.internal"`,
 	)
-	cmd.PersistentFlags().StringVar(
+	cmd.PersistentFlags().Var(
-		&cfg.CloudProvider, "cloud-provider", cfg.CloudProvider,
+		flags.NewCloudProviderFlag(&cfg.CloudProvider), "cloud-provider",
-		`Enable cloud provider features (external load-balancers, storage, etc), e.g. "gce"`,
+		`Enable cloud provider features (external load-balancers, storage, etc). Note that you have to configure all kubelets manually`,
 	)
 
 	cmd.PersistentFlags().StringVar(
@@ -205,14 +204,12 @@ func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight
 	cfg.KubernetesVersion = ver
 	fmt.Println("Using Kubernetes version:", ver)
 
-	// TODO(phase1+) create a custom flag
+	// Warn about the limitations with the current cloudprovider solution.
 	if cfg.CloudProvider != "" {
-		if cloudprovider.IsCloudProvider(cfg.CloudProvider) {
+		fmt.Println("WARNING: For cloudprovider integrations to work --cloud-provider must be set for all kubelets in the cluster.")
-			fmt.Printf("cloud provider %q initialized for the control plane. Remember to set the same cloud provider flag on the kubelet.\n", cfg.CloudProvider)
+		fmt.Println("\t(/etc/systemd/system/kubelet.service.d/10-kubeadm.conf should be edited for this purpose)")
-		} else {
-			return nil, fmt.Errorf("cloud provider %q is not supported, you can use any of %v, or leave it unset.\n", cfg.CloudProvider, cloudprovider.CloudProviders())
-		}
 	}
+
 	return &Init{cfg: cfg}, nil
 }
 

cmd/kubeadm/app/images/images.go

@@ -31,17 +31,19 @@ const (
 	KubeSchedulerImage         = "scheduler"
 	KubeProxyImage             = "proxy"
 
-	KubeDNSImage         = "kube-dns"
+	KubeDNSImage            = "kubedns"
-	KubeDNSmasqImage     = "dnsmasq"
+	KubeDNSmasqImage        = "kube-dnsmasq"
+	KubeDNSmasqMetricsImage = "dnsmasq-metrics"
 	KubeExechealthzImage    = "exechealthz"
 	Pause                   = "pause"
 
 	gcrPrefix   = "gcr.io/google_containers"
 	etcdVersion = "3.0.14-kubeadm"
 
-	kubeDNSVersion     = "1.7"
+	kubeDNSVersion        = "1.9"
-	dnsmasqVersion     = "1.3"
+	dnsmasqVersion        = "1.4"
-	exechealthzVersion = "1.1"
+	exechealthzVersion    = "1.2"
+	dnsmasqMetricsVersion = "1.0"
 	pauseVersion          = "3.0"
 )
 
@@ -64,6 +66,7 @@ func GetAddonImage(image string) string {
 	return map[string]string{
 		KubeDNSImage:            fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "kubedns", runtime.GOARCH, kubeDNSVersion),
 		KubeDNSmasqImage:        fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "kube-dnsmasq", runtime.GOARCH, dnsmasqVersion),
+		KubeDNSmasqMetricsImage: fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "dnsmasq-metrics", runtime.GOARCH, dnsmasqMetricsVersion),
 		KubeExechealthzImage:    fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "exechealthz", runtime.GOARCH, exechealthzVersion),
 		Pause:                   fmt.Sprintf("%s/%s-%s:%s", repoPrefix, "pause", runtime.GOARCH, pauseVersion),
 	}[image]

cmd/kubeadm/app/master/addons.go

@@ -27,7 +27,7 @@ import (
 	"k8s.io/kubernetes/pkg/api/resource"
 	"k8s.io/kubernetes/pkg/api/v1"
 	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5"
-	ipallocator "k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
+	"k8s.io/kubernetes/pkg/registry/core/service/ipallocator"
 	"k8s.io/kubernetes/pkg/util/intstr"
 )
 
@@ -83,26 +83,9 @@ func createKubeProxyPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec {
 
 func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec {
 
-	dnsPodResources := v1.ResourceList{
-		v1.ResourceName(v1.ResourceCPU):    resource.MustParse("100m"),
-		v1.ResourceName(v1.ResourceMemory): resource.MustParse("170Mi"),
-	}
-
-	healthzPodResources := v1.ResourceList{
-		v1.ResourceName(v1.ResourceCPU):    resource.MustParse("10m"),
-		v1.ResourceName(v1.ResourceMemory): resource.MustParse("50Mi"),
-	}
-
 	kubeDNSPort := int32(10053)
 	dnsmasqPort := int32(53)
 
-	nslookup := fmt.Sprintf("nslookup kubernetes.default.svc.%s 127.0.0.1", cfg.Networking.DNSDomain)
-
-	nslookup = fmt.Sprintf("-cmd=%s:%d >/dev/null && %s:%d >/dev/null",
-		nslookup, dnsmasqPort,
-		nslookup, kubeDNSPort,
-	)
-
 	return v1.PodSpec{
 		Containers: []v1.Container{
 			// DNS server
@@ -110,18 +93,18 @@ func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec {
 				Name:  "kube-dns",
 				Image: images.GetAddonImage(images.KubeDNSImage),
 				Resources: v1.ResourceRequirements{
-					Limits:   dnsPodResources,
+					Limits: v1.ResourceList{
-					Requests: dnsPodResources,
+						v1.ResourceName(v1.ResourceMemory): resource.MustParse("170Mi"),
+					},
+					Requests: v1.ResourceList{
+						v1.ResourceName(v1.ResourceCPU):    resource.MustParse("100m"),
+						v1.ResourceName(v1.ResourceMemory): resource.MustParse("70Mi"),
 					},
-				Args: []string{
-					fmt.Sprintf("--domain=%s", cfg.Networking.DNSDomain),
-					fmt.Sprintf("--dns-port=%d", kubeDNSPort),
-					// TODO __PILLAR__FEDERATIONS__DOMAIN__MAP__
 				},
 				LivenessProbe: &v1.Probe{
 					Handler: v1.Handler{
 						HTTPGet: &v1.HTTPGetAction{
-							Path:   "/healthz",
+							Path:   "/healthz-kubedns",
 							Port:   intstr.FromInt(8080),
 							Scheme: v1.URISchemeHTTP,
 						},
@@ -129,7 +112,7 @@ func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec {
 					InitialDelaySeconds: 60,
 					TimeoutSeconds:      5,
 					SuccessThreshold:    1,
-					FailureThreshold:    1,
+					FailureThreshold:    5,
 				},
 				// # we poll on pod startup for the Kubernetes master service and
 				// # only setup the /readiness HTTP server once that's available.
@@ -141,9 +124,20 @@ func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec {
 							Scheme: v1.URISchemeHTTP,
 						},
 					},
-					InitialDelaySeconds: 30,
+					InitialDelaySeconds: 3,
 					TimeoutSeconds:      5,
 				},
+				Args: []string{
+					fmt.Sprintf("--domain=%s", cfg.Networking.DNSDomain),
+					fmt.Sprintf("--dns-port=%d", kubeDNSPort),
+					"--config-map=kube-dns",
+				},
+				Env: []v1.EnvVar{
+					{
+						Name:  "PROMETHEUS_PORT",
+						Value: "10055",
+					},
+				},
 				Ports: []v1.ContainerPort{
 					{
 						ContainerPort: kubeDNSPort,
@@ -155,20 +149,35 @@ func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec {
 						Name:          "dns-tcp-local",
 						Protocol:      v1.ProtocolTCP,
 					},
+					{
+						ContainerPort: 10055,
+						Name:          "metrics",
+						Protocol:      v1.ProtocolTCP,
+					},
 				},
 			},
 			// dnsmasq
 			{
 				Name:  "dnsmasq",
 				Image: images.GetAddonImage(images.KubeDNSmasqImage),
-				Resources: v1.ResourceRequirements{
+				LivenessProbe: &v1.Probe{
-					Limits:   dnsPodResources,
+					Handler: v1.Handler{
-					Requests: dnsPodResources,
+						HTTPGet: &v1.HTTPGetAction{
+							Path:   "/healthz-dnsmasq",
+							Port:   intstr.FromInt(8080),
+							Scheme: v1.URISchemeHTTP,
+						},
+					},
+					InitialDelaySeconds: 60,
+					TimeoutSeconds:      5,
+					SuccessThreshold:    1,
+					FailureThreshold:    5,
 				},
 				Args: []string{
 					"--cache-size=1000",
 					"--no-resolv",
 					fmt.Sprintf("--server=127.0.0.1#%d", kubeDNSPort),
+					"--log-facility=-",
 				},
 				Ports: []v1.ContainerPort{
 					{
@@ -182,19 +191,66 @@ func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec {
 						Protocol:      v1.ProtocolTCP,
 					},
 				},
+				Resources: v1.ResourceRequirements{
+					Requests: v1.ResourceList{
+						v1.ResourceName(v1.ResourceCPU):    resource.MustParse("150m"),
+						v1.ResourceName(v1.ResourceMemory): resource.MustParse("10Mi"),
+					},
+				},
+			},
+			{
+				Name:  "dnsmasq-metrics",
+				Image: images.GetAddonImage(images.KubeDNSmasqMetricsImage),
+				LivenessProbe: &v1.Probe{
+					Handler: v1.Handler{
+						HTTPGet: &v1.HTTPGetAction{
+							Path:   "/metrics",
+							Port:   intstr.FromInt(10054),
+							Scheme: v1.URISchemeHTTP,
+						},
+					},
+					InitialDelaySeconds: 60,
+					TimeoutSeconds:      5,
+					SuccessThreshold:    1,
+					FailureThreshold:    5,
+				},
+				Args: []string{
+					"--v=2",
+					"--logtostderr",
+				},
+				Ports: []v1.ContainerPort{
+					{
+						ContainerPort: 10054,
+						Name:          "metrics",
+						Protocol:      v1.ProtocolTCP,
+					},
+				},
+				Resources: v1.ResourceRequirements{
+					Requests: v1.ResourceList{
+						v1.ResourceName(v1.ResourceMemory): resource.MustParse("10Mi"),
+					},
+				},
 			},
 			// healthz
 			{
 				Name:  "healthz",
 				Image: images.GetAddonImage(images.KubeExechealthzImage),
 				Resources: v1.ResourceRequirements{
-					Limits:   healthzPodResources,
+					Limits: v1.ResourceList{
-					Requests: healthzPodResources,
+						v1.ResourceName(v1.ResourceMemory): resource.MustParse("50Mi"),
+					},
+					Requests: v1.ResourceList{
+						v1.ResourceName(v1.ResourceCPU):    resource.MustParse("10m"),
+						v1.ResourceName(v1.ResourceMemory): resource.MustParse("50Mi"),
+					},
 				},
 				Args: []string{
-					nslookup,
+					fmt.Sprintf("--cmd=nslookup kubernetes.default.svc.%s 127.0.0.1 >/dev/null", cfg.Networking.DNSDomain),
-					"-port=8080",
+					"--url=/healthz-dnsmasq",
-					"-quiet",
+					fmt.Sprintf("--cmd=nslookup kubernetes.default.svc.%s 127.0.0.1:%d >/dev/null", cfg.Networking.DNSDomain, kubeDNSPort),
+					"--url=/healthz-kubedns",
+					"--port=8080",
+					"--quiet",
 				},
 				Ports: []v1.ContainerPort{{
 					ContainerPort: 8080,
@@ -204,7 +260,6 @@ func createKubeDNSPodSpec(cfg *kubeadmapi.MasterConfiguration) v1.PodSpec {
 		},
 		DNSPolicy: v1.DNSDefault,
 	}
-
 }
 
 func createKubeDNSServiceSpec(cfg *kubeadmapi.MasterConfiguration) (*v1.ServiceSpec, error) {
@@ -217,16 +272,14 @@ func createKubeDNSServiceSpec(cfg *kubeadmapi.MasterConfiguration) (*v1.ServiceS
 		return nil, fmt.Errorf("unable to allocate IP address for kube-dns addon from the given CIDR (%q) [%v]", cfg.Networking.ServiceSubnet, err)
 	}
 
-	svc := &v1.ServiceSpec{
+	return &v1.ServiceSpec{
 		Selector: map[string]string{"name": "kube-dns"},
 		Ports: []v1.ServicePort{
 			{Name: "dns", Port: 53, Protocol: v1.ProtocolUDP},
 			{Name: "dns-tcp", Port: 53, Protocol: v1.ProtocolTCP},
 		},
 		ClusterIP: ip.String(),
-	}
+	}, nil
-
-	return svc, nil
 }
 
 func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error {
@@ -250,10 +303,11 @@ func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientse
 
 	kubeDNSServiceSpec, err := createKubeDNSServiceSpec(cfg)
 	if err != nil {
-		return fmt.Errorf("<master/addons> failed creating essential kube-dns addon - %v", err)
+		return fmt.Errorf("failed creating essential kube-dns addon [%v]", err)
 	}
 
 	kubeDNSService := NewService("kube-dns", *kubeDNSServiceSpec)
+	kubeDNSService.ObjectMeta.Labels["kubernetes.io/name"] = "KubeDNS"
 	if _, err := client.Services(api.NamespaceSystem).Create(kubeDNSService); err != nil {
 		return fmt.Errorf("<master/addons> failed creating essential kube-dns addon [%v]", err)
 	}

cmd/kubeadm/app/master/manifests.go

@@ -34,10 +34,9 @@ import (
 )
 
 // Static pod definitions in golang form are included below so that `kubeadm init` can get going.
-
 const (
 	DefaultClusterName     = "kubernetes"
-	DefaultCloudConfigPath = "/etc/kubernetes/cloud-config.json"
+	DefaultCloudConfigPath = "/etc/kubernetes/cloud-config"
 
 	etcd                  = "etcd"
 	apiServer             = "apiserver"
@@ -312,6 +311,15 @@ func getAPIServerCommand(cfg *kubeadmapi.MasterConfiguration) (command []string)
 		command = append(command, etcdClientFileArg, etcdKeyFileArg)
 	}
 
+	if cfg.CloudProvider != "" {
+		command = append(command, "--cloud-provider="+cfg.CloudProvider)
+
+		// Only append the --cloud-config option if there's a such file
+		if _, err := os.Stat(DefaultCloudConfigPath); err == nil {
+			command = append(command, "--cloud-config="+DefaultCloudConfigPath)
+		}
+	}
+
 	return
 }