Basic E2E tests for kubeadm

Run with `go run test/e2e_kubeadm/runner/local/run_local.go -build -test-flags="--kubeconfig=$KUBECONFIG"`

build/BUILD

@@ -148,7 +148,6 @@ filegroup(
         "//cmd/kubemark",  # TODO: server platforms only
         "//cmd/linkcheck",
         "//test/e2e:e2e.test",
-        "//test/e2e_kubeadm:e2e_kubeadm.test",  # TODO: server platforms only
         "//test/e2e_node:e2e_node.test",  # TODO: server platforms only
         "//vendor/github.com/onsi/ginkgo/ginkgo",
     ],

hack/.golint_failures

@@ -1,3 +1,4 @@
+
 cluster/images/etcd-version-monitor
 cmd/hyperkube
 cmd/kube-apiserver/app
@@ -699,6 +700,7 @@ test/e2e/ui
 test/e2e/upgrades
 test/e2e/upgrades/apps
 test/e2e/upgrades/storage
+test/e2e_kubeadm
 test/e2e_node
 test/e2e_node/builder
 test/e2e_node/environment

test/e2e_kubeadm/BUILD

@@ -1,3 +1,5 @@
+package(default_visibility = ["//visibility:public"])
+
 load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_test(
@@ -7,6 +9,7 @@ go_test(
         "kubeadm_test.go",
     ],
     embed = [":go_default_library"],
+    tags = ["e2e"],
     deps = [
         "//test/e2e/framework:go_default_library",
         "//vendor/github.com/onsi/ginkgo:go_default_library",

test/e2e_kubeadm/e2e_kubeadm_suite_test.go

@@ -0,0 +1,48 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_kubeadm
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+	"github.com/spf13/pflag"
+
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+func init() {
+	framework.RegisterCommonFlags()
+	framework.RegisterClusterFlags()
+
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+}
+
+func TestMain(m *testing.M) {
+	pflag.Parse()
+	framework.AfterReadingAllFlags(&framework.TestContext)
+	os.Exit(m.Run())
+}
+
+func TestE2E(t *testing.T) {
+	reporters := []ginkgo.Reporter{}
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "E2EKubadm suite", reporters)
+}

test/e2e_kubeadm/kubeadm_test.go

@@ -0,0 +1,136 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_kubeadm
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	bootstrapapi "k8s.io/client-go/tools/bootstrap/token/api"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	masterTaint                  = "node-role.kubernetes.io/master"
+	kubeadmConfigNamespace       = "kube-system"
+	kubeadmConfigName            = "kubeadm-config"
+	clusterInfoNamespace         = "kube-public"
+	clusterInfoName              = "cluster-info"
+	bootstrapSignerRoleNamespace = "kube-system"
+	bootstrapSignerRoleName      = "system:controller:bootstrap-signer"
+)
+
+var _ = framework.KubeDescribe("Kubeadm [Feature:Kubeadm]", func() {
+	f := framework.NewDefaultFramework("kubeadm")
+
+	Describe("kubeadm master", func() {
+		It("should be labelled and tainted", func() {
+			selector := labels.Set{masterTaint: ""}.AsSelector()
+			master, err := f.ClientSet.CoreV1().Nodes().
+				List(metav1.ListOptions{LabelSelector: selector.String()})
+			framework.ExpectNoError(err, "couldn't find a master node")
+			Expect(master.Items).NotTo(BeEmpty())
+			for _, master := range master.Items {
+				Expect(master.Spec.Taints).To(
+					ContainElement(taint(masterTaint, corev1.TaintEffectNoSchedule)),
+				)
+			}
+		})
+	})
+
+	Describe("kubeadm-config config map", func() {
+		It("should exist", func() {
+			_, err := f.ClientSet.CoreV1().
+				ConfigMaps(kubeadmConfigNamespace).
+				Get(kubeadmConfigName, metav1.GetOptions{})
+			framework.ExpectNoError(err)
+		})
+	})
+
+	Describe("cluster-info", func() {
+		It("should have expected keys", func() {
+			clientInfo, err := f.ClientSet.CoreV1().
+				ConfigMaps(clusterInfoNamespace).
+				Get(clusterInfoName, metav1.GetOptions{})
+			framework.ExpectNoError(err, "couldn't find config map")
+
+			Expect(clientInfo.Data).To(HaveKey(HavePrefix(bootstrapapi.JWSSignatureKeyPrefix)))
+			Expect(clientInfo.Data).To(HaveKey(bootstrapapi.KubeConfigKey))
+		})
+
+		It("should be public", func() {
+			cfg, err := framework.LoadConfig()
+			framework.ExpectNoError(err, "couldn't get config")
+			cfg = rest.AnonymousClientConfig(cfg)
+			client, err := kubernetes.NewForConfig(cfg)
+			framework.ExpectNoError(err, "couldn't create client")
+
+			_, err = client.CoreV1().ConfigMaps(clusterInfoNamespace).
+				Get(clusterInfoName, metav1.GetOptions{})
+			framework.ExpectNoError(err, "couldn't anonymously access config")
+		})
+	})
+
+	Describe("bootstrap signer RBAC role", func() {
+		It("should exist", func() {
+			_, err := f.ClientSet.RbacV1().
+				Roles(bootstrapSignerRoleNamespace).
+				Get(bootstrapSignerRoleName, metav1.GetOptions{})
+			framework.ExpectNoError(err, "doesn't exist")
+		})
+	})
+
+	Describe("kubeadm:kubelet-bootstrap cluster role binding", func() {
+		It("should exist", func() {
+			binding, err := f.ClientSet.RbacV1().
+				ClusterRoleBindings().
+				Get("kubeadm:kubelet-bootstrap", metav1.GetOptions{})
+			framework.ExpectNoError(err, "couldn't get clusterrolebinding")
+			Expect(binding.Subjects).To(
+				ContainElement(subject(
+					"system:bootstrappers:kubeadm:default-node-token",
+					rbacv1.GroupKind,
+				)),
+			)
+			Expect(binding.RoleRef.Name).To(Equal("system:node-bootstrapper"))
+		})
+	})
+
+	Describe("autoapproval for new bootstrap token", func() {
+		It("should create a clusterrolebinding", func() {
+			binding, err := f.ClientSet.RbacV1().
+				ClusterRoleBindings().
+				Get("kubeadm:node-autoapprove-bootstrap", metav1.GetOptions{})
+			framework.ExpectNoError(err, "couldn't get clusterrolebinding")
+			Expect(binding.Subjects).To(
+				ContainElement(subject(
+					"system:bootstrappers:kubeadm:default-node-token",
+					rbacv1.GroupKind,
+				)),
+			)
+			Expect(binding.RoleRef.Name).To(
+				Equal("system:certificates.k8s.io:certificatesigningrequests:nodeclient"),
+			)
+		})
+	})
+})

test/e2e_kubeadm/matchers.go

@@ -0,0 +1,37 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_kubeadm
+
+import (
+	"github.com/onsi/gomega"
+	"github.com/onsi/gomega/gstruct"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func subject(name, kind string) gomega.OmegaMatcher {
+	return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+		"Name": gomega.Equal(name),
+		"Kind": gomega.Equal(kind),
+	})
+}
+
+func taint(key string, effect corev1.TaintEffect) gomega.OmegaMatcher {
+	return gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
+		"Key":    gomega.Equal(key),
+		"Effect": gomega.Equal(effect),
+	})
+}

test/e2e_kubeadm/runner/local/run_local.go

@@ -0,0 +1,106 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/test/utils"
+)
+
+func bazelBuild() error {
+	targets := []string{
+		"//vendor/github.com/onsi/ginkgo/ginkgo",
+		"//test/e2e_kubeadm:e2e_kubeadm.test",
+	}
+
+	args := append([]string{"build"}, targets...)
+
+	return execCommand("bazel", args...)
+}
+
+var ginkgoFlags = flag.String("ginkgo-flags", "", "Space-separated list of arguments to pass to Ginkgo test runner.")
+var testFlags = flag.String("test-flags", "", "Space-separated list of arguments to pass to node e2e test.")
+var build = flag.Bool("build", false, "use Bazel to build binaries before testing")
+
+func main() {
+	flag.Parse()
+
+	if *build {
+		if err := bazelBuild(); err != nil {
+			glog.Exitf("couldn't build with bazel: %v", err)
+		}
+	}
+
+	ginkgo, err := getBazelGinkgo()
+	if err != nil {
+		glog.Fatalf("Failed to get ginkgo binary: %v", err)
+	}
+
+	test, err := getBazelTestBin()
+	if err != nil {
+		glog.Fatalf("Failed to get test file: %v", err)
+	}
+
+	args := append(strings.Split(*ginkgoFlags, " "), test, "--")
+	args = append(args, strings.Split(*testFlags, " ")...)
+
+	if execCommand(ginkgo, args...); err != nil {
+		glog.Exitf("Test failed: %v", err)
+	}
+
+}
+
+func getBazelTestBin() (string, error) {
+	k8sRoot, err := utils.GetK8sRootDir()
+	if err != nil {
+		return "", err
+	}
+	buildFile := filepath.Join(k8sRoot, "bazel-bin/test/e2e_kubeadm/e2e_kubeadm.test")
+	if _, err := os.Stat(buildFile); err != nil {
+		return "", err
+	}
+	return buildFile, nil
+
+}
+
+func getBazelGinkgo() (string, error) {
+	k8sRoot, err := utils.GetK8sRootDir()
+	if err != nil {
+		return "", err
+	}
+	buildOutputDir := filepath.Join(k8sRoot, "bazel-bin", "vendor/github.com/onsi/ginkgo/ginkgo", fmt.Sprintf("%s_%s_stripped", runtime.GOOS, runtime.GOARCH), "ginkgo")
+	if _, err := os.Stat(buildOutputDir); err != nil {
+		return "", err
+	}
+	return buildOutputDir, nil
+}
+
+func execCommand(binary string, args ...string) error {
+	fmt.Printf("Running command: %v %v\n", binary, strings.Join(args, " "))
+	cmd := exec.Command(binary, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}