Merge pull request #41160 from pipejakob/webhooksigner-pr

Automatic merge from submit-queue (batch tested with PRs 42058, 41160, 42065, 42076, 39338)

New command for stand-alone GKE certificates controller

New stand-alone certificates controller for GKE. Rather than requiring the CA's private key on disk, this allows making external calls to GKE in order to sign cluster certificates.
    
**Which issue this PR fixes**: fixes #39761

**Release note**:

```release-note
New GKE certificates controller.
```

CC @mikedanese @jcbsmpsn

cmd/BUILD

@@ -22,6 +22,7 @@ filegroup(
         "//cmd/genswaggertypedocs:all-srcs",
         "//cmd/genutils:all-srcs",
         "//cmd/genyaml:all-srcs",
+        "//cmd/gke-certificates-controller:all-srcs",
         "//cmd/hyperkube:all-srcs",
         "//cmd/kube-apiserver:all-srcs",
         "//cmd/kube-controller-manager:all-srcs",

cmd/gke-certificates-controller/BUILD

@@ -0,0 +1,44 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    tags = ["automanaged"],
+    deps = [
+        "//cmd/gke-certificates-controller/app:go_default_library",
+        "//pkg/util/logs:go_default_library",
+        "//pkg/version/verflag:go_default_library",
+        "//vendor:github.com/spf13/pflag",
+        "//vendor:k8s.io/apiserver/pkg/util/flag",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//cmd/gke-certificates-controller/app:all-srcs",
+    ],
+    tags = ["automanaged"],
+)
+
+go_binary(
+    name = "gke-certificates-controller",
+    library = ":go_default_library",
+    tags = ["automanaged"],
+)

cmd/gke-certificates-controller/OWNERS

@@ -0,0 +1,8 @@
+approvers:
+- pipejakob
+- mikedanese
+- roberthbailey
+reviewers:
+- pipejakob
+- mikedanese
+- roberthbailey

cmd/gke-certificates-controller/app/BUILD

@@ -0,0 +1,60 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "gke_certificates_controller.go",
+        "gke_signer.go",
+        "options.go",
+    ],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/apis/certificates/install:go_default_library",
+        "//pkg/apis/certificates/v1beta1:go_default_library",
+        "//pkg/client/clientset_generated/clientset:go_default_library",
+        "//pkg/client/informers/informers_generated/externalversions:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//pkg/controller/certificates:go_default_library",
+        "//vendor:github.com/golang/glog",
+        "//vendor:github.com/spf13/cobra",
+        "//vendor:github.com/spf13/pflag",
+        "//vendor:k8s.io/apimachinery/pkg/apis/meta/v1",
+        "//vendor:k8s.io/apimachinery/pkg/runtime/schema",
+        "//vendor:k8s.io/apiserver/pkg/util/webhook",
+        "//vendor:k8s.io/client-go/kubernetes/typed/core/v1",
+        "//vendor:k8s.io/client-go/plugin/pkg/client/auth",
+        "//vendor:k8s.io/client-go/rest",
+        "//vendor:k8s.io/client-go/tools/clientcmd",
+        "//vendor:k8s.io/client-go/tools/record",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["gke_signer_test.go"],
+    library = ":go_default_library",
+    tags = ["automanaged"],
+    deps = ["//pkg/apis/certificates/v1beta1:go_default_library"],
+)

cmd/gke-certificates-controller/app/gke_certificates_controller.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package app implements a server that runs a stand-alone version of the
+// certificates controller for GKE clusters.
+package app
+
+import (
+	"time"
+
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
+	informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/externalversions"
+	"k8s.io/kubernetes/pkg/controller"
+	"k8s.io/kubernetes/pkg/controller/certificates"
+
+	// Install all auth plugins
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+
+	"github.com/golang/glog"
+	"github.com/spf13/cobra"
+)
+
+// NewGKECertificatesControllerCommand creates a new *cobra.Command with default parameters.
+func NewGKECertificatesControllerCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use: "gke-certificates-controller",
+		Long: `The Kubernetes GKE certificates controller is a daemon that
+handles auto-approving and signing certificates for GKE clusters.`,
+	}
+
+	return cmd
+}
+
+// Run runs the GKECertificatesController. This should never exit.
+func Run(s *GKECertificatesController) error {
+	kubeconfig, err := clientcmd.BuildConfigFromFlags("", s.Kubeconfig)
+	if err != nil {
+		return err
+	}
+
+	kubeClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, "gke-certificates-controller"))
+	if err != nil {
+		return err
+	}
+
+	eventBroadcaster := record.NewBroadcaster()
+	eventBroadcaster.StartLogging(glog.Infof)
+	eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.Core().RESTClient()).Events("")})
+
+	clientBuilder := controller.SimpleControllerClientBuilder{ClientConfig: kubeconfig}
+	client := clientBuilder.ClientOrDie("certificate-controller")
+
+	sharedInformers := informers.NewSharedInformerFactory(client, time.Duration(12)*time.Hour)
+
+	signer, err := NewGKESigner(s.ClusterSigningGKEKubeconfig, s.ClusterSigningGKERetryBackoff.Duration)
+	if err != nil {
+		return err
+	}
+
+	controller, err := certificates.NewCertificateController(
+		client,
+		sharedInformers.Certificates().V1beta1().CertificateSigningRequests(),
+		signer,
+		certificates.NewGroupApprover(s.ApproveAllKubeletCSRsForGroup),
+	)
+	if err != nil {
+		return err
+	}
+
+	sharedInformers.Start(nil)
+	controller.Run(1, nil) // runs forever
+	return nil
+}

cmd/gke-certificates-controller/app/gke_signer.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/golang/glog"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/util/webhook"
+	"k8s.io/client-go/rest"
+	"k8s.io/kubernetes/pkg/api"
+	_ "k8s.io/kubernetes/pkg/apis/certificates/install"
+	certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1"
+)
+
+var (
+	groupVersions = []schema.GroupVersion{certificates.SchemeGroupVersion}
+)
+
+// GKESigner uses external calls to GKE in order to sign certificate signing
+// requests.
+type GKESigner struct {
+	webhook        *webhook.GenericWebhook
+	kubeConfigFile string
+	retryBackoff   time.Duration
+}
+
+// NewGKESigner will create a new instance of a GKESigner.
+func NewGKESigner(kubeConfigFile string, retryBackoff time.Duration) (*GKESigner, error) {
+	webhook, err := webhook.NewGenericWebhook(api.Registry, api.Codecs, kubeConfigFile, groupVersions, retryBackoff)
+	if err != nil {
+		return nil, err
+	}
+
+	return &GKESigner{
+		webhook:        webhook,
+		kubeConfigFile: kubeConfigFile,
+		retryBackoff:   retryBackoff,
+	}, nil
+}
+
+// Sign will make an external call to GKE order to sign the given
+// *certificates.CertificateSigningRequest, using the GKESigner's
+// kubeConfigFile.
+func (s *GKESigner) Sign(csr *certificates.CertificateSigningRequest) (*certificates.CertificateSigningRequest, error) {
+	result := s.webhook.WithExponentialBackoff(func() rest.Result {
+		return s.webhook.RestClient.Post().Body(csr).Do()
+	})
+
+	if err := result.Error(); err != nil {
+		return nil, s.webhookError(csr, err)
+	}
+
+	var statusCode int
+	if result.StatusCode(&statusCode); statusCode < 200 || statusCode >= 300 {
+		return nil, s.webhookError(csr, fmt.Errorf("received unsuccessful response code from webhook: %d", statusCode))
+	}
+
+	result_csr := &certificates.CertificateSigningRequest{}
+
+	if err := result.Into(result_csr); err != nil {
+		return nil, s.webhookError(result_csr, err)
+	}
+
+	// Keep the original CSR intact, and only update fields we expect to change.
+	csr.Status.Certificate = result_csr.Status.Certificate
+	return csr, nil
+}
+
+func (s *GKESigner) webhookError(csr *certificates.CertificateSigningRequest, err error) error {
+	glog.V(2).Infof("error contacting webhook backend: %s", err)
+	return err
+}

cmd/gke-certificates-controller/app/gke_signer_test.go

@@ -0,0 +1,153 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"text/template"
+	"time"
+
+	certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1"
+)
+
+const kubeConfigTmpl = `
+clusters:
+- cluster:
+    server: {{ .Server }}
+    name: testcluster
+users:
+- user:
+    username: admin
+    password: mypass
+`
+
+func TestGKESigner(t *testing.T) {
+	goodResponse := &certificates.CertificateSigningRequest{
+		Status: certificates.CertificateSigningRequestStatus{
+			Certificate: []byte("fake certificate"),
+		},
+	}
+
+	invalidResponse := "{ \"status\": \"Not a properly formatted CSR response\" }"
+
+	cases := []struct {
+		mockResponse interface{}
+		expected     []byte
+		failCalls    int
+		wantErr      bool
+	}{
+		{
+			mockResponse: goodResponse,
+			expected:     goodResponse.Status.Certificate,
+			wantErr:      false,
+		},
+		{
+			mockResponse: goodResponse,
+			expected:     goodResponse.Status.Certificate,
+			failCalls:    3,
+			wantErr:      false,
+		},
+		{
+			mockResponse: goodResponse,
+			failCalls:    20,
+			wantErr:      true,
+		},
+		{
+			mockResponse: invalidResponse,
+			wantErr:      true,
+		},
+	}
+
+	for _, c := range cases {
+		server, err := newTestServer(c.mockResponse, c.failCalls)
+		if err != nil {
+			t.Fatalf("error creating test server")
+		}
+
+		kubeConfig, err := ioutil.TempFile("", "kubeconfig")
+		if err != nil {
+			t.Fatalf("error creating kubeconfig tempfile: %v", err)
+		}
+
+		tmpl, err := template.New("kubeconfig").Parse(kubeConfigTmpl)
+		if err != nil {
+			t.Fatalf("error creating kubeconfig template: %v", err)
+		}
+
+		data := struct{ Server string }{server.httpserver.URL}
+
+		if err := tmpl.Execute(kubeConfig, data); err != nil {
+			t.Fatalf("error executing kubeconfig template: %v", err)
+		}
+
+		if err := kubeConfig.Close(); err != nil {
+			t.Fatalf("error closing kubeconfig template: %v", err)
+		}
+
+		signer, err := NewGKESigner(kubeConfig.Name(), time.Duration(500)*time.Millisecond)
+		if err != nil {
+			t.Fatalf("error creating GKESigner: %v", err)
+		}
+
+		cert, err := signer.Sign(&certificates.CertificateSigningRequest{})
+
+		if c.wantErr {
+			if err == nil {
+				t.Errorf("wanted error during GKE.Sign() call, got not none")
+			}
+		} else {
+			if err != nil {
+				t.Errorf("error while signing: %v", err)
+			}
+
+			if !bytes.Equal(cert.Status.Certificate, c.expected) {
+				t.Errorf("response certificate didn't match expected %v: %v", c.expected, cert)
+			}
+		}
+	}
+}
+
+type testServer struct {
+	httpserver *httptest.Server
+	failCalls  int
+	response   interface{}
+}
+
+func newTestServer(response interface{}, failCalls int) (*testServer, error) {
+	server := &testServer{
+		response:  response,
+		failCalls: failCalls,
+	}
+
+	server.httpserver = httptest.NewServer(server)
+	return server, nil
+}
+
+func (s *testServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if s.failCalls > 0 {
+		http.Error(w, "Service unavailable", 500)
+		s.failCalls--
+	} else {
+		w.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(w).Encode(s.response)
+	}
+}

cmd/gke-certificates-controller/app/options.go

@@ -0,0 +1,54 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package app implements a server that runs a stand-alone version of the
+// certificates controller.
+package app
+
+import (
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/spf13/pflag"
+)
+
+// GKECertificatesController is the main context object for the package.
+type GKECertificatesController struct {
+	Kubeconfig                    string
+	ClusterSigningGKEKubeconfig   string
+	ClusterSigningGKERetryBackoff metav1.Duration
+	ApproveAllKubeletCSRsForGroup string
+}
+
+// Create a new instance of a GKECertificatesController with default parameters.
+func NewGKECertificatesController() *GKECertificatesController {
+	s := &GKECertificatesController{
+		ClusterSigningGKERetryBackoff: metav1.Duration{Duration: 500 * time.Millisecond},
+	}
+	return s
+}
+
+// AddFlags adds flags for a specific GKECertificatesController to the
+// specified FlagSet.
+func (s *GKECertificatesController) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
+
+	fs.StringVar(&s.ClusterSigningGKEKubeconfig, "cluster-signing-gke-kubeconfig", s.ClusterSigningGKEKubeconfig, "If set, use the kubeconfig file to call GKE to sign cluster-scoped certificates instead of using a local private key.")
+	fs.DurationVar(&s.ClusterSigningGKERetryBackoff.Duration, "cluster-signing-gke-retry-backoff", s.ClusterSigningGKERetryBackoff.Duration, "The initial backoff to use when retrying requests to GKE. Additional attempts will use exponential backoff.")
+
+	fs.StringVar(&s.ApproveAllKubeletCSRsForGroup, "insecure-experimental-approve-all-kubelet-csrs-for-group", s.ApproveAllKubeletCSRsForGroup, "The group for which the controller-manager will auto approve all CSRs for kubelet client certificates.")
+}

cmd/gke-certificates-controller/main.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// The GKE certificates controller is responsible for monitoring certificate
+// signing requests and (potentially) auto-approving and signing them within
+// GKE.
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"k8s.io/apiserver/pkg/util/flag"
+	"k8s.io/kubernetes/cmd/gke-certificates-controller/app"
+	"k8s.io/kubernetes/pkg/util/logs"
+	"k8s.io/kubernetes/pkg/version/verflag"
+
+	"github.com/spf13/pflag"
+)
+
+// TODO(pipejakob): Move this entire cmd directory into its own repo
+func main() {
+	s := app.NewGKECertificatesController()
+	s.AddFlags(pflag.CommandLine)
+
+	flag.InitFlags()
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	verflag.PrintAndExitIfRequested()
+
+	if err := app.Run(s); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}

hack/.linted_packages

@@ -8,6 +8,7 @@ cmd/genkubedocs
 cmd/genman
 cmd/genswaggertypedocs
 cmd/genyaml
+cmd/gke-certificates-controller
 cmd/kube-apiserver
 cmd/kube-apiserver/app
 cmd/kube-apiserver/app/options

hack/lib/golang.sh

@@ -19,8 +19,6 @@ readonly KUBE_GO_PACKAGE=k8s.io/kubernetes
 readonly KUBE_GOPATH="${KUBE_OUTPUT}/go"
 
 # The set of server targets that we are only building for Linux
-# Note: if you are adding something here, you might need to add it to
-# kube::build::source_targets in build/common.sh as well.
 # If you update this list, please also update build/release-tars/BUILD.
 kube::golang::server_targets() {
   local targets=(
@@ -174,11 +172,15 @@ readonly KUBE_TEST_SERVER_PLATFORMS=("${KUBE_SERVER_PLATFORMS[@]}")
 # laptops-versus-not.
 readonly KUBE_PARALLEL_BUILD_MEMORY=11
 
+# TODO(pipejakob) gke-certificates-controller is included here to exercise its
+# compilation, but it doesn't need to be distributed in any of our tars. Its
+# code is only living in this repo temporarily until it finds a new home.
 readonly KUBE_ALL_TARGETS=(
   "${KUBE_SERVER_TARGETS[@]}"
   "${KUBE_CLIENT_TARGETS[@]}"
   "${KUBE_TEST_TARGETS[@]}"
   "${KUBE_TEST_SERVER_TARGETS[@]}"
+  cmd/gke-certificates-controller
 )
 readonly KUBE_ALL_BINARIES=("${KUBE_ALL_TARGETS[@]##*/}")
 

hack/verify-flags/known-flags.txt

@@ -89,6 +89,8 @@ cluster-ip
 cluster-monitor-period
 cluster-name
 cluster-signing-cert-file
+cluster-signing-gke-kubeconfig
+cluster-signing-gke-retry-backoff
 cluster-signing-key-file
 cluster-tag
 cni-bin-dir