feat: kubeconfig and certificates rotation

cmd/manager/cmd.go

@@ -31,6 +31,7 @@ import (
 	"github.com/clastix/kamaji/internal/webhook/routes"
 )
 
+//nolint:maintidx
 func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 	// CLI flags
 	var (
@@ -105,7 +106,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 
-			tcpChannel := make(controllers.TenantControlPlaneChannel)
+			tcpChannel, certChannel := make(controllers.TenantControlPlaneChannel), make(controllers.CertificateChannel)
 
 			if err = (&controllers.DataStore{TenantControlPlaneTrigger: tcpChannel}).SetupWithManager(mgr); err != nil {
 				setupLog.Error(err, "unable to create controller", "controller", "DataStore")
@@ -122,6 +123,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 					KineContainerImage:   kineImage,
 					TmpBaseDirectory:     tmpDirectory,
 				},
+				CertificateChan:         certChannel,
 				TriggerChan:             tcpChannel,
 				KamajiNamespace:         managerNamespace,
 				KamajiServiceAccount:    managerServiceAccountName,
@@ -136,6 +138,12 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
 				return err
 			}
 
+			if err = (&controllers.CertificateLifecycle{Channel: certChannel}).SetupWithManager(mgr); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "CertificateLifecycle")
+
+				return err
+			}
+
 			if err = (&kamajiv1alpha1.DatastoreUsedSecret{}).SetupWithManager(ctx, mgr); err != nil {
 				setupLog.Error(err, "unable to create indexer", "indexer", "DatastoreUsedSecret")
 

controllers/cert_channel.go

@@ -0,0 +1,10 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"sigs.k8s.io/controller-runtime/pkg/event"
+)
+
+type CertificateChannel chan event.GenericEvent

controllers/secret_controller.go

@@ -0,0 +1,160 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controllers
+
+import (
+	"context"
+	"crypto/x509"
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/constants"
+	"github.com/clastix/kamaji/internal/crypto"
+	"github.com/clastix/kamaji/internal/utilities"
+)
+
+type CertificateLifecycle struct {
+	Channel CertificateChannel
+	client  client.Client
+}
+
+func (s *CertificateLifecycle) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
+	logger := log.FromContext(ctx)
+
+	logger.Info("starting CertificateLifecycle handling")
+
+	secret := corev1.Secret{}
+	if err := s.client.Get(ctx, request.NamespacedName, &secret); err != nil {
+		if k8serrors.IsNotFound(err) {
+			logger.Info("resource may have been deleted, skipping")
+
+			return reconcile.Result{}, nil
+		}
+	}
+
+	checkType, ok := secret.GetLabels()[constants.ControllerLabelResource]
+	if !ok {
+		logger.Info("missing controller label, shouldn't happen")
+
+		return reconcile.Result{}, nil
+	}
+
+	var crt *x509.Certificate
+	var err error
+
+	switch checkType {
+	case "x509":
+		crt, err = s.extractCertificateFromBareSecret(secret)
+	case "kubeconfig":
+		crt, err = s.extractCertificateFromKubeconfig(secret)
+	default:
+		err = fmt.Errorf("unsupported strategy, %s", checkType)
+	}
+
+	if err != nil {
+		logger.Error(err, "skipping reconciliation")
+
+		return reconcile.Result{}, nil
+	}
+
+	deadline := time.Now().AddDate(0, 0, 1)
+
+	if deadline.After(crt.NotAfter) {
+		logger.Info("certificate near expiration, must be rotated")
+
+		s.Channel <- event.GenericEvent{Object: &kamajiv1alpha1.TenantControlPlane{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      secret.GetOwnerReferences()[0].Name,
+				Namespace: secret.Namespace,
+			},
+		}}
+
+		logger.Info("certificate rotation triggered")
+
+		return reconcile.Result{}, nil
+	}
+
+	after := crt.NotAfter.Sub(deadline)
+
+	logger.Info("certificate is still valid, enqueuing back", "after", after.String())
+
+	return reconcile.Result{Requeue: true, RequeueAfter: after}, nil
+}
+
+func (s *CertificateLifecycle) extractCertificateFromBareSecret(secret corev1.Secret) (*x509.Certificate, error) {
+	var crt *x509.Certificate
+	var err error
+
+	for _, v := range secret.Data {
+		if crt, err = crypto.ParseCertificateBytes(v); err == nil {
+			break
+		}
+	}
+
+	if crt == nil {
+		return nil, fmt.Errorf("none of the provided keys is containing a valid x509 certificate")
+	}
+
+	return crt, nil
+}
+
+func (s *CertificateLifecycle) extractCertificateFromKubeconfig(secret corev1.Secret) (*x509.Certificate, error) {
+	var kc *clientcmdapiv1.Config
+	var err error
+
+	for k := range secret.Data {
+		if kc, err = utilities.DecodeKubeconfig(secret, k); err == nil {
+			break
+		}
+	}
+
+	if kc == nil {
+		return nil, fmt.Errorf("none of the provided keys is containing a valid kubeconfig")
+	}
+
+	crt, err := crypto.ParseCertificateBytes(kc.AuthInfos[0].AuthInfo.ClientCertificateData)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot parse kubeconfig certificate bytes")
+	}
+
+	return crt, nil
+}
+
+func (s *CertificateLifecycle) SetupWithManager(mgr controllerruntime.Manager) error {
+	s.client = mgr.GetClient()
+
+	supportedStrategies := sets.New[string]("x509", "kubeconfig")
+
+	return controllerruntime.NewControllerManagedBy(mgr).
+		For(&corev1.Secret{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
+			labels := object.GetLabels()
+
+			if labels == nil {
+				return false
+			}
+
+			value, ok := labels[constants.ControllerLabelResource]
+			if !ok {
+				return false
+			}
+
+			return supportedStrategies.Has(value)
+		}))).
+		Complete(s)
+}

controllers/tenantcontrolplane_controller.go

@@ -50,6 +50,10 @@ type TenantControlPlaneReconciler struct {
 	KamajiService           string
 	KamajiMigrateImage      string
 	MaxConcurrentReconciles int
+	// CertificateChan is the channel used by the CertificateLifecycleController that is checking for
+	// certificates and kubeconfig user certs validity: a generic event for the given TCP will be triggered
+	// once the validity threshold for the given certificate is reached.
+	CertificateChan CertificateChannel
 
 	clock mutex.Clock
 }
@@ -223,6 +227,14 @@ func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error
 	r.clock = clock.RealClock{}
 
 	return ctrl.NewControllerManagedBy(mgr).
+		Watches(&source.Channel{Source: r.CertificateChan}, handler.Funcs{GenericFunc: func(genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) {
+			limitingInterface.AddRateLimited(ctrl.Request{
+				NamespacedName: k8stypes.NamespacedName{
+					Namespace: genericEvent.Object.GetNamespace(),
+					Name:      genericEvent.Object.GetName(),
+				},
+			})
+		}}).
 		Watches(&source.Channel{Source: r.TriggerChan}, handler.Funcs{GenericFunc: func(genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) {
 			limitingInterface.AddRateLimited(ctrl.Request{
 				NamespacedName: k8stypes.NamespacedName{

internal/resources/api_server_certificate.go

@@ -103,6 +103,8 @@ func (r *APIServerCertificate) mutate(ctx context.Context, tenantControlPlane *k
 
 		if err := ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil {
 			logger.Error(err, "cannot set controller reference", "resource", r.GetName())
+
+			return err
 		}
 
 		if checksum := tenantControlPlane.Status.Certificates.APIServer.Checksum; len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0 {

internal/resources/api_server_kubelet_client_certificate.go

@@ -103,6 +103,8 @@ func (r *APIServerKubeletClientCertificate) mutate(ctx context.Context, tenantCo
 
 		if err := ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil {
 			logger.Error(err, "cannot set controller reference", "resource", r.GetName())
+
+			return err
 		}
 
 		if checksum := tenantControlPlane.Status.Certificates.APIServerKubeletClient.Checksum; len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0 {

internal/resources/datastore/datastore_certificate.go

@@ -16,6 +16,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/constants"
 	"github.com/clastix/kamaji/internal/crypto"
 	"github.com/clastix/kamaji/internal/utilities"
 )
@@ -88,6 +89,19 @@ func (r *Certificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1al
 
 		r.resource.Data["ca.crt"] = ca
 
+		r.resource.SetLabels(utilities.MergeMaps(
+			utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()),
+			map[string]string{
+				constants.ControllerLabelResource: "x509",
+			},
+		))
+
+		if err = ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil {
+			logger.Error(err, "cannot set controller reference", "resource", r.GetName())
+
+			return err
+		}
+
 		if utilities.GetObjectChecksum(r.resource) == utilities.CalculateMapChecksum(r.resource.Data) {
 			if r.DataStore.Spec.Driver == kamajiv1alpha1.EtcdDriver {
 				if isValid, _ := crypto.IsValidCertificateKeyPairBytes(r.resource.Data["server.crt"], r.resource.Data["server.key"]); isValid {
@@ -141,11 +155,6 @@ func (r *Certificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1al
 
 		utilities.SetObjectChecksum(r.resource, r.resource.Data)
 
-		r.resource.SetLabels(utilities.MergeMaps(
-			utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()),
-			r.resource.GetLabels(),
-		))
-
-		return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
+		return nil
 	}
 }

internal/resources/front-proxy-client-certificate.go

@@ -103,6 +103,8 @@ func (r *FrontProxyClientCertificate) mutate(ctx context.Context, tenantControlP
 
 		if err := ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil {
 			logger.Error(err, "cannot set controller reference", "resource", r.GetName())
+
+			return err
 		}
 
 		if checksum := tenantControlPlane.Status.Certificates.FrontProxyClient.Checksum; len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0 {

internal/resources/konnectivity/certificate_resource.go

@@ -99,6 +99,8 @@ func (r *CertificateResource) mutate(ctx context.Context, tenantControlPlane *ka
 
 		if err := ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil {
 			logger.Error(err, "cannot set controller reference", "resource", r.GetName())
+
+			return err
 		}
 
 		if checksum := tenantControlPlane.Status.Addons.Konnectivity.Certificate.Checksum; len(checksum) > 0 && checksum == utilities.CalculateMapChecksum(r.resource.Data) {

internal/utilities/kubeconfig.go

@@ -0,0 +1,29 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package utilities
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1"
+)
+
+func DecodeKubeconfig(secret corev1.Secret, key string) (*clientcmdapiv1.Config, error) {
+	bytes, ok := secret.Data[key]
+	if !ok {
+		return nil, fmt.Errorf("%s is not into kubeconfig secret", key)
+	}
+
+	return DecodeKubeconfigYAML(bytes)
+}
+
+func DecodeKubeconfigYAML(bytes []byte) (*clientcmdapiv1.Config, error) {
+	kubeconfig := &clientcmdapiv1.Config{}
+	if err := DecodeFromYAML(string(bytes), kubeconfig); err != nil {
+		return nil, err
+	}
+
+	return kubeconfig, nil
+}

internal/utilities/tenant_client.go

@@ -44,17 +44,7 @@ func GetTenantKubeconfig(ctx context.Context, client client.Client, tenantContro
 		return nil, err
 	}
 
-	bytes, ok := secretKubeconfig.Data[kubeadmconstants.AdminKubeConfigFileName]
-	if !ok {
-		return nil, fmt.Errorf("%s is not into kubeconfig secret", kubeadmconstants.AdminKubeConfigFileName)
-	}
-
-	kubeconfig := &clientcmdapiv1.Config{}
-	if err := DecodeFromYAML(string(bytes), kubeconfig); err != nil {
-		return nil, err
-	}
-
-	return kubeconfig, nil
+	return DecodeKubeconfig(*secretKubeconfig, kubeadmconstants.AdminKubeConfigFileName)
 }
 
 func GetRESTClientConfig(ctx context.Context, client client.Client, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*restclient.Config, error) {