fix: certificate authority rotation handling

controllers/soot/manager.go

@@ -123,22 +123,27 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
 	// the soot manager if this is already registered.
 	v, ok := m.sootMap[request.String()]
 	if ok {
-		// The TenantControlPlane is in non-ready mode, or marked for deletion:
-		// we don't want to pollute with messages due to broken connection.
-		// Once the TCP will be ready again, the event will be intercepted and the manager started back.
-		if tcpStatus == kamajiv1alpha1.VersionNotReady {
+		switch {
+		case tcpStatus == kamajiv1alpha1.VersionCARotating:
+			// The TenantControlPlane CA has been rotated, it means the running manager
+			// must be restarted to avoid certificate signed by unknown authority errors.
 			return reconcile.Result{}, m.cleanup(ctx, request, tcp)
-		}
-
-		for _, trigger := range v.triggers {
-			trigger <- event.GenericEvent{Object: tcp}
+		case tcpStatus == kamajiv1alpha1.VersionNotReady:
+			// The TenantControlPlane is in non-ready mode, or marked for deletion:
+			// we don't want to pollute with messages due to broken connection.
+			// Once the TCP will be ready again, the event will be intercepted and the manager started back.
+			return reconcile.Result{}, m.cleanup(ctx, request, tcp)
+		default:
+			for _, trigger := range v.triggers {
+				trigger <- event.GenericEvent{Object: tcp}
+			}
 		}
 
 		return reconcile.Result{}, nil
 	}
 	// No need to start a soot manager if the TenantControlPlane is not ready:
 	// enqueuing back is not required since we're going to get that event once ready.
-	if tcpStatus == kamajiv1alpha1.VersionNotReady {
+	if tcpStatus == kamajiv1alpha1.VersionNotReady || tcpStatus == kamajiv1alpha1.VersionCARotating {
 		log.FromContext(ctx).Info("skipping start of the soot manager for a not ready instance")
 
 		return reconcile.Result{}, nil

internal/crypto/crypto.go

@@ -135,6 +135,32 @@ func IsValidCertificateKeyPairBytes(certificateBytes []byte, privateKeyBytes []b
 	}
 }
 
+func VerifyCertificate(cert, ca []byte, usages ...x509.ExtKeyUsage) (bool, error) {
+	if len(usages) == 0 {
+		return false, fmt.Errorf("missing usages for certificate verification")
+	}
+
+	crt, err := ParseCertificateBytes(cert)
+	if err != nil {
+		return false, err
+	}
+
+	caCrt, err := ParseCertificateBytes(ca)
+	if err != nil {
+		return false, err
+	}
+
+	roots := x509.NewCertPool()
+	roots.AddCert(caCrt)
+
+	chains, err := crt.Verify(x509.VerifyOptions{
+		Roots:     roots,
+		KeyUsages: usages,
+	})
+
+	return len(chains) > 0, err
+}
+
 func generateCertificateKeyPairBytes(template *x509.Certificate, caCert *x509.Certificate, caKey *rsa.PrivateKey) (*bytes.Buffer, *bytes.Buffer, error) {
 	certPrivKey, err := rsa.GenerateKey(cryptorand.Reader, 2048)
 	if err != nil {

internal/resources/api_server_certificate.go

@@ -5,6 +5,7 @@ package resources
 
 import (
 	"context"
+	"crypto/x509"
 	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
@@ -83,16 +84,31 @@ func (r *APIServerCertificate) UpdateTenantControlPlaneStatus(_ context.Context,
 func (r *APIServerCertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
 	return func() error {
 		logger := log.FromContext(ctx, "resource", r.GetName())
+		// Retrieving the TenantControlPlane CA:
+		// this is required to trigger a new generation in case of Certificate Authority rotation.
+		namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.CA.SecretName}
+		secretCA := &corev1.Secret{}
+		if err := r.Client.Get(ctx, namespacedName, secretCA); err != nil {
+			logger.Error(err, "cannot retrieve CA secret")
+
+			return err
+		}
 
 		if checksum := tenantControlPlane.Status.Certificates.APIServer.Checksum; len(checksum) > 0 && checksum == r.resource.GetAnnotations()[constants.Checksum] {
-			isValid, err := crypto.CheckCertificateAndPrivateKeyPairValidity(
+			isCAValid, err := crypto.VerifyCertificate(r.resource.Data[kubeadmconstants.APIServerCertName], secretCA.Data[kubeadmconstants.CACertName], x509.ExtKeyUsageServerAuth)
+			if err != nil {
+				logger.Info(fmt.Sprintf("certificate-authority verify failed: %s", err.Error()))
+			}
+
+			isCertValid, err := crypto.CheckCertificateAndPrivateKeyPairValidity(
 				r.resource.Data[kubeadmconstants.APIServerCertName],
 				r.resource.Data[kubeadmconstants.APIServerKeyName],
 			)
 			if err != nil {
 				logger.Info(fmt.Sprintf("%s certificate-private_key pair is not valid: %s", kubeadmconstants.APIServerCertAndKeyBaseName, err.Error()))
 			}
-			if isValid {
+
+			if isCAValid && isCertValid {
 				return nil
 			}
 		}
@@ -104,14 +120,6 @@ func (r *APIServerCertificate) mutate(ctx context.Context, tenantControlPlane *k
 			return err
 		}
 
-		namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.CA.SecretName}
-		secretCA := &corev1.Secret{}
-		if err = r.Client.Get(ctx, namespacedName, secretCA); err != nil {
-			logger.Error(err, "cannot retrieve CA secret")
-
-			return err
-		}
-
 		ca := kubeadm.CertificatePrivateKeyPair{
 			Name:        kubeadmconstants.CACertAndKeyBaseName,
 			Certificate: secretCA.Data[kubeadmconstants.CACertName],

internal/resources/api_server_kubelet_client_certificate.go

@@ -5,6 +5,7 @@ package resources
 
 import (
 	"context"
+	"crypto/x509"
 	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
@@ -83,8 +84,22 @@ func (r *APIServerKubeletClientCertificate) UpdateTenantControlPlaneStatus(_ con
 func (r *APIServerKubeletClientCertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
 	return func() error {
 		logger := log.FromContext(ctx, "resource", r.GetName())
+		// Retrieving the TenantControlPlane CA:
+		// this is required to trigger a new generation in case of Certificate Authority rotation.
+		namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.CA.SecretName}
+		secretCA := &corev1.Secret{}
+		if err := r.Client.Get(ctx, namespacedName, secretCA); err != nil {
+			logger.Error(err, "cannot retrieve CA secret")
+
+			return err
+		}
 
 		if checksum := tenantControlPlane.Status.Certificates.APIServerKubeletClient.Checksum; len(checksum) > 0 && checksum == r.resource.GetAnnotations()[constants.Checksum] {
+			isCAValid, err := crypto.VerifyCertificate(r.resource.Data[kubeadmconstants.APIServerKubeletClientCertName], secretCA.Data[kubeadmconstants.CACertName], x509.ExtKeyUsageClientAuth)
+			if err != nil {
+				logger.Info(fmt.Sprintf("certificate-authority verify failed: %s", err.Error()))
+			}
+
 			isValid, err := crypto.CheckCertificateAndPrivateKeyPairValidity(
 				r.resource.Data[kubeadmconstants.APIServerKubeletClientCertName],
 				r.resource.Data[kubeadmconstants.APIServerKubeletClientKeyName],
@@ -92,7 +107,8 @@ func (r *APIServerKubeletClientCertificate) mutate(ctx context.Context, tenantCo
 			if err != nil {
 				logger.Info(fmt.Sprintf("%s certificate-private_key pair is not valid: %s", kubeadmconstants.APIServerKubeletClientCertAndKeyBaseName, err.Error()))
 			}
-			if isValid {
+
+			if isValid && isCAValid {
 				return nil
 			}
 		}
@@ -104,15 +120,6 @@ func (r *APIServerKubeletClientCertificate) mutate(ctx context.Context, tenantCo
 			return err
 		}
 
-		namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.CA.SecretName}
-
-		secretCA := &corev1.Secret{}
-		if err = r.Client.Get(ctx, namespacedName, secretCA); err != nil {
-			logger.Error(err, "cannot retrieve CA secret")
-
-			return err
-		}
-
 		ca := kubeadm.CertificatePrivateKeyPair{
 			Name:        kubeadmconstants.CACertAndKeyBaseName,
 			Certificate: secretCA.Data[kubeadmconstants.CACertName],

internal/resources/ca_certificate.go

@@ -24,12 +24,14 @@ import (
 
 type CACertificate struct {
 	resource     *corev1.Secret
+	isRotatingCA bool
+
 	Client       client.Client
 	TmpDirectory string
 }
 
 func (r *CACertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
-	return tenantControlPlane.Status.Certificates.CA.SecretName != r.resource.GetName() ||
+	return r.isRotatingCA || tenantControlPlane.Status.Certificates.CA.SecretName != r.resource.GetName() ||
 		tenantControlPlane.Status.Certificates.CA.Checksum != r.resource.GetAnnotations()[constants.Checksum]
 }
 
@@ -76,6 +78,9 @@ func (r *CACertificate) UpdateTenantControlPlaneStatus(_ context.Context, tenant
 	tenantControlPlane.Status.Certificates.CA.LastUpdate = metav1.Now()
 	tenantControlPlane.Status.Certificates.CA.SecretName = r.resource.GetName()
 	tenantControlPlane.Status.Certificates.CA.Checksum = r.resource.GetAnnotations()[constants.Checksum]
+	if r.isRotatingCA {
+		tenantControlPlane.Status.Kubernetes.Version.Status = &kamajiv1alpha1.VersionCARotating
+	}
 
 	return nil
 }
@@ -97,6 +102,10 @@ func (r *CACertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1
 			}
 		}
 
+		if tenantControlPlane.Status.Kubernetes.Version.Status != nil && *tenantControlPlane.Status.Kubernetes.Version.Status != kamajiv1alpha1.VersionProvisioning {
+			r.isRotatingCA = true
+		}
+
 		config, err := getStoredKubeadmConfiguration(ctx, r.Client, r.TmpDirectory, tenantControlPlane)
 		if err != nil {
 			logger.Error(err, "cannot retrieve kubeadm configuration")

internal/resources/front-proxy-client-certificate.go

@@ -5,6 +5,7 @@ package resources
 
 import (
 	"context"
+	"crypto/x509"
 	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
@@ -83,8 +84,21 @@ func (r *FrontProxyClientCertificate) UpdateTenantControlPlaneStatus(_ context.C
 func (r *FrontProxyClientCertificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
 	return func() error {
 		logger := log.FromContext(ctx, "resource", r.GetName())
+		// Retrieving the TenantControlPlane CA:
+		// this is required to trigger a new generation in case of Certificate Authority rotation.
+		namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName}
+		secretCA := &corev1.Secret{}
+		if err := r.Client.Get(ctx, namespacedName, secretCA); err != nil {
+			logger.Error(err, "cannot retrieve CA secret")
 
+			return err
+		}
 		if checksum := tenantControlPlane.Status.Certificates.FrontProxyClient.Checksum; len(checksum) > 0 && checksum == r.resource.GetAnnotations()[constants.Checksum] {
+			isCAValid, err := crypto.VerifyCertificate(r.resource.Data[kubeadmconstants.FrontProxyClientCertName], secretCA.Data[kubeadmconstants.FrontProxyCACertName], x509.ExtKeyUsageClientAuth)
+			if err != nil {
+				logger.Info(fmt.Sprintf("certificate-authority verify failed: %s", err.Error()))
+			}
+
 			isValid, err := crypto.CheckCertificateAndPrivateKeyPairValidity(
 				r.resource.Data[kubeadmconstants.FrontProxyClientCertName],
 				r.resource.Data[kubeadmconstants.FrontProxyClientKeyName],
@@ -92,7 +106,8 @@ func (r *FrontProxyClientCertificate) mutate(ctx context.Context, tenantControlP
 			if err != nil {
 				logger.Info(fmt.Sprintf("%s certificate-private_key pair is not valid: %s", kubeadmconstants.FrontProxyClientCertAndKeyBaseName, err.Error()))
 			}
-			if isValid {
+
+			if isValid && isCAValid {
 				return nil
 			}
 		}
@@ -104,14 +119,6 @@ func (r *FrontProxyClientCertificate) mutate(ctx context.Context, tenantControlP
 			return err
 		}
 
-		namespacedName := k8stypes.NamespacedName{Namespace: tenantControlPlane.GetNamespace(), Name: tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName}
-		secretCA := &corev1.Secret{}
-		if err = r.Client.Get(ctx, namespacedName, secretCA); err != nil {
-			logger.Error(err, "cannot retrieve CA secret")
-
-			return err
-		}
-
 		ca := kubeadm.CertificatePrivateKeyPair{
 			Name:        kubeadmconstants.FrontProxyCACertAndKeyBaseName,
 			Certificate: secretCA.Data[kubeadmconstants.FrontProxyCACertName],