diff --git a/cmd/manager/cmd.go b/cmd/manager/cmd.go index 81bd805..79803de 100644 --- a/cmd/manager/cmd.go +++ b/cmd/manager/cmd.go @@ -31,6 +31,7 @@ import ( "github.com/clastix/kamaji/internal/webhook/routes" ) +//nolint:maintidx func NewCmd(scheme *runtime.Scheme) *cobra.Command { // CLI flags var ( @@ -105,7 +106,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { return err } - tcpChannel := make(controllers.TenantControlPlaneChannel) + tcpChannel, certChannel := make(controllers.TenantControlPlaneChannel), make(controllers.CertificateChannel) if err = (&controllers.DataStore{TenantControlPlaneTrigger: tcpChannel}).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DataStore") @@ -122,6 +123,7 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { KineContainerImage: kineImage, TmpBaseDirectory: tmpDirectory, }, + CertificateChan: certChannel, TriggerChan: tcpChannel, KamajiNamespace: managerNamespace, KamajiServiceAccount: managerServiceAccountName, @@ -136,6 +138,12 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { return err } + if err = (&controllers.CertificateLifecycle{Channel: certChannel}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CertificateLifecycle") + + return err + } + if err = (&kamajiv1alpha1.DatastoreUsedSecret{}).SetupWithManager(ctx, mgr); err != nil { setupLog.Error(err, "unable to create indexer", "indexer", "DatastoreUsedSecret") diff --git a/controllers/cert_channel.go b/controllers/cert_channel.go new file mode 100644 index 0000000..8ca83e0 --- /dev/null +++ b/controllers/cert_channel.go @@ -0,0 +1,10 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "sigs.k8s.io/controller-runtime/pkg/event" +) + +type CertificateChannel chan event.GenericEvent diff --git a/controllers/secret_controller.go b/controllers/secret_controller.go new file mode 100644 index 0000000..9e18c9c --- /dev/null +++ b/controllers/secret_controller.go @@ -0,0 +1,160 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "context" + "crypto/x509" + "fmt" + "time" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/constants" + "github.com/clastix/kamaji/internal/crypto" + "github.com/clastix/kamaji/internal/utilities" +) + +type CertificateLifecycle struct { + Channel CertificateChannel + client client.Client +} + +func (s *CertificateLifecycle) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + logger := log.FromContext(ctx) + + logger.Info("starting CertificateLifecycle handling") + + secret := corev1.Secret{} + if err := s.client.Get(ctx, request.NamespacedName, &secret); err != nil { + if k8serrors.IsNotFound(err) { + logger.Info("resource may have been deleted, skipping") + + return reconcile.Result{}, nil + } + } + + checkType, ok := secret.GetLabels()[constants.ControllerLabelResource] + if !ok { + logger.Info("missing controller label, shouldn't happen") + + return reconcile.Result{}, nil + } + + var crt *x509.Certificate + var err error + + switch checkType { + case "x509": + crt, err = s.extractCertificateFromBareSecret(secret) + case "kubeconfig": + crt, err = s.extractCertificateFromKubeconfig(secret) + default: + err = fmt.Errorf("unsupported strategy, %s", checkType) + } + + if err != nil { + logger.Error(err, "skipping reconciliation") + + return reconcile.Result{}, nil + } + + deadline := time.Now().AddDate(0, 0, 1) + + if deadline.After(crt.NotAfter) { + logger.Info("certificate near expiration, must be rotated") + + s.Channel <- event.GenericEvent{Object: &kamajiv1alpha1.TenantControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: secret.GetOwnerReferences()[0].Name, + Namespace: secret.Namespace, + }, + }} + + logger.Info("certificate rotation triggered") + + return reconcile.Result{}, nil + } + + after := crt.NotAfter.Sub(deadline) + + logger.Info("certificate is still valid, enqueuing back", "after", after.String()) + + return reconcile.Result{Requeue: true, RequeueAfter: after}, nil +} + +func (s *CertificateLifecycle) extractCertificateFromBareSecret(secret corev1.Secret) (*x509.Certificate, error) { + var crt *x509.Certificate + var err error + + for _, v := range secret.Data { + if crt, err = crypto.ParseCertificateBytes(v); err == nil { + break + } + } + + if crt == nil { + return nil, fmt.Errorf("none of the provided keys is containing a valid x509 certificate") + } + + return crt, nil +} + +func (s *CertificateLifecycle) extractCertificateFromKubeconfig(secret corev1.Secret) (*x509.Certificate, error) { + var kc *clientcmdapiv1.Config + var err error + + for k := range secret.Data { + if kc, err = utilities.DecodeKubeconfig(secret, k); err == nil { + break + } + } + + if kc == nil { + return nil, fmt.Errorf("none of the provided keys is containing a valid kubeconfig") + } + + crt, err := crypto.ParseCertificateBytes(kc.AuthInfos[0].AuthInfo.ClientCertificateData) + if err != nil { + return nil, errors.Wrap(err, "cannot parse kubeconfig certificate bytes") + } + + return crt, nil +} + +func (s *CertificateLifecycle) SetupWithManager(mgr controllerruntime.Manager) error { + s.client = mgr.GetClient() + + supportedStrategies := sets.New[string]("x509", "kubeconfig") + + return controllerruntime.NewControllerManagedBy(mgr). + For(&corev1.Secret{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { + labels := object.GetLabels() + + if labels == nil { + return false + } + + value, ok := labels[constants.ControllerLabelResource] + if !ok { + return false + } + + return supportedStrategies.Has(value) + }))). + Complete(s) +} diff --git a/controllers/tenantcontrolplane_controller.go b/controllers/tenantcontrolplane_controller.go index 8588a35..068bb78 100644 --- a/controllers/tenantcontrolplane_controller.go +++ b/controllers/tenantcontrolplane_controller.go @@ -50,6 +50,10 @@ type TenantControlPlaneReconciler struct { KamajiService string KamajiMigrateImage string MaxConcurrentReconciles int + // CertificateChan is the channel used by the CertificateLifecycleController that is checking for + // certificates and kubeconfig user certs validity: a generic event for the given TCP will be triggered + // once the validity threshold for the given certificate is reached. + CertificateChan CertificateChannel clock mutex.Clock } @@ -223,6 +227,14 @@ func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error r.clock = clock.RealClock{} return ctrl.NewControllerManagedBy(mgr). + Watches(&source.Channel{Source: r.CertificateChan}, handler.Funcs{GenericFunc: func(genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) { + limitingInterface.AddRateLimited(ctrl.Request{ + NamespacedName: k8stypes.NamespacedName{ + Namespace: genericEvent.Object.GetNamespace(), + Name: genericEvent.Object.GetName(), + }, + }) + }}). Watches(&source.Channel{Source: r.TriggerChan}, handler.Funcs{GenericFunc: func(genericEvent event.GenericEvent, limitingInterface workqueue.RateLimitingInterface) { limitingInterface.AddRateLimited(ctrl.Request{ NamespacedName: k8stypes.NamespacedName{ diff --git a/internal/resources/api_server_certificate.go b/internal/resources/api_server_certificate.go index f8926ee..a056dad 100644 --- a/internal/resources/api_server_certificate.go +++ b/internal/resources/api_server_certificate.go @@ -103,6 +103,8 @@ func (r *APIServerCertificate) mutate(ctx context.Context, tenantControlPlane *k if err := ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil { logger.Error(err, "cannot set controller reference", "resource", r.GetName()) + + return err } if checksum := tenantControlPlane.Status.Certificates.APIServer.Checksum; len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0 { diff --git a/internal/resources/api_server_kubelet_client_certificate.go b/internal/resources/api_server_kubelet_client_certificate.go index 2fcea50..85b4d42 100644 --- a/internal/resources/api_server_kubelet_client_certificate.go +++ b/internal/resources/api_server_kubelet_client_certificate.go @@ -103,6 +103,8 @@ func (r *APIServerKubeletClientCertificate) mutate(ctx context.Context, tenantCo if err := ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil { logger.Error(err, "cannot set controller reference", "resource", r.GetName()) + + return err } if checksum := tenantControlPlane.Status.Certificates.APIServerKubeletClient.Checksum; len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0 { diff --git a/internal/resources/datastore/datastore_certificate.go b/internal/resources/datastore/datastore_certificate.go index 6deda70..a1fb766 100644 --- a/internal/resources/datastore/datastore_certificate.go +++ b/internal/resources/datastore/datastore_certificate.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/constants" "github.com/clastix/kamaji/internal/crypto" "github.com/clastix/kamaji/internal/utilities" ) @@ -88,6 +89,19 @@ func (r *Certificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1al r.resource.Data["ca.crt"] = ca + r.resource.SetLabels(utilities.MergeMaps( + utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()), + map[string]string{ + constants.ControllerLabelResource: "x509", + }, + )) + + if err = ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil { + logger.Error(err, "cannot set controller reference", "resource", r.GetName()) + + return err + } + if utilities.GetObjectChecksum(r.resource) == utilities.CalculateMapChecksum(r.resource.Data) { if r.DataStore.Spec.Driver == kamajiv1alpha1.EtcdDriver { if isValid, _ := crypto.IsValidCertificateKeyPairBytes(r.resource.Data["server.crt"], r.resource.Data["server.key"]); isValid { @@ -141,11 +155,6 @@ func (r *Certificate) mutate(ctx context.Context, tenantControlPlane *kamajiv1al utilities.SetObjectChecksum(r.resource, r.resource.Data) - r.resource.SetLabels(utilities.MergeMaps( - utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()), - r.resource.GetLabels(), - )) - - return ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) + return nil } } diff --git a/internal/resources/front-proxy-client-certificate.go b/internal/resources/front-proxy-client-certificate.go index 72577e8..f5ed67c 100644 --- a/internal/resources/front-proxy-client-certificate.go +++ b/internal/resources/front-proxy-client-certificate.go @@ -103,6 +103,8 @@ func (r *FrontProxyClientCertificate) mutate(ctx context.Context, tenantControlP if err := ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil { logger.Error(err, "cannot set controller reference", "resource", r.GetName()) + + return err } if checksum := tenantControlPlane.Status.Certificates.FrontProxyClient.Checksum; len(checksum) > 0 && checksum == utilities.GetObjectChecksum(r.resource) || len(r.resource.UID) > 0 { diff --git a/internal/resources/konnectivity/certificate_resource.go b/internal/resources/konnectivity/certificate_resource.go index 72d80ea..97fd35e 100644 --- a/internal/resources/konnectivity/certificate_resource.go +++ b/internal/resources/konnectivity/certificate_resource.go @@ -99,6 +99,8 @@ func (r *CertificateResource) mutate(ctx context.Context, tenantControlPlane *ka if err := ctrl.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()); err != nil { logger.Error(err, "cannot set controller reference", "resource", r.GetName()) + + return err } if checksum := tenantControlPlane.Status.Addons.Konnectivity.Certificate.Checksum; len(checksum) > 0 && checksum == utilities.CalculateMapChecksum(r.resource.Data) { diff --git a/internal/utilities/kubeconfig.go b/internal/utilities/kubeconfig.go new file mode 100644 index 0000000..422a082 --- /dev/null +++ b/internal/utilities/kubeconfig.go @@ -0,0 +1,29 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package utilities + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" +) + +func DecodeKubeconfig(secret corev1.Secret, key string) (*clientcmdapiv1.Config, error) { + bytes, ok := secret.Data[key] + if !ok { + return nil, fmt.Errorf("%s is not into kubeconfig secret", key) + } + + return DecodeKubeconfigYAML(bytes) +} + +func DecodeKubeconfigYAML(bytes []byte) (*clientcmdapiv1.Config, error) { + kubeconfig := &clientcmdapiv1.Config{} + if err := DecodeFromYAML(string(bytes), kubeconfig); err != nil { + return nil, err + } + + return kubeconfig, nil +} diff --git a/internal/utilities/tenant_client.go b/internal/utilities/tenant_client.go index 3e6e884..20603c2 100644 --- a/internal/utilities/tenant_client.go +++ b/internal/utilities/tenant_client.go @@ -44,17 +44,7 @@ func GetTenantKubeconfig(ctx context.Context, client client.Client, tenantContro return nil, err } - bytes, ok := secretKubeconfig.Data[kubeadmconstants.AdminKubeConfigFileName] - if !ok { - return nil, fmt.Errorf("%s is not into kubeconfig secret", kubeadmconstants.AdminKubeConfigFileName) - } - - kubeconfig := &clientcmdapiv1.Config{} - if err := DecodeFromYAML(string(bytes), kubeconfig); err != nil { - return nil, err - } - - return kubeconfig, nil + return DecodeKubeconfig(*secretKubeconfig, kubeadmconstants.AdminKubeConfigFileName) } func GetRESTClientConfig(ctx context.Context, client client.Client, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (*restclient.Config, error) {