diff --git a/controllers/resources.go b/controllers/resources.go index c0569a2..a5830f9 100644 --- a/controllers/resources.go +++ b/controllers/resources.go @@ -14,6 +14,7 @@ import ( kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/internal/resources" "github.com/clastix/kamaji/internal/resources/konnectivity" + "github.com/clastix/kamaji/internal/types" ) const ( @@ -179,27 +180,32 @@ func getKubeconfigResources(c client.Client, log logr.Logger, tcpReconcilerConfi } func getKubernetesStorageResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { - return []resources.Resource{ - &resources.ETCDCACertificatesResource{ - Name: "etcd-ca-certificates", - Client: c, - Log: log, - ETCDCASecretName: tcpReconcilerConfig.ETCDCASecretName, - ETCDCASecretNamespace: tcpReconcilerConfig.ETCDCASecretNamespace, - }, - &resources.ETCDCertificatesResource{ - Name: "etcd-certificates", - Client: c, - Log: log, - }, - &resources.ETCDSetupResource{ - Name: "etcd-setup", - Client: c, - Log: log, - ETCDClientCertsSecret: getNamespacedName(tcpReconcilerConfig.ETCDClientSecretNamespace, tcpReconcilerConfig.ETCDClientSecretName), - ETCDCACertsSecret: getNamespacedName(tcpReconcilerConfig.ETCDCASecretNamespace, tcpReconcilerConfig.ETCDCASecretName), - Endpoints: getArrayFromString(tcpReconcilerConfig.ETCDEndpoints), - }, + switch tcpReconcilerConfig.ETCDStorageType { + case types.ETCD: + return []resources.Resource{ + &resources.ETCDCACertificatesResource{ + Name: "etcd-ca-certificates", + Client: c, + Log: log, + ETCDCASecretName: tcpReconcilerConfig.ETCDCASecretName, + ETCDCASecretNamespace: tcpReconcilerConfig.ETCDCASecretNamespace, + }, + &resources.ETCDCertificatesResource{ + Name: "etcd-certificates", + Client: c, + Log: log, + }, + &resources.ETCDSetupResource{ + Name: "etcd-setup", + Client: c, + Log: log, + ETCDClientCertsSecret: getNamespacedName(tcpReconcilerConfig.ETCDClientSecretNamespace, tcpReconcilerConfig.ETCDClientSecretName), + ETCDCACertsSecret: getNamespacedName(tcpReconcilerConfig.ETCDCASecretNamespace, tcpReconcilerConfig.ETCDCASecretName), + Endpoints: getArrayFromString(tcpReconcilerConfig.ETCDEndpoints), + }, + } + default: + return []resources.Resource{} } } diff --git a/controllers/tenantcontrolplane_controller.go b/controllers/tenantcontrolplane_controller.go index 60859ea..2389d60 100644 --- a/controllers/tenantcontrolplane_controller.go +++ b/controllers/tenantcontrolplane_controller.go @@ -21,6 +21,7 @@ import ( kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" kamajierrors "github.com/clastix/kamaji/internal/errors" "github.com/clastix/kamaji/internal/resources" + "github.com/clastix/kamaji/internal/types" ) const ( @@ -36,6 +37,7 @@ type TenantControlPlaneReconciler struct { // TenantControlPlaneReconcilerConfig gives the necessary configuration for TenantControlPlaneReconciler. type TenantControlPlaneReconcilerConfig struct { + ETCDStorageType types.ETCDStorageType ETCDCASecretName string ETCDCASecretNamespace string ETCDClientSecretName string @@ -83,14 +85,13 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R registeredDeleteableResources := GetDeleteableResources(groupDeleteableResourceBuilderConfiguration) for _, resource := range registeredDeleteableResources { - if err := resource.Delete(ctx, tenantControlPlane); err != nil { + if err := resources.HandleDeletion(ctx, resource, tenantControlPlane); err != nil { return ctrl.Result{}, err } } if hasFinalizer { - controllerutil.RemoveFinalizer(tenantControlPlane, finalizer) - if err := r.Update(ctx, tenantControlPlane); err != nil { + if err := r.RemoveFinalizer(ctx, tenantControlPlane); err != nil { return ctrl.Result{}, err } } @@ -99,12 +100,7 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R } if !hasFinalizer { - controllerutil.AddFinalizer(tenantControlPlane, finalizer) - if err := r.Update(ctx, tenantControlPlane); err != nil { - return ctrl.Result{}, err - } - - return ctrl.Result{}, nil + return ctrl.Result{}, r.AddFinalizer(ctx, tenantControlPlane) } groupResourceBuilderConfiguration := GroupResourceBuilderConfiguration{ @@ -200,3 +196,15 @@ func hasFinalizer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) bool { return false } + +func (r *TenantControlPlaneReconciler) AddFinalizer(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + controllerutil.AddFinalizer(tenantControlPlane, finalizer) + + return r.Update(ctx, tenantControlPlane) +} + +func (r *TenantControlPlaneReconciler) RemoveFinalizer(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + controllerutil.RemoveFinalizer(tenantControlPlane, finalizer) + + return r.Update(ctx, tenantControlPlane) +} diff --git a/docs/reference.md b/docs/reference.md index a27ec13..358b107 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -21,7 +21,7 @@ Available flags are the following: --etcd-client-secret-name Name of the secret which contains ETCD client certificates. (default: "root-client-certs") --etcd-client-secret-namespace Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji") --etcd-compaction-interval ETCD Compaction interval (i.e. "5m0s"). (default: "0" (disabled)) ---etcd-endpoints Comma-separated list with ETCD endpoints (i.e. etcd-0.etcd.kamaji.svc.cluster.local,etcd-1.etcd.kamaji.svc.cluster.local,etcd-2.etcd.kamaji.svc.cluster.local) +--etcd-endpoints Comma-separated list with ETCD endpoints (i.e. https://etcd-0.etcd.kamaji.svc.cluster.local,https://etcd-1.etcd.kamaji.svc.cluster.local,https://etcd-2.etcd.kamaji.svc.cluster.local) --health-probe-bind-address string The address the probe endpoint binds to. (default ":8081") --kubeconfig string Paths to a kubeconfig. Only required if out-of-cluster. --leader-elect Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager. diff --git a/e2e/suite_test.go b/e2e/suite_test.go index 13ea3e9..8743ef4 100644 --- a/e2e/suite_test.go +++ b/e2e/suite_test.go @@ -23,9 +23,11 @@ import ( // These tests use Ginkgo (BDD-style Go testing framework). Refer to // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. -var cfg *rest.Config // nolint -var k8sClient client.Client -var testEnv *envtest.Environment +var ( + cfg *rest.Config // nolint + k8sClient client.Client + testEnv *envtest.Environment +) func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) diff --git a/e2e/tenant_control_plane_ready_test.go b/e2e/tenant_control_plane_ready_test.go index 122db92..016a3b0 100644 --- a/e2e/tenant_control_plane_ready_test.go +++ b/e2e/tenant_control_plane_ready_test.go @@ -76,7 +76,7 @@ var _ = Describe("Deploy a TenantControlPlane resource", func() { } // Check if Status field has been created on TenantControlPlane struct - if *&tcp.Status.Kubernetes.Version.Status == nil { + if tcp.Status.Kubernetes.Version.Status == nil { return "" } diff --git a/e2e/worker_kubeadm_join_test.go b/e2e/worker_kubeadm_join_test.go index 87a6155..cfc982b 100644 --- a/e2e/worker_kubeadm_join_test.go +++ b/e2e/worker_kubeadm_join_test.go @@ -109,6 +109,10 @@ var _ = Describe("starting a kind worker with kubeadm", func() { return "" } + if tcp.Status.Kubernetes.Version.Status == nil { + return "" + } + return *tcp.Status.Kubernetes.Version.Status }, 5*time.Minute, time.Second).Should(Equal(kamajiv1alpha1.VersionReady)) }) diff --git a/helm/kamaji/README.md b/helm/kamaji/README.md index 26665e9..96f4fe5 100644 --- a/helm/kamaji/README.md +++ b/helm/kamaji/README.md @@ -64,7 +64,7 @@ Kubernetes: `>=1.18` | etcd.clientSecret.name | string | `"root-client-certs"` | Name of the secret which contains ETCD client certificates. (default: "root-client-certs") | | etcd.clientSecret.namespace | string | `"kamaji-system"` | Name of the namespace where the secret which contains ETCD client certificates is. (default: "kamaji") | | etcd.compactionInterval | int | `0` | ETCD Compaction interval (e.g. "5m0s"). (default: "0" (disabled)) | -| etcd.endpoints | string | `"etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379"` | (string) Comma-separated list of the endpoints of the etcd cluster's members. | +| etcd.endpoints | string | `"https://etcd-0.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-1.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-2.etcd.kamaji-system.svc.cluster.local:2379"` | (string) Comma-separated list of the endpoints of the etcd cluster's members. | | extraArgs | list | `[]` | A list of extra arguments to add to the kamaji controller default ones | | fullnameOverride | string | `""` | | | healthProbeBindAddress | string | `":8081"` | The address the probe endpoint binds to. (default ":8081") | diff --git a/helm/kamaji/values.sample.yaml b/helm/kamaji/values.sample.yaml index aa04ad4..a37f140 100644 --- a/helm/kamaji/values.sample.yaml +++ b/helm/kamaji/values.sample.yaml @@ -1,2 +1,2 @@ etcd: - endpoints: "etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379" + endpoints: "https://etcd-0.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-1.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-2.etcd.kamaji-system.svc.cluster.local:2379" diff --git a/helm/kamaji/values.yaml b/helm/kamaji/values.yaml index b92d4fc..c523ead 100644 --- a/helm/kamaji/values.yaml +++ b/helm/kamaji/values.yaml @@ -35,7 +35,7 @@ etcd: compactionInterval: 0 # -- (string) Comma-separated list of the endpoints of the etcd cluster's members. - endpoints: "etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379" + endpoints: "https://etcd-0.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-1.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-2.etcd.kamaji-system.svc.cluster.local:2379" # -- The address the probe endpoint binds to. (default ":8081") healthProbeBindAddress: ":8081" diff --git a/internal/config/config.go b/internal/config/config.go index 3c9b10c..efab01d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -22,6 +22,7 @@ var ( const ( envPrefix = "KAMAJI" + defaultETCDStorageType = "etcd" defaultETCDCASecretName = "etcd-certs" defaultETCDCASecretNamespace = "kamaji-system" defaultETCDEndpoints = "etcd-server:2379" @@ -40,11 +41,12 @@ func InitConfig() (*viper.Viper, error) { flag.String("health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.Bool("leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + flag.String("etcd-storage-type", defaultETCDStorageType, "Type of storage for ETCD (i.e etcd, kine-mysql, kine-postgres)") flag.String("etcd-ca-secret-name", defaultETCDCASecretName, "Name of the secret which contains CA's certificate and private key.") flag.String("etcd-ca-secret-namespace", defaultETCDCASecretNamespace, "Namespace of the secret which contains CA's certificate and private key.") flag.String("etcd-client-secret-name", defaultETCDClientSecretName, "Name of the secret which contains ETCD client certificates") flag.String("etcd-client-secret-namespace", defaultETCDClientSecretNamespace, "Name of the namespace where the secret which contains ETCD client certificates is") - flag.String("etcd-endpoints", defaultETCDEndpoints, "Comma-separated list with ETCD endpoints (i.e. etcd-0.etcd.kamaji-system.svc.cluster.local,etcd-1.etcd.kamaji-system.svc.cluster.local,etcd-2.etcd.kamaji-system.svc.cluster.local)") + flag.String("etcd-endpoints", defaultETCDEndpoints, "Comma-separated list with ETCD endpoints (i.e. https://etcd-0.etcd.kamaji-system.svc.cluster.local,https://etcd-1.etcd.kamaji-system.svc.cluster.local,https://etcd-2.etcd.kamaji-system.svc.cluster.local)") flag.String("etcd-compaction-interval", defaultETCDCompactionInterval, "ETCD Compaction interval (i.e. \"5m0s\"). (default: \"0\" (disabled))") flag.String("tmp-directory", defaultTmpDirectory, "Directory which will be used to work with temporary files.") @@ -74,6 +76,9 @@ func InitConfig() (*viper.Viper, error) { if err := config.BindEnv("leader-elect", fmt.Sprintf("%s_LEADER_ELECTION", envPrefix)); err != nil { return nil, err } + if err := config.BindEnv("etcd-storage-type", fmt.Sprintf("%s_ETCD_STORAGE_TYPE", envPrefix)); err != nil { + return nil, err + } if err := config.BindEnv("etcd-ca-secret-name", fmt.Sprintf("%s_ETCD_CA_SECRET_NAME", envPrefix)); err != nil { return nil, err } diff --git a/internal/kubeadm/configuration.go b/internal/kubeadm/configuration.go index 01105fd..90ec5c4 100644 --- a/internal/kubeadm/configuration.go +++ b/internal/kubeadm/configuration.go @@ -6,6 +6,7 @@ package kubeadm import ( "encoding/json" "fmt" + "strings" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -14,6 +15,12 @@ import ( kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" ) +const ( + defaultCAFile = "/etc/kubernetes/pki/etcd/ca.crt" + defaultCertFile = "/etc/kubernetes/pki/apiserver-etcd-client.crt" + defaultKeyFile = "/etc/kubernetes/pki/apiserver-etcd-client.key" +) + func CreateKubeadmInitConfiguration(params Parameters) Configuration { config := kubeadmapi.InitConfiguration{ ClusterConfiguration: getKubeadmClusterConfiguration(params), @@ -40,7 +47,16 @@ func CreateKubeadmInitConfiguration(params Parameters) Configuration { return Configuration{InitConfiguration: config} } +func isHTTPS(url string) bool { + return strings.HasPrefix(url, "https") +} + func getKubeadmClusterConfiguration(params Parameters) kubeadmapi.ClusterConfiguration { + caFile, certFile, keyFile := "", "", "" + if isHTTPS(params.ETCDs[0]) { + caFile, certFile, keyFile = defaultCAFile, defaultCertFile, defaultKeyFile + } + return kubeadmapi.ClusterConfiguration{ KubernetesVersion: params.TenantControlPlaneVersion, ClusterName: params.TenantControlPlaneName, @@ -57,10 +73,10 @@ func getKubeadmClusterConfiguration(params Parameters) kubeadmapi.ClusterConfigu ControlPlaneEndpoint: params.TenantControlPlaneEndpoint, Etcd: kubeadmapi.Etcd{ External: &kubeadmapi.ExternalEtcd{ - Endpoints: formatETCDEndpoints(params.ETCDs), - CAFile: "/etc/kubernetes/pki/etcd/ca.crt", - CertFile: "/etc/kubernetes/pki/apiserver-etcd-client.crt", - KeyFile: "/etc/kubernetes/pki/apiserver-etcd-client.key", + Endpoints: params.ETCDs, + CAFile: caFile, + CertFile: certFile, + KeyFile: keyFile, }, }, APIServer: kubeadmapi.APIServer{ @@ -131,12 +147,3 @@ func getJSONStringFromStruct(i interface{}) (string, error) { return string(b), nil } - -func formatETCDEndpoints(etcds []string) []string { - formatedETCDs := make([]string, 0, len(etcds)) - for _, etcd := range etcds { - formatedETCDs = append(formatedETCDs, fmt.Sprintf("https://%s/", etcd)) - } - - return formatedETCDs -} diff --git a/internal/resources/etcd_setup.go b/internal/resources/etcd_setup.go index f7496b4..2b5ebf4 100644 --- a/internal/resources/etcd_setup.go +++ b/internal/resources/etcd_setup.go @@ -22,13 +22,13 @@ const ( caKeyName = kubeadmconstants.CACertName ) -type resource struct { +type etcdSetupResource struct { role etcd.Role user etcd.User } type ETCDSetupResource struct { - resource *resource + resource *etcdSetupResource Client client.Client Log logr.Logger Name string @@ -55,7 +55,7 @@ func (r *ETCDSetupResource) CleanUp(ctx context.Context, tenantControlPlane *kam } func (r *ETCDSetupResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { - r.resource = &resource{ + r.resource = &etcdSetupResource{ role: etcd.Role{Name: tenantControlPlane.Name, Exists: false}, user: etcd.User{Name: tenantControlPlane.Name, Exists: false}, } diff --git a/internal/resources/k8s_deployment_resource.go b/internal/resources/k8s_deployment_resource.go index ab7033b..608bf94 100644 --- a/internal/resources/k8s_deployment_resource.go +++ b/internal/resources/k8s_deployment_resource.go @@ -16,7 +16,7 @@ import ( corev1 "k8s.io/api/core/v1" quantity "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" + apimachinerytypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" "k8s.io/kubernetes/cmd/kubeadm/app/constants" @@ -26,6 +26,7 @@ import ( kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/internal/resources/konnectivity" + "github.com/clastix/kamaji/internal/types" "github.com/clastix/kamaji/internal/utilities" ) @@ -39,6 +40,7 @@ const ( type KubernetesDeploymentResource struct { resource *appsv1.Deployment Client client.Client + ETCDStorageType types.ETCDStorageType ETCDEndpoints []string ETCDCompactionInterval string Name string @@ -79,7 +81,7 @@ func (r *KubernetesDeploymentResource) Define(ctx context.Context, tenantControl func (r *KubernetesDeploymentResource) secretHashValue(ctx context.Context, namespace, name string) (string, error) { secret := &corev1.Secret{} - if err := r.Client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { + if err := r.Client.Get(ctx, apimachinerytypes.NamespacedName{Namespace: namespace, Name: name}, secret); err != nil { return "", errors.Wrap(err, "cannot retrieve *corev1.Secret for resource version retrieval") } // Go access map values in random way, it means we have to sort them @@ -105,11 +107,6 @@ func (r *KubernetesDeploymentResource) CreateOrUpdate(ctx context.Context, tenan maxUnavailable := intstr.FromInt(0) - etcdEndpoints := make([]string, len(r.ETCDEndpoints)) - for i, v := range r.ETCDEndpoints { - etcdEndpoints[i] = fmt.Sprintf("https://%s", v) - } - address, err := tenantControlPlane.GetControlPlaneAddress(ctx, r.Client) if err != nil { return controllerutil.OperationResultNone, errors.Wrap(err, "cannot create TenantControlPlane Deployment") @@ -151,16 +148,6 @@ func (r *KubernetesDeploymentResource) CreateOrUpdate(ctx context.Context, tenan return }(), - "component.kamaji.clastix.io/etcd-ca-certificates": func() (hash string) { - hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.CA.SecretName) - - return - }(), - "component.kamaji.clastix.io/etcd-certificates": func() (hash string) { - hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName) - - return - }(), "component.kamaji.clastix.io/front-proxy-ca-certificate": func() (hash string) { hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName) @@ -207,22 +194,6 @@ func (r *KubernetesDeploymentResource) CreateOrUpdate(ctx context.Context, tenan { Secret: secretProjection(tenantControlPlane.Status.Certificates.SA.SecretName, constants.ServiceAccountPublicKeyName, constants.ServiceAccountPrivateKeyName), }, - { - Secret: secretProjection(tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName, constants.APIServerEtcdClientCertName, constants.APIServerEtcdClientKeyName), - }, - { - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: tenantControlPlane.Status.Certificates.ETCD.CA.SecretName, - }, - Items: []corev1.KeyToPath{ - { - Key: constants.CACertName, - Path: constants.EtcdCACertName, - }, - }, - }, - }, }, DefaultMode: pointer.Int32Ptr(420), }, @@ -296,12 +267,6 @@ func (r *KubernetesDeploymentResource) CreateOrUpdate(ctx context.Context, tenan fmt.Sprintf("--client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)), fmt.Sprintf("--enable-admission-plugins=%s", strings.Join(tenantControlPlane.Spec.Kubernetes.AdmissionControllers.ToSlice(), ",")), "--enable-bootstrap-token-auth=true", - fmt.Sprintf("--etcd-compaction-interval=%s", r.ETCDCompactionInterval), - fmt.Sprintf("--etcd-cafile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.EtcdCACertName)), - fmt.Sprintf("--etcd-certfile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientCertName)), - fmt.Sprintf("--etcd-keyfile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientKeyName)), - fmt.Sprintf("--etcd-servers=%s", strings.Join(etcdEndpoints, ",")), - fmt.Sprintf("--etcd-prefix=/%s", tenantControlPlane.GetName()), fmt.Sprintf("--service-cluster-ip-range=%s", tenantControlPlane.Spec.NetworkProfile.ServiceCIDR), fmt.Sprintf("--kubelet-client-certificate=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientCertName)), fmt.Sprintf("--kubelet-client-key=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientKeyName)), @@ -556,6 +521,8 @@ func (r *KubernetesDeploymentResource) CreateOrUpdate(ctx context.Context, tenan }, } + r.customizeStorage(ctx, &r.resource.Spec.Template, *tenantControlPlane) + if err := r.reconcileKonnectivity(&r.resource.Spec.Template.Spec, *tenantControlPlane); err != nil { return err } @@ -776,3 +743,62 @@ func (r *KubernetesDeploymentResource) buildKonnectivityServerContainer(tenantCo ImagePullPolicy: corev1.PullIfNotPresent, } } + +func (r *KubernetesDeploymentResource) customizeStorage(ctx context.Context, podTemplate *corev1.PodTemplateSpec, tenantControlPlane kamajiv1alpha1.TenantControlPlane) { + switch r.ETCDStorageType { + case types.ETCD: + r.customizeETCDStorage(ctx, podTemplate, tenantControlPlane) + default: + return + } +} + +func (r *KubernetesDeploymentResource) customizeETCDStorage(ctx context.Context, podTemplate *corev1.PodTemplateSpec, tenantControlPlane kamajiv1alpha1.TenantControlPlane) { + labels := map[string]string{ + "component.kamaji.clastix.io/etcd-ca-certificates": func() (hash string) { + hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.CA.SecretName) + + return + }(), + "component.kamaji.clastix.io/etcd-certificates": func() (hash string) { + hash, _ = r.secretHashValue(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName) + + return + }(), + } + + podTemplate.SetLabels( + utilities.MergeMaps(labels, podTemplate.Labels), + ) + + commands := []string{fmt.Sprintf("--etcd-compaction-interval=%s", r.ETCDCompactionInterval), + fmt.Sprintf("--etcd-cafile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.EtcdCACertName)), + fmt.Sprintf("--etcd-certfile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientCertName)), + fmt.Sprintf("--etcd-keyfile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientKeyName)), + fmt.Sprintf("--etcd-servers=%s", strings.Join(r.ETCDEndpoints, ",")), + fmt.Sprintf("--etcd-prefix=/%s", tenantControlPlane.GetName()), + } + + podTemplate.Spec.Containers[0].Command = append(podTemplate.Spec.Containers[0].Command, commands...) + + volumeProjections := []corev1.VolumeProjection{ + { + Secret: secretProjection(tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName, constants.APIServerEtcdClientCertName, constants.APIServerEtcdClientKeyName), + }, + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: tenantControlPlane.Status.Certificates.ETCD.CA.SecretName, + }, + Items: []corev1.KeyToPath{ + { + Key: constants.CACertName, + Path: constants.EtcdCACertName, + }, + }, + }, + }, + } + + podTemplate.Spec.Volumes[0].VolumeSource.Projected.Sources = append(podTemplate.Spec.Volumes[0].VolumeSource.Projected.Sources, volumeProjections...) +} diff --git a/internal/resources/resource.go b/internal/resources/resource.go index c753023..a70e307 100644 --- a/internal/resources/resource.go +++ b/internal/resources/resource.go @@ -28,6 +28,7 @@ type Resource interface { } type DeleteableResource interface { + Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error Delete(context.Context, *kamajiv1alpha1.TenantControlPlane) error } @@ -72,6 +73,15 @@ func Handle(ctx context.Context, resource Resource, tenantControlPlane *kamajiv1 return controllerutil.OperationResultNone, err } +// HandleDeletion handles the deletion of the given resource +func HandleDeletion(ctx context.Context, resource DeleteableResource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { + if err := resource.Define(ctx, tenantControlPlane); err != nil { + return err + } + + return resource.Delete(ctx, tenantControlPlane) +} + func createOrUpdate(ctx context.Context, resource Resource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { result, err := resource.CreateOrUpdate(ctx, tenantControlPlane) if err != nil { diff --git a/internal/types/etcd_storage.go b/internal/types/etcd_storage.go new file mode 100644 index 0000000..301dd56 --- /dev/null +++ b/internal/types/etcd_storage.go @@ -0,0 +1,27 @@ +package types + +type ETCDStorageType int + +const ( + ETCD ETCDStorageType = iota +) + +const ( + defaultETCDStorageType = ETCD +) + +var ( + etcdStorageTypeString = map[string]ETCDStorageType{ + "etcd": ETCD, + } +) + +// ParseETCDStorageType returns the ETCDStorageType given a string representation of the type +func ParseETCDStorageType(s string) ETCDStorageType { + if storageType, ok := etcdStorageTypeString[s]; ok { + return storageType + } + + // TODO: we have to decide what to do in this situation + return defaultETCDStorageType +} diff --git a/kamaji.yaml b/kamaji.yaml index 5fd8faf..77b3927 100755 --- a/kamaji.yaml +++ b/kamaji.yaml @@ -1,6 +1,6 @@ etcd-ca-secret-name: "etcd-certs" etcd-ca-secret-namespace: kamaji-system -etcd-endpoints: etcd-0.etcd.kamaji-system.svc.cluster.local:2379,etcd-1.etcd.kamaji-system.svc.cluster.local:2379,etcd-2.etcd.kamaji-system.svc.cluster.local:2379 +etcd-endpoints: https://etcd-0.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-1.etcd.kamaji-system.svc.cluster.local:2379,https://etcd-2.etcd.kamaji-system.svc.cluster.local:2379 etcd-client-secret-name: root-client-certs etcd-client-secret-namespaced: kamaji-system etcd-compaction: "0" diff --git a/main.go b/main.go index 760d867..8ed2379 100644 --- a/main.go +++ b/main.go @@ -33,6 +33,7 @@ import ( "github.com/clastix/kamaji/controllers" "github.com/clastix/kamaji/internal" "github.com/clastix/kamaji/internal/config" + "github.com/clastix/kamaji/internal/types" ) var ( @@ -76,6 +77,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: controllers.TenantControlPlaneReconcilerConfig{ + ETCDStorageType: types.ParseETCDStorageType(conf.GetString("etcd-storage-type")), ETCDCASecretName: conf.GetString("etcd-ca-secret-name"), ETCDCASecretNamespace: conf.GetString("etcd-ca-secret-namespace"), ETCDClientSecretName: conf.GetString("etcd-client-secret-name"),