From d3fb03a75242270cd75bb3f27690f73ab84831ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9onard=20Suslian?= <58216931+synthe102@users.noreply.github.com> Date: Fri, 12 Dec 2025 12:10:02 +0100 Subject: [PATCH] feat: add support for multiple Datastores (#961) * feat: add support for multiple Datastores * docs: add guide for datastore overrides * feat(datastore): add e2e test for dataStoreOverrides * ci: reclaim disk space from runner to fix flaky tests --- .github/workflows/e2e.yaml | 3 + Makefile | 10 +- api/v1alpha1/tenantcontrolplane_types.go | 14 ++- api/v1alpha1/zz_generated.deepcopy.go | 20 ++++ ...i.clastix.io_tenantcontrolplanes_spec.yaml | 13 +++ ...kamaji.clastix.io_tenantcontrolplanes.yaml | 13 +++ controllers/resources.go | 61 +++++++++--- controllers/tenantcontrolplane_controller.go | 62 +++++++++--- docs/content/guides/datastore-overrides.md | 78 +++++++++++++++ docs/content/reference/api.md | 39 ++++++++ docs/mkdocs.yml | 1 + e2e/tcp_datastore_overrides_test.go | 68 ++++++++++++++ internal/builders/controlplane/deployment.go | 24 +++++ .../builders/controlplane/deployment_test.go | 46 +++++++++ .../datastore/datastore_storage_config.go | 11 ++- internal/resources/k8s_deployment_resource.go | 2 + internal/webhook/handlers/tcp_datastore.go | 18 +++- .../webhook/handlers/tcp_datastore_test.go | 94 +++++++++++++++++++ internal/webhook/handlers/tcp_deployment.go | 14 +++ 19 files changed, 555 insertions(+), 36 deletions(-) create mode 100644 docs/content/guides/datastore-overrides.md create mode 100644 e2e/tcp_datastore_overrides_test.go create mode 100644 internal/builders/controlplane/deployment_test.go create mode 100644 internal/webhook/handlers/tcp_datastore_test.go diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 29131e0..1edea2c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -41,6 +41,9 @@ jobs: - uses: actions/setup-go@v6 with: go-version-file: go.mod + - name: reclaim disk space from runner + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL - run: | sudo apt-get update sudo apt-get install -y golang-cfssl diff --git a/Makefile b/Makefile index f410573..7c61fe5 100644 --- a/Makefile +++ b/Makefile @@ -177,7 +177,7 @@ datastore-postgres: $(MAKE) NAME=gold _datastore-postgres _datastore-etcd: - $(HELM) upgrade --install etcd-$(NAME) clastix/kamaji-etcd --create-namespace -n etcd-system --set datastore.enabled=true --set fullnameOverride=etcd-$(NAME) + $(HELM) upgrade --install etcd-$(NAME) clastix/kamaji-etcd --create-namespace -n $(NAMESPACE) --set datastore.enabled=true --set fullnameOverride=etcd-$(NAME) $(EXTRA_ARGS) _datastore-nats: $(MAKE) NAME=$(NAME) NAMESPACE=nats-system -C deploy/kine/nats nats @@ -186,9 +186,11 @@ _datastore-nats: datastore-etcd: helm $(HELM) repo add clastix https://clastix.github.io/charts $(HELM) repo update - $(MAKE) NAME=bronze _datastore-etcd - $(MAKE) NAME=silver _datastore-etcd - $(MAKE) NAME=gold _datastore-etcd + $(MAKE) NAME=bronze NAMESPACE=etcd-system _datastore-etcd + $(MAKE) NAME=silver NAMESPACE=etcd-system _datastore-etcd + $(MAKE) NAME=gold NAMESPACE=etcd-system _datastore-etcd + $(MAKE) NAME=primary NAMESPACE=kamaji-system EXTRA_ARGS='--set certManager.enabled=true --set certManager.issuerRef.kind=Issuer --set certManager.issuerRef.name=kamaji-selfsigned-issuer --set selfSignedCertificates.enabled=false' _datastore-etcd + $(MAKE) NAME=secondary NAMESPACE=kamaji-system EXTRA_ARGS='--set certManager.enabled=true --set certManager.ca.create=false --set certManager.ca.nameOverride=etcd-primary-ca --set certManager.issuerRef.kind=Issuer --set certManager.issuerRef.name=kamaji-selfsigned-issuer --set selfSignedCertificates.enabled=false' _datastore-etcd datastore-nats: helm $(HELM) repo add nats https://nats-io.github.io/k8s/helm/charts/ diff --git a/api/v1alpha1/tenantcontrolplane_types.go b/api/v1alpha1/tenantcontrolplane_types.go index 0a8037d..cda9511 100644 --- a/api/v1alpha1/tenantcontrolplane_types.go +++ b/api/v1alpha1/tenantcontrolplane_types.go @@ -355,6 +355,14 @@ func (p *Permissions) HasAnyLimitation() bool { return false } +// DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore. +type DataStoreOverride struct { + // Resource specifies which kubernetes resource to target. + Resource string `json:"resource,omitempty"` + // DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource. + DataStore string `json:"dataStore,omitempty"` +} + // TenantControlPlaneSpec defines the desired state of TenantControlPlane. // +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStore) || has(self.dataStore)", message="unsetting the dataStore is not supported" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStoreSchema) || has(self.dataStoreSchema)", message="unsetting the dataStoreSchema is not supported" @@ -389,8 +397,10 @@ type TenantControlPlaneSpec struct { // to the user to avoid clashes between different TenantControlPlanes. If not set upon creation, Kamaji will default the // DataStoreUsername by concatenating the namespace and name of the TenantControlPlane. // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the dataStoreUsername is not supported" - DataStoreUsername string `json:"dataStoreUsername,omitempty"` - ControlPlane ControlPlane `json:"controlPlane"` + DataStoreUsername string `json:"dataStoreUsername,omitempty"` + // DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores. + DataStoreOverrides []DataStoreOverride `json:"dataStoreOverrides,omitempty"` + ControlPlane ControlPlane `json:"controlPlane"` // Kubernetes specification for tenant control plane Kubernetes KubernetesSpec `json:"kubernetes"` // NetworkProfile specifies how the network is diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 478b149..d47eb1e 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -538,6 +538,21 @@ func (in *DataStoreList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataStoreOverride) DeepCopyInto(out *DataStoreOverride) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreOverride. +func (in *DataStoreOverride) DeepCopy() *DataStoreOverride { + if in == nil { + return nil + } + out := new(DataStoreOverride) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DataStoreSetupStatus) DeepCopyInto(out *DataStoreSetupStatus) { *out = *in @@ -1591,6 +1606,11 @@ func (in *TenantControlPlaneList) DeepCopyObject() runtime.Object { func (in *TenantControlPlaneSpec) DeepCopyInto(out *TenantControlPlaneSpec) { *out = *in out.WritePermissions = in.WritePermissions + if in.DataStoreOverrides != nil { + in, out := &in.DataStoreOverrides, &out.DataStoreOverrides + *out = make([]DataStoreOverride, len(*in)) + copy(*out, *in) + } in.ControlPlane.DeepCopyInto(&out.ControlPlane) in.Kubernetes.DeepCopyInto(&out.Kubernetes) in.NetworkProfile.DeepCopyInto(&out.NetworkProfile) diff --git a/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml b/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml index 9c3508b..e07834e 100644 --- a/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml +++ b/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml @@ -6985,6 +6985,19 @@ versions: Migration from one DataStore to another backed by the same Driver is possible. See: https://kamaji.clastix.io/guides/datastore-migration/ Migration from one DataStore to another backed by a different Driver is not supported. type: string + dataStoreOverrides: + description: DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores. + items: + description: DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore. + properties: + dataStore: + description: DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource. + type: string + resource: + description: Resource specifies which kubernetes resource to target. + type: string + type: object + type: array dataStoreSchema: description: |- DataStoreSchema allows to specify the name of the database (for relational DataStores) or the key prefix (for etcd). This diff --git a/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml b/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml index 18c1e86..b34de10 100644 --- a/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml +++ b/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml @@ -6993,6 +6993,19 @@ spec: Migration from one DataStore to another backed by the same Driver is possible. See: https://kamaji.clastix.io/guides/datastore-migration/ Migration from one DataStore to another backed by a different Driver is not supported. type: string + dataStoreOverrides: + description: DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores. + items: + description: DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore. + properties: + dataStore: + description: DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource. + type: string + resource: + description: Resource specifies which kubernetes resource to target. + type: string + type: object + type: array dataStoreSchema: description: |- DataStoreSchema allows to specify the name of the database (for relational DataStores) or the key prefix (for etcd). This diff --git a/controllers/resources.go b/controllers/resources.go index 31385f9..f2c3737 100644 --- a/controllers/resources.go +++ b/controllers/resources.go @@ -26,18 +26,20 @@ import ( ) type GroupResourceBuilderConfiguration struct { - client client.Client - log logr.Logger - tcpReconcilerConfig TenantControlPlaneReconcilerConfig - tenantControlPlane kamajiv1alpha1.TenantControlPlane - ExpirationThreshold time.Duration - Connection datastore.Connection - DataStore kamajiv1alpha1.DataStore - KamajiNamespace string - KamajiServiceAccount string - KamajiService string - KamajiMigrateImage string - DiscoveryClient discovery.DiscoveryInterface + client client.Client + log logr.Logger + tcpReconcilerConfig TenantControlPlaneReconcilerConfig + tenantControlPlane kamajiv1alpha1.TenantControlPlane + ExpirationThreshold time.Duration + Connection datastore.Connection + DataStore kamajiv1alpha1.DataStore + DataStoreOverrides []builder.DataStoreOverrides + DataStoreOverriedsConnections map[string]datastore.Connection + KamajiNamespace string + KamajiServiceAccount string + KamajiService string + KamajiMigrateImage string + DiscoveryClient discovery.DiscoveryInterface } type GroupDeletableResourceBuilderConfiguration struct { @@ -62,8 +64,9 @@ func GetResources(ctx context.Context, config GroupResourceBuilderConfiguration) resources = append(resources, getKubernetesCertificatesResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...) resources = append(resources, getKubeconfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...) resources = append(resources, getKubernetesStorageResources(config.client, config.Connection, config.DataStore, config.ExpirationThreshold)...) + resources = append(resources, getKubernetesAdditionalStorageResources(config.client, config.DataStoreOverriedsConnections, config.DataStoreOverrides, config.ExpirationThreshold)...) resources = append(resources, getKonnectivityServerRequirementsResources(config.client, config.ExpirationThreshold)...) - resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.DataStore)...) + resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.DataStore, config.DataStoreOverrides)...) resources = append(resources, getKonnectivityServerPatchResources(config.client)...) resources = append(resources, getDataStoreMigratingCleanup(config.client, config.KamajiNamespace)...) resources = append(resources, getKubernetesIngressResources(config.client)...) @@ -252,12 +255,42 @@ func getKubernetesStorageResources(c client.Client, dbConnection datastore.Conne } } -func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dataStore kamajiv1alpha1.DataStore) []resources.Resource { +func getKubernetesAdditionalStorageResources(c client.Client, dbConnections map[string]datastore.Connection, dataStoreOverrides []builder.DataStoreOverrides, threshold time.Duration) []resources.Resource { + res := make([]resources.Resource, 0, len(dataStoreOverrides)) + for _, dso := range dataStoreOverrides { + datastore := dso.DataStore + res = append(res, + &ds.MultiTenancy{ + DataStore: datastore, + }, + &ds.Config{ + Client: c, + ConnString: dbConnections[dso.Resource].GetConnectionString(), + DataStore: datastore, + IsOverride: true, + }, + &ds.Setup{ + Client: c, + Connection: dbConnections[dso.Resource], + DataStore: datastore, + }, + &ds.Certificate{ + Client: c, + DataStore: datastore, + CertExpirationThreshold: threshold, + }) + } + + return res +} + +func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dataStore kamajiv1alpha1.DataStore, dataStoreOverrides []builder.DataStoreOverrides) []resources.Resource { return []resources.Resource{ &resources.KubernetesDeploymentResource{ Client: c, DataStore: dataStore, KineContainerImage: tcpReconcilerConfig.KineContainerImage, + DataStoreOverrides: dataStoreOverrides, }, } } diff --git a/controllers/tenantcontrolplane_controller.go b/controllers/tenantcontrolplane_controller.go index 9e830d9..1651c99 100644 --- a/controllers/tenantcontrolplane_controller.go +++ b/controllers/tenantcontrolplane_controller.go @@ -37,6 +37,7 @@ import ( kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/controllers/finalizers" "github.com/clastix/kamaji/controllers/utils" + controlplanebuilder "github.com/clastix/kamaji/internal/builders/controlplane" "github.com/clastix/kamaji/internal/datastore" kamajierrors "github.com/clastix/kamaji/internal/errors" "github.com/clastix/kamaji/internal/resources" @@ -157,6 +158,25 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R } defer dsConnection.Close() + dso, err := r.dataStoreOverride(ctx, tenantControlPlane) + if err != nil { + log.Error(err, "cannot retrieve the DataStoreOverrides for the given instance") + + return ctrl.Result{}, err + } + dsoConnections := make(map[string]datastore.Connection, len(dso)) + for _, ds := range dso { + dsoConnection, err := datastore.NewStorageConnection(ctx, r.Client, ds.DataStore) + if err != nil { + log.Error(err, "cannot generate the DataStoreOverride connection for the given instance") + + return ctrl.Result{}, err + } + defer dsoConnection.Close() + + dsoConnections[ds.Resource] = dsoConnection + } + if markedToBeDeleted && controllerutil.ContainsFinalizer(tenantControlPlane, finalizers.DatastoreFinalizer) { log.Info("marked for deletion, performing clean-up") @@ -183,17 +203,19 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R } groupResourceBuilderConfiguration := GroupResourceBuilderConfiguration{ - client: r.Client, - log: log, - tcpReconcilerConfig: r.Config, - tenantControlPlane: *tenantControlPlane, - Connection: dsConnection, - DataStore: *ds, - KamajiNamespace: r.KamajiNamespace, - KamajiServiceAccount: r.KamajiServiceAccount, - KamajiService: r.KamajiService, - KamajiMigrateImage: r.KamajiMigrateImage, - DiscoveryClient: r.DiscoveryClient, + client: r.Client, + log: log, + tcpReconcilerConfig: r.Config, + tenantControlPlane: *tenantControlPlane, + Connection: dsConnection, + DataStore: *ds, + DataStoreOverrides: dso, + DataStoreOverriedsConnections: dsoConnections, + KamajiNamespace: r.KamajiNamespace, + KamajiServiceAccount: r.KamajiServiceAccount, + KamajiService: r.KamajiService, + KamajiMigrateImage: r.KamajiMigrateImage, + DiscoveryClient: r.DiscoveryClient, } registeredResources := GetResources(ctx, groupResourceBuilderConfiguration) @@ -362,3 +384,21 @@ func (r *TenantControlPlaneReconciler) dataStore(ctx context.Context, tenantCont return &ds, nil } + +func (r *TenantControlPlaneReconciler) dataStoreOverride(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) ([]controlplanebuilder.DataStoreOverrides, error) { + datastores := make([]controlplanebuilder.DataStoreOverrides, 0, len(tenantControlPlane.Spec.DataStoreOverrides)) + + for _, dso := range tenantControlPlane.Spec.DataStoreOverrides { + var ds kamajiv1alpha1.DataStore + if err := r.Client.Get(ctx, k8stypes.NamespacedName{Name: dso.DataStore}, &ds); err != nil { + return nil, errors.Wrap(err, "cannot retrieve *kamajiv1alpha.DataStore object") + } + if ds.Spec.Driver != kamajiv1alpha1.EtcdDriver { + return nil, errors.New("DataStoreOverrides can only use ETCD driver") + } + + datastores = append(datastores, controlplanebuilder.DataStoreOverrides{Resource: dso.Resource, DataStore: ds}) + } + + return datastores, nil +} diff --git a/docs/content/guides/datastore-overrides.md b/docs/content/guides/datastore-overrides.md new file mode 100644 index 0000000..15a77fe --- /dev/null +++ b/docs/content/guides/datastore-overrides.md @@ -0,0 +1,78 @@ +# Datastore Overrides + +Kamaji offers the possibility of having multiple ETCD clusters backing different resources of the k8s api server by configuring the [`--etcd-servers-overrides`](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/#:~:text=%2D%2Detcd%2Dservers%2Doverrides%20strings) flag. This feature can be useful for massive clusters to store resources with high churn in a dedicated ETCD cluster. + +## Install Datastores + +Create a self-signed cert-manager `ClusterIssuer`. +```bash +echo 'apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: self-signed +spec: + selfSigned: {} +' | kubectl apply -f - +``` + +Install two Datastores, a primary and a secondary that will be used for `/events` resources. +```bash + helm install etcd-primary clastix/kamaji-etcd -n kamaji-etcd --create-namespace \ + --set selfSignedCertificates.enabled=false \ + --set certManager.enabled=true \ + --set certManager.issuerRef.kind=ClusterIssuer \ + --set certManager.issuerRef.name=self-signed +``` + +For the secondary Datastore, use the cert-manager CA created by the `etcd-primary` helm release. +```bash + helm install etcd-secondary clastix/kamaji-etcd -n kamaji-etcd --create-namespace \ + --set selfSignedCertificates.enabled=false \ + --set certManager.enabled=true \ + --set certManager.ca.create=false \ + --set certManager.ca.nameOverride=etcd-primary-kamaji-etcd-ca \ + --set certManager.issuerRef.kind=ClusterIssuer \ + --set certManager.issuerRef.name=self-signed +``` + +## Create a Tenant Control Plane + +Using the `spec.dataStoreOverrides` field, Datastores different from the one used in `spec.dataStore` can be used to store specific resources. + +```bash +echo 'apiVersion: kamaji.clastix.io/v1alpha1 +kind: TenantControlPlane +metadata: + name: k8s-133 + labels: + tenant.clastix.io: k8s-133 +spec: + controlPlane: + deployment: + replicas: 2 + service: + serviceType: LoadBalancer + kubernetes: + version: "v1.33.1" + kubelet: + cgroupfs: systemd + dataStore: etcd-primary-kamaji-etcd + dataStoreOverrides: + - resource: "/events" # Store events in the secondary ETCD + dataStore: etcd-secondary-kamaji-etcd + networkProfile: + port: 6443 + addons: + coreDNS: {} + kubeProxy: {} + konnectivity: + server: + port: 8132 + agent: + mode: DaemonSet +' | k apply -f - +``` + +## Considerations + +Only built-in resources can be tagetted by `--etcd-servers-overrides`, it is currently not possible to target Custom Resources. diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index 13bbf51..4e3d61f 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -28500,6 +28500,13 @@ Migration from one DataStore to another backed by the same Driver is possible. S Migration from one DataStore to another backed by a different Driver is not supported.
false + + dataStoreOverrides + []object + + DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
+ + false dataStoreSchema string @@ -42214,6 +42221,38 @@ In case this value is set, kubeadm does not change automatically the version of +`TenantControlPlane.spec.dataStoreOverrides[index]` + + +DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
dataStorestring + DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
+ 		false
resourcestring + Resource specifies which kubernetes resource to target.
+ 		false
+ + `TenantControlPlane.spec.networkProfile` diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 3c6048a..6326092 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -76,6 +76,7 @@ nav: - guides/pausing.md - guides/write-permissions.md - guides/datastore-migration.md + - guides/datastore-overrides.md - guides/gitops.md - guides/console.md - guides/kubeconfig-generator.md diff --git a/e2e/tcp_datastore_overrides_test.go b/e2e/tcp_datastore_overrides_test.go new file mode 100644 index 0000000..5f2d476 --- /dev/null +++ b/e2e/tcp_datastore_overrides_test.go @@ -0,0 +1,68 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + pointer "k8s.io/utils/ptr" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +var _ = Describe("Deploy a TenantControlPlane resource with DataStoreOverrides", func() { + // Fill TenantControlPlane object + tcp := &kamajiv1alpha1.TenantControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tcp-clusterip", + Namespace: "default", + }, + Spec: kamajiv1alpha1.TenantControlPlaneSpec{ + DataStore: "etcd-primary", + DataStoreOverrides: []kamajiv1alpha1.DataStoreOverride{{ + Resource: "/events", + DataStore: "etcd-secondary", + }}, + ControlPlane: kamajiv1alpha1.ControlPlane{ + Deployment: kamajiv1alpha1.DeploymentSpec{ + Replicas: pointer.To(int32(1)), + }, + Service: kamajiv1alpha1.ServiceSpec{ + ServiceType: "ClusterIP", + }, + }, + NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{ + Address: "172.18.0.2", + }, + Kubernetes: kamajiv1alpha1.KubernetesSpec{ + Version: "v1.23.6", + Kubelet: kamajiv1alpha1.KubeletSpec{ + CGroupFS: "cgroupfs", + }, + AdmissionControllers: kamajiv1alpha1.AdmissionControllers{ + "LimitRanger", + "ResourceQuota", + }, + }, + Addons: kamajiv1alpha1.AddonsSpec{}, + }, + } + + // Create a TenantControlPlane resource into the cluster + JustBeforeEach(func() { + Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred()) + }) + + // Delete the TenantControlPlane resource after test is finished + JustAfterEach(func() { + Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed()) + }) + // Check if TenantControlPlane resource has been created + It("Should be Ready", func() { + StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady) + }) +}) diff --git a/internal/builders/controlplane/deployment.go b/internal/builders/controlplane/deployment.go index a998e55..8f72785 100644 --- a/internal/builders/controlplane/deployment.go +++ b/internal/builders/controlplane/deployment.go @@ -52,9 +52,15 @@ const ( kineInitContainerName = "chmod" ) +type DataStoreOverrides struct { + Resource string + DataStore kamajiv1alpha1.DataStore +} + type Deployment struct { KineContainerImage string DataStore kamajiv1alpha1.DataStore + DataStoreOverrides []DataStoreOverrides Client client.Client } @@ -711,11 +717,29 @@ func (d Deployment) buildKubeAPIServerCommand(tenantControlPlane kamajiv1alpha1. desiredArgs["--etcd-keyfile"] = "/etc/kubernetes/pki/etcd/server.key" } + if len(d.DataStoreOverrides) != 0 { + desiredArgs["--etcd-servers-overrides"] = d.etcdServersOverrides() + } + // Order matters, here: extraArgs could try to overwrite some arguments managed by Kamaji and that would be crucial. // Adding as first element of the array of maps, we're sure that these overrides will be sanitized by our configuration. return utilities.MergeMaps(current, desiredArgs, extraArgs) } +func (d Deployment) etcdServersOverrides() string { + dataStoreOverridesEndpoints := make([]string, 0, len(d.DataStoreOverrides)) + for _, dso := range d.DataStoreOverrides { + httpsEndpoints := make([]string, 0, len(dso.DataStore.Spec.Endpoints)) + + for _, ep := range dso.DataStore.Spec.Endpoints { + httpsEndpoints = append(httpsEndpoints, fmt.Sprintf("https://%s", ep)) + } + dataStoreOverridesEndpoints = append(dataStoreOverridesEndpoints, fmt.Sprintf("%s#%s", dso.Resource, strings.Join(httpsEndpoints, ";"))) + } + + return strings.Join(dataStoreOverridesEndpoints, ",") +} + func (d Deployment) secretProjection(secretName, certKeyName, keyName string) *corev1.SecretProjection { return &corev1.SecretProjection{ LocalObjectReference: corev1.LocalObjectReference{ diff --git a/internal/builders/controlplane/deployment_test.go b/internal/builders/controlplane/deployment_test.go new file mode 100644 index 0000000..4030ff4 --- /dev/null +++ b/internal/builders/controlplane/deployment_test.go @@ -0,0 +1,46 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package controlplane + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +var _ = Describe("Controlplane Deployment", func() { + var d Deployment + BeforeEach(func() { + d = Deployment{ + DataStoreOverrides: []DataStoreOverrides{{ + Resource: "/events", + DataStore: kamajiv1alpha1.DataStore{ + Spec: kamajiv1alpha1.DataStoreSpec{ + Endpoints: kamajiv1alpha1.Endpoints{"etcd-0", "etcd-1", "etcd-2"}, + }, + }, + }}, + } + }) + + Describe("DataStoreOverrides flag generation", func() { + It("should generate valid --etcd-servers-overrides value", func() { + etcdSerVersOverrides := d.etcdServersOverrides() + Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2")) + }) + It("should generate valid --etcd-servers-overrides value with 2 DataStoreOverrides", func() { + d.DataStoreOverrides = append(d.DataStoreOverrides, DataStoreOverrides{ + Resource: "/pods", + DataStore: kamajiv1alpha1.DataStore{ + Spec: kamajiv1alpha1.DataStoreSpec{ + Endpoints: kamajiv1alpha1.Endpoints{"etcd-3", "etcd-4", "etcd-5"}, + }, + }, + }) + etcdSerVersOverrides := d.etcdServersOverrides() + Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2,/pods#https://etcd-3;https://etcd-4;https://etcd-5")) + }) + }) +}) diff --git a/internal/resources/datastore/datastore_storage_config.go b/internal/resources/datastore/datastore_storage_config.go index 492a498..c786547 100644 --- a/internal/resources/datastore/datastore_storage_config.go +++ b/internal/resources/datastore/datastore_storage_config.go @@ -30,6 +30,7 @@ type Config struct { Client client.Client ConnString string DataStore kamajiv1alpha1.DataStore + IsOverride bool } func (r *Config) GetHistogram() prometheus.Histogram { @@ -103,10 +104,12 @@ func (r *Config) GetName() string { } func (r *Config) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { - tenantControlPlane.Status.Storage.Driver = string(r.DataStore.Spec.Driver) - tenantControlPlane.Status.Storage.DataStoreName = r.DataStore.GetName() - tenantControlPlane.Status.Storage.Config.SecretName = r.resource.GetName() - tenantControlPlane.Status.Storage.Config.Checksum = utilities.GetObjectChecksum(r.resource) + if !r.IsOverride { + tenantControlPlane.Status.Storage.Driver = string(r.DataStore.Spec.Driver) + tenantControlPlane.Status.Storage.DataStoreName = r.DataStore.GetName() + tenantControlPlane.Status.Storage.Config.SecretName = r.resource.GetName() + tenantControlPlane.Status.Storage.Config.Checksum = utilities.GetObjectChecksum(r.resource) + } return nil } diff --git a/internal/resources/k8s_deployment_resource.go b/internal/resources/k8s_deployment_resource.go index ff51508..25134fb 100644 --- a/internal/resources/k8s_deployment_resource.go +++ b/internal/resources/k8s_deployment_resource.go @@ -22,6 +22,7 @@ type KubernetesDeploymentResource struct { resource *appsv1.Deployment Client client.Client DataStore kamajiv1alpha1.DataStore + DataStoreOverrides []builder.DataStoreOverrides KineContainerImage string } @@ -66,6 +67,7 @@ func (r *KubernetesDeploymentResource) mutate(ctx context.Context, tenantControl Client: r.Client, DataStore: r.DataStore, KineContainerImage: r.KineContainerImage, + DataStoreOverrides: r.DataStoreOverrides, }).Build(ctx, r.resource, *tenantControlPlane) return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) diff --git a/internal/webhook/handlers/tcp_datastore.go b/internal/webhook/handlers/tcp_datastore.go index c049bdb..73c26ae 100644 --- a/internal/webhook/handlers/tcp_datastore.go +++ b/internal/webhook/handlers/tcp_datastore.go @@ -30,7 +30,7 @@ func (t TenantControlPlaneDataStore) OnCreate(object runtime.Object) AdmissionRe return nil, t.check(ctx, tcp.Spec.DataStore) } - return nil, nil + return nil, t.checkDataStoreOverrides(ctx, tcp) } } @@ -61,3 +61,19 @@ func (t TenantControlPlaneDataStore) check(ctx context.Context, dataStoreName st return nil } + +func (t TenantControlPlaneDataStore) checkDataStoreOverrides(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) error { + overrideCheck := make(map[string]struct{}, 0) + for _, ds := range tcp.Spec.DataStoreOverrides { + if _, exists := overrideCheck[ds.Resource]; !exists { + overrideCheck[ds.Resource] = struct{}{} + } else { + return fmt.Errorf("duplicate resource override in Spec.DataStoreOverrides") + } + if err := t.check(ctx, ds.DataStore); err != nil { + return err + } + } + + return nil +} diff --git a/internal/webhook/handlers/tcp_datastore_test.go b/internal/webhook/handlers/tcp_datastore_test.go new file mode 100644 index 0000000..8a9f5f2 --- /dev/null +++ b/internal/webhook/handlers/tcp_datastore_test.go @@ -0,0 +1,94 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package handlers + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +var _ = Describe("TCP Datastore webhook", func() { + var ( + ctx context.Context + t TenantControlPlaneDataStore + tcp *kamajiv1alpha1.TenantControlPlane + ) + BeforeEach(func() { + scheme := runtime.NewScheme() + utilruntime.Must(kamajiv1alpha1.AddToScheme(scheme)) + + ctx = context.Background() + t = TenantControlPlaneDataStore{ + Client: fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(&kamajiv1alpha1.DataStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + }, &kamajiv1alpha1.DataStore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bar", + }, + }).Build(), + } + tcp = &kamajiv1alpha1.TenantControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tcp", + Namespace: "default", + }, + Spec: kamajiv1alpha1.TenantControlPlaneSpec{}, + } + }) + Describe("validation should succeed without DataStoreOverrides", func() { + It("should validate TCP without DataStoreOverrides", func() { + err := t.checkDataStoreOverrides(ctx, tcp) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Describe("validation should fail with duplicate resources in DataStoreOverrides", func() { + It("should fail to validate TCP with duplicate resources in DataStoreOverrides", func() { + tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{ + Resource: "/event", + DataStore: "foo", + }, { + Resource: "/event", + DataStore: "bar", + }} + err := t.checkDataStoreOverrides(ctx, tcp) + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("validation should succeed with valid DataStoreOverrides", func() { + It("should validate TCP with valid DataStoreOverrides", func() { + tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{ + Resource: "/leases", + DataStore: "foo", + }, { + Resource: "/event", + DataStore: "bar", + }} + err := t.checkDataStoreOverrides(ctx, tcp) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Describe("validation should fail with nonexistent DataStoreOverrides", func() { + It("should fail to validate TCP with nonexistent DataStoreOverrides", func() { + tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{ + Resource: "/leases", + DataStore: "baz", + }} + err := t.checkDataStoreOverrides(ctx, tcp) + Expect(err).To(HaveOccurred()) + }) + }) +}) diff --git a/internal/webhook/handlers/tcp_deployment.go b/internal/webhook/handlers/tcp_deployment.go index bca7016..959d1ed 100644 --- a/internal/webhook/handlers/tcp_deployment.go +++ b/internal/webhook/handlers/tcp_deployment.go @@ -67,6 +67,20 @@ func (t TenantControlPlaneDeployment) OnUpdate(newObject runtime.Object, oldObje } t.DeploymentBuilder.DataStore = ds + dataStoreOverrides := make([]controlplane.DataStoreOverrides, 0, len(tcp.Spec.DataStoreOverrides)) + + for _, dso := range tcp.Spec.DataStoreOverrides { + ds := kamajiv1alpha1.DataStore{} + if err := t.Client.Get(ctx, types.NamespacedName{Name: dso.DataStore}, &ds); err != nil { + return nil, err + } + dataStoreOverrides = append(dataStoreOverrides, controlplane.DataStoreOverrides{ + Resource: dso.Resource, + DataStore: ds, + }) + } + t.DeploymentBuilder.DataStoreOverrides = dataStoreOverrides + deployment := appsv1.Deployment{} deployment.Name = tcp.Name deployment.Namespace = tcp.Namespace