feat: add support for multiple Datastores (#961)

* feat: add support for multiple Datastores

* docs: add guide for datastore overrides

* feat(datastore): add e2e test for dataStoreOverrides

* ci: reclaim disk space from runner to fix flaky tests

.github/workflows/e2e.yaml

@@ -41,6 +41,9 @@ jobs:
       - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
+      - name: reclaim disk space from runner
+        run: |
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
       - run: |
           sudo apt-get update
           sudo apt-get install -y golang-cfssl

Makefile

@@ -177,7 +177,7 @@ datastore-postgres:
 	$(MAKE) NAME=gold _datastore-postgres
 
 _datastore-etcd:
-	$(HELM) upgrade --install etcd-$(NAME) clastix/kamaji-etcd --create-namespace -n etcd-system --set datastore.enabled=true --set fullnameOverride=etcd-$(NAME)
+	$(HELM) upgrade --install etcd-$(NAME) clastix/kamaji-etcd --create-namespace -n $(NAMESPACE) --set datastore.enabled=true --set fullnameOverride=etcd-$(NAME) $(EXTRA_ARGS)
 
 _datastore-nats:
 	$(MAKE) NAME=$(NAME) NAMESPACE=nats-system -C deploy/kine/nats nats
@@ -186,9 +186,11 @@ _datastore-nats:
 datastore-etcd: helm
 	$(HELM) repo add clastix https://clastix.github.io/charts
 	$(HELM) repo update
-	$(MAKE) NAME=bronze _datastore-etcd
-	$(MAKE) NAME=silver _datastore-etcd
-	$(MAKE) NAME=gold _datastore-etcd
+	$(MAKE) NAME=bronze NAMESPACE=etcd-system _datastore-etcd
+	$(MAKE) NAME=silver NAMESPACE=etcd-system _datastore-etcd
+	$(MAKE) NAME=gold NAMESPACE=etcd-system _datastore-etcd
+	$(MAKE) NAME=primary NAMESPACE=kamaji-system EXTRA_ARGS='--set certManager.enabled=true --set certManager.issuerRef.kind=Issuer --set certManager.issuerRef.name=kamaji-selfsigned-issuer --set selfSignedCertificates.enabled=false' _datastore-etcd
+	$(MAKE) NAME=secondary NAMESPACE=kamaji-system EXTRA_ARGS='--set certManager.enabled=true --set certManager.ca.create=false --set certManager.ca.nameOverride=etcd-primary-ca --set certManager.issuerRef.kind=Issuer --set certManager.issuerRef.name=kamaji-selfsigned-issuer --set selfSignedCertificates.enabled=false' _datastore-etcd
 
 datastore-nats: helm
 	$(HELM) repo add nats https://nats-io.github.io/k8s/helm/charts/

api/v1alpha1/tenantcontrolplane_types.go

@@ -355,6 +355,14 @@ func (p *Permissions) HasAnyLimitation() bool {
 	return false
 }
 
+// DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
+type DataStoreOverride struct {
+	// Resource specifies which kubernetes resource to target.
+	Resource string `json:"resource,omitempty"`
+	// DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
+	DataStore string `json:"dataStore,omitempty"`
+}
+
 // TenantControlPlaneSpec defines the desired state of TenantControlPlane.
 // +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStore) || has(self.dataStore)", message="unsetting the dataStore is not supported"
 // +kubebuilder:validation:XValidation:rule="!has(oldSelf.dataStoreSchema) || has(self.dataStoreSchema)", message="unsetting the dataStoreSchema is not supported"
@@ -389,8 +397,10 @@ type TenantControlPlaneSpec struct {
 	// to the user to avoid clashes between different TenantControlPlanes. If not set upon creation, Kamaji will default the
 	// DataStoreUsername by concatenating the namespace and name of the TenantControlPlane.
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="changing the dataStoreUsername is not supported"
-	DataStoreUsername string       `json:"dataStoreUsername,omitempty"`
-	ControlPlane      ControlPlane `json:"controlPlane"`
+	DataStoreUsername string `json:"dataStoreUsername,omitempty"`
+	// DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
+	DataStoreOverrides []DataStoreOverride `json:"dataStoreOverrides,omitempty"`
+	ControlPlane       ControlPlane        `json:"controlPlane"`
 	// Kubernetes specification for tenant control plane
 	Kubernetes KubernetesSpec `json:"kubernetes"`
 	// NetworkProfile specifies how the network is

api/v1alpha1/zz_generated.deepcopy.go

@@ -538,6 +538,21 @@ func (in *DataStoreList) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DataStoreOverride) DeepCopyInto(out *DataStoreOverride) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataStoreOverride.
+func (in *DataStoreOverride) DeepCopy() *DataStoreOverride {
+	if in == nil {
+		return nil
+	}
+	out := new(DataStoreOverride)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DataStoreSetupStatus) DeepCopyInto(out *DataStoreSetupStatus) {
 	*out = *in
@@ -1591,6 +1606,11 @@ func (in *TenantControlPlaneList) DeepCopyObject() runtime.Object {
 func (in *TenantControlPlaneSpec) DeepCopyInto(out *TenantControlPlaneSpec) {
 	*out = *in
 	out.WritePermissions = in.WritePermissions
+	if in.DataStoreOverrides != nil {
+		in, out := &in.DataStoreOverrides, &out.DataStoreOverrides
+		*out = make([]DataStoreOverride, len(*in))
+		copy(*out, *in)
+	}
 	in.ControlPlane.DeepCopyInto(&out.ControlPlane)
 	in.Kubernetes.DeepCopyInto(&out.Kubernetes)
 	in.NetworkProfile.DeepCopyInto(&out.NetworkProfile)

charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml

@@ -6985,6 +6985,19 @@ versions:
                   Migration from one DataStore to another backed by the same Driver is possible. See: https://kamaji.clastix.io/guides/datastore-migration/
                   Migration from one DataStore to another backed by a different Driver is not supported.
                 type: string
+              dataStoreOverrides:
+                description: DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
+                items:
+                  description: DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
+                  properties:
+                    dataStore:
+                      description: DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
+                      type: string
+                    resource:
+                      description: Resource specifies which kubernetes resource to target.
+                      type: string
+                  type: object
+                type: array
               dataStoreSchema:
                 description: |-
                   DataStoreSchema allows to specify the name of the database (for relational DataStores) or the key prefix (for etcd). This

charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml

@@ -6993,6 +6993,19 @@ spec:
                     Migration from one DataStore to another backed by the same Driver is possible. See: https://kamaji.clastix.io/guides/datastore-migration/
                     Migration from one DataStore to another backed by a different Driver is not supported.
                   type: string
+                dataStoreOverrides:
+                  description: DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.
+                  items:
+                    description: DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
+                    properties:
+                      dataStore:
+                        description: DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.
+                        type: string
+                      resource:
+                        description: Resource specifies which kubernetes resource to target.
+                        type: string
+                    type: object
+                  type: array
                 dataStoreSchema:
                   description: |-
                     DataStoreSchema allows to specify the name of the database (for relational DataStores) or the key prefix (for etcd). This

controllers/resources.go

@@ -26,18 +26,20 @@ import (
 )
 
 type GroupResourceBuilderConfiguration struct {
-	client               client.Client
-	log                  logr.Logger
-	tcpReconcilerConfig  TenantControlPlaneReconcilerConfig
-	tenantControlPlane   kamajiv1alpha1.TenantControlPlane
-	ExpirationThreshold  time.Duration
-	Connection           datastore.Connection
-	DataStore            kamajiv1alpha1.DataStore
-	KamajiNamespace      string
-	KamajiServiceAccount string
-	KamajiService        string
-	KamajiMigrateImage   string
-	DiscoveryClient      discovery.DiscoveryInterface
+	client                        client.Client
+	log                           logr.Logger
+	tcpReconcilerConfig           TenantControlPlaneReconcilerConfig
+	tenantControlPlane            kamajiv1alpha1.TenantControlPlane
+	ExpirationThreshold           time.Duration
+	Connection                    datastore.Connection
+	DataStore                     kamajiv1alpha1.DataStore
+	DataStoreOverrides            []builder.DataStoreOverrides
+	DataStoreOverriedsConnections map[string]datastore.Connection
+	KamajiNamespace               string
+	KamajiServiceAccount          string
+	KamajiService                 string
+	KamajiMigrateImage            string
+	DiscoveryClient               discovery.DiscoveryInterface
 }
 
 type GroupDeletableResourceBuilderConfiguration struct {
@@ -62,8 +64,9 @@ func GetResources(ctx context.Context, config GroupResourceBuilderConfiguration)
 	resources = append(resources, getKubernetesCertificatesResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
 	resources = append(resources, getKubeconfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...)
 	resources = append(resources, getKubernetesStorageResources(config.client, config.Connection, config.DataStore, config.ExpirationThreshold)...)
+	resources = append(resources, getKubernetesAdditionalStorageResources(config.client, config.DataStoreOverriedsConnections, config.DataStoreOverrides, config.ExpirationThreshold)...)
 	resources = append(resources, getKonnectivityServerRequirementsResources(config.client, config.ExpirationThreshold)...)
-	resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.DataStore)...)
+	resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.DataStore, config.DataStoreOverrides)...)
 	resources = append(resources, getKonnectivityServerPatchResources(config.client)...)
 	resources = append(resources, getDataStoreMigratingCleanup(config.client, config.KamajiNamespace)...)
 	resources = append(resources, getKubernetesIngressResources(config.client)...)
@@ -252,12 +255,42 @@ func getKubernetesStorageResources(c client.Client, dbConnection datastore.Conne
 	}
 }
 
-func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dataStore kamajiv1alpha1.DataStore) []resources.Resource {
+func getKubernetesAdditionalStorageResources(c client.Client, dbConnections map[string]datastore.Connection, dataStoreOverrides []builder.DataStoreOverrides, threshold time.Duration) []resources.Resource {
+	res := make([]resources.Resource, 0, len(dataStoreOverrides))
+	for _, dso := range dataStoreOverrides {
+		datastore := dso.DataStore
+		res = append(res,
+			&ds.MultiTenancy{
+				DataStore: datastore,
+			},
+			&ds.Config{
+				Client:     c,
+				ConnString: dbConnections[dso.Resource].GetConnectionString(),
+				DataStore:  datastore,
+				IsOverride: true,
+			},
+			&ds.Setup{
+				Client:     c,
+				Connection: dbConnections[dso.Resource],
+				DataStore:  datastore,
+			},
+			&ds.Certificate{
+				Client:                  c,
+				DataStore:               datastore,
+				CertExpirationThreshold: threshold,
+			})
+	}
+
+	return res
+}
+
+func getKubernetesDeploymentResources(c client.Client, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, dataStore kamajiv1alpha1.DataStore, dataStoreOverrides []builder.DataStoreOverrides) []resources.Resource {
 	return []resources.Resource{
 		&resources.KubernetesDeploymentResource{
 			Client:             c,
 			DataStore:          dataStore,
 			KineContainerImage: tcpReconcilerConfig.KineContainerImage,
+			DataStoreOverrides: dataStoreOverrides,
 		},
 	}
 }

controllers/tenantcontrolplane_controller.go

@@ -37,6 +37,7 @@ import (
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/controllers/finalizers"
 	"github.com/clastix/kamaji/controllers/utils"
+	controlplanebuilder "github.com/clastix/kamaji/internal/builders/controlplane"
 	"github.com/clastix/kamaji/internal/datastore"
 	kamajierrors "github.com/clastix/kamaji/internal/errors"
 	"github.com/clastix/kamaji/internal/resources"
@@ -157,6 +158,25 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 	}
 	defer dsConnection.Close()
 
+	dso, err := r.dataStoreOverride(ctx, tenantControlPlane)
+	if err != nil {
+		log.Error(err, "cannot retrieve the DataStoreOverrides for the given instance")
+
+		return ctrl.Result{}, err
+	}
+	dsoConnections := make(map[string]datastore.Connection, len(dso))
+	for _, ds := range dso {
+		dsoConnection, err := datastore.NewStorageConnection(ctx, r.Client, ds.DataStore)
+		if err != nil {
+			log.Error(err, "cannot generate the DataStoreOverride connection for the given instance")
+
+			return ctrl.Result{}, err
+		}
+		defer dsoConnection.Close()
+
+		dsoConnections[ds.Resource] = dsoConnection
+	}
+
 	if markedToBeDeleted && controllerutil.ContainsFinalizer(tenantControlPlane, finalizers.DatastoreFinalizer) {
 		log.Info("marked for deletion, performing clean-up")
 
@@ -183,17 +203,19 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
 	}
 
 	groupResourceBuilderConfiguration := GroupResourceBuilderConfiguration{
-		client:               r.Client,
-		log:                  log,
-		tcpReconcilerConfig:  r.Config,
-		tenantControlPlane:   *tenantControlPlane,
-		Connection:           dsConnection,
-		DataStore:            *ds,
-		KamajiNamespace:      r.KamajiNamespace,
-		KamajiServiceAccount: r.KamajiServiceAccount,
-		KamajiService:        r.KamajiService,
-		KamajiMigrateImage:   r.KamajiMigrateImage,
-		DiscoveryClient:      r.DiscoveryClient,
+		client:                        r.Client,
+		log:                           log,
+		tcpReconcilerConfig:           r.Config,
+		tenantControlPlane:            *tenantControlPlane,
+		Connection:                    dsConnection,
+		DataStore:                     *ds,
+		DataStoreOverrides:            dso,
+		DataStoreOverriedsConnections: dsoConnections,
+		KamajiNamespace:               r.KamajiNamespace,
+		KamajiServiceAccount:          r.KamajiServiceAccount,
+		KamajiService:                 r.KamajiService,
+		KamajiMigrateImage:            r.KamajiMigrateImage,
+		DiscoveryClient:               r.DiscoveryClient,
 	}
 	registeredResources := GetResources(ctx, groupResourceBuilderConfiguration)
 
@@ -362,3 +384,21 @@ func (r *TenantControlPlaneReconciler) dataStore(ctx context.Context, tenantCont
 
 	return &ds, nil
 }
+
+func (r *TenantControlPlaneReconciler) dataStoreOverride(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) ([]controlplanebuilder.DataStoreOverrides, error) {
+	datastores := make([]controlplanebuilder.DataStoreOverrides, 0, len(tenantControlPlane.Spec.DataStoreOverrides))
+
+	for _, dso := range tenantControlPlane.Spec.DataStoreOverrides {
+		var ds kamajiv1alpha1.DataStore
+		if err := r.Client.Get(ctx, k8stypes.NamespacedName{Name: dso.DataStore}, &ds); err != nil {
+			return nil, errors.Wrap(err, "cannot retrieve *kamajiv1alpha.DataStore object")
+		}
+		if ds.Spec.Driver != kamajiv1alpha1.EtcdDriver {
+			return nil, errors.New("DataStoreOverrides can only use ETCD driver")
+		}
+
+		datastores = append(datastores, controlplanebuilder.DataStoreOverrides{Resource: dso.Resource, DataStore: ds})
+	}
+
+	return datastores, nil
+}

docs/content/guides/datastore-overrides.md

@@ -0,0 +1,78 @@
+# Datastore Overrides
+
+Kamaji offers the possibility of having multiple ETCD clusters backing different resources of the k8s api server by configuring the [`--etcd-servers-overrides`](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/#:~:text=%2D%2Detcd%2Dservers%2Doverrides%20strings) flag. This feature can be useful for massive clusters to store resources with high churn in a dedicated ETCD cluster.
+
+## Install Datastores
+
+Create a self-signed cert-manager `ClusterIssuer`.
+```bash
+echo 'apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: self-signed
+spec:
+  selfSigned: {}
+' | kubectl apply -f -
+```
+
+Install two Datastores, a primary and a secondary that will be used for `/events` resources.
+```bash
+ helm install etcd-primary clastix/kamaji-etcd -n kamaji-etcd --create-namespace \
+   --set selfSignedCertificates.enabled=false \
+   --set certManager.enabled=true \
+   --set certManager.issuerRef.kind=ClusterIssuer \
+   --set certManager.issuerRef.name=self-signed
+```
+
+For the secondary Datastore, use the cert-manager CA created by the `etcd-primary` helm release.
+```bash
+ helm install etcd-secondary clastix/kamaji-etcd -n kamaji-etcd --create-namespace \
+   --set selfSignedCertificates.enabled=false \
+   --set certManager.enabled=true \
+   --set certManager.ca.create=false \
+   --set certManager.ca.nameOverride=etcd-primary-kamaji-etcd-ca \
+   --set certManager.issuerRef.kind=ClusterIssuer \
+   --set certManager.issuerRef.name=self-signed
+```
+
+## Create a Tenant Control Plane
+
+Using the `spec.dataStoreOverrides` field, Datastores different from the one used in `spec.dataStore` can be used to store specific resources.
+
+```bash
+echo 'apiVersion: kamaji.clastix.io/v1alpha1
+kind: TenantControlPlane
+metadata:
+  name: k8s-133
+  labels:
+    tenant.clastix.io: k8s-133
+spec:
+  controlPlane:
+    deployment:
+      replicas: 2
+    service:
+      serviceType: LoadBalancer
+  kubernetes:
+    version: "v1.33.1"
+    kubelet:
+      cgroupfs: systemd
+  dataStore: etcd-primary-kamaji-etcd
+  dataStoreOverrides:
+    - resource: "/events" # Store events in the secondary ETCD
+      dataStore: etcd-secondary-kamaji-etcd
+  networkProfile:
+    port: 6443
+  addons:
+    coreDNS: {}
+    kubeProxy: {}
+    konnectivity:
+      server:
+        port: 8132
+      agent:
+        mode: DaemonSet
+' | k apply -f -
+```
+
+## Considerations
+
+Only built-in resources can be tagetted by `--etcd-servers-overrides`, it is currently not possible to target Custom Resources.

docs/content/reference/api.md

@@ -28500,6 +28500,13 @@ Migration from one DataStore to another backed by the same Driver is possible. S
 Migration from one DataStore to another backed by a different Driver is not supported.<br/>
         </td>
         <td>false</td>
+      </tr><tr>
+        <td><b><a href="#tenantcontrolplanespecdatastoreoverridesindex">dataStoreOverrides</a></b></td>
+        <td>[]object</td>
+        <td>
+          DataStoreOverride defines which kubernetes resources will be stored in dedicated datastores.<br/>
+        </td>
+        <td>false</td>
       </tr><tr>
         <td><b>dataStoreSchema</b></td>
         <td>string</td>
@@ -42214,6 +42221,38 @@ In case this value is set, kubeadm does not change automatically the version of
 </table>
 
 
+<span id="tenantcontrolplanespecdatastoreoverridesindex">`TenantControlPlane.spec.dataStoreOverrides[index]`</span>
+
+
+DataStoreOverride defines which kubernetes resource will be stored in a dedicated datastore.
+
+<table>
+    <thead>
+        <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+            <th>Required</th>
+        </tr>
+    </thead>
+    <tbody><tr>
+        <td><b>dataStore</b></td>
+        <td>string</td>
+        <td>
+          DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Resource.<br/>
+        </td>
+        <td>false</td>
+      </tr><tr>
+        <td><b>resource</b></td>
+        <td>string</td>
+        <td>
+          Resource specifies which kubernetes resource to target.<br/>
+        </td>
+        <td>false</td>
+      </tr></tbody>
+</table>
+
+
 <span id="tenantcontrolplanespecnetworkprofile">`TenantControlPlane.spec.networkProfile`</span>
 
 

docs/mkdocs.yml

@@ -76,6 +76,7 @@ nav:
   - guides/pausing.md
   - guides/write-permissions.md
   - guides/datastore-migration.md
+  - guides/datastore-overrides.md
   - guides/gitops.md
   - guides/console.md
   - guides/kubeconfig-generator.md

e2e/tcp_datastore_overrides_test.go

@@ -0,0 +1,68 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	pointer "k8s.io/utils/ptr"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Deploy a TenantControlPlane resource with DataStoreOverrides", func() {
+	// Fill TenantControlPlane object
+	tcp := &kamajiv1alpha1.TenantControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "tcp-clusterip",
+			Namespace: "default",
+		},
+		Spec: kamajiv1alpha1.TenantControlPlaneSpec{
+			DataStore: "etcd-primary",
+			DataStoreOverrides: []kamajiv1alpha1.DataStoreOverride{{
+				Resource:  "/events",
+				DataStore: "etcd-secondary",
+			}},
+			ControlPlane: kamajiv1alpha1.ControlPlane{
+				Deployment: kamajiv1alpha1.DeploymentSpec{
+					Replicas: pointer.To(int32(1)),
+				},
+				Service: kamajiv1alpha1.ServiceSpec{
+					ServiceType: "ClusterIP",
+				},
+			},
+			NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{
+				Address: "172.18.0.2",
+			},
+			Kubernetes: kamajiv1alpha1.KubernetesSpec{
+				Version: "v1.23.6",
+				Kubelet: kamajiv1alpha1.KubeletSpec{
+					CGroupFS: "cgroupfs",
+				},
+				AdmissionControllers: kamajiv1alpha1.AdmissionControllers{
+					"LimitRanger",
+					"ResourceQuota",
+				},
+			},
+			Addons: kamajiv1alpha1.AddonsSpec{},
+		},
+	}
+
+	// Create a TenantControlPlane resource into the cluster
+	JustBeforeEach(func() {
+		Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred())
+	})
+
+	// Delete the TenantControlPlane resource after test is finished
+	JustAfterEach(func() {
+		Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed())
+	})
+	// Check if TenantControlPlane resource has been created
+	It("Should be Ready", func() {
+		StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady)
+	})
+})

internal/builders/controlplane/deployment.go

@@ -52,9 +52,15 @@ const (
 	kineInitContainerName     = "chmod"
 )
 
+type DataStoreOverrides struct {
+	Resource  string
+	DataStore kamajiv1alpha1.DataStore
+}
+
 type Deployment struct {
 	KineContainerImage string
 	DataStore          kamajiv1alpha1.DataStore
+	DataStoreOverrides []DataStoreOverrides
 	Client             client.Client
 }
 
@@ -711,11 +717,29 @@ func (d Deployment) buildKubeAPIServerCommand(tenantControlPlane kamajiv1alpha1.
 		desiredArgs["--etcd-keyfile"] = "/etc/kubernetes/pki/etcd/server.key"
 	}
 
+	if len(d.DataStoreOverrides) != 0 {
+		desiredArgs["--etcd-servers-overrides"] = d.etcdServersOverrides()
+	}
+
 	// Order matters, here: extraArgs could try to overwrite some arguments managed by Kamaji and that would be crucial.
 	// Adding as first element of the array of maps, we're sure that these overrides will be sanitized by our configuration.
 	return utilities.MergeMaps(current, desiredArgs, extraArgs)
 }
 
+func (d Deployment) etcdServersOverrides() string {
+	dataStoreOverridesEndpoints := make([]string, 0, len(d.DataStoreOverrides))
+	for _, dso := range d.DataStoreOverrides {
+		httpsEndpoints := make([]string, 0, len(dso.DataStore.Spec.Endpoints))
+
+		for _, ep := range dso.DataStore.Spec.Endpoints {
+			httpsEndpoints = append(httpsEndpoints, fmt.Sprintf("https://%s", ep))
+		}
+		dataStoreOverridesEndpoints = append(dataStoreOverridesEndpoints, fmt.Sprintf("%s#%s", dso.Resource, strings.Join(httpsEndpoints, ";")))
+	}
+
+	return strings.Join(dataStoreOverridesEndpoints, ",")
+}
+
 func (d Deployment) secretProjection(secretName, certKeyName, keyName string) *corev1.SecretProjection {
 	return &corev1.SecretProjection{
 		LocalObjectReference: corev1.LocalObjectReference{

internal/builders/controlplane/deployment_test.go

@@ -0,0 +1,46 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package controlplane
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("Controlplane Deployment", func() {
+	var d Deployment
+	BeforeEach(func() {
+		d = Deployment{
+			DataStoreOverrides: []DataStoreOverrides{{
+				Resource: "/events",
+				DataStore: kamajiv1alpha1.DataStore{
+					Spec: kamajiv1alpha1.DataStoreSpec{
+						Endpoints: kamajiv1alpha1.Endpoints{"etcd-0", "etcd-1", "etcd-2"},
+					},
+				},
+			}},
+		}
+	})
+
+	Describe("DataStoreOverrides flag generation", func() {
+		It("should generate valid --etcd-servers-overrides value", func() {
+			etcdSerVersOverrides := d.etcdServersOverrides()
+			Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2"))
+		})
+		It("should generate valid --etcd-servers-overrides value with 2 DataStoreOverrides", func() {
+			d.DataStoreOverrides = append(d.DataStoreOverrides, DataStoreOverrides{
+				Resource: "/pods",
+				DataStore: kamajiv1alpha1.DataStore{
+					Spec: kamajiv1alpha1.DataStoreSpec{
+						Endpoints: kamajiv1alpha1.Endpoints{"etcd-3", "etcd-4", "etcd-5"},
+					},
+				},
+			})
+			etcdSerVersOverrides := d.etcdServersOverrides()
+			Expect(etcdSerVersOverrides).To(Equal("/events#https://etcd-0;https://etcd-1;https://etcd-2,/pods#https://etcd-3;https://etcd-4;https://etcd-5"))
+		})
+	})
+})

internal/resources/datastore/datastore_storage_config.go

@@ -30,6 +30,7 @@ type Config struct {
 	Client     client.Client
 	ConnString string
 	DataStore  kamajiv1alpha1.DataStore
+	IsOverride bool
 }
 
 func (r *Config) GetHistogram() prometheus.Histogram {
@@ -103,10 +104,12 @@ func (r *Config) GetName() string {
 }
 
 func (r *Config) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
-	tenantControlPlane.Status.Storage.Driver = string(r.DataStore.Spec.Driver)
-	tenantControlPlane.Status.Storage.DataStoreName = r.DataStore.GetName()
-	tenantControlPlane.Status.Storage.Config.SecretName = r.resource.GetName()
-	tenantControlPlane.Status.Storage.Config.Checksum = utilities.GetObjectChecksum(r.resource)
+	if !r.IsOverride {
+		tenantControlPlane.Status.Storage.Driver = string(r.DataStore.Spec.Driver)
+		tenantControlPlane.Status.Storage.DataStoreName = r.DataStore.GetName()
+		tenantControlPlane.Status.Storage.Config.SecretName = r.resource.GetName()
+		tenantControlPlane.Status.Storage.Config.Checksum = utilities.GetObjectChecksum(r.resource)
+	}
 
 	return nil
 }

internal/resources/k8s_deployment_resource.go

@@ -22,6 +22,7 @@ type KubernetesDeploymentResource struct {
 	resource           *appsv1.Deployment
 	Client             client.Client
 	DataStore          kamajiv1alpha1.DataStore
+	DataStoreOverrides []builder.DataStoreOverrides
 	KineContainerImage string
 }
 
@@ -66,6 +67,7 @@ func (r *KubernetesDeploymentResource) mutate(ctx context.Context, tenantControl
 			Client:             r.Client,
 			DataStore:          r.DataStore,
 			KineContainerImage: r.KineContainerImage,
+			DataStoreOverrides: r.DataStoreOverrides,
 		}).Build(ctx, r.resource, *tenantControlPlane)
 
 		return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())

internal/webhook/handlers/tcp_datastore.go

@@ -30,7 +30,7 @@ func (t TenantControlPlaneDataStore) OnCreate(object runtime.Object) AdmissionRe
 			return nil, t.check(ctx, tcp.Spec.DataStore)
 		}
 
-		return nil, nil
+		return nil, t.checkDataStoreOverrides(ctx, tcp)
 	}
 }
 
@@ -61,3 +61,19 @@ func (t TenantControlPlaneDataStore) check(ctx context.Context, dataStoreName st
 
 	return nil
 }
+
+func (t TenantControlPlaneDataStore) checkDataStoreOverrides(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) error {
+	overrideCheck := make(map[string]struct{}, 0)
+	for _, ds := range tcp.Spec.DataStoreOverrides {
+		if _, exists := overrideCheck[ds.Resource]; !exists {
+			overrideCheck[ds.Resource] = struct{}{}
+		} else {
+			return fmt.Errorf("duplicate resource override in Spec.DataStoreOverrides")
+		}
+		if err := t.check(ctx, ds.DataStore); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/webhook/handlers/tcp_datastore_test.go

@@ -0,0 +1,94 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package handlers
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+)
+
+var _ = Describe("TCP Datastore webhook", func() {
+	var (
+		ctx context.Context
+		t   TenantControlPlaneDataStore
+		tcp *kamajiv1alpha1.TenantControlPlane
+	)
+	BeforeEach(func() {
+		scheme := runtime.NewScheme()
+		utilruntime.Must(kamajiv1alpha1.AddToScheme(scheme))
+
+		ctx = context.Background()
+		t = TenantControlPlaneDataStore{
+			Client: fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(&kamajiv1alpha1.DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "foo",
+				},
+			}, &kamajiv1alpha1.DataStore{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "bar",
+				},
+			}).Build(),
+		}
+		tcp = &kamajiv1alpha1.TenantControlPlane{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "tcp",
+				Namespace: "default",
+			},
+			Spec: kamajiv1alpha1.TenantControlPlaneSpec{},
+		}
+	})
+	Describe("validation should succeed without DataStoreOverrides", func() {
+		It("should validate TCP without DataStoreOverrides", func() {
+			err := t.checkDataStoreOverrides(ctx, tcp)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+
+	Describe("validation should fail with duplicate resources in DataStoreOverrides", func() {
+		It("should fail to validate TCP with duplicate resources in DataStoreOverrides", func() {
+			tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
+				Resource:  "/event",
+				DataStore: "foo",
+			}, {
+				Resource:  "/event",
+				DataStore: "bar",
+			}}
+			err := t.checkDataStoreOverrides(ctx, tcp)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+
+	Describe("validation should succeed with valid DataStoreOverrides", func() {
+		It("should validate TCP with valid DataStoreOverrides", func() {
+			tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
+				Resource:  "/leases",
+				DataStore: "foo",
+			}, {
+				Resource:  "/event",
+				DataStore: "bar",
+			}}
+			err := t.checkDataStoreOverrides(ctx, tcp)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+
+	Describe("validation should fail with nonexistent DataStoreOverrides", func() {
+		It("should fail to validate TCP with nonexistent DataStoreOverrides", func() {
+			tcp.Spec.DataStoreOverrides = []kamajiv1alpha1.DataStoreOverride{{
+				Resource:  "/leases",
+				DataStore: "baz",
+			}}
+			err := t.checkDataStoreOverrides(ctx, tcp)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+})

internal/webhook/handlers/tcp_deployment.go

@@ -67,6 +67,20 @@ func (t TenantControlPlaneDeployment) OnUpdate(newObject runtime.Object, oldObje
 		}
 		t.DeploymentBuilder.DataStore = ds
 
+		dataStoreOverrides := make([]controlplane.DataStoreOverrides, 0, len(tcp.Spec.DataStoreOverrides))
+
+		for _, dso := range tcp.Spec.DataStoreOverrides {
+			ds := kamajiv1alpha1.DataStore{}
+			if err := t.Client.Get(ctx, types.NamespacedName{Name: dso.DataStore}, &ds); err != nil {
+				return nil, err
+			}
+			dataStoreOverrides = append(dataStoreOverrides, controlplane.DataStoreOverrides{
+				Resource:  dso.Resource,
+				DataStore: ds,
+			})
+		}
+		t.DeploymentBuilder.DataStoreOverrides = dataStoreOverrides
+
 		deployment := appsv1.Deployment{}
 		deployment.Name = tcp.Name
 		deployment.Namespace = tcp.Namespace