From d06affc216626ec6863feeebb2627a49778f5469 Mon Sep 17 00:00:00 2001 From: Matteo Ruina Date: Tue, 30 May 2023 14:18:38 +0200 Subject: [PATCH] feat: tcp deployment mangling --- controllers/resources.go | 3 +- internal/builders/controlplane/deployment.go | 563 +++++++++++------- .../controlplane/konnectivity_server.go | 251 ++++++++ internal/resources/k8s_deployment_resource.go | 5 + .../konnectivity/deployment_resource.go | 254 +------- 5 files changed, 610 insertions(+), 466 deletions(-) create mode 100644 internal/builders/controlplane/konnectivity_server.go diff --git a/controllers/resources.go b/controllers/resources.go index 32df3ae..18875be 100644 --- a/controllers/resources.go +++ b/controllers/resources.go @@ -14,6 +14,7 @@ import ( kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/controllers/finalizers" + builder "github.com/clastix/kamaji/internal/builders/controlplane" "github.com/clastix/kamaji/internal/datastore" "github.com/clastix/kamaji/internal/resources" ds "github.com/clastix/kamaji/internal/resources/datastore" @@ -245,7 +246,7 @@ func getKonnectivityServerRequirementsResources(c client.Client) []resources.Res func getKonnectivityServerPatchResources(c client.Client) []resources.Resource { return []resources.Resource{ - &konnectivity.KubernetesDeploymentResource{Client: c}, + &konnectivity.KubernetesDeploymentResource{Builder: builder.Konnectivity{}, Client: c}, &konnectivity.ServiceResource{Client: c}, } } diff --git a/internal/builders/controlplane/deployment.go b/internal/builders/controlplane/deployment.go index 283d4c4..cf579e5 100644 --- a/internal/builders/controlplane/deployment.go +++ b/internal/builders/controlplane/deployment.go @@ -21,29 +21,27 @@ import ( "github.com/clastix/kamaji/internal/utilities" ) -type orderedIndex int - +// Volume names. const ( - apiServerIndex orderedIndex = iota - schedulerIndex - controllerManagerIndex -) - -const ( - etcKubernetesPKIVolume orderedIndex = iota - etcCACertificates - etcSSLCerts - usrShareCACertificates - usrLocalShareCACertificates - schedulerKubeconfig - controllerManagerKubeconfig + kubernetesPKIVolumeName = "etc-kubernetes-pki" + caCertificatesVolumeName = "etc-ca-certificates" + sslCertsVolumeName = "etc-ssl-certs" + usrShareCACertificatesVolumeName = "usr-share-ca-certificates" + usrLocalShareCaCertificateVolumeName = "usr-local-share-ca-certificates" + schedulerKubeconfigVolumeName = "scheduler-kubeconfig" + controllerManagerKubeconfigVolumeName = "controller-manager-kubeconfig" + dataStoreCertsVolumeName = "kine-config" + kineVolumeCertName = "kine-certs" ) const ( apiServerFlagsAnnotation = "kube-apiserver.kamaji.clastix.io/args" - kineContainerName = "kine" - dataStoreCerts = "kine-config" - kineVolumeCertName = "kine-certs" + // Kamaji container names. + apiServerContainerName = "kube-apiserver" + controlPlaneContainerName = "kube-controller-manager" + schedulerContainerName = "kube-scheduler" + kineContainerName = "kine" + kineInitContainerName = "chmod" ) type Deployment struct { @@ -59,6 +57,36 @@ func (d *Deployment) SetContainers(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1. d.buildKine(podSpec, tcp) } +// SetInitContainers allows adding extra init containers from the user-space: +// this function must be called priorit the SetContainers to ensure the idempotency of podSpec building. +func (d *Deployment) SetInitContainers(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) { + initContainers := tcp.Spec.ControlPlane.Deployment.AdditionalInitContainers + + if d.DataStore.Spec.Driver == kamajiv1alpha1.EtcdDriver { + podSpec.InitContainers = initContainers + } + + found, index := utilities.HasNamedContainer(podSpec.InitContainers, kineInitContainerName) + if found { + initContainers = append(initContainers, podSpec.InitContainers[index:]...) + } + + podSpec.InitContainers = initContainers +} + +// SetAdditionalContainers must be called before SetContainers: the user-space ones are going to be prepended +// to simplify the management of the Kamaji ones during the create or update action. +func (d *Deployment) SetAdditionalContainers(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) { + containers := tcp.Spec.ControlPlane.Deployment.AdditionalContainers + + found, index := utilities.HasNamedContainer(podSpec.Containers, apiServerContainerName) + if found { + containers = append(containers, podSpec.Containers[index:]...) + } + + podSpec.Containers = containers +} + func (d *Deployment) SetStrategy(deployment *appsv1.DeploymentSpec, tcp *kamajiv1alpha1.TenantControlPlane) { deployment.Strategy = appsv1.DeploymentStrategy{ Type: tcp.Spec.ControlPlane.Deployment.Strategy.Type, @@ -112,7 +140,9 @@ func (d *Deployment) SetVolumes(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.Ten } func (d *Deployment) buildPKIVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) { - if index := int(etcKubernetesPKIVolume) + 1; len(podSpec.Volumes) < index { + found, index := utilities.HasNamedVolume(podSpec.Volumes, kubernetesPKIVolumeName) + if !found { + index = len(podSpec.Volumes) podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) } @@ -161,115 +191,128 @@ func (d *Deployment) buildPKIVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1 }) } - podSpec.Volumes[etcKubernetesPKIVolume] = corev1.Volume{ - Name: "etc-kubernetes-pki", - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - Sources: sources, - DefaultMode: pointer.Int32(420), - }, + podSpec.Volumes[index].Name = kubernetesPKIVolumeName + podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: sources, + DefaultMode: pointer.Int32(420), }, } } func (d *Deployment) buildCAVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) { - if index := int(etcCACertificates) + 1; len(podSpec.Volumes) < index { + found, index := utilities.HasNamedVolume(podSpec.Volumes, caCertificatesVolumeName) + if !found { + index = len(podSpec.Volumes) podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) } - podSpec.Volumes[etcCACertificates] = corev1.Volume{ - Name: "etc-ca-certificates", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: tcp.Status.Certificates.CA.SecretName, - DefaultMode: pointer.Int32(420), - }, + podSpec.Volumes[index].Name = caCertificatesVolumeName + podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tcp.Status.Certificates.CA.SecretName, + DefaultMode: pointer.Int32(420), }, } } func (d *Deployment) buildSSLCertsVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) { - if index := int(etcSSLCerts) + 1; len(podSpec.Volumes) < index { + found, index := utilities.HasNamedVolume(podSpec.Volumes, sslCertsVolumeName) + if !found { + index = len(podSpec.Volumes) podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) } - podSpec.Volumes[etcSSLCerts] = corev1.Volume{ - Name: "etc-ssl-certs", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: tcp.Status.Certificates.CA.SecretName, - DefaultMode: pointer.Int32(420), - }, + podSpec.Volumes[index].Name = sslCertsVolumeName + podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tcp.Status.Certificates.CA.SecretName, + DefaultMode: pointer.Int32(420), }, } } func (d *Deployment) buildShareCAVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) { - if index := int(usrShareCACertificates) + 1; len(podSpec.Volumes) < index { + found, index := utilities.HasNamedVolume(podSpec.Volumes, usrShareCACertificatesVolumeName) + if !found { + index = len(podSpec.Volumes) podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) } - podSpec.Volumes[usrShareCACertificates] = corev1.Volume{ - Name: "usr-share-ca-certificates", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: tcp.Status.Certificates.CA.SecretName, - DefaultMode: pointer.Int32(420), - }, + podSpec.Volumes[index].Name = usrShareCACertificatesVolumeName + podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tcp.Status.Certificates.CA.SecretName, + DefaultMode: pointer.Int32(420), }, } } func (d *Deployment) buildLocalShareCAVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) { - if index := int(usrLocalShareCACertificates) + 1; len(podSpec.Volumes) < index { + found, index := utilities.HasNamedVolume(podSpec.Volumes, usrLocalShareCaCertificateVolumeName) + if !found { + index = len(podSpec.Volumes) podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) } - podSpec.Volumes[usrLocalShareCACertificates] = corev1.Volume{ - Name: "usr-local-share-ca-certificates", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: tcp.Status.Certificates.CA.SecretName, - DefaultMode: pointer.Int32(420), - }, + podSpec.Volumes[index].Name = usrLocalShareCaCertificateVolumeName + podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tcp.Status.Certificates.CA.SecretName, + DefaultMode: pointer.Int32(420), }, } } func (d *Deployment) buildSchedulerVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) { - if index := int(schedulerKubeconfig) + 1; len(podSpec.Volumes) < index { + found, index := utilities.HasNamedVolume(podSpec.Volumes, schedulerKubeconfigVolumeName) + if !found { + index = len(podSpec.Volumes) podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) } - podSpec.Volumes[schedulerKubeconfig] = corev1.Volume{ - Name: "scheduler-kubeconfig", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: tcp.Status.KubeConfig.Scheduler.SecretName, - DefaultMode: pointer.Int32(420), - }, + podSpec.Volumes[index].Name = schedulerKubeconfigVolumeName + podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tcp.Status.KubeConfig.Scheduler.SecretName, + DefaultMode: pointer.Int32(420), }, } } func (d *Deployment) buildControllerManagerVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) { - if index := int(controllerManagerKubeconfig) + 1; len(podSpec.Volumes) < index { + found, index := utilities.HasNamedVolume(podSpec.Volumes, controllerManagerKubeconfigVolumeName) + if !found { + index = len(podSpec.Volumes) podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) } - podSpec.Volumes[controllerManagerKubeconfig] = corev1.Volume{ - Name: "controller-manager-kubeconfig", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: tcp.Status.KubeConfig.ControllerManager.SecretName, - DefaultMode: pointer.Int32(420), - }, + podSpec.Volumes[index].Name = controllerManagerKubeconfigVolumeName + podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: tcp.Status.KubeConfig.ControllerManager.SecretName, + DefaultMode: pointer.Int32(420), }, } } +// SetAdditionalVolumes must be called before SetVolumes: the user-space ones are going to be prepended +// to simplify the management of the Kamaji ones during the create or update action. +func (d *Deployment) SetAdditionalVolumes(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) { + volumes := tcp.Spec.ControlPlane.Deployment.AdditionalVolumes + + found, index := utilities.HasNamedVolume(podSpec.Volumes, kubernetesPKIVolumeName) + if found { + volumes = append(volumes, podSpec.Volumes[index:]...) + } + + podSpec.Volumes = volumes +} + func (d *Deployment) BuildScheduler(podSpec *corev1.PodSpec, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) { - if index := int(schedulerIndex) + 1; len(podSpec.Containers) < index { + found, index := utilities.HasNamedContainer(podSpec.Containers, schedulerContainerName) + if !found { + index = len(podSpec.Containers) podSpec.Containers = append(podSpec.Containers, corev1.Container{}) } @@ -287,18 +330,11 @@ func (d *Deployment) BuildScheduler(podSpec *corev1.PodSpec, tenantControlPlane args["--kubeconfig"] = kubeconfig args["--leader-elect"] = "true" //nolint:goconst - podSpec.Containers[schedulerIndex].Name = "kube-scheduler" - podSpec.Containers[schedulerIndex].Image = fmt.Sprintf("registry.k8s.io/kube-scheduler:%s", tenantControlPlane.Spec.Kubernetes.Version) - podSpec.Containers[schedulerIndex].Command = []string{"kube-scheduler"} - podSpec.Containers[schedulerIndex].Args = utilities.ArgsFromMapToSlice(args) - podSpec.Containers[schedulerIndex].VolumeMounts = []corev1.VolumeMount{ - { - Name: "scheduler-kubeconfig", - ReadOnly: true, - MountPath: "/etc/kubernetes", - }, - } - podSpec.Containers[schedulerIndex].LivenessProbe = &corev1.Probe{ + podSpec.Containers[index].Name = schedulerContainerName + podSpec.Containers[index].Image = fmt.Sprintf("registry.k8s.io/kube-scheduler:%s", tenantControlPlane.Spec.Kubernetes.Version) + podSpec.Containers[index].Command = []string{"kube-scheduler"} + podSpec.Containers[index].Args = utilities.ArgsFromMapToSlice(args) + podSpec.Containers[index].LivenessProbe = &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", @@ -312,8 +348,7 @@ func (d *Deployment) BuildScheduler(podSpec *corev1.PodSpec, tenantControlPlane SuccessThreshold: 1, FailureThreshold: 3, } - - podSpec.Containers[schedulerIndex].StartupProbe = &corev1.Probe{ + podSpec.Containers[index].StartupProbe = &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", @@ -327,25 +362,44 @@ func (d *Deployment) BuildScheduler(podSpec *corev1.PodSpec, tenantControlPlane SuccessThreshold: 1, FailureThreshold: 3, } - podSpec.Containers[schedulerIndex].ImagePullPolicy = corev1.PullAlways - podSpec.Containers[schedulerIndex].Resources = corev1.ResourceRequirements{ + podSpec.Containers[index].ImagePullPolicy = corev1.PullAlways + podSpec.Containers[index].Resources = corev1.ResourceRequirements{ Limits: nil, Requests: nil, } + // Volume mounts + var extraVolumeMounts []corev1.VolumeMount + + if additionalVolumeMounts := tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalVolumeMounts; additionalVolumeMounts != nil { + extraVolumeMounts = append(extraVolumeMounts, additionalVolumeMounts.Scheduler...) + } + + volumeMounts := d.initVolumeMounts(schedulerKubeconfigVolumeName, podSpec.Containers[index].VolumeMounts, extraVolumeMounts...) + + d.ensureVolumeMount(&volumeMounts, corev1.VolumeMount{ + Name: schedulerKubeconfigVolumeName, + ReadOnly: true, + MountPath: "/etc/kubernetes", + }) + + podSpec.Containers[index].VolumeMounts = volumeMounts if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil { if resource := componentsResources.Scheduler; resource != nil { - podSpec.Containers[schedulerIndex].Resources.Limits = resource.Limits - podSpec.Containers[schedulerIndex].Resources.Requests = resource.Requests + podSpec.Containers[index].Resources.Limits = resource.Limits + podSpec.Containers[index].Resources.Requests = resource.Requests } } } func (d *Deployment) buildControllerManager(podSpec *corev1.PodSpec, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) { - if index := int(controllerManagerIndex) + 1; len(podSpec.Containers) < index { + found, index := utilities.HasNamedContainer(podSpec.Containers, controlPlaneContainerName) + if !found { + index = len(podSpec.Containers) podSpec.Containers = append(podSpec.Containers, corev1.Container{}) } - + // Configuring the arguments of the container, + // taking in consideration the extra args from the user-space. args := map[string]string{} if tenantControlPlane.Spec.ControlPlane.Deployment.ExtraArgs != nil { @@ -372,44 +426,11 @@ func (d *Deployment) buildControllerManager(podSpec *corev1.PodSpec, tenantContr args["--service-account-private-key-file"] = path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName) args["--use-service-account-credentials"] = "true" - podSpec.Containers[controllerManagerIndex].Name = "kube-controller-manager" - podSpec.Containers[controllerManagerIndex].Image = fmt.Sprintf("registry.k8s.io/kube-controller-manager:%s", tenantControlPlane.Spec.Kubernetes.Version) - podSpec.Containers[controllerManagerIndex].Command = []string{"kube-controller-manager"} - podSpec.Containers[controllerManagerIndex].Args = utilities.ArgsFromMapToSlice(args) - podSpec.Containers[controllerManagerIndex].VolumeMounts = []corev1.VolumeMount{ - { - Name: "controller-manager-kubeconfig", - ReadOnly: true, - MountPath: "/etc/kubernetes", - }, - { - Name: "etc-kubernetes-pki", - ReadOnly: true, - MountPath: v1beta3.DefaultCertificatesDir, - }, - { - Name: "etc-ca-certificates", - ReadOnly: true, - MountPath: "/etc/ca-certificates", - }, - { - Name: "etc-ssl-certs", - ReadOnly: true, - MountPath: "/etc/ssl/certs", - }, - { - Name: "usr-share-ca-certificates", - ReadOnly: true, - MountPath: "/usr/share/ca-certificates", - }, - { - Name: "usr-local-share-ca-certificates", - ReadOnly: true, - MountPath: "/usr/local/share/ca-certificates", - }, - } - - podSpec.Containers[controllerManagerIndex].LivenessProbe = &corev1.Probe{ + podSpec.Containers[index].Name = "kube-controller-manager" + podSpec.Containers[index].Image = fmt.Sprintf("registry.k8s.io/kube-controller-manager:%s", tenantControlPlane.Spec.Kubernetes.Version) + podSpec.Containers[index].Command = []string{"kube-controller-manager"} + podSpec.Containers[index].Args = utilities.ArgsFromMapToSlice(args) + podSpec.Containers[index].LivenessProbe = &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", @@ -423,8 +444,7 @@ func (d *Deployment) buildControllerManager(podSpec *corev1.PodSpec, tenantContr SuccessThreshold: 1, FailureThreshold: 3, } - - podSpec.Containers[controllerManagerIndex].StartupProbe = &corev1.Probe{ + podSpec.Containers[index].StartupProbe = &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/healthz", @@ -438,32 +458,106 @@ func (d *Deployment) buildControllerManager(podSpec *corev1.PodSpec, tenantContr SuccessThreshold: 1, FailureThreshold: 3, } - podSpec.Containers[controllerManagerIndex].ImagePullPolicy = corev1.PullAlways - podSpec.Containers[controllerManagerIndex].Resources = corev1.ResourceRequirements{ + // Volume mounts + var extraVolumeMounts []corev1.VolumeMount + + if additionalVolumeMounts := tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalVolumeMounts; additionalVolumeMounts != nil { + extraVolumeMounts = append(extraVolumeMounts, additionalVolumeMounts.ControllerManager...) + } + + volumeMounts := d.initVolumeMounts(controllerManagerKubeconfigVolumeName, podSpec.Containers[index].VolumeMounts, extraVolumeMounts...) + + d.ensureVolumeMount(&volumeMounts, corev1.VolumeMount{ + Name: controllerManagerKubeconfigVolumeName, + ReadOnly: true, + MountPath: "/etc/kubernetes", + }) + d.ensureVolumeMount(&volumeMounts, corev1.VolumeMount{ + Name: kubernetesPKIVolumeName, + ReadOnly: true, + MountPath: v1beta3.DefaultCertificatesDir, + }) + d.ensureVolumeMount(&volumeMounts, corev1.VolumeMount{ + Name: caCertificatesVolumeName, + ReadOnly: true, + MountPath: "/etc/ca-certificates", + }) + d.ensureVolumeMount(&volumeMounts, corev1.VolumeMount{ + Name: sslCertsVolumeName, + ReadOnly: true, + MountPath: "/etc/ssl/certs", + }) + d.ensureVolumeMount(&volumeMounts, corev1.VolumeMount{ + Name: usrShareCACertificatesVolumeName, + ReadOnly: true, + MountPath: "/usr/share/ca-certificates", + }) + d.ensureVolumeMount(&volumeMounts, corev1.VolumeMount{ + Name: usrLocalShareCaCertificateVolumeName, + ReadOnly: true, + MountPath: "/usr/local/share/ca-certificates", + }) + + podSpec.Containers[index].VolumeMounts = volumeMounts + + // Managing container resources + podSpec.Containers[index].Resources = corev1.ResourceRequirements{ Limits: nil, Requests: nil, } if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil { if resource := componentsResources.ControllerManager; resource != nil { - podSpec.Containers[controllerManagerIndex].Resources.Limits = resource.Limits - podSpec.Containers[controllerManagerIndex].Resources.Requests = resource.Requests + podSpec.Containers[index].Resources.Limits = resource.Limits + podSpec.Containers[index].Resources.Requests = resource.Requests } } } +// ensureVolumeMount retrieve the index for the named volumeMount, in case of missing it's going to be appended. +func (d *Deployment) ensureVolumeMount(in *[]corev1.VolumeMount, desired corev1.VolumeMount) { + list := *in + + found, index := utilities.HasNamedVolumeMount(*in, desired.Name) + if !found { + index = len(list) + list = append(list, corev1.VolumeMount{}) + } + + list[index] = desired + + *in = list +} + +// initVolumeMounts is responsible to create the idempotent slice of corev1.VolumeMount: +// firstSystemVolumeMountName must refer to the first Kamaji-space volume mount to detect properly user-space ones. +func (d *Deployment) initVolumeMounts(firstSystemVolumeMountName string, actual []corev1.VolumeMount, extra ...corev1.VolumeMount) []corev1.VolumeMount { + var volumeMounts []corev1.VolumeMount + + volumeMounts = append(volumeMounts, extra...) + // Retrieve the first safe volume mount to pick up from: + // this is required to be sure to delete all the extra containers from the user space. + if vmFound, vmIndex := utilities.HasNamedVolumeMount(actual, firstSystemVolumeMountName); vmFound { + volumeMounts = append(volumeMounts, actual[vmIndex:]...) + } + + return volumeMounts +} + func (d *Deployment) buildKubeAPIServer(podSpec *corev1.PodSpec, tenantControlPlane *kamajiv1alpha1.TenantControlPlane, address string) { - if index := int(apiServerIndex) + 1; len(podSpec.Containers) < index { + found, index := utilities.HasNamedContainer(podSpec.Containers, apiServerContainerName) + if !found { + index = len(podSpec.Containers) podSpec.Containers = append(podSpec.Containers, corev1.Container{}) } - args := d.buildKubeAPIServerCommand(tenantControlPlane, address, utilities.ArgsFromSliceToMap(podSpec.Containers[apiServerIndex].Args)) + args := d.buildKubeAPIServerCommand(tenantControlPlane, address, utilities.ArgsFromSliceToMap(podSpec.Containers[index].Args)) - podSpec.Containers[apiServerIndex].Name = "kube-apiserver" - podSpec.Containers[apiServerIndex].Args = utilities.ArgsFromMapToSlice(args) - podSpec.Containers[apiServerIndex].Image = fmt.Sprintf("registry.k8s.io/kube-apiserver:%s", tenantControlPlane.Spec.Kubernetes.Version) - podSpec.Containers[apiServerIndex].Command = []string{"kube-apiserver"} - podSpec.Containers[apiServerIndex].LivenessProbe = &corev1.Probe{ + podSpec.Containers[index].Name = apiServerContainerName + podSpec.Containers[index].Args = utilities.ArgsFromMapToSlice(args) + podSpec.Containers[index].Image = fmt.Sprintf("registry.k8s.io/kube-apiserver:%s", tenantControlPlane.Spec.Kubernetes.Version) + podSpec.Containers[index].Command = []string{"kube-apiserver"} + podSpec.Containers[index].LivenessProbe = &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/livez", @@ -477,7 +571,7 @@ func (d *Deployment) buildKubeAPIServer(podSpec *corev1.PodSpec, tenantControlPl SuccessThreshold: 1, FailureThreshold: 3, } - podSpec.Containers[apiServerIndex].ReadinessProbe = &corev1.Probe{ + podSpec.Containers[index].ReadinessProbe = &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/readyz", @@ -491,7 +585,7 @@ func (d *Deployment) buildKubeAPIServer(podSpec *corev1.PodSpec, tenantControlPl SuccessThreshold: 1, FailureThreshold: 3, } - podSpec.Containers[apiServerIndex].StartupProbe = &corev1.Probe{ + podSpec.Containers[index].StartupProbe = &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Path: "/livez", @@ -505,45 +599,53 @@ func (d *Deployment) buildKubeAPIServer(podSpec *corev1.PodSpec, tenantControlPl SuccessThreshold: 1, FailureThreshold: 3, } - podSpec.Containers[apiServerIndex].ImagePullPolicy = corev1.PullAlways + podSpec.Containers[index].ImagePullPolicy = corev1.PullAlways + // Volume mounts + var extraVolumeMounts []corev1.VolumeMount - if len(podSpec.Containers[apiServerIndex].VolumeMounts) < 5 { - podSpec.Containers[apiServerIndex].VolumeMounts = make([]corev1.VolumeMount, 5) + if additionalVolumeMounts := tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalVolumeMounts; additionalVolumeMounts != nil { + extraVolumeMounts = append(extraVolumeMounts, additionalVolumeMounts.APIServer...) } - podSpec.Containers[apiServerIndex].VolumeMounts[0] = corev1.VolumeMount{ - Name: "etc-kubernetes-pki", + + volumeMounts := d.initVolumeMounts(kubernetesPKIVolumeName, podSpec.Containers[index].VolumeMounts, extraVolumeMounts...) + + d.ensureVolumeMount(&volumeMounts, corev1.VolumeMount{ + Name: kubernetesPKIVolumeName, ReadOnly: true, MountPath: v1beta3.DefaultCertificatesDir, - } - podSpec.Containers[apiServerIndex].VolumeMounts[1] = corev1.VolumeMount{ - Name: "etc-ca-certificates", + }) + d.ensureVolumeMount(&volumeMounts, corev1.VolumeMount{ + Name: caCertificatesVolumeName, ReadOnly: true, MountPath: "/etc/ca-certificates", - } - podSpec.Containers[apiServerIndex].VolumeMounts[2] = corev1.VolumeMount{ - Name: "etc-ssl-certs", + }) + d.ensureVolumeMount(&volumeMounts, corev1.VolumeMount{ + Name: sslCertsVolumeName, ReadOnly: true, MountPath: "/etc/ssl/certs", - } - podSpec.Containers[apiServerIndex].VolumeMounts[3] = corev1.VolumeMount{ - Name: "usr-share-ca-certificates", + }) + d.ensureVolumeMount(&volumeMounts, corev1.VolumeMount{ + Name: usrShareCACertificatesVolumeName, ReadOnly: true, MountPath: "/usr/share/ca-certificates", - } - podSpec.Containers[apiServerIndex].VolumeMounts[4] = corev1.VolumeMount{ - Name: "usr-local-share-ca-certificates", + }) + d.ensureVolumeMount(&volumeMounts, corev1.VolumeMount{ + Name: usrLocalShareCaCertificateVolumeName, ReadOnly: true, MountPath: "/usr/local/share/ca-certificates", - } - podSpec.Containers[apiServerIndex].Resources = corev1.ResourceRequirements{ + }) + + podSpec.Containers[index].VolumeMounts = volumeMounts + // Managing container resource requirements + podSpec.Containers[index].Resources = corev1.ResourceRequirements{ Limits: nil, Requests: nil, } if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil { if resource := componentsResources.APIServer; resource != nil { - podSpec.Containers[apiServerIndex].Resources.Limits = resource.Limits - podSpec.Containers[apiServerIndex].Resources.Requests = resource.Requests + podSpec.Containers[index].Resources.Limits = resource.Limits + podSpec.Containers[index].Resources.Requests = resource.Requests } } } @@ -629,41 +731,42 @@ func (d *Deployment) secretProjection(secretName, certKeyName, keyName string) * } func (d *Deployment) removeKineVolumes(podSpec *corev1.PodSpec) { - if found, index := utilities.HasNamedVolume(podSpec.Volumes, kineVolumeCertName); found { - var volumes []corev1.Volume + for _, volumeName := range []string{kineVolumeCertName, dataStoreCertsVolumeName} { + if found, index := utilities.HasNamedVolume(podSpec.Volumes, volumeName); found { + var volumes []corev1.Volume - volumes = append(volumes, podSpec.Volumes[:index]...) - volumes = append(volumes, podSpec.Volumes[index+1:]...) + volumes = append(volumes, podSpec.Volumes[:index]...) + volumes = append(volumes, podSpec.Volumes[index+1:]...) - podSpec.Volumes = volumes + podSpec.Volumes = volumes + } } } func (d *Deployment) buildKineVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) { - // Adding the volume for chmod'ed Kine certificates. - found, index := utilities.HasNamedVolume(podSpec.Volumes, dataStoreCerts) - if !found { - podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) - index = len(podSpec.Volumes) - 1 + if d.DataStore.Spec.Driver == kamajiv1alpha1.EtcdDriver { + return } - podSpec.Volumes[index].Name = dataStoreCerts + found, index := utilities.HasNamedVolume(podSpec.Volumes, dataStoreCertsVolumeName) + if !found { + index = len(podSpec.Volumes) + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) + } + + podSpec.Volumes[index].Name = dataStoreCertsVolumeName podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: tcp.Status.Storage.Certificate.SecretName, DefaultMode: pointer.Int32(420), }, } - if d.DataStore.Spec.Driver == kamajiv1alpha1.EtcdDriver { - d.removeKineVolumes(podSpec) - - return - } // Adding the volume to read Kine certificates: // these must be subsequently fixed with a chmod due to pg issues with private key. - if found, index = utilities.HasNamedVolume(podSpec.Volumes, kineVolumeCertName); !found { + found, index = utilities.HasNamedVolume(podSpec.Volumes, kineVolumeCertName) + if !found { + index = len(podSpec.Volumes) podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) - index = len(podSpec.Volumes) - 1 } podSpec.Volumes[index].Name = kineVolumeCertName @@ -673,8 +776,8 @@ func (d *Deployment) buildKineVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha } func (d *Deployment) removeKineContainers(podSpec *corev1.PodSpec) { - found, index := utilities.HasNamedContainer(podSpec.Containers, kineContainerName) - if found { + // Removing the kine container, if present + if found, index := utilities.HasNamedContainer(podSpec.Containers, kineContainerName); found { var containers []corev1.Container containers = append(containers, podSpec.Containers[:index]...) @@ -682,24 +785,59 @@ func (d *Deployment) removeKineContainers(podSpec *corev1.PodSpec) { podSpec.Containers = containers } + // Removing the kine init-container, if present + if found, index := utilities.HasNamedContainer(podSpec.InitContainers, kineInitContainerName); found { + var initContainers []corev1.Container - podSpec.InitContainers = nil + initContainers = append(initContainers, podSpec.InitContainers[:index]...) + initContainers = append(initContainers, podSpec.InitContainers[index+1:]...) + + podSpec.InitContainers = initContainers + } } func (d *Deployment) buildKine(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) { if d.DataStore.Spec.Driver == kamajiv1alpha1.EtcdDriver { d.removeKineContainers(podSpec) + d.removeKineVolumes(podSpec) return } - // Kine is expecting an additional container, and it must be removed before proceeding with the additional one - // in order to make this function idempotent. - found, index := utilities.HasNamedContainer(podSpec.Containers, kineContainerName) + // Ensuring the init container required for kine is present: + // a chmod is required for kine in order to read the certificates to connect to the secured datastore. + found, index := utilities.HasNamedContainer(podSpec.InitContainers, kineInitContainerName) if !found { - podSpec.Containers = append(podSpec.Containers, corev1.Container{}) - index = len(podSpec.Containers) - 1 + index = len(podSpec.InitContainers) + podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{}) } + podSpec.InitContainers[index].Name = kineInitContainerName + podSpec.InitContainers[index].Image = d.KineContainerImage + podSpec.InitContainers[index].Command = []string{"sh"} + podSpec.InitContainers[index].Args = []string{ + "-c", + "cp /kine/*.* /certs && chmod -R 600 /certs/*.*", + } + podSpec.InitContainers[index].VolumeMounts = []corev1.VolumeMount{ + { + Name: dataStoreCertsVolumeName, + ReadOnly: true, + MountPath: "/kine", + }, + { + Name: kineVolumeCertName, + MountPath: "/certs", + ReadOnly: false, + }, + } + // Kine is expecting an additional container, and it must be removed before proceeding with the additional one + // in order to make this function idempotent. + found, index = utilities.HasNamedContainer(podSpec.Containers, kineContainerName) + if !found { + index = len(podSpec.Containers) + podSpec.Containers = append(podSpec.Containers, corev1.Container{}) + } + // Building kine arguments, taking in consideration the user-space ones if provided. args := map[string]string{} if tcp.Spec.ControlPlane.Deployment.ExtraArgs != nil { @@ -717,33 +855,6 @@ func (d *Deployment) buildKine(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.Tena args["--cert-file"] = "/certs/server.crt" args["--key-file"] = "/certs/server.key" - podSpec.InitContainers = []corev1.Container{ - { - Name: "chmod", - Image: d.KineContainerImage, - ImagePullPolicy: corev1.PullAlways, - TerminationMessagePath: corev1.TerminationMessagePathDefault, - TerminationMessagePolicy: corev1.TerminationMessageReadFile, - Command: []string{"sh"}, - Args: []string{ - "-c", - "cp /kine/*.* /certs && chmod -R 600 /certs/*.*", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: dataStoreCerts, - ReadOnly: true, - MountPath: "/kine", - }, - { - Name: kineVolumeCertName, - MountPath: "/certs", - ReadOnly: false, - }, - }, - }, - } - podSpec.Containers[index].Name = kineContainerName podSpec.Containers[index].Image = d.KineContainerImage podSpec.Containers[index].Command = []string{"/bin/kine"} @@ -755,8 +866,6 @@ func (d *Deployment) buildKine(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.Tena ReadOnly: false, }, } - podSpec.Containers[index].TerminationMessagePath = corev1.TerminationMessagePathDefault - podSpec.Containers[index].TerminationMessagePolicy = corev1.TerminationMessageReadFile podSpec.Containers[index].Env = []corev1.EnvVar{ { Name: "GODEBUG", @@ -779,7 +888,6 @@ func (d *Deployment) buildKine(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.Tena Protocol: corev1.ProtocolTCP, }, } - podSpec.Containers[index].ImagePullPolicy = corev1.PullAlways } func (d *Deployment) SetSelector(deploymentSpec *appsv1.DeploymentSpec, tcp *kamajiv1alpha1.TenantControlPlane) { @@ -835,7 +943,7 @@ func (d *Deployment) ResetKubeAPIServerFlags(resource *appsv1.Deployment, tcp *k return } // kube-apiserver container is not still there, we can skip the hashing - if found, _ := utilities.HasNamedContainer(resource.Spec.Template.Spec.Containers, "kube-apiserver"); !found { + if found, _ := utilities.HasNamedContainer(resource.Spec.Template.Spec.Containers, apiServerContainerName); !found { return } // setting up annotation to avoid assignment to a nil one @@ -855,7 +963,8 @@ func (d *Deployment) ResetKubeAPIServerFlags(resource *appsv1.Deployment, tcp *k } // there's a mismatch in the count from the previous hash: let's reset and store the desired extra args count. if count != len(tcp.Spec.ControlPlane.Deployment.ExtraArgs.APIServer) { - resource.Spec.Template.Spec.Containers[apiServerIndex].Args = []string{} + _, index := utilities.HasNamedContainer(resource.Spec.Template.Spec.Containers, apiServerContainerName) + resource.Spec.Template.Spec.Containers[index].Args = []string{} } resource.GetAnnotations()[apiServerFlagsAnnotation] = fmt.Sprintf("%d", len(tcp.Spec.ControlPlane.Deployment.ExtraArgs.APIServer)) diff --git a/internal/builders/controlplane/konnectivity_server.go b/internal/builders/controlplane/konnectivity_server.go new file mode 100644 index 0000000..efbadc1 --- /dev/null +++ b/internal/builders/controlplane/konnectivity_server.go @@ -0,0 +1,251 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package controlplane + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/utilities" +) + +const ( + AgentName = "konnectivity-agent" + CertCommonName = "system:konnectivity-server" + + konnectivityEgressSelectorConfigurationPath = "/etc/kubernetes/konnectivity/configurations/egress-selector-configuration.yaml" + konnectivityServerName = "konnectivity-server" + konnectivityServerPath = "/run/konnectivity" + + egressSelectorConfigurationVolume = "egress-selector-configuration" + konnectivityUDSVolume = "konnectivity-uds" + konnectivityServerKubeconfigVolume = "konnectivity-server-kubeconfig" +) + +type Konnectivity struct{} + +func (k Konnectivity) BuildKonnectivityContainer(addon *kamajiv1alpha1.KonnectivitySpec, replicas int32, podSpec *corev1.PodSpec) { + found, index := utilities.HasNamedContainer(podSpec.Containers, konnectivityServerName) + if !found { + index = len(podSpec.Containers) + podSpec.Containers = append(podSpec.Containers, corev1.Container{}) + } + + podSpec.Containers[index].Name = konnectivityServerName + podSpec.Containers[index].Image = fmt.Sprintf("%s:%s", addon.KonnectivityServerSpec.Image, addon.KonnectivityServerSpec.Version) + podSpec.Containers[index].Command = []string{"/proxy-server"} + + args := utilities.ArgsFromSliceToMap(addon.KonnectivityServerSpec.ExtraArgs) + + args["--uds-name"] = fmt.Sprintf("%s/konnectivity-server.socket", konnectivityServerPath) + args["--cluster-cert"] = "/etc/kubernetes/pki/apiserver.crt" + args["--cluster-key"] = "/etc/kubernetes/pki/apiserver.key" + args["--mode"] = "grpc" + args["--server-port"] = "0" + args["--agent-port"] = fmt.Sprintf("%d", addon.KonnectivityServerSpec.Port) + args["--admin-port"] = "8133" + args["--health-port"] = "8134" + args["--agent-namespace"] = "kube-system" + args["--agent-service-account"] = AgentName + args["--kubeconfig"] = "/etc/kubernetes/konnectivity-server.conf" + args["--authentication-audience"] = CertCommonName + args["--server-count"] = fmt.Sprintf("%d", replicas) + + podSpec.Containers[index].Args = utilities.ArgsFromMapToSlice(args) + podSpec.Containers[index].LivenessProbe = &corev1.Probe{ + InitialDelaySeconds: 30, + TimeoutSeconds: 60, + PeriodSeconds: 10, + SuccessThreshold: 1, + FailureThreshold: 3, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromInt(8134), + Scheme: corev1.URISchemeHTTP, + }, + }, + } + podSpec.Containers[index].Ports = []corev1.ContainerPort{ + { + Name: "agentport", + ContainerPort: addon.KonnectivityServerSpec.Port, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "adminport", + ContainerPort: 8133, + Protocol: corev1.ProtocolTCP, + }, + { + Name: "healthport", + ContainerPort: 8134, + Protocol: corev1.ProtocolTCP, + }, + } + podSpec.Containers[index].VolumeMounts = []corev1.VolumeMount{ + { + Name: "etc-kubernetes-pki", + MountPath: "/etc/kubernetes/pki", + ReadOnly: true, + }, + { + Name: "konnectivity-server-kubeconfig", + MountPath: "/etc/kubernetes/konnectivity-server.conf", + SubPath: "konnectivity-server.conf", + ReadOnly: true, + }, + { + Name: konnectivityUDSVolume, + MountPath: konnectivityServerPath, + ReadOnly: false, + }, + } + podSpec.Containers[index].ImagePullPolicy = corev1.PullAlways + podSpec.Containers[index].Resources = corev1.ResourceRequirements{ + Limits: nil, + Requests: nil, + } + + if resources := addon.KonnectivityServerSpec.Resources; resources != nil { + podSpec.Containers[index].Resources.Limits = resources.Limits + podSpec.Containers[index].Resources.Requests = resources.Requests + } +} + +func (k Konnectivity) RemovingVolumeMounts(podSpec *corev1.PodSpec) { + found, index := utilities.HasNamedContainer(podSpec.Containers, apiServerContainerName) + if !found { + return + } + + for _, volumeMountName := range []string{konnectivityUDSVolume, egressSelectorConfigurationVolume, konnectivityServerKubeconfigVolume} { + if ok, i := utilities.HasNamedVolumeMount(podSpec.Containers[index].VolumeMounts, volumeMountName); ok { + var volumesMounts []corev1.VolumeMount + + volumesMounts = append(volumesMounts, podSpec.Containers[index].VolumeMounts[:i]...) + volumesMounts = append(volumesMounts, podSpec.Containers[index].VolumeMounts[i+1:]...) + + podSpec.Containers[index].VolumeMounts = volumesMounts + } + } +} + +func (k Konnectivity) RemovingVolumes(podSpec *corev1.PodSpec) { + for _, volumeName := range []string{konnectivityUDSVolume, egressSelectorConfigurationVolume} { + if volumeFound, volumeIndex := utilities.HasNamedVolume(podSpec.Volumes, volumeName); volumeFound { + var volumes []corev1.Volume + + volumes = append(volumes, podSpec.Volumes[:volumeIndex]...) + volumes = append(volumes, podSpec.Volumes[volumeIndex+1:]...) + + podSpec.Volumes = volumes + } + } +} + +func (k Konnectivity) RemovingKubeAPIServerContainerArg(podSpec *corev1.PodSpec) { + if found, index := utilities.HasNamedContainer(podSpec.Containers, apiServerContainerName); found { + argsMap := utilities.ArgsFromSliceToMap(podSpec.Containers[index].Args) + + if utilities.ArgsRemoveFlag(argsMap, "--egress-selector-config-file") { + podSpec.Containers[index].Args = utilities.ArgsFromMapToSlice(argsMap) + } + } +} + +func (k Konnectivity) RemovingContainer(podSpec *corev1.PodSpec) { + if found, index := utilities.HasNamedContainer(podSpec.Containers, konnectivityServerName); found { + var containers []corev1.Container + + containers = append(containers, podSpec.Containers[:index]...) + containers = append(containers, podSpec.Containers[index+1:]...) + + podSpec.Containers = containers + } +} + +func (k Konnectivity) BuildVolumeMounts(podSpec *corev1.PodSpec) { + found, index := utilities.HasNamedContainer(podSpec.Containers, apiServerContainerName) + if !found { + return + } + // Adding the egress selector config file flag + args := utilities.ArgsFromSliceToMap(podSpec.Containers[index].Args) + + utilities.ArgsAddFlagValue(args, "--egress-selector-config-file", konnectivityEgressSelectorConfigurationPath) + + podSpec.Containers[index].Args = utilities.ArgsFromMapToSlice(args) + + vFound, vIndex := false, 0 + // Patching the volume mounts + if vFound, vIndex = utilities.HasNamedVolumeMount(podSpec.Containers[index].VolumeMounts, konnectivityUDSVolume); !vFound { + vIndex = len(podSpec.Containers[index].VolumeMounts) + podSpec.Containers[index].VolumeMounts = append(podSpec.Containers[index].VolumeMounts, corev1.VolumeMount{}) + } + + podSpec.Containers[index].VolumeMounts[vIndex].Name = konnectivityUDSVolume + podSpec.Containers[index].VolumeMounts[vIndex].ReadOnly = false + podSpec.Containers[index].VolumeMounts[vIndex].MountPath = konnectivityServerPath + + if vFound, vIndex = utilities.HasNamedVolumeMount(podSpec.Containers[index].VolumeMounts, egressSelectorConfigurationVolume); !vFound { + vIndex = len(podSpec.Containers[index].VolumeMounts) + podSpec.Containers[index].VolumeMounts = append(podSpec.Containers[index].VolumeMounts, corev1.VolumeMount{}) + } + + podSpec.Containers[index].VolumeMounts[vIndex].Name = egressSelectorConfigurationVolume + podSpec.Containers[index].VolumeMounts[vIndex].ReadOnly = false + podSpec.Containers[index].VolumeMounts[vIndex].MountPath = "/etc/kubernetes/konnectivity/configurations" +} + +func (k Konnectivity) BuildVolumes(status kamajiv1alpha1.KonnectivityStatus, podSpec *corev1.PodSpec) { + found, index := false, 0 + // Defining volumes for the UDS socket + found, index = utilities.HasNamedVolume(podSpec.Volumes, konnectivityUDSVolume) + if !found { + index = len(podSpec.Volumes) + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) + } + + podSpec.Volumes[index].Name = konnectivityUDSVolume + podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: "Memory", + }, + } + // Defining volumes for the egress selector configuration + found, index = utilities.HasNamedVolume(podSpec.Volumes, egressSelectorConfigurationVolume) + if !found { + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) + index = len(podSpec.Volumes) - 1 + } + + podSpec.Volumes[index].Name = egressSelectorConfigurationVolume + podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: status.ConfigMap.Name, + }, + DefaultMode: pointer.Int32(420), + }, + } + // Defining volume for the Konnectivity kubeconfig + found, index = utilities.HasNamedVolume(podSpec.Volumes, konnectivityServerKubeconfigVolume) + if !found { + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{}) + index = len(podSpec.Volumes) - 1 + } + + podSpec.Volumes[index].Name = konnectivityServerKubeconfigVolume + podSpec.Volumes[index].VolumeSource = corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: status.Kubeconfig.SecretName, + DefaultMode: pointer.Int32(420), + }, + } +} diff --git a/internal/resources/k8s_deployment_resource.go b/internal/resources/k8s_deployment_resource.go index 35929cc..652044a 100644 --- a/internal/resources/k8s_deployment_resource.go +++ b/internal/resources/k8s_deployment_resource.go @@ -88,9 +88,14 @@ func (r *KubernetesDeploymentResource) mutate(ctx context.Context, tenantControl d.SetRuntimeClass(&r.resource.Spec.Template.Spec, tenantControlPlane) d.SetReplicas(&r.resource.Spec, tenantControlPlane) d.ResetKubeAPIServerFlags(r.resource, tenantControlPlane) + d.SetInitContainers(&r.resource.Spec.Template.Spec, tenantControlPlane) + d.SetAdditionalContainers(&r.resource.Spec.Template.Spec, tenantControlPlane) d.SetContainers(&r.resource.Spec.Template.Spec, tenantControlPlane, address) + d.SetAdditionalVolumes(&r.resource.Spec.Template.Spec, tenantControlPlane) d.SetVolumes(&r.resource.Spec.Template.Spec, tenantControlPlane) + r.Client.Scheme().Default(r.resource) + return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) } } diff --git a/internal/resources/konnectivity/deployment_resource.go b/internal/resources/konnectivity/deployment_resource.go index 6a9d236..179c3a6 100644 --- a/internal/resources/konnectivity/deployment_resource.go +++ b/internal/resources/konnectivity/deployment_resource.go @@ -7,33 +7,22 @@ import ( "context" "fmt" - "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + builder "github.com/clastix/kamaji/internal/builders/controlplane" "github.com/clastix/kamaji/internal/utilities" ) -const ( - konnectivityEgressSelectorConfigurationPath = "/etc/kubernetes/konnectivity/configurations/egress-selector-configuration.yaml" - konnectivityServerName = "konnectivity-server" - konnectivityServerPath = "/run/konnectivity" - - egressSelectorConfigurationVolume = "egress-selector-configuration" - konnectivityUDSVolume = "konnectivity-uds" - konnectivityServerKubeconfigVolume = "konnectivity-server-kubeconfig" -) - type KubernetesDeploymentResource struct { resource *appsv1.Deployment - Client client.Client + + Builder builder.Konnectivity + Client client.Client } func (r *KubernetesDeploymentResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { @@ -57,52 +46,17 @@ func (r *KubernetesDeploymentResource) CleanUp(ctx context.Context, _ *kamajiv1a logger.Info("performing clean-up from Deployment of Konnectivity") res, err := utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, func() error { - if found, index := utilities.HasNamedContainer(r.resource.Spec.Template.Spec.Containers, konnectivityServerName); found { - logger.Info("removing Konnectivity container") + logger.Info("removing Konnectivity container") + r.Builder.RemovingContainer(&r.resource.Spec.Template.Spec) - var containers []corev1.Container + logger.Info("removing egress selector configuration file from kube-apiserver container") + r.Builder.RemovingKubeAPIServerContainerArg(&r.resource.Spec.Template.Spec) - containers = append(containers, r.resource.Spec.Template.Spec.Containers[:index]...) - containers = append(containers, r.resource.Spec.Template.Spec.Containers[index+1:]...) + logger.Info("removing Konnectivity volumes") + r.Builder.RemovingVolumes(&r.resource.Spec.Template.Spec) - r.resource.Spec.Template.Spec.Containers = containers - } - - if found, index := utilities.HasNamedContainer(r.resource.Spec.Template.Spec.Containers, "kube-apiserver"); found { - argsMap := utilities.ArgsFromSliceToMap(r.resource.Spec.Template.Spec.Containers[index].Args) - - if utilities.ArgsRemoveFlag(argsMap, "--egress-selector-config-file") { - logger.Info("removing egress selector configuration file from kube-apiserver container") - - r.resource.Spec.Template.Spec.Containers[index].Args = utilities.ArgsFromMapToSlice(argsMap) - } - - for _, volumeName := range []string{konnectivityUDSVolume, egressSelectorConfigurationVolume} { - if volumeFound, volumeIndex := utilities.HasNamedVolume(r.resource.Spec.Template.Spec.Volumes, volumeName); volumeFound { - logger.Info("removing Konnectivity volume " + volumeName) - - var volumes []corev1.Volume - - volumes = append(volumes, r.resource.Spec.Template.Spec.Volumes[:volumeIndex]...) - volumes = append(volumes, r.resource.Spec.Template.Spec.Volumes[volumeIndex+1:]...) - - r.resource.Spec.Template.Spec.Volumes = volumes - } - } - - for _, volumeMountName := range []string{konnectivityUDSVolume, egressSelectorConfigurationVolume, konnectivityServerKubeconfigVolume} { - if ok, i := utilities.HasNamedVolumeMount(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts, volumeMountName); ok { - logger.Info("removing Konnectivity volume mount " + volumeMountName) - - var volumesMounts []corev1.VolumeMount - - volumesMounts = append(volumesMounts, r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[:i]...) - volumesMounts = append(volumesMounts, r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[i+1:]...) - - r.resource.Spec.Template.Spec.Containers[index].VolumeMounts = volumesMounts - } - } - } + logger.Info("removing Konnectivity volume mounts") + r.Builder.RemovingVolumeMounts(&r.resource.Spec.Template.Spec) return nil }) @@ -121,95 +75,6 @@ func (r *KubernetesDeploymentResource) Define(_ context.Context, tenantControlPl return nil } -func (r *KubernetesDeploymentResource) syncContainer(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) { - found, index := utilities.HasNamedContainer(r.resource.Spec.Template.Spec.Containers, konnectivityServerName) - if !found { - r.resource.Spec.Template.Spec.Containers = append(r.resource.Spec.Template.Spec.Containers, corev1.Container{}) - index = len(r.resource.Spec.Template.Spec.Containers) - 1 - } - - r.resource.Spec.Template.Spec.Containers[index].Name = konnectivityServerName - r.resource.Spec.Template.Spec.Containers[index].Image = fmt.Sprintf("%s:%s", tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityServerSpec.Image, tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityServerSpec.Version) - r.resource.Spec.Template.Spec.Containers[index].Command = []string{"/proxy-server"} - - args := utilities.ArgsFromSliceToMap(tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityServerSpec.ExtraArgs) - - args["--uds-name"] = fmt.Sprintf("%s/konnectivity-server.socket", konnectivityServerPath) - args["--cluster-cert"] = "/etc/kubernetes/pki/apiserver.crt" - args["--cluster-key"] = "/etc/kubernetes/pki/apiserver.key" - args["--mode"] = "grpc" - args["--server-port"] = "0" - args["--agent-port"] = fmt.Sprintf("%d", tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityServerSpec.Port) - args["--admin-port"] = "8133" - args["--health-port"] = "8134" - args["--agent-namespace"] = "kube-system" - args["--agent-service-account"] = AgentName - args["--kubeconfig"] = "/etc/kubernetes/konnectivity-server.conf" - args["--authentication-audience"] = CertCommonName - args["--server-count"] = fmt.Sprintf("%d", tenantControlPlane.Spec.ControlPlane.Deployment.Replicas) - - r.resource.Spec.Template.Spec.Containers[index].Args = utilities.ArgsFromMapToSlice(args) - r.resource.Spec.Template.Spec.Containers[index].LivenessProbe = &corev1.Probe{ - InitialDelaySeconds: 30, - TimeoutSeconds: 60, - PeriodSeconds: 10, - SuccessThreshold: 1, - FailureThreshold: 3, - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/healthz", - Port: intstr.FromInt(8134), - Scheme: corev1.URISchemeHTTP, - }, - }, - } - r.resource.Spec.Template.Spec.Containers[index].Ports = []corev1.ContainerPort{ - { - Name: "agentport", - ContainerPort: tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityServerSpec.Port, - Protocol: corev1.ProtocolTCP, - }, - { - Name: "adminport", - ContainerPort: 8133, - Protocol: corev1.ProtocolTCP, - }, - { - Name: "healthport", - ContainerPort: 8134, - Protocol: corev1.ProtocolTCP, - }, - } - r.resource.Spec.Template.Spec.Containers[index].VolumeMounts = []corev1.VolumeMount{ - { - Name: "etc-kubernetes-pki", - MountPath: "/etc/kubernetes/pki", - ReadOnly: true, - }, - { - Name: "konnectivity-server-kubeconfig", - MountPath: "/etc/kubernetes/konnectivity-server.conf", - SubPath: "konnectivity-server.conf", - ReadOnly: true, - }, - { - Name: konnectivityUDSVolume, - MountPath: konnectivityServerPath, - ReadOnly: false, - }, - } - r.resource.Spec.Template.Spec.Containers[index].ImagePullPolicy = corev1.PullAlways - r.resource.Spec.Template.Spec.Containers[index].Resources = corev1.ResourceRequirements{ - Limits: nil, - Requests: nil, - } - - if resources := tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityServerSpec.Resources; resources != nil { - r.resource.Spec.Template.Spec.Containers[index].Resources.Limits = resources.Limits - r.resource.Spec.Template.Spec.Containers[index].Resources.Requests = resources.Requests - } -} - func (r *KubernetesDeploymentResource) mutate(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { return func() (err error) { // If konnectivity is disabled, no operation is required: @@ -222,13 +87,11 @@ func (r *KubernetesDeploymentResource) mutate(_ context.Context, tenantControlPl return fmt.Errorf("the Deployment resource is not ready to be mangled for Konnectivity server enrichment") } - r.syncContainer(tenantControlPlane) + r.Builder.BuildKonnectivityContainer(tenantControlPlane.Spec.Addons.Konnectivity, tenantControlPlane.Spec.ControlPlane.Deployment.Replicas, &r.resource.Spec.Template.Spec) + r.Builder.BuildVolumeMounts(&r.resource.Spec.Template.Spec) + r.Builder.BuildVolumes(tenantControlPlane.Status.Addons.Konnectivity, &r.resource.Spec.Template.Spec) - if err = r.patchKubeAPIServerContainer(); err != nil { - return errors.Wrap(err, "cannot sync patch kube-apiserver container") - } - - r.syncVolumes(tenantControlPlane) + r.Client.Scheme().Default(r.resource) return nil } @@ -247,88 +110,3 @@ func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(_ context. return nil } - -func (r *KubernetesDeploymentResource) patchKubeAPIServerContainer() error { - // Patching VolumesMounts - found, index := false, 0 - - found, index = utilities.HasNamedContainer(r.resource.Spec.Template.Spec.Containers, "kube-apiserver") - if !found { - return fmt.Errorf("missing kube-apiserver container, cannot patch arguments") - } - // Adding the egress selector config file flag - args := utilities.ArgsFromSliceToMap(r.resource.Spec.Template.Spec.Containers[index].Args) - - utilities.ArgsAddFlagValue(args, "--egress-selector-config-file", konnectivityEgressSelectorConfigurationPath) - - r.resource.Spec.Template.Spec.Containers[index].Args = utilities.ArgsFromMapToSlice(args) - - vFound, vIndex := false, 0 - // Patching the volume mounts - if vFound, vIndex = utilities.HasNamedVolumeMount(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts, konnectivityUDSVolume); !vFound { - r.resource.Spec.Template.Spec.Containers[index].VolumeMounts = append(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts, corev1.VolumeMount{}) - vIndex = len(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts) - 1 - } - - r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].Name = konnectivityUDSVolume - r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].ReadOnly = false - r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].MountPath = konnectivityServerPath - - if vFound, vIndex = utilities.HasNamedVolumeMount(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts, egressSelectorConfigurationVolume); !vFound { - r.resource.Spec.Template.Spec.Containers[index].VolumeMounts = append(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts, corev1.VolumeMount{}) - vIndex = len(r.resource.Spec.Template.Spec.Containers[index].VolumeMounts) - 1 - } - - r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].Name = egressSelectorConfigurationVolume - r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].ReadOnly = false - r.resource.Spec.Template.Spec.Containers[index].VolumeMounts[vIndex].MountPath = "/etc/kubernetes/konnectivity/configurations" - - return nil -} - -func (r *KubernetesDeploymentResource) syncVolumes(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) { - found, index := false, 0 - // Defining volumes for the UDS socket - found, index = utilities.HasNamedVolume(r.resource.Spec.Template.Spec.Volumes, konnectivityUDSVolume) - if !found { - r.resource.Spec.Template.Spec.Volumes = append(r.resource.Spec.Template.Spec.Volumes, corev1.Volume{}) - index = len(r.resource.Spec.Template.Spec.Volumes) - 1 - } - - r.resource.Spec.Template.Spec.Volumes[index].Name = konnectivityUDSVolume - r.resource.Spec.Template.Spec.Volumes[index].VolumeSource = corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: "Memory", - }, - } - // Defining volumes for the egress selector configuration - found, index = utilities.HasNamedVolume(r.resource.Spec.Template.Spec.Volumes, egressSelectorConfigurationVolume) - if !found { - r.resource.Spec.Template.Spec.Volumes = append(r.resource.Spec.Template.Spec.Volumes, corev1.Volume{}) - index = len(r.resource.Spec.Template.Spec.Volumes) - 1 - } - - r.resource.Spec.Template.Spec.Volumes[index].Name = egressSelectorConfigurationVolume - r.resource.Spec.Template.Spec.Volumes[index].VolumeSource = corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: tenantControlPlane.Status.Addons.Konnectivity.ConfigMap.Name, - }, - DefaultMode: pointer.Int32(420), - }, - } - // Defining volume for the Konnectivity kubeconfig - found, index = utilities.HasNamedVolume(r.resource.Spec.Template.Spec.Volumes, konnectivityServerKubeconfigVolume) - if !found { - r.resource.Spec.Template.Spec.Volumes = append(r.resource.Spec.Template.Spec.Volumes, corev1.Volume{}) - index = len(r.resource.Spec.Template.Spec.Volumes) - 1 - } - - r.resource.Spec.Template.Spec.Volumes[index].Name = konnectivityServerKubeconfigVolume - r.resource.Spec.Template.Spec.Volumes[index].VolumeSource = corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig.SecretName, - DefaultMode: pointer.Int32(420), - }, - } -}