mirror of
https://github.com/outbackdingo/kamaji.git
synced 2026-01-27 10:19:29 +00:00
refactor: using builder strategy for tcp deployment
This commit is contained in:
Dario Tranchitella
663
internal/builders/controlplane/deployment.go
Normal file
663
internal/builders/controlplane/deployment.go
Normal file
@@ -0,0 +1,663 @@
|
||||
// Copyright 2022 Clastix Labs
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package controlplane
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
"k8s.io/utils/pointer"
|
||||
|
||||
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
|
||||
"github.com/clastix/kamaji/internal/types"
|
||||
"github.com/clastix/kamaji/internal/utilities"
|
||||
)
|
||||
|
||||
type orderedIndex int
|
||||
|
||||
const (
|
||||
apiServerIndex orderedIndex = iota
|
||||
schedulerIndex
|
||||
controllerManagerIndex
|
||||
kineIndex
|
||||
)
|
||||
|
||||
const (
|
||||
etcKubernetesPKIVolume orderedIndex = iota
|
||||
etcCACertificates
|
||||
etcSSLCerts
|
||||
usrShareCACertificates
|
||||
usrLocalShareCACertificates
|
||||
schedulerKubeconfig
|
||||
controllerManagerKubeconfig
|
||||
kineConfig
|
||||
|
||||
kineVolumeName = "kine-config"
|
||||
)
|
||||
|
||||
type Deployment struct {
|
||||
Address string
|
||||
ETCDEndpoints []string
|
||||
ETCDCompactionInterval string
|
||||
ETCDStorageType types.ETCDStorageType
|
||||
}
|
||||
|
||||
func (d *Deployment) SetContainers(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane, address string) {
|
||||
d.buildKubeAPIServer(podSpec, tcp, address)
|
||||
d.BuildScheduler(podSpec, tcp)
|
||||
d.buildControllerManager(podSpec, tcp)
|
||||
d.buildKine(podSpec, tcp)
|
||||
}
|
||||
|
||||
func (d *Deployment) SetStrategy(deployment *appsv1.DeploymentSpec) {
|
||||
maxSurge := intstr.FromString("100%")
|
||||
|
||||
maxUnavailable := intstr.FromInt(0)
|
||||
|
||||
deployment.Strategy = appsv1.DeploymentStrategy{
|
||||
Type: appsv1.RollingUpdateDeploymentStrategyType,
|
||||
RollingUpdate: &appsv1.RollingUpdateDeployment{
|
||||
MaxUnavailable: &maxUnavailable,
|
||||
MaxSurge: &maxSurge,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) SetVolumes(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
|
||||
for _, fn := range []func(*corev1.PodSpec, *kamajiv1alpha1.TenantControlPlane){
|
||||
d.buildPKIVolume,
|
||||
d.buildCAVolume,
|
||||
d.buildSSLCertsVolume,
|
||||
d.buildShareCAVolume,
|
||||
d.buildLocalShareCAVolume,
|
||||
d.buildSchedulerVolume,
|
||||
d.buildControllerManagerVolume,
|
||||
d.buildKineVolume,
|
||||
} {
|
||||
fn(podSpec, tcp)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) buildPKIVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
|
||||
if index := int(etcKubernetesPKIVolume) + 1; len(podSpec.Volumes) < index {
|
||||
podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
|
||||
}
|
||||
|
||||
sources := []corev1.VolumeProjection{
|
||||
{
|
||||
Secret: d.secretProjection(tcp.Status.Certificates.APIServer.SecretName, constants.APIServerCertName, constants.APIServerKeyName),
|
||||
},
|
||||
{
|
||||
Secret: d.secretProjection(tcp.Status.Certificates.CA.SecretName, constants.CACertName, constants.CAKeyName),
|
||||
},
|
||||
{
|
||||
Secret: d.secretProjection(tcp.Status.Certificates.APIServerKubeletClient.SecretName, constants.APIServerKubeletClientCertName, constants.APIServerKubeletClientKeyName),
|
||||
},
|
||||
{
|
||||
Secret: d.secretProjection(tcp.Status.Certificates.FrontProxyCA.SecretName, constants.FrontProxyCACertName, constants.FrontProxyCAKeyName),
|
||||
},
|
||||
{
|
||||
Secret: d.secretProjection(tcp.Status.Certificates.FrontProxyClient.SecretName, constants.FrontProxyClientCertName, constants.FrontProxyClientKeyName),
|
||||
},
|
||||
{
|
||||
Secret: d.secretProjection(tcp.Status.Certificates.SA.SecretName, constants.ServiceAccountPublicKeyName, constants.ServiceAccountPrivateKeyName),
|
||||
},
|
||||
}
|
||||
|
||||
if d.ETCDStorageType == types.ETCD {
|
||||
sources = append(sources, corev1.VolumeProjection{
|
||||
Secret: d.secretProjection(tcp.Status.Certificates.ETCD.APIServer.SecretName, constants.APIServerEtcdClientCertName, constants.APIServerEtcdClientKeyName),
|
||||
})
|
||||
sources = append(sources, corev1.VolumeProjection{
|
||||
Secret: &corev1.SecretProjection{
|
||||
LocalObjectReference: corev1.LocalObjectReference{
|
||||
Name: tcp.Status.Certificates.ETCD.CA.SecretName,
|
||||
},
|
||||
Items: []corev1.KeyToPath{
|
||||
{
|
||||
Key: constants.CACertName,
|
||||
Path: constants.EtcdCACertName,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
podSpec.Volumes[etcKubernetesPKIVolume] = corev1.Volume{
|
||||
Name: "etc-kubernetes-pki",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Projected: &corev1.ProjectedVolumeSource{
|
||||
Sources: sources,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) buildCAVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
|
||||
if index := int(etcCACertificates) + 1; len(podSpec.Volumes) < index {
|
||||
podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
|
||||
}
|
||||
|
||||
podSpec.Volumes[etcCACertificates] = corev1.Volume{
|
||||
Name: "etc-ca-certificates",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tcp.Status.Certificates.CA.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) buildSSLCertsVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
|
||||
if index := int(etcSSLCerts) + 1; len(podSpec.Volumes) < index {
|
||||
podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
|
||||
}
|
||||
|
||||
podSpec.Volumes[etcSSLCerts] = corev1.Volume{
|
||||
Name: "etc-ssl-certs",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tcp.Status.Certificates.CA.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) buildShareCAVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
|
||||
if index := int(usrShareCACertificates) + 1; len(podSpec.Volumes) < index {
|
||||
podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
|
||||
}
|
||||
|
||||
podSpec.Volumes[usrShareCACertificates] = corev1.Volume{
|
||||
Name: "usr-share-ca-certificates",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tcp.Status.Certificates.CA.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) buildLocalShareCAVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
|
||||
if index := int(usrLocalShareCACertificates) + 1; len(podSpec.Volumes) < index {
|
||||
podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
|
||||
}
|
||||
|
||||
podSpec.Volumes[usrLocalShareCACertificates] = corev1.Volume{
|
||||
Name: "usr-local-share-ca-certificates",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tcp.Status.Certificates.CA.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) buildSchedulerVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
|
||||
if index := int(schedulerKubeconfig) + 1; len(podSpec.Volumes) < index {
|
||||
podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
|
||||
}
|
||||
|
||||
podSpec.Volumes[schedulerKubeconfig] = corev1.Volume{
|
||||
Name: "scheduler-kubeconfig",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tcp.Status.KubeConfig.Scheduler.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) buildControllerManagerVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
|
||||
if index := int(controllerManagerKubeconfig) + 1; len(podSpec.Volumes) < index {
|
||||
podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
|
||||
}
|
||||
|
||||
podSpec.Volumes[controllerManagerKubeconfig] = corev1.Volume{
|
||||
Name: "controller-manager-kubeconfig",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tcp.Status.KubeConfig.ControllerManager.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) BuildScheduler(podSpec *corev1.PodSpec, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) {
|
||||
if index := int(schedulerIndex) + 1; len(podSpec.Containers) < index {
|
||||
podSpec.Containers = append(podSpec.Containers, corev1.Container{})
|
||||
}
|
||||
|
||||
podSpec.Containers[schedulerIndex].Name = "kube-scheduler"
|
||||
podSpec.Containers[schedulerIndex].Image = fmt.Sprintf("k8s.gcr.io/kube-scheduler:%s", tenantControlPlane.Spec.Kubernetes.Version)
|
||||
podSpec.Containers[schedulerIndex].Command = []string{
|
||||
"kube-scheduler",
|
||||
"--authentication-kubeconfig=/etc/kubernetes/scheduler.conf",
|
||||
"--authorization-kubeconfig=/etc/kubernetes/scheduler.conf",
|
||||
"--bind-address=0.0.0.0",
|
||||
"--kubeconfig=/etc/kubernetes/scheduler.conf",
|
||||
"--leader-elect=true",
|
||||
}
|
||||
podSpec.Containers[schedulerIndex].VolumeMounts = []corev1.VolumeMount{
|
||||
{
|
||||
Name: "scheduler-kubeconfig",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/kubernetes",
|
||||
},
|
||||
}
|
||||
podSpec.Containers[schedulerIndex].LivenessProbe = &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/healthz",
|
||||
Port: intstr.FromInt(10259),
|
||||
Scheme: corev1.URISchemeHTTPS,
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 0,
|
||||
TimeoutSeconds: 1,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
}
|
||||
podSpec.Containers[schedulerIndex].StartupProbe = &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/healthz",
|
||||
Port: intstr.FromInt(10259),
|
||||
Scheme: corev1.URISchemeHTTPS,
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 0,
|
||||
TimeoutSeconds: 1,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
}
|
||||
podSpec.Containers[schedulerIndex].ImagePullPolicy = corev1.PullAlways
|
||||
podSpec.Containers[schedulerIndex].Resources = corev1.ResourceRequirements{
|
||||
Limits: nil,
|
||||
Requests: nil,
|
||||
}
|
||||
|
||||
if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil {
|
||||
if resource := componentsResources.Scheduler; resource != nil {
|
||||
podSpec.Containers[schedulerIndex].Resources = *resource
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) buildControllerManager(podSpec *corev1.PodSpec, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) {
|
||||
if index := int(controllerManagerIndex) + 1; len(podSpec.Containers) < index {
|
||||
podSpec.Containers = append(podSpec.Containers, corev1.Container{})
|
||||
}
|
||||
|
||||
podSpec.Containers[controllerManagerIndex].Name = "kube-controller-manager"
|
||||
podSpec.Containers[controllerManagerIndex].Image = fmt.Sprintf("k8s.gcr.io/kube-controller-manager:%s", tenantControlPlane.Spec.Kubernetes.Version)
|
||||
podSpec.Containers[controllerManagerIndex].Command = []string{
|
||||
"kube-controller-manager",
|
||||
"--allocate-node-cidrs=true",
|
||||
"--authentication-kubeconfig=/etc/kubernetes/controller-manager.conf",
|
||||
"--authorization-kubeconfig=/etc/kubernetes/controller-manager.conf",
|
||||
"--bind-address=0.0.0.0",
|
||||
fmt.Sprintf("--client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
|
||||
fmt.Sprintf("--cluster-name=%s", tenantControlPlane.GetName()),
|
||||
fmt.Sprintf("--cluster-signing-cert-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
|
||||
fmt.Sprintf("--cluster-signing-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CAKeyName)),
|
||||
"--controllers=*,bootstrapsigner,tokencleaner",
|
||||
"--kubeconfig=/etc/kubernetes/controller-manager.conf",
|
||||
"--leader-elect=true",
|
||||
fmt.Sprintf("--service-cluster-ip-range=%s", tenantControlPlane.Spec.NetworkProfile.ServiceCIDR),
|
||||
fmt.Sprintf("--cluster-cidr=%s", tenantControlPlane.Spec.NetworkProfile.PodCIDR),
|
||||
fmt.Sprintf("--requestheader-client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyCACertName)),
|
||||
fmt.Sprintf("--root-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
|
||||
fmt.Sprintf("--service-account-private-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName)),
|
||||
"--use-service-account-credentials=true",
|
||||
}
|
||||
podSpec.Containers[controllerManagerIndex].VolumeMounts = []corev1.VolumeMount{
|
||||
{
|
||||
Name: "controller-manager-kubeconfig",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/kubernetes",
|
||||
},
|
||||
{
|
||||
Name: "etc-kubernetes-pki",
|
||||
ReadOnly: true,
|
||||
MountPath: v1beta3.DefaultCertificatesDir,
|
||||
},
|
||||
{
|
||||
Name: "etc-ca-certificates",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/ca-certificates",
|
||||
},
|
||||
{
|
||||
Name: "etc-ssl-certs",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/ssl/certs",
|
||||
},
|
||||
{
|
||||
Name: "usr-share-ca-certificates",
|
||||
ReadOnly: true,
|
||||
MountPath: "/usr/share/ca-certificates",
|
||||
},
|
||||
{
|
||||
Name: "usr-local-share-ca-certificates",
|
||||
ReadOnly: true,
|
||||
MountPath: "/usr/local/share/ca-certificates",
|
||||
},
|
||||
}
|
||||
podSpec.Containers[controllerManagerIndex].LivenessProbe = &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/healthz",
|
||||
Port: intstr.FromInt(10257),
|
||||
Scheme: corev1.URISchemeHTTPS,
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 0,
|
||||
TimeoutSeconds: 1,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
}
|
||||
podSpec.Containers[controllerManagerIndex].StartupProbe = &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/healthz",
|
||||
Port: intstr.FromInt(10257),
|
||||
Scheme: corev1.URISchemeHTTPS,
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 0,
|
||||
TimeoutSeconds: 1,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
}
|
||||
podSpec.Containers[controllerManagerIndex].ImagePullPolicy = corev1.PullAlways
|
||||
podSpec.Containers[controllerManagerIndex].Resources = corev1.ResourceRequirements{
|
||||
Limits: nil,
|
||||
Requests: nil,
|
||||
}
|
||||
|
||||
if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil {
|
||||
if resource := componentsResources.ControllerManager; resource != nil {
|
||||
podSpec.Containers[controllerManagerIndex].Resources = *resource
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) buildKubeAPIServer(podSpec *corev1.PodSpec, tenantControlPlane *kamajiv1alpha1.TenantControlPlane, address string) {
|
||||
if index := int(apiServerIndex) + 1; len(podSpec.Containers) < index {
|
||||
podSpec.Containers = append(podSpec.Containers, corev1.Container{})
|
||||
}
|
||||
|
||||
args := d.buildKubeAPIServerCommand(tenantControlPlane, address, utilities.ArgsFromSliceToMap(podSpec.Containers[apiServerIndex].Args))
|
||||
|
||||
podSpec.Containers[apiServerIndex].Name = "kube-apiserver"
|
||||
podSpec.Containers[apiServerIndex].Args = utilities.ArgsFromMapToSlice(args)
|
||||
podSpec.Containers[apiServerIndex].Image = fmt.Sprintf("k8s.gcr.io/kube-apiserver:%s", tenantControlPlane.Spec.Kubernetes.Version)
|
||||
podSpec.Containers[apiServerIndex].Command = []string{"kube-apiserver"}
|
||||
podSpec.Containers[apiServerIndex].LivenessProbe = &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/livez",
|
||||
Port: intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
|
||||
Scheme: corev1.URISchemeHTTPS,
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 0,
|
||||
TimeoutSeconds: 1,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
}
|
||||
podSpec.Containers[apiServerIndex].ReadinessProbe = &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/readyz",
|
||||
Port: intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
|
||||
Scheme: corev1.URISchemeHTTPS,
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 0,
|
||||
TimeoutSeconds: 1,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
}
|
||||
podSpec.Containers[apiServerIndex].StartupProbe = &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/livez",
|
||||
Port: intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
|
||||
Scheme: corev1.URISchemeHTTPS,
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 0,
|
||||
TimeoutSeconds: 1,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
}
|
||||
podSpec.Containers[apiServerIndex].ImagePullPolicy = corev1.PullAlways
|
||||
|
||||
if len(podSpec.Containers[apiServerIndex].VolumeMounts) < 5 {
|
||||
podSpec.Containers[apiServerIndex].VolumeMounts = make([]corev1.VolumeMount, 5)
|
||||
}
|
||||
podSpec.Containers[apiServerIndex].VolumeMounts[0] = corev1.VolumeMount{
|
||||
Name: "etc-kubernetes-pki",
|
||||
ReadOnly: true,
|
||||
MountPath: v1beta3.DefaultCertificatesDir,
|
||||
}
|
||||
podSpec.Containers[apiServerIndex].VolumeMounts[1] = corev1.VolumeMount{
|
||||
Name: "etc-ca-certificates",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/ca-certificates",
|
||||
}
|
||||
podSpec.Containers[apiServerIndex].VolumeMounts[2] = corev1.VolumeMount{
|
||||
Name: "etc-ssl-certs",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/ssl/certs",
|
||||
}
|
||||
podSpec.Containers[apiServerIndex].VolumeMounts[3] = corev1.VolumeMount{
|
||||
Name: "usr-share-ca-certificates",
|
||||
ReadOnly: true,
|
||||
MountPath: "/usr/share/ca-certificates",
|
||||
}
|
||||
podSpec.Containers[apiServerIndex].VolumeMounts[4] = corev1.VolumeMount{
|
||||
Name: "usr-local-share-ca-certificates",
|
||||
ReadOnly: true,
|
||||
MountPath: "/usr/local/share/ca-certificates",
|
||||
}
|
||||
podSpec.Containers[apiServerIndex].Resources = corev1.ResourceRequirements{
|
||||
Limits: nil,
|
||||
Requests: nil,
|
||||
}
|
||||
|
||||
if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil {
|
||||
if resource := componentsResources.APIServer; resource != nil {
|
||||
podSpec.Containers[apiServerIndex].Resources = *resource
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) buildKubeAPIServerCommand(tenantControlPlane *kamajiv1alpha1.TenantControlPlane, address string, current map[string]string) map[string]string {
|
||||
desiredArgs := map[string]string{
|
||||
"--allow-privileged": "true",
|
||||
"--authorization-mode": "Node,RBAC",
|
||||
"--advertise-address": address,
|
||||
"--client-ca-file": path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName),
|
||||
"--enable-admission-plugins": strings.Join(tenantControlPlane.Spec.Kubernetes.AdmissionControllers.ToSlice(), ","),
|
||||
"--enable-bootstrap-token-auth": "true",
|
||||
"--etcd-servers": strings.Join(d.ETCDEndpoints, ","),
|
||||
"--service-cluster-ip-range": tenantControlPlane.Spec.NetworkProfile.ServiceCIDR,
|
||||
"--kubelet-client-certificate": path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientCertName),
|
||||
"--kubelet-client-key": path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientKeyName),
|
||||
"--kubelet-preferred-address-types": "Hostname,InternalIP,ExternalIP",
|
||||
"--proxy-client-cert-file": path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyClientCertName),
|
||||
"--proxy-client-key-file": path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyClientKeyName),
|
||||
"--requestheader-allowed-names": "front-proxy-client",
|
||||
"--requestheader-extra-headers-prefix": "X-Remote-Extra-",
|
||||
"--requestheader-group-headers": "X-Remote-Group",
|
||||
"--requestheader-username-headers": "X-Remote-User",
|
||||
"--secure-port": fmt.Sprintf("%d", tenantControlPlane.Spec.NetworkProfile.Port),
|
||||
"--service-account-issuer": fmt.Sprintf("https://localhost:%d", tenantControlPlane.Spec.NetworkProfile.Port),
|
||||
"--service-account-key-file": path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPublicKeyName),
|
||||
"--service-account-signing-key-file": path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName),
|
||||
"--tls-cert-file": path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerCertName),
|
||||
"--tls-private-key-file": path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKeyName),
|
||||
}
|
||||
|
||||
if d.ETCDStorageType == types.ETCD {
|
||||
desiredArgs["--etcd-compaction-interval"] = d.ETCDCompactionInterval
|
||||
desiredArgs["--etcd-cafile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.EtcdCACertName)
|
||||
desiredArgs["--etcd-certfile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientCertName)
|
||||
desiredArgs["--etcd-keyfile"] = path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientKeyName)
|
||||
desiredArgs["--etcd-prefix"] = fmt.Sprintf("/%s", tenantControlPlane.GetName())
|
||||
}
|
||||
|
||||
return utilities.MergeMaps(current, desiredArgs)
|
||||
}
|
||||
|
||||
func (d *Deployment) secretProjection(secretName, certKeyName, keyName string) *corev1.SecretProjection {
|
||||
return &corev1.SecretProjection{
|
||||
LocalObjectReference: corev1.LocalObjectReference{
|
||||
Name: secretName,
|
||||
},
|
||||
Items: []corev1.KeyToPath{
|
||||
{
|
||||
Key: certKeyName,
|
||||
Path: certKeyName,
|
||||
},
|
||||
{
|
||||
Key: keyName,
|
||||
Path: keyName,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) buildKineVolume(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
|
||||
// Kine is expecting an additional volume for its configuration, and it must be removed before proceeding with the
|
||||
// customized storage that is idempotent
|
||||
if found, index := utilities.HasNamedVolume(podSpec.Volumes, kineVolumeName); found {
|
||||
var volumes []corev1.Volume
|
||||
|
||||
volumes = append(volumes, podSpec.Volumes[:index]...)
|
||||
volumes = append(volumes, podSpec.Volumes[index+1:]...)
|
||||
|
||||
podSpec.Volumes = volumes
|
||||
}
|
||||
|
||||
if d.ETCDStorageType == types.KineMySQL {
|
||||
if index := int(kineConfig) + 1; len(podSpec.Volumes) < index {
|
||||
podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{})
|
||||
}
|
||||
|
||||
podSpec.Volumes[kineConfig].Name = kineVolumeName
|
||||
podSpec.Volumes[kineConfig].VolumeSource = corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tcp.Status.Storage.KineMySQL.Certificate.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) buildKine(podSpec *corev1.PodSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
|
||||
const kineContainerName = "kine"
|
||||
// Kine is expecting an additional container, and it must be removed before proceeding with the additional one
|
||||
// in order to make this function idempotent.
|
||||
if found, index := utilities.HasNamedContainer(podSpec.Containers, kineContainerName); found {
|
||||
var containers []corev1.Container
|
||||
|
||||
containers = append(containers, podSpec.Containers[:index]...)
|
||||
containers = append(containers, podSpec.Containers[index+1:]...)
|
||||
|
||||
podSpec.Containers = containers
|
||||
}
|
||||
|
||||
if d.ETCDStorageType == types.KineMySQL {
|
||||
if index := int(kineIndex) + 1; len(podSpec.Containers) < index {
|
||||
podSpec.Containers = append(podSpec.Containers, corev1.Container{})
|
||||
}
|
||||
|
||||
podSpec.Containers[kineIndex].Name = kineContainerName
|
||||
podSpec.Containers[kineIndex].Image = fmt.Sprintf("%s:%s", "rancher/kine", "v0.9.2-amd64") // TODO: parameter.
|
||||
podSpec.Containers[kineIndex].Args = []string{
|
||||
"--endpoint=mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@tcp($(MYSQL_HOST):$(MYSQL_PORT))/$(MYSQL_SCHEMA)",
|
||||
"--ca-file=/kine/ca.crt",
|
||||
"--cert-file=/kine/server.crt",
|
||||
"--key-file=/kine/server.key",
|
||||
}
|
||||
podSpec.Containers[kineIndex].VolumeMounts = []corev1.VolumeMount{
|
||||
{
|
||||
Name: kineVolumeName,
|
||||
MountPath: "/kine",
|
||||
ReadOnly: true,
|
||||
},
|
||||
}
|
||||
podSpec.Containers[kineIndex].Env = []corev1.EnvVar{
|
||||
{
|
||||
Name: "GODEBUG",
|
||||
Value: "x509ignoreCN=0",
|
||||
},
|
||||
}
|
||||
podSpec.Containers[kineIndex].EnvFrom = []corev1.EnvFromSource{
|
||||
{
|
||||
SecretRef: &corev1.SecretEnvSource{
|
||||
LocalObjectReference: corev1.LocalObjectReference{
|
||||
Name: tcp.Status.Storage.KineMySQL.Config.SecretName,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
podSpec.Containers[kineIndex].Ports = []corev1.ContainerPort{
|
||||
{
|
||||
ContainerPort: 2379,
|
||||
Name: "server",
|
||||
Protocol: corev1.ProtocolTCP,
|
||||
},
|
||||
}
|
||||
podSpec.Containers[kineIndex].ImagePullPolicy = corev1.PullAlways
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) SetSelector(deploymentSpec *appsv1.DeploymentSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
|
||||
deploymentSpec.Selector = &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"kamaji.clastix.io/soot": tcp.GetName(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Deployment) SetReplicas(deploymentSpec *appsv1.DeploymentSpec, tcp *kamajiv1alpha1.TenantControlPlane) {
|
||||
deploymentSpec.Replicas = pointer.Int32(tcp.Spec.ControlPlane.Deployment.Replicas)
|
||||
}
|
||||
|
||||
func (d *Deployment) SetTemplateLabels(template *corev1.PodTemplateSpec, labels map[string]string) {
|
||||
template.SetLabels(labels)
|
||||
}
|
||||
|
||||
func (d *Deployment) SetLabels(resource *appsv1.Deployment, labels map[string]string) {
|
||||
resource.SetLabels(labels)
|
||||
}
|
||||
|
||||
func (d *Deployment) SetAnnotations(resource *appsv1.Deployment, annotations map[string]string) {
|
||||
resource.SetAnnotations(annotations)
|
||||
}
|
||||
@@ -5,35 +5,19 @@ package resources
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
quantity "k8s.io/apimachinery/pkg/api/resource"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
|
||||
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
"k8s.io/utils/pointer"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
||||
|
||||
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
|
||||
"github.com/clastix/kamaji/internal/resources/konnectivity"
|
||||
builder "github.com/clastix/kamaji/internal/builders/controlplane"
|
||||
"github.com/clastix/kamaji/internal/types"
|
||||
"github.com/clastix/kamaji/internal/utilities"
|
||||
)
|
||||
|
||||
const (
|
||||
konnectivityEgressSelectorConfigurationPath = "/etc/kubernetes/konnectivity/configurations/egress-selector-configuration.yaml"
|
||||
konnectivityServerName = "konnectivity-server"
|
||||
konnectivityServerPath = "/run/konnectivity"
|
||||
konnectivityUDSName = "konnectivity-uds"
|
||||
)
|
||||
|
||||
type KubernetesDeploymentResource struct {
|
||||
resource *appsv1.Deployment
|
||||
Client client.Client
|
||||
@@ -47,24 +31,23 @@ func (r *KubernetesDeploymentResource) isStatusEqual(tenantControlPlane *kamajiv
|
||||
return r.resource.Status.String() == tenantControlPlane.Status.Kubernetes.Deployment.DeploymentStatus.String()
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
|
||||
func (r *KubernetesDeploymentResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
|
||||
return !r.isStatusEqual(tenantControlPlane) || tenantControlPlane.Spec.Kubernetes.Version != tenantControlPlane.Status.Kubernetes.Version.Version
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool {
|
||||
func (r *KubernetesDeploymentResource) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) {
|
||||
return tenantControlPlane.Spec.Addons.Konnectivity == nil, nil
|
||||
func (r *KubernetesDeploymentResource) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
|
||||
func (r *KubernetesDeploymentResource) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
|
||||
r.resource = &appsv1.Deployment{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: tenantControlPlane.GetName(),
|
||||
Namespace: tenantControlPlane.GetNamespace(),
|
||||
Labels: utilities.CommonLabels(tenantControlPlane.GetName()),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -74,181 +57,26 @@ func (r *KubernetesDeploymentResource) Define(ctx context.Context, tenantControl
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn {
|
||||
maxSurge := intstr.FromString("100%")
|
||||
|
||||
maxUnavailable := intstr.FromInt(0)
|
||||
|
||||
address, err := tenantControlPlane.GetControlPlaneAddress(ctx, r.Client)
|
||||
if err != nil {
|
||||
return func() error {
|
||||
return func() error {
|
||||
address, err := tenantControlPlane.GetControlPlaneAddress(ctx, r.Client)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "cannot create TenantControlPlane Deployment")
|
||||
}
|
||||
}
|
||||
|
||||
return func() error {
|
||||
labels := utilities.MergeMaps(r.resource.GetLabels(), tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Labels)
|
||||
r.resource.SetLabels(labels)
|
||||
|
||||
annotations := utilities.MergeMaps(r.resource.GetAnnotations(), tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Annotations)
|
||||
r.resource.SetAnnotations(annotations)
|
||||
|
||||
r.resource.Spec.Replicas = pointer.Int32(tenantControlPlane.Spec.ControlPlane.Deployment.Replicas)
|
||||
r.resource.Spec.Selector = &metav1.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"kamaji.clastix.io/soot": tenantControlPlane.GetName(),
|
||||
},
|
||||
}
|
||||
r.resource.Spec.Template.ObjectMeta = metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"kamaji.clastix.io/soot": tenantControlPlane.GetName(),
|
||||
"component.kamaji.clastix.io/api-server-certificate": func() (hash string) {
|
||||
hash, _ = utilities.SecretHashValue(ctx, r.Client, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.APIServer.SecretName)
|
||||
|
||||
return
|
||||
}(),
|
||||
"component.kamaji.clastix.io/api-server-kubelet-client-certificate": func() (hash string) {
|
||||
hash, _ = utilities.SecretHashValue(ctx, r.Client, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.APIServerKubeletClient.SecretName)
|
||||
|
||||
return
|
||||
}(),
|
||||
"component.kamaji.clastix.io/ca": func() (hash string) {
|
||||
hash, _ = utilities.SecretHashValue(ctx, r.Client, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.CA.SecretName)
|
||||
|
||||
return
|
||||
}(),
|
||||
"component.kamaji.clastix.io/controller-manager-kubeconfig": func() (hash string) {
|
||||
hash, _ = utilities.SecretHashValue(ctx, r.Client, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.ControllerManager.SecretName)
|
||||
|
||||
return
|
||||
}(),
|
||||
"component.kamaji.clastix.io/front-proxy-ca-certificate": func() (hash string) {
|
||||
hash, _ = utilities.SecretHashValue(ctx, r.Client, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName)
|
||||
|
||||
return
|
||||
}(),
|
||||
"component.kamaji.clastix.io/front-proxy-client-certificate": func() (hash string) {
|
||||
hash, _ = utilities.SecretHashValue(ctx, r.Client, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyClient.SecretName)
|
||||
|
||||
return
|
||||
}(),
|
||||
"component.kamaji.clastix.io/service-account": func() (hash string) {
|
||||
hash, _ = utilities.SecretHashValue(ctx, r.Client, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.SA.SecretName)
|
||||
|
||||
return
|
||||
}(),
|
||||
"component.kamaji.clastix.io/scheduler-kubeconfig": func() (hash string) {
|
||||
hash, _ = utilities.SecretHashValue(ctx, r.Client, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.Scheduler.SecretName)
|
||||
|
||||
return
|
||||
}(),
|
||||
},
|
||||
}
|
||||
r.resource.Spec.Template.Spec.Volumes = []corev1.Volume{
|
||||
{
|
||||
Name: "etc-kubernetes-pki",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Projected: &corev1.ProjectedVolumeSource{
|
||||
Sources: []corev1.VolumeProjection{
|
||||
{
|
||||
Secret: secretProjection(tenantControlPlane.Status.Certificates.APIServer.SecretName, constants.APIServerCertName, constants.APIServerKeyName),
|
||||
},
|
||||
{
|
||||
Secret: secretProjection(tenantControlPlane.Status.Certificates.CA.SecretName, constants.CACertName, constants.CAKeyName),
|
||||
},
|
||||
{
|
||||
Secret: secretProjection(tenantControlPlane.Status.Certificates.APIServerKubeletClient.SecretName, constants.APIServerKubeletClientCertName, constants.APIServerKubeletClientKeyName),
|
||||
},
|
||||
{
|
||||
Secret: secretProjection(tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName, constants.FrontProxyCACertName, constants.FrontProxyCAKeyName),
|
||||
},
|
||||
{
|
||||
Secret: secretProjection(tenantControlPlane.Status.Certificates.FrontProxyClient.SecretName, constants.FrontProxyClientCertName, constants.FrontProxyClientKeyName),
|
||||
},
|
||||
{
|
||||
Secret: secretProjection(tenantControlPlane.Status.Certificates.SA.SecretName, constants.ServiceAccountPublicKeyName, constants.ServiceAccountPrivateKeyName),
|
||||
},
|
||||
},
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "etc-ca-certificates",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tenantControlPlane.Status.Certificates.CA.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "etc-ssl-certs",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tenantControlPlane.Status.Certificates.CA.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "usr-share-ca-certificates",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tenantControlPlane.Status.Certificates.CA.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "usr-local-share-ca-certificates",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tenantControlPlane.Status.Certificates.CA.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "scheduler-kubeconfig",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tenantControlPlane.Status.KubeConfig.Scheduler.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "controller-manager-kubeconfig",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tenantControlPlane.Status.KubeConfig.ControllerManager.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if len(r.resource.Spec.Template.Spec.Containers) < 3 {
|
||||
r.resource.Spec.Template.Spec.Containers = make([]corev1.Container, 3)
|
||||
}
|
||||
|
||||
r.syncKubeApiServer(tenantControlPlane, address)
|
||||
r.syncScheduler(tenantControlPlane)
|
||||
r.syncControllerManager(tenantControlPlane)
|
||||
|
||||
r.resource.Spec.Strategy = appsv1.DeploymentStrategy{
|
||||
Type: appsv1.RollingUpdateDeploymentStrategyType,
|
||||
RollingUpdate: &appsv1.RollingUpdateDeployment{
|
||||
MaxUnavailable: &maxUnavailable,
|
||||
MaxSurge: &maxSurge,
|
||||
},
|
||||
}
|
||||
|
||||
r.customizeStorage(ctx, &r.resource.Spec.Template, *tenantControlPlane)
|
||||
|
||||
if err := r.reconcileKonnectivity(&r.resource.Spec.Template.Spec, *tenantControlPlane); err != nil {
|
||||
return err
|
||||
d := builder.Deployment{
|
||||
Address: address,
|
||||
ETCDEndpoints: r.ETCDEndpoints,
|
||||
ETCDCompactionInterval: r.ETCDCompactionInterval,
|
||||
ETCDStorageType: r.ETCDStorageType,
|
||||
}
|
||||
d.SetLabels(r.resource, utilities.MergeMaps(utilities.CommonLabels(tenantControlPlane.GetName()), tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Labels))
|
||||
d.SetAnnotations(r.resource, utilities.MergeMaps(r.resource.Annotations, tenantControlPlane.Spec.ControlPlane.Deployment.AdditionalMetadata.Annotations))
|
||||
d.SetTemplateLabels(&r.resource.Spec.Template, r.deploymentTemplateLabels(ctx, tenantControlPlane))
|
||||
d.SetStrategy(&r.resource.Spec)
|
||||
d.SetSelector(&r.resource.Spec, tenantControlPlane)
|
||||
d.SetReplicas(&r.resource.Spec, tenantControlPlane)
|
||||
d.SetContainers(&r.resource.Spec.Template.Spec, tenantControlPlane, address)
|
||||
d.SetVolumes(&r.resource.Spec.Template.Spec, tenantControlPlane)
|
||||
|
||||
return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme())
|
||||
}
|
||||
@@ -262,7 +90,7 @@ func (r *KubernetesDeploymentResource) GetName() string {
|
||||
return r.Name
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
|
||||
func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
|
||||
switch {
|
||||
case !r.isProgressingUpgrade():
|
||||
tenantControlPlane.Status.Kubernetes.Version.Status = &kamajiv1alpha1.VersionReady
|
||||
@@ -285,6 +113,33 @@ func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(ctx contex
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) deploymentTemplateLabels(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (labels map[string]string) {
|
||||
hash := func(ctx context.Context, namespace, secretName string) (hash string) {
|
||||
hash, _ = utilities.SecretHashValue(ctx, r.Client, namespace, secretName)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
labels = map[string]string{
|
||||
"kamaji.clastix.io/soot": tenantControlPlane.GetName(),
|
||||
"component.kamaji.clastix.io/api-server-certificate": hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.APIServer.SecretName),
|
||||
"component.kamaji.clastix.io/api-server-kubelet-client-certificate": hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.APIServerKubeletClient.SecretName),
|
||||
"component.kamaji.clastix.io/ca": hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.CA.SecretName),
|
||||
"component.kamaji.clastix.io/controller-manager-kubeconfig": hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.ControllerManager.SecretName),
|
||||
"component.kamaji.clastix.io/front-proxy-ca-certificate": hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyCA.SecretName),
|
||||
"component.kamaji.clastix.io/front-proxy-client-certificate": hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.FrontProxyClient.SecretName),
|
||||
"component.kamaji.clastix.io/service-account": hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.SA.SecretName),
|
||||
"component.kamaji.clastix.io/scheduler-kubeconfig": hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.KubeConfig.Scheduler.SecretName),
|
||||
}
|
||||
|
||||
if r.ETCDStorageType == types.ETCD {
|
||||
labels["component.kamaji.clastix.io/etcd-ca-certificates"] = hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.CA.SecretName)
|
||||
labels["component.kamaji.clastix.io/etcd-certificates"] = hash(ctx, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName)
|
||||
}
|
||||
|
||||
return labels
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) isProgressingUpgrade() bool {
|
||||
if r.resource.ObjectMeta.GetGeneration() != r.resource.Status.ObservedGeneration {
|
||||
return true
|
||||
@@ -310,535 +165,3 @@ func (r *KubernetesDeploymentResource) isProvisioning(tenantControlPlane *kamaji
|
||||
func (r *KubernetesDeploymentResource) isNotReady() bool {
|
||||
return r.resource.Status.ReadyReplicas == 0
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) reconcileKonnectivity(podSpec *corev1.PodSpec, tenantControlPlane kamajiv1alpha1.TenantControlPlane) error {
|
||||
if tenantControlPlane.Spec.Addons.Konnectivity == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return r.addKonnectivity(podSpec, tenantControlPlane)
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) addKonnectivity(podSpec *corev1.PodSpec, tenantControlPlane kamajiv1alpha1.TenantControlPlane) error {
|
||||
flags := r.buildKonnectivityFlags()
|
||||
podSpec.Containers[0].Command = append(podSpec.Containers[0].Command, flags...)
|
||||
|
||||
volumes := r.buildKonnectivityVolumes(tenantControlPlane)
|
||||
podSpec.Volumes = append(podSpec.Volumes, volumes...)
|
||||
|
||||
volumeMounts := r.buildKonnectivityVolumeMounts()
|
||||
podSpec.Containers[0].VolumeMounts = append(podSpec.Containers[0].VolumeMounts, volumeMounts...)
|
||||
|
||||
container := r.buildKonnectivityServerContainer(tenantControlPlane)
|
||||
podSpec.Containers = append(podSpec.Containers, container)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) buildKonnectivityFlags() []string {
|
||||
return []string{
|
||||
fmt.Sprintf("--egress-selector-config-file=%s", konnectivityEgressSelectorConfigurationPath),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) buildKonnectivityVolumes(tenantControlPlane kamajiv1alpha1.TenantControlPlane) []corev1.Volume {
|
||||
return []corev1.Volume{
|
||||
{
|
||||
Name: konnectivityUDSName,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
EmptyDir: &corev1.EmptyDirVolumeSource{
|
||||
Medium: "Memory",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "egress-selector-configuration",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
ConfigMap: &corev1.ConfigMapVolumeSource{
|
||||
LocalObjectReference: corev1.LocalObjectReference{
|
||||
Name: tenantControlPlane.Status.Addons.Konnectivity.EgressSelectorConfiguration,
|
||||
},
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "konnectivity-server-kubeconfig",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) buildKonnectivityVolumeMounts() []corev1.VolumeMount {
|
||||
return []corev1.VolumeMount{
|
||||
{
|
||||
Name: konnectivityUDSName,
|
||||
ReadOnly: false,
|
||||
MountPath: konnectivityServerPath,
|
||||
},
|
||||
{
|
||||
Name: "egress-selector-configuration",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/kubernetes/konnectivity/configurations",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) buildKonnectivityServerContainer(tenantControlPlane kamajiv1alpha1.TenantControlPlane) corev1.Container {
|
||||
return corev1.Container{
|
||||
Name: konnectivityServerName,
|
||||
Image: fmt.Sprintf("%s:%s", tenantControlPlane.Spec.Addons.Konnectivity.ServerImage, tenantControlPlane.Spec.Addons.Konnectivity.Version),
|
||||
Command: []string{"/proxy-server"},
|
||||
Args: []string{
|
||||
"-v=8",
|
||||
"--logtostderr=true",
|
||||
fmt.Sprintf("--uds-name=%s/konnectivity-server.socket", konnectivityServerPath),
|
||||
"--cluster-cert=/etc/kubernetes/pki/apiserver.crt",
|
||||
"--cluster-key=/etc/kubernetes/pki/apiserver.key",
|
||||
"--mode=grpc",
|
||||
"--server-port=0",
|
||||
fmt.Sprintf("--agent-port=%d", tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort),
|
||||
"--admin-port=8133",
|
||||
"--health-port=8134",
|
||||
"--agent-namespace=kube-system",
|
||||
fmt.Sprintf("--agent-service-account=%s", konnectivity.AgentName),
|
||||
"--kubeconfig=/etc/kubernetes/konnectivity-server.conf",
|
||||
fmt.Sprintf("--authentication-audience=%s", konnectivity.CertCommonName),
|
||||
fmt.Sprintf("--server-count=%d", tenantControlPlane.Spec.ControlPlane.Deployment.Replicas),
|
||||
},
|
||||
LivenessProbe: &corev1.Probe{
|
||||
InitialDelaySeconds: 30,
|
||||
TimeoutSeconds: 60,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/healthz",
|
||||
Port: intstr.FromInt(8134),
|
||||
Scheme: corev1.URISchemeHTTP,
|
||||
},
|
||||
},
|
||||
},
|
||||
Resources: corev1.ResourceRequirements{
|
||||
Requests: corev1.ResourceList{
|
||||
corev1.ResourceCPU: quantity.MustParse("100m"),
|
||||
},
|
||||
},
|
||||
Ports: []corev1.ContainerPort{
|
||||
{
|
||||
Name: "agentport",
|
||||
ContainerPort: tenantControlPlane.Spec.Addons.Konnectivity.ProxyPort,
|
||||
Protocol: corev1.ProtocolTCP,
|
||||
},
|
||||
{
|
||||
Name: "adminport",
|
||||
ContainerPort: 8133,
|
||||
Protocol: corev1.ProtocolTCP,
|
||||
},
|
||||
{
|
||||
Name: "healthport",
|
||||
ContainerPort: 8134,
|
||||
Protocol: corev1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{
|
||||
Name: "etc-kubernetes-pki",
|
||||
MountPath: "/etc/kubernetes/pki",
|
||||
ReadOnly: true,
|
||||
},
|
||||
{
|
||||
Name: "konnectivity-server-kubeconfig",
|
||||
MountPath: "/etc/kubernetes/konnectivity-server.conf",
|
||||
SubPath: "konnectivity-server.conf",
|
||||
ReadOnly: true,
|
||||
},
|
||||
{
|
||||
Name: "konnectivity-uds",
|
||||
MountPath: konnectivityServerPath,
|
||||
ReadOnly: false,
|
||||
},
|
||||
},
|
||||
TerminationMessagePath: "/dev/termination-log",
|
||||
TerminationMessagePolicy: "File",
|
||||
ImagePullPolicy: corev1.PullAlways,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) customizeStorage(ctx context.Context, podTemplate *corev1.PodTemplateSpec, tenantControlPlane kamajiv1alpha1.TenantControlPlane) {
|
||||
switch r.ETCDStorageType {
|
||||
case types.ETCD:
|
||||
r.customizeETCDStorage(ctx, podTemplate, tenantControlPlane)
|
||||
case types.KineMySQL:
|
||||
r.customizeKineMySQLStorage(ctx, podTemplate, tenantControlPlane)
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) customizeETCDStorage(ctx context.Context, podTemplate *corev1.PodTemplateSpec, tenantControlPlane kamajiv1alpha1.TenantControlPlane) {
|
||||
labels := map[string]string{
|
||||
"component.kamaji.clastix.io/etcd-ca-certificates": func() (hash string) {
|
||||
hash, _ = utilities.SecretHashValue(ctx, r.Client, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.CA.SecretName)
|
||||
|
||||
return
|
||||
}(),
|
||||
"component.kamaji.clastix.io/etcd-certificates": func() (hash string) {
|
||||
hash, _ = utilities.SecretHashValue(ctx, r.Client, tenantControlPlane.GetNamespace(), tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName)
|
||||
|
||||
return
|
||||
}(),
|
||||
}
|
||||
|
||||
podTemplate.SetLabels(
|
||||
utilities.MergeMaps(labels, podTemplate.Labels),
|
||||
)
|
||||
|
||||
commands := []string{
|
||||
fmt.Sprintf("--etcd-compaction-interval=%s", r.ETCDCompactionInterval),
|
||||
fmt.Sprintf("--etcd-cafile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.EtcdCACertName)),
|
||||
fmt.Sprintf("--etcd-certfile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientCertName)),
|
||||
fmt.Sprintf("--etcd-keyfile=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerEtcdClientKeyName)),
|
||||
fmt.Sprintf("--etcd-prefix=/%s", tenantControlPlane.GetName()),
|
||||
}
|
||||
|
||||
podTemplate.Spec.Containers[0].Command = append(podTemplate.Spec.Containers[0].Command, commands...)
|
||||
|
||||
volumeProjections := []corev1.VolumeProjection{
|
||||
{
|
||||
Secret: secretProjection(tenantControlPlane.Status.Certificates.ETCD.APIServer.SecretName, constants.APIServerEtcdClientCertName, constants.APIServerEtcdClientKeyName),
|
||||
},
|
||||
{
|
||||
Secret: &corev1.SecretProjection{
|
||||
LocalObjectReference: corev1.LocalObjectReference{
|
||||
Name: tenantControlPlane.Status.Certificates.ETCD.CA.SecretName,
|
||||
},
|
||||
Items: []corev1.KeyToPath{
|
||||
{
|
||||
Key: constants.CACertName,
|
||||
Path: constants.EtcdCACertName,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
podTemplate.Spec.Volumes[0].VolumeSource.Projected.Sources = append(podTemplate.Spec.Volumes[0].VolumeSource.Projected.Sources, volumeProjections...)
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) customizeKineMySQLStorage(ctx context.Context, podTemplate *corev1.PodTemplateSpec, tenantControlPlane kamajiv1alpha1.TenantControlPlane) {
|
||||
volume := corev1.Volume{
|
||||
Name: "mysql-config",
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: tenantControlPlane.Status.Storage.KineMySQL.Certificate.SecretName,
|
||||
DefaultMode: pointer.Int32Ptr(420),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
podTemplate.Spec.Volumes = append(podTemplate.Spec.Volumes, volume)
|
||||
|
||||
container := corev1.Container{
|
||||
Name: "kine",
|
||||
// TODO: parameter.
|
||||
Image: fmt.Sprintf("%s:%s", "rancher/kine", "v0.9.2-amd64"),
|
||||
Args: []string{
|
||||
"--endpoint=mysql://$(MYSQL_USER):$(MYSQL_PASSWORD)@tcp($(MYSQL_HOST):$(MYSQL_PORT))/$(MYSQL_SCHEMA)",
|
||||
"--ca-file=/kine/ca.crt",
|
||||
"--cert-file=/kine/server.crt",
|
||||
"--key-file=/kine/server.key",
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{
|
||||
{
|
||||
Name: volume.Name,
|
||||
MountPath: "/kine",
|
||||
ReadOnly: true,
|
||||
},
|
||||
},
|
||||
Env: []corev1.EnvVar{
|
||||
{Name: "GODEBUG", Value: "x509ignoreCN=0"},
|
||||
},
|
||||
EnvFrom: []corev1.EnvFromSource{
|
||||
{
|
||||
SecretRef: &corev1.SecretEnvSource{
|
||||
LocalObjectReference: corev1.LocalObjectReference{
|
||||
Name: tenantControlPlane.Status.Storage.KineMySQL.Config.SecretName,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
Ports: []corev1.ContainerPort{
|
||||
{
|
||||
ContainerPort: 2379,
|
||||
Name: "server",
|
||||
Protocol: corev1.ProtocolTCP,
|
||||
},
|
||||
},
|
||||
TerminationMessagePath: "/dev/termination-log",
|
||||
TerminationMessagePolicy: "File",
|
||||
ImagePullPolicy: corev1.PullAlways,
|
||||
}
|
||||
|
||||
podTemplate.Spec.Containers = append(podTemplate.Spec.Containers, container)
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) syncKubeApiServer(tenantControlPlane *kamajiv1alpha1.TenantControlPlane, address string) {
|
||||
r.resource.Spec.Template.Spec.Containers[0].Name = "kube-apiserver"
|
||||
r.resource.Spec.Template.Spec.Containers[0].Image = fmt.Sprintf("k8s.gcr.io/kube-apiserver:%s", tenantControlPlane.Spec.Kubernetes.Version)
|
||||
r.resource.Spec.Template.Spec.Containers[0].Command = []string{
|
||||
"kube-apiserver",
|
||||
"--allow-privileged=true",
|
||||
"--authorization-mode=Node,RBAC",
|
||||
fmt.Sprintf("--advertise-address=%s", address),
|
||||
fmt.Sprintf("--client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
|
||||
fmt.Sprintf("--enable-admission-plugins=%s", strings.Join(tenantControlPlane.Spec.Kubernetes.AdmissionControllers.ToSlice(), ",")),
|
||||
"--enable-bootstrap-token-auth=true",
|
||||
fmt.Sprintf("--etcd-servers=%s", strings.Join(r.ETCDEndpoints, ",")),
|
||||
fmt.Sprintf("--service-cluster-ip-range=%s", tenantControlPlane.Spec.NetworkProfile.ServiceCIDR),
|
||||
fmt.Sprintf("--kubelet-client-certificate=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientCertName)),
|
||||
fmt.Sprintf("--kubelet-client-key=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKubeletClientKeyName)),
|
||||
"--kubelet-preferred-address-types=Hostname,InternalIP,ExternalIP",
|
||||
fmt.Sprintf("--proxy-client-cert-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyClientCertName)),
|
||||
fmt.Sprintf("--proxy-client-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyClientKeyName)),
|
||||
"--requestheader-allowed-names=front-proxy-client",
|
||||
"--requestheader-extra-headers-prefix=X-Remote-Extra-",
|
||||
"--requestheader-group-headers=X-Remote-Group",
|
||||
"--requestheader-username-headers=X-Remote-User",
|
||||
fmt.Sprintf("--secure-port=%d", tenantControlPlane.Spec.NetworkProfile.Port),
|
||||
fmt.Sprintf("--service-account-issuer=https://localhost:%d", tenantControlPlane.Spec.NetworkProfile.Port),
|
||||
fmt.Sprintf("--service-account-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPublicKeyName)),
|
||||
fmt.Sprintf("--service-account-signing-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName)),
|
||||
fmt.Sprintf("--tls-cert-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerCertName)),
|
||||
fmt.Sprintf("--tls-private-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.APIServerKeyName)),
|
||||
}
|
||||
r.resource.Spec.Template.Spec.Containers[0].LivenessProbe = &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/livez",
|
||||
Port: intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
|
||||
Scheme: corev1.URISchemeHTTPS,
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 0,
|
||||
TimeoutSeconds: 1,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
}
|
||||
r.resource.Spec.Template.Spec.Containers[0].ReadinessProbe = &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/readyz",
|
||||
Port: intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
|
||||
Scheme: corev1.URISchemeHTTPS,
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 0,
|
||||
TimeoutSeconds: 1,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
}
|
||||
r.resource.Spec.Template.Spec.Containers[0].StartupProbe = &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/livez",
|
||||
Port: intstr.FromInt(int(tenantControlPlane.Spec.NetworkProfile.Port)),
|
||||
Scheme: corev1.URISchemeHTTPS,
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 0,
|
||||
TimeoutSeconds: 1,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
}
|
||||
r.resource.Spec.Template.Spec.Containers[0].ImagePullPolicy = corev1.PullAlways
|
||||
r.resource.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
|
||||
{
|
||||
Name: "etc-kubernetes-pki",
|
||||
ReadOnly: true,
|
||||
MountPath: v1beta3.DefaultCertificatesDir,
|
||||
},
|
||||
{
|
||||
Name: "etc-ca-certificates",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/ca-certificates",
|
||||
},
|
||||
{
|
||||
Name: "etc-ssl-certs",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/ssl/certs",
|
||||
},
|
||||
{
|
||||
Name: "usr-share-ca-certificates",
|
||||
ReadOnly: true,
|
||||
MountPath: "/usr/share/ca-certificates",
|
||||
},
|
||||
{
|
||||
Name: "usr-local-share-ca-certificates",
|
||||
ReadOnly: true,
|
||||
MountPath: "/usr/local/share/ca-certificates",
|
||||
},
|
||||
}
|
||||
|
||||
if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil {
|
||||
if resource := componentsResources.APIServer; resource != nil {
|
||||
r.resource.Spec.Template.Spec.Containers[0].Resources = *resource
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) syncScheduler(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) {
|
||||
r.resource.Spec.Template.Spec.Containers[1].Name = "kube-scheduler"
|
||||
r.resource.Spec.Template.Spec.Containers[1].Image = fmt.Sprintf("k8s.gcr.io/kube-scheduler:%s", tenantControlPlane.Spec.Kubernetes.Version)
|
||||
r.resource.Spec.Template.Spec.Containers[1].Command = []string{
|
||||
"kube-scheduler",
|
||||
"--authentication-kubeconfig=/etc/kubernetes/scheduler.conf",
|
||||
"--authorization-kubeconfig=/etc/kubernetes/scheduler.conf",
|
||||
"--bind-address=0.0.0.0",
|
||||
"--kubeconfig=/etc/kubernetes/scheduler.conf",
|
||||
"--leader-elect=true",
|
||||
}
|
||||
r.resource.Spec.Template.Spec.Containers[1].VolumeMounts = []corev1.VolumeMount{
|
||||
{
|
||||
Name: "scheduler-kubeconfig",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/kubernetes",
|
||||
},
|
||||
}
|
||||
r.resource.Spec.Template.Spec.Containers[1].LivenessProbe = &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/healthz",
|
||||
Port: intstr.FromInt(10259),
|
||||
Scheme: corev1.URISchemeHTTPS,
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 0,
|
||||
TimeoutSeconds: 1,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
}
|
||||
r.resource.Spec.Template.Spec.Containers[1].StartupProbe = &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/healthz",
|
||||
Port: intstr.FromInt(10259),
|
||||
Scheme: corev1.URISchemeHTTPS,
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 0,
|
||||
TimeoutSeconds: 1,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
}
|
||||
r.resource.Spec.Template.Spec.Containers[1].ImagePullPolicy = corev1.PullAlways
|
||||
|
||||
if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil {
|
||||
if resource := componentsResources.Scheduler; resource != nil {
|
||||
r.resource.Spec.Template.Spec.Containers[1].Resources = *resource
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *KubernetesDeploymentResource) syncControllerManager(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) {
|
||||
r.resource.Spec.Template.Spec.Containers[2].Name = "kube-controller-manager"
|
||||
r.resource.Spec.Template.Spec.Containers[2].Image = fmt.Sprintf("k8s.gcr.io/kube-controller-manager:%s", tenantControlPlane.Spec.Kubernetes.Version)
|
||||
r.resource.Spec.Template.Spec.Containers[2].Command = []string{
|
||||
"kube-controller-manager",
|
||||
"--allocate-node-cidrs=true",
|
||||
"--authentication-kubeconfig=/etc/kubernetes/controller-manager.conf",
|
||||
"--authorization-kubeconfig=/etc/kubernetes/controller-manager.conf",
|
||||
"--bind-address=0.0.0.0",
|
||||
fmt.Sprintf("--client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
|
||||
fmt.Sprintf("--cluster-name=%s", tenantControlPlane.GetName()),
|
||||
fmt.Sprintf("--cluster-signing-cert-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
|
||||
fmt.Sprintf("--cluster-signing-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CAKeyName)),
|
||||
"--controllers=*,bootstrapsigner,tokencleaner",
|
||||
"--kubeconfig=/etc/kubernetes/controller-manager.conf",
|
||||
"--leader-elect=true",
|
||||
fmt.Sprintf("--service-cluster-ip-range=%s", tenantControlPlane.Spec.NetworkProfile.ServiceCIDR),
|
||||
fmt.Sprintf("--cluster-cidr=%s", tenantControlPlane.Spec.NetworkProfile.PodCIDR),
|
||||
fmt.Sprintf("--requestheader-client-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.FrontProxyCACertName)),
|
||||
fmt.Sprintf("--root-ca-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.CACertName)),
|
||||
fmt.Sprintf("--service-account-private-key-file=%s", path.Join(v1beta3.DefaultCertificatesDir, constants.ServiceAccountPrivateKeyName)),
|
||||
"--use-service-account-credentials=true",
|
||||
}
|
||||
r.resource.Spec.Template.Spec.Containers[2].VolumeMounts = []corev1.VolumeMount{
|
||||
{
|
||||
Name: "controller-manager-kubeconfig",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/kubernetes",
|
||||
},
|
||||
{
|
||||
Name: "etc-kubernetes-pki",
|
||||
ReadOnly: true,
|
||||
MountPath: v1beta3.DefaultCertificatesDir,
|
||||
},
|
||||
{
|
||||
Name: "etc-ca-certificates",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/ca-certificates",
|
||||
},
|
||||
{
|
||||
Name: "etc-ssl-certs",
|
||||
ReadOnly: true,
|
||||
MountPath: "/etc/ssl/certs",
|
||||
},
|
||||
{
|
||||
Name: "usr-share-ca-certificates",
|
||||
ReadOnly: true,
|
||||
MountPath: "/usr/share/ca-certificates",
|
||||
},
|
||||
{
|
||||
Name: "usr-local-share-ca-certificates",
|
||||
ReadOnly: true,
|
||||
MountPath: "/usr/local/share/ca-certificates",
|
||||
},
|
||||
}
|
||||
r.resource.Spec.Template.Spec.Containers[2].LivenessProbe = &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/healthz",
|
||||
Port: intstr.FromInt(10257),
|
||||
Scheme: corev1.URISchemeHTTPS,
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 0,
|
||||
TimeoutSeconds: 1,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
}
|
||||
r.resource.Spec.Template.Spec.Containers[2].StartupProbe = &corev1.Probe{
|
||||
ProbeHandler: corev1.ProbeHandler{
|
||||
HTTPGet: &corev1.HTTPGetAction{
|
||||
Path: "/healthz",
|
||||
Port: intstr.FromInt(10257),
|
||||
Scheme: corev1.URISchemeHTTPS,
|
||||
},
|
||||
},
|
||||
InitialDelaySeconds: 0,
|
||||
TimeoutSeconds: 1,
|
||||
PeriodSeconds: 10,
|
||||
SuccessThreshold: 1,
|
||||
FailureThreshold: 3,
|
||||
}
|
||||
r.resource.Spec.Template.Spec.Containers[2].ImagePullPolicy = corev1.PullAlways
|
||||
|
||||
if componentsResources := tenantControlPlane.Spec.ControlPlane.Deployment.Resources; componentsResources != nil {
|
||||
if resource := componentsResources.ControllerManager; resource != nil {
|
||||
r.resource.Spec.Template.Spec.Containers[1].Resources = *resource
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user