diff --git a/api/v1alpha1/tenantcontrolplane_types.go b/api/v1alpha1/tenantcontrolplane_types.go index 9e651d6..16c91f0 100644 --- a/api/v1alpha1/tenantcontrolplane_types.go +++ b/api/v1alpha1/tenantcontrolplane_types.go @@ -226,7 +226,9 @@ type KonnectivityServerSpec struct { // The port which Konnectivity server is listening to. Port int32 `json:"port"` // Container image version of the Konnectivity server. - //+kubebuilder:default=v0.28.6 + // If left empty, Kamaji will automatically inflect the version from the deployed Tenant Control Plane. + // + // WARNING: for last cut-off releases, the container image could be not available. Version string `json:"version,omitempty"` // Container image used by the Konnectivity server. //+kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-server @@ -250,7 +252,9 @@ type KonnectivityAgentSpec struct { //+kubebuilder:default=registry.k8s.io/kas-network-proxy/proxy-agent Image string `json:"image,omitempty"` // Version for Konnectivity agent. - //+kubebuilder:default=v0.28.6 + // If left empty, Kamaji will automatically inflect the version from the deployed Tenant Control Plane. + // + // WARNING: for last cut-off releases, the container image could be not available. Version string `json:"version,omitempty"` // Tolerations for the deployed agent. // Can be customized to start the konnectivity-agent even if the nodes are not ready or tainted. @@ -275,9 +279,9 @@ type KonnectivityAgentSpec struct { // KonnectivitySpec defines the spec for Konnectivity. type KonnectivitySpec struct { - //+kubebuilder:default={version:"v0.28.6",image:"registry.k8s.io/kas-network-proxy/proxy-server",port:8132} + //+kubebuilder:default={image:"registry.k8s.io/kas-network-proxy/proxy-server",port:8132} KonnectivityServerSpec KonnectivityServerSpec `json:"server,omitempty"` - //+kubebuilder:default={version:"v0.28.6",image:"registry.k8s.io/kas-network-proxy/proxy-agent",mode:"DaemonSet"} + //+kubebuilder:default={image:"registry.k8s.io/kas-network-proxy/proxy-agent",mode:"DaemonSet"} KonnectivityAgentSpec KonnectivityAgentSpec `json:"agent,omitempty"` } diff --git a/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml b/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml index 2c0abb1..e74efe9 100644 --- a/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml +++ b/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml @@ -89,7 +89,6 @@ versions: default: image: registry.k8s.io/kas-network-proxy/proxy-agent mode: DaemonSet - version: v0.28.6 properties: extraArgs: description: |- @@ -170,8 +169,11 @@ versions: type: object type: array version: - default: v0.28.6 - description: Version for Konnectivity agent. + description: |- + Version for Konnectivity agent. + If left empty, Kamaji will automatically inflect the version from the deployed Tenant Control Plane. + + WARNING: for last cut-off releases, the container image could be not available. type: string type: object x-kubernetes-validations: @@ -181,7 +183,6 @@ versions: default: image: registry.k8s.io/kas-network-proxy/proxy-server port: 8132 - version: v0.28.6 properties: extraArgs: description: |- @@ -260,8 +261,11 @@ versions: type: object type: object version: - default: v0.28.6 - description: Container image version of the Konnectivity server. + description: |- + Container image version of the Konnectivity server. + If left empty, Kamaji will automatically inflect the version from the deployed Tenant Control Plane. + + WARNING: for last cut-off releases, the container image could be not available. type: string required: - port diff --git a/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml b/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml index 609a7b9..5b60d25 100644 --- a/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml +++ b/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml @@ -97,7 +97,6 @@ spec: default: image: registry.k8s.io/kas-network-proxy/proxy-agent mode: DaemonSet - version: v0.28.6 properties: extraArgs: description: |- @@ -178,8 +177,11 @@ spec: type: object type: array version: - default: v0.28.6 - description: Version for Konnectivity agent. + description: |- + Version for Konnectivity agent. + If left empty, Kamaji will automatically inflect the version from the deployed Tenant Control Plane. + + WARNING: for last cut-off releases, the container image could be not available. type: string type: object x-kubernetes-validations: @@ -189,7 +191,6 @@ spec: default: image: registry.k8s.io/kas-network-proxy/proxy-server port: 8132 - version: v0.28.6 properties: extraArgs: description: |- @@ -268,8 +269,11 @@ spec: type: object type: object version: - default: v0.28.6 - description: Container image version of the Konnectivity server. + description: |- + Container image version of the Konnectivity server. + If left empty, Kamaji will automatically inflect the version from the deployed Tenant Control Plane. + + WARNING: for last cut-off releases, the container image could be not available. type: string required: - port diff --git a/docs/content/concepts/konnectivity.md b/docs/content/concepts/konnectivity.md index e8e2d73..f7920de 100644 --- a/docs/content/concepts/konnectivity.md +++ b/docs/content/concepts/konnectivity.md @@ -1,14 +1,15 @@ # Konnectivity -In traditional Kubernetes deployments, the control plane components need to communicate directly with worker nodes for various operations -like executing commands in pods, retrieving logs, or managing port forwards. +In traditional Kubernetes deployments, the control plane components need to communicate directly with worker nodes for various operations like: +executing commands in pods, retrieving logs, or managing port forwards. + However, in many real-world environments, especially those spanning multiple networks or cloud providers, direct communication isn't always possible or desirable. This is where Konnectivity comes in. ## Understanding Konnectivity in Kamaji Kamaji integrates [Konnectivity](https://kubernetes.io/docs/concepts/architecture/control-plane-node-communication/) as a core component of its architecture. -Each Tenant Control Plane pod includes a konnectivity-server running as a sidecar container, +Each Tenant Control Plane pod includes a `konnectivity-server` running as a sidecar container, which establishes and maintains secure tunnels with agents running on the worker nodes. This design ensures reliable communication even in complex network environments. @@ -86,3 +87,68 @@ Available strategies are the following: By integrating Konnectivity as a core feature, Kamaji ensures that your Tenant Clusters can operate reliably and securely across any network topology, making it easier to build and manage distributed Kubernetes environments at scale. + +## Version compatibility between API Server and Konnectivity + +In recent Kubernetes releases, Konnectivity has aligned its versioning with the Kubernetes API Server. + +This means that for example: +- Kubernetes v1.34.0 pairs with Konnectivity v0.34.0 +- Kubernetes v1.33.0 pairs with Konnectivity v0.33.0 + +Within Kamaji, this version matching happens automatically. + +The field `TenantControlPlane.spec.addons.konnectivity` determines the proper Konnectivity version for both the server and the agent, +ensuring compatibility with the tenant control plane's API Server version. + +!!! warning "Konnectivity images could not be available!" + For the most recent Kubernetes releases, the corresponding Konnectivity image artifacts _may not yet be built and published_ by the upstream community. + In these cases, you may need to override the automatic pairing and configure a previous Konnectivity version that is available. + +You can still have a version skew between the Kubernetes API Server for the given Tenant Control Plane, and the Konnectivity components. + +```yaml +apiVersion: kamaji.clastix.io/v1alpha1 +kind: TenantControlPlane +metadata: + name: konnectivity + namespace: default +spec: + addons: + coreDNS: {} + konnectivity: + agent: + hostNetwork: false + image: registry.k8s.io/kas-network-proxy/proxy-agent + mode: DaemonSet + tolerations: + - key: CriticalAddonsOnly + operator: Exists + version: v0.33.0 + server: + image: registry.k8s.io/kas-network-proxy/proxy-server + port: 8132 + version: v0.33.0 + kubeProxy: {} + controlPlane: + deployment: + replicas: 2 + service: + serviceType: LoadBalancer + dataStore: etcd-kamaji-etcd + kubernetes: + kubelet: + cgroupfs: systemd + preferredAddressTypes: + - InternalIP + - ExternalIP + - Hostname + version: v1.34.0 + networkProfile: + clusterDomain: cluster.local + dnsServiceIPs: + - 10.96.0.10 + podCidr: 10.244.0.0/16 + port: 6443 + serviceCidr: 10.96.0.0/16 +``` diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index 5413c40..eeb62bd 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -41113,7 +41113,7 @@ Enables the Konnectivity addon in the Tenant Cluster, required if the worker nod

- Default: map[image:registry.k8s.io/kas-network-proxy/proxy-agent mode:DaemonSet version:v0.28.6]
+ Default: map[image:registry.k8s.io/kas-network-proxy/proxy-agent mode:DaemonSet]
false @@ -41122,7 +41122,7 @@ Enables the Konnectivity addon in the Tenant Cluster, required if the worker nod

- Default: map[image:registry.k8s.io/kas-network-proxy/proxy-server port:8132 version:v0.28.6]
+ Default: map[image:registry.k8s.io/kas-network-proxy/proxy-server port:8132]
false @@ -41208,9 +41208,10 @@ Can be customized to start the konnectivity-agent even if the nodes are not read version string - Version for Konnectivity agent.
-
- Default: v0.28.6
+ Version for Konnectivity agent. +If left empty, Kamaji will automatically inflect the version from the deployed Tenant Control Plane. + +WARNING: for last cut-off releases, the container image could be not available.
false @@ -41335,9 +41336,10 @@ unxpected ways. Only modify if you know what you are doing.
version string - Container image version of the Konnectivity server.
-
- Default: v0.28.6
+ Container image version of the Konnectivity server. +If left empty, Kamaji will automatically inflect the version from the deployed Tenant Control Plane. + +WARNING: for last cut-off releases, the container image could be not available.
false diff --git a/internal/builders/controlplane/konnectivity_server.go b/internal/builders/controlplane/konnectivity_server.go index 5de3af2..8b9d93f 100644 --- a/internal/builders/controlplane/konnectivity_server.go +++ b/internal/builders/controlplane/konnectivity_server.go @@ -6,6 +6,7 @@ package controlplane import ( "fmt" + "github.com/blang/semver" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -33,7 +34,20 @@ type Konnectivity struct { Scheme runtime.Scheme } -func (k Konnectivity) buildKonnectivityContainer(addon *kamajiv1alpha1.KonnectivitySpec, replicas int32, podSpec *corev1.PodSpec) { +func (k Konnectivity) serverVersion(tcpVersion, addonVersion string) string { + if addonVersion != "" { + return addonVersion + } + + version, parsedErr := semver.ParseTolerant(tcpVersion) + if parsedErr != nil { + return "" + } + + return fmt.Sprintf("v0.%d.0", version.Minor) +} + +func (k Konnectivity) buildKonnectivityContainer(tcpVersion string, addon *kamajiv1alpha1.KonnectivitySpec, replicas int32, podSpec *corev1.PodSpec) { found, index := utilities.HasNamedContainer(podSpec.Containers, konnectivityServerName) if !found { index = len(podSpec.Containers) @@ -41,7 +55,7 @@ func (k Konnectivity) buildKonnectivityContainer(addon *kamajiv1alpha1.Konnectiv } podSpec.Containers[index].Name = konnectivityServerName - podSpec.Containers[index].Image = fmt.Sprintf("%s:%s", addon.KonnectivityServerSpec.Image, addon.KonnectivityServerSpec.Version) + podSpec.Containers[index].Image = fmt.Sprintf("%s:%s", addon.KonnectivityServerSpec.Image, k.serverVersion(tcpVersion, addon.KonnectivityServerSpec.Version)) podSpec.Containers[index].Command = []string{"/proxy-server"} args := utilities.ArgsFromSliceToMap(addon.KonnectivityServerSpec.ExtraArgs) @@ -254,7 +268,7 @@ func (k Konnectivity) buildVolumes(status kamajiv1alpha1.KonnectivityStatus, pod } func (k Konnectivity) Build(deployment *appsv1.Deployment, tenantControlPlane kamajiv1alpha1.TenantControlPlane) { - k.buildKonnectivityContainer(tenantControlPlane.Spec.Addons.Konnectivity, *tenantControlPlane.Spec.ControlPlane.Deployment.Replicas, &deployment.Spec.Template.Spec) + k.buildKonnectivityContainer(tenantControlPlane.Spec.Kubernetes.Version, tenantControlPlane.Spec.Addons.Konnectivity, *tenantControlPlane.Spec.ControlPlane.Deployment.Replicas, &deployment.Spec.Template.Spec) k.buildVolumeMounts(&deployment.Spec.Template.Spec) k.buildVolumes(tenantControlPlane.Status.Addons.Konnectivity, &deployment.Spec.Template.Spec) diff --git a/internal/resources/konnectivity/agent.go b/internal/resources/konnectivity/agent.go index 37bf657..74ea964 100644 --- a/internal/resources/konnectivity/agent.go +++ b/internal/resources/konnectivity/agent.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/blang/semver" "github.com/prometheus/client_golang/prometheus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -36,6 +37,19 @@ func (r *Agent) GetHistogram() prometheus.Histogram { return agentCollector } +func (r *Agent) agentVersion(tcp *kamajiv1alpha1.TenantControlPlane) string { + if tcp.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Version != "" { + return tcp.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Version + } + + version, parsedErr := semver.ParseTolerant(tcp.Spec.Kubernetes.Version) + if parsedErr != nil { + return "" + } + + return fmt.Sprintf("v0.%d.0", version.Minor) +} + func (r *Agent) ShouldStatusBeUpdated(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) bool { return tcp.Spec.Addons.Konnectivity == nil && (tcp.Status.Addons.Konnectivity.Agent.Namespace != "" || tcp.Status.Addons.Konnectivity.Agent.Name != "") || tcp.Spec.Addons.Konnectivity != nil && (tcp.Status.Addons.Konnectivity.Agent.Namespace != r.resource.GetNamespace() || tcp.Status.Addons.Konnectivity.Agent.Name != r.resource.GetName()) || @@ -219,7 +233,7 @@ func (r *Agent) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.T podTemplateSpec.Spec.Containers = make([]corev1.Container, 1) } - podTemplateSpec.Spec.Containers[0].Image = fmt.Sprintf("%s:%s", tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Image, tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Version) + podTemplateSpec.Spec.Containers[0].Image = fmt.Sprintf("%s:%s", tenantControlPlane.Spec.Addons.Konnectivity.KonnectivityAgentSpec.Image, r.agentVersion(tenantControlPlane)) podTemplateSpec.Spec.Containers[0].Name = AgentName podTemplateSpec.Spec.Containers[0].Command = []string{"/proxy-agent"}