From 3be6cf1c4f37e516cdd1f3c48a674311af2df8d5 Mon Sep 17 00:00:00 2001 From: mendrugory Date: Thu, 26 May 2022 13:06:46 +0200 Subject: [PATCH] feat: konnectivity --- api/v1alpha1/tenantcontrolplane_funcs.go | 29 + api/v1alpha1/tenantcontrolplane_types.go | 35 +- api/v1alpha1/zz_generated.deepcopy.go | 18 + ...kamaji.clastix.io_tenantcontrolplanes.yaml | 268 ++- config/install.yaml | 242 +- ...maji_v1alpha1_tenantcontrolplane_kind.yaml | 53 - ...pha1_tenantcontrolplane_kind_nodeport.yaml | 48 - ...v1alpha1_tenantcontrolplane_kind_test.yaml | 53 - ...amaji_v1alpha1_tenantcontrolplane_lab.yaml | 40 - ..._v1alpha1_tenantcontrolplane_microk8s.yaml | 49 - ...olplane_microk8s_raspberrypi_nodeport.yaml | 41 - controllers/resources.go | 41 +- deploy/kind/Makefile | 3 + deploy/kind/join-node-konnectivity.bash | 34 + deploy/kind/kind-kamaji.yaml | 6 +- docs/kamaji-tenant-deployment-guide.md | 17 + docs/reference.md | 13 + go.mod | 1 + helm/kamaji/crds/tenantcontrolplane.yaml | 1961 ++++++++++------- internal/resources/ca_certificate.go | 4 +- internal/resources/k8s_deployment_resource.go | 17 +- internal/resources/konnectivity/agent.go | 2 +- .../konnectivity/certificate_resource.go | 2 +- .../cluster_role_binding_resource.go | 2 +- internal/resources/konnectivity/constants.go | 19 + .../egress_selector_configuration_resource.go | 2 +- .../konnectivity/kubeconfig_resource.go | 2 +- .../konnectivity/service_account_resource.go | 2 +- .../konnectivity/service_resource.go | 24 +- internal/utilities/utilities.go | 23 + 30 files changed, 1875 insertions(+), 1176 deletions(-) delete mode 100644 config/samples/kamaji_v1alpha1_tenantcontrolplane_kind.yaml delete mode 100644 config/samples/kamaji_v1alpha1_tenantcontrolplane_kind_nodeport.yaml delete mode 100644 config/samples/kamaji_v1alpha1_tenantcontrolplane_kind_test.yaml delete mode 100644 config/samples/kamaji_v1alpha1_tenantcontrolplane_lab.yaml delete mode 100644 config/samples/kamaji_v1alpha1_tenantcontrolplane_microk8s.yaml delete mode 100644 config/samples/kamaji_v1alpha1_tenantcontrolplane_microk8s_raspberrypi_nodeport.yaml create mode 100755 deploy/kind/join-node-konnectivity.bash create mode 100644 internal/resources/konnectivity/constants.go diff --git a/api/v1alpha1/tenantcontrolplane_funcs.go b/api/v1alpha1/tenantcontrolplane_funcs.go index 3d80473..62b569d 100644 --- a/api/v1alpha1/tenantcontrolplane_funcs.go +++ b/api/v1alpha1/tenantcontrolplane_funcs.go @@ -42,3 +42,32 @@ func (in *TenantControlPlane) GetControlPlaneAddress(ctx context.Context, client return "", kamajierrors.MissingValidIPError{} } + +func (in *TenantControlPlane) GetKonnectivityServerAddress(ctx context.Context, client client.Client) (string, error) { + var loadBalancerStatus corev1.LoadBalancerStatus + + svc := &corev1.Service{} + err := client.Get(ctx, types.NamespacedName{Namespace: in.GetNamespace(), Name: in.Status.Addons.Konnectivity.Service.Name}, svc) + if err != nil { + return "", errors.Wrap(err, "cannot retrieve Service for Konnectivity") + } + + switch { + case len(in.Spec.Addons.Konnectivity.ProxyHost) > 0: + // Returning the hard-coded value in the specification in case of non LoadBalanced resources + return in.Spec.Addons.Konnectivity.ProxyHost, nil + case svc.Spec.Type == corev1.ServiceTypeLoadBalancer: + loadBalancerStatus = svc.Status.LoadBalancer + if len(loadBalancerStatus.Ingress) == 0 { + return "", kamajierrors.NonExposedLoadBalancerError{} + } + + for _, lb := range loadBalancerStatus.Ingress { + if ip := lb.IP; len(ip) > 0 { + return ip, nil + } + } + } + + return "", kamajierrors.MissingValidIPError{} +} diff --git a/api/v1alpha1/tenantcontrolplane_types.go b/api/v1alpha1/tenantcontrolplane_types.go index 6baca68..2f865bf 100644 --- a/api/v1alpha1/tenantcontrolplane_types.go +++ b/api/v1alpha1/tenantcontrolplane_types.go @@ -98,13 +98,14 @@ type AddonSpec struct{} // KonnectivitySpec defines the spec for Konnectivity. type KonnectivitySpec struct { // Port of Konnectivity proxy server. - // +kubebuilder:default=8132 ProxyPort int32 `json:"proxyPort"` // Host of Konnectivity proxy server. ProxyHost string `json:"proxyHost,omitempty"` AllowAddressAsExternalIP bool `json:"allowAddressAsExternalIP,omitempty"` + // ServiceType allows specifying how to expose the Konnectivity Proxy Server. + ServiceType ServiceType `json:"serviceType"` // Version for Konnectivity server and agent. - // +kubebuilder:default=v0.0.16 + // +kubebuilder:default=v0.0.31 Version string `json:"version,omitempty"` // ServerImage defines the container image for Konnectivity's server. // +kubebuilder:default=us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server @@ -155,8 +156,9 @@ type ETCDCertificatesStatus struct { // CertificatePrivateKeyPair defines the status. type CertificatePrivateKeyPairStatus struct { - SecretName string `json:"secretName,omitempty"` - LastUpdate metav1.Time `json:"lastUpdate,omitempty"` + SecretName string `json:"secretName,omitempty"` + LastUpdate metav1.Time `json:"lastUpdate,omitempty"` + ResourceVersion string `json:"resourceVersion,omitempty"` } // CertificatePrivateKeyPair defines the status. @@ -228,6 +230,27 @@ type KubeadmPhasesStatus struct { BootstrapToken KubeadmPhaseStatus `json:"bootstrapToken"` } +type ExternalKubernetesObjectStatus struct { + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` + // Resource version of k8s object + RV string `json:"resourceVersion,omitempty"` + // Last time when k8s object was updated + LastUpdate metav1.Time `json:"lastUpdate,omitempty"` +} + +// KonnectivityStatus defines the status of Konnectivity as Addon. +type KonnectivityStatus struct { + Enabled bool `json:"enabled"` + EgressSelectorConfiguration string `json:"egressSelectorConfiguration,omitempty"` + Certificate CertificatePrivateKeyPairStatus `json:"certificate,omitempty"` + Kubeconfig KubeconfigStatus `json:"kubeconfig,omitempty"` + ServiceAccount ExternalKubernetesObjectStatus `json:"sa,omitempty"` + ClusterRoleBinding ExternalKubernetesObjectStatus `json:"clusterrolebinding,omitempty"` + Agent ExternalKubernetesObjectStatus `json:"agent,omitempty"` + Service KubernetesServiceStatus `json:"service,omitempty"` +} + // AddonStatus defines the observed state of an Addon. type AddonStatus struct { Enabled bool `json:"enabled"` @@ -247,6 +270,8 @@ func (d *AddonStatus) SetKubeadmConfigResourceVersion(rv string) { type AddonsStatus struct { CoreDNS AddonStatus `json:"coreDNS,omitempty"` KubeProxy AddonStatus `json:"kubeProxy,omitempty"` + + Konnectivity KonnectivityStatus `json:"konnectivity,omitempty"` } // TenantControlPlaneStatus defines the observed state of TenantControlPlane. @@ -305,6 +330,8 @@ type KubernetesDeploymentStatus struct { Name string `json:"name"` // The namespace which the Deployment for the given cluster is deployed. Namespace string `json:"namespace"` + // Last time when deployment was updated + LastUpdate metav1.Time `json:"lastUpdate,omitempty"` } // KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index e0b1363..0036b4f 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -123,6 +123,7 @@ func (in *AddonsStatus) DeepCopyInto(out *AddonsStatus) { *out = *in in.CoreDNS.DeepCopyInto(&out.CoreDNS) in.KubeProxy.DeepCopyInto(&out.KubeProxy) + in.Konnectivity.DeepCopyInto(&out.Konnectivity) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AddonsStatus. @@ -280,6 +281,22 @@ func (in *ETCDStatus) DeepCopy() *ETCDStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalKubernetesObjectStatus) DeepCopyInto(out *ExternalKubernetesObjectStatus) { + *out = *in + in.LastUpdate.DeepCopyInto(&out.LastUpdate) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalKubernetesObjectStatus. +func (in *ExternalKubernetesObjectStatus) DeepCopy() *ExternalKubernetesObjectStatus { + if in == nil { + return nil + } + out := new(ExternalKubernetesObjectStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *IngressSpec) DeepCopyInto(out *IngressSpec) { *out = *in @@ -435,6 +452,7 @@ func (in *KubeletSpec) DeepCopy() *KubeletSpec { func (in *KubernetesDeploymentStatus) DeepCopyInto(out *KubernetesDeploymentStatus) { *out = *in in.DeploymentStatus.DeepCopyInto(&out.DeploymentStatus) + in.LastUpdate.DeepCopyInto(&out.LastUpdate) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesDeploymentStatus. diff --git a/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml b/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml index ade7f8a..58c745b 100644 --- a/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml +++ b/config/crd/bases/kamaji.clastix.io_tenantcontrolplanes.yaml @@ -80,7 +80,6 @@ spec: description: Host of Konnectivity proxy server. type: string proxyPort: - default: 8132 description: Port of Konnectivity proxy server. format: int32 type: integer @@ -89,12 +88,21 @@ spec: description: ServerImage defines the container image for Konnectivity's server. type: string + serviceType: + description: ServiceType allows specifying how to expose the + Konnectivity Proxy Server. + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string version: - default: v0.0.16 + default: v0.0.31 description: Version for Konnectivity server and agent. type: string required: - proxyPort + - serviceType type: object kubeProxy: description: AddonSpec defines the spec for every addon. @@ -334,6 +342,248 @@ spec: required: - enabled type: object + konnectivity: + description: KonnectivityStatus defines the status of Konnectivity + as Addon. + properties: + agent: + properties: + lastUpdate: + description: Last time when k8s object was updated + format: date-time + type: string + name: + type: string + namespace: + type: string + resourceVersion: + description: Resource version of k8s object + type: string + type: object + certificate: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + resourceVersion: + type: string + secretName: + type: string + type: object + clusterrolebinding: + properties: + lastUpdate: + description: Last time when k8s object was updated + format: date-time + type: string + name: + type: string + namespace: + type: string + resourceVersion: + description: Resource version of k8s object + type: string + type: object + egressSelectorConfiguration: + type: string + enabled: + type: boolean + kubeconfig: + description: TenantControlPlaneKubeconfigsStatus contains + information about a the generated kubeconfig. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + sa: + properties: + lastUpdate: + description: Last time when k8s object was updated + format: date-time + type: string + name: + type: string + namespace: + type: string + resourceVersion: + description: Resource version of k8s object + type: string + type: object + service: + description: KubernetesServiceStatus defines the status for + the Tenant Control Plane Service in the management cluster. + properties: + conditions: + description: Current service state + items: + description: "Condition contains details for one aspect + of the current state of this API Resource. --- This + struct is intended for direct use as an array at the + field path .status.conditions. For example, type + FooStatus struct{ // Represents the observations + of a foo's current state. // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\" + \ // +patchMergeKey=type // +patchStrategy=merge + \ // +listType=map // +listMapKey=type Conditions + []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` + \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time + the condition transitioned from one status to + another. This should be when the underlying condition + changed. If that is not known, then using the + time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message + indicating details about the transition. This + may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, + if .metadata.generation is currently 12, but the + .status.conditions[x].observedGeneration is 9, + the condition is out of date with respect to the + current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier + indicating the reason for the condition's last + transition. Producers of specific condition types + may define expected values and meanings for this + field, and whether the values are considered a + guaranteed API. The value should be a CamelCase + string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, + False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in + foo.example.com/CamelCase. --- Many .condition.type + values are consistent across resources like Available, + but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to + deconflict is important. The regex it matches + is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + loadBalancer: + description: LoadBalancer contains the current status + of the load-balancer, if one is present. + properties: + ingress: + description: Ingress is a list containing ingress + points for the load-balancer. Traffic intended for + the service should be sent to these ingress points. + items: + description: 'LoadBalancerIngress represents the + status of a load-balancer ingress point: traffic + intended for the service should be sent to an + ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer + ingress points that are DNS based (typically + AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress + points that are IP based (typically GCE or + OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service + ports If used, every port defined in the service + should have an entry in it + items: + properties: + error: + description: 'Error is to record the problem + with the service port The format of + the error shall comply with the following + rules: - built-in error values shall + be specified in this file and those + shall use CamelCase names - cloud + provider specific error values must + have names that comply with the format + foo.example.com/CamelCase. --- The regex + it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of + the service port of which status is + recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol + of the service port of which status + is recorded here The supported values + are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + name: + description: The name of the Service for the given cluster. + type: string + namespace: + description: The namespace which the Service for the given + cluster is deployed. + type: string + port: + description: The port where the service is running + format: int32 + type: integer + required: + - name + - namespace + - port + type: object + required: + - enabled + type: object kubeProxy: description: AddonStatus defines the observed state of an Addon. properties: @@ -358,6 +608,8 @@ spec: lastUpdate: format: date-time type: string + resourceVersion: + type: string secretName: type: string type: object @@ -367,6 +619,8 @@ spec: lastUpdate: format: date-time type: string + resourceVersion: + type: string secretName: type: string type: object @@ -376,6 +630,8 @@ spec: lastUpdate: format: date-time type: string + resourceVersion: + type: string secretName: type: string type: object @@ -410,6 +666,8 @@ spec: lastUpdate: format: date-time type: string + resourceVersion: + type: string secretName: type: string type: object @@ -419,6 +677,8 @@ spec: lastUpdate: format: date-time type: string + resourceVersion: + type: string secretName: type: string type: object @@ -579,6 +839,10 @@ spec: - type type: object type: array + lastUpdate: + description: Last time when deployment was updated + format: date-time + type: string name: description: The name of the Deployment for the given cluster. type: string diff --git a/config/install.yaml b/config/install.yaml index 20c1707..df17d52 100644 --- a/config/install.yaml +++ b/config/install.yaml @@ -61,39 +61,48 @@ spec: description: TenantControlPlaneSpec defines the desired state of TenantControlPlane. properties: addons: - default: - coreDNS: - enabled: true - kubeProxy: - enabled: true description: Addons contain which addons are enabled properties: coreDNS: - default: - enabled: true description: AddonSpec defines the spec for every addon. - properties: - enabled: - default: true - type: boolean type: object konnectivity: - default: - enabled: true - description: AddonSpec defines the spec for every addon. + description: KonnectivitySpec defines the spec for Konnectivity. properties: - enabled: - default: true + agentImage: + default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent + description: AgentImage defines the container image for Konnectivity's agent. + type: string + allowAddressAsExternalIP: type: boolean + proxyHost: + description: Host of Konnectivity proxy server. + type: string + proxyPort: + description: Port of Konnectivity proxy server. + format: int32 + type: integer + serverImage: + default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server + description: ServerImage defines the container image for Konnectivity's server. + type: string + serviceType: + description: ServiceType allows specifying how to expose the Konnectivity Proxy Server. + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + version: + default: v0.0.31 + description: Version for Konnectivity server and agent. + type: string + required: + - proxyPort + - serviceType type: object kubeProxy: - default: - enabled: true description: AddonSpec defines the spec for every addon. - properties: - enabled: - default: true - type: boolean type: object type: object controlPlane: @@ -310,10 +319,181 @@ spec: - enabled type: object konnectivity: - description: KonnectivityStatus defines the status of Konnectivity as Addon + description: KonnectivityStatus defines the status of Konnectivity as Addon. properties: - egressSelectorConfigurationStatus: + agent: + properties: + lastUpdate: + description: Last time when k8s object was updated + format: date-time + type: string + name: + type: string + namespace: + type: string + resourceVersion: + description: Resource version of k8s object + type: string + type: object + certificate: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + resourceVersion: + type: string + secretName: + type: string + type: object + clusterrolebinding: + properties: + lastUpdate: + description: Last time when k8s object was updated + format: date-time + type: string + name: + type: string + namespace: + type: string + resourceVersion: + description: Resource version of k8s object + type: string + type: object + egressSelectorConfiguration: + type: string + enabled: type: boolean + kubeconfig: + description: TenantControlPlaneKubeconfigsStatus contains information about a the generated kubeconfig. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + sa: + properties: + lastUpdate: + description: Last time when k8s object was updated + format: date-time + type: string + name: + type: string + namespace: + type: string + resourceVersion: + description: Resource version of k8s object + type: string + type: object + service: + description: KubernetesServiceStatus defines the status for the Tenant Control Plane Service in the management cluster. + properties: + conditions: + description: Current service state + items: + description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ // Represents the observations of a foo's current state. // Known .status.conditions.type are: \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge // +listType=map // +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating details about the transition. This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + loadBalancer: + description: LoadBalancer contains the current status of the load-balancer, if one is present. + properties: + ingress: + description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service ports If used, every port defined in the service should have an entry in it + items: + properties: + error: + description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use CamelCase names - cloud provider specific error values must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the service port of which status is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + name: + description: The name of the Service for the given cluster. + type: string + namespace: + description: The namespace which the Service for the given cluster is deployed. + type: string + port: + description: The port where the service is running + format: int32 + type: integer + required: + - name + - namespace + - port + type: object + required: + - enabled type: object kubeProxy: description: AddonStatus defines the observed state of an Addon. @@ -338,6 +518,8 @@ spec: lastUpdate: format: date-time type: string + resourceVersion: + type: string secretName: type: string type: object @@ -347,6 +529,8 @@ spec: lastUpdate: format: date-time type: string + resourceVersion: + type: string secretName: type: string type: object @@ -356,6 +540,8 @@ spec: lastUpdate: format: date-time type: string + resourceVersion: + type: string secretName: type: string type: object @@ -387,6 +573,8 @@ spec: lastUpdate: format: date-time type: string + resourceVersion: + type: string secretName: type: string type: object @@ -396,6 +584,8 @@ spec: lastUpdate: format: date-time type: string + resourceVersion: + type: string secretName: type: string type: object @@ -535,6 +725,10 @@ spec: - type type: object type: array + lastUpdate: + description: Last time when deployment was updated + format: date-time + type: string name: description: The name of the Deployment for the given cluster. type: string diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind.yaml deleted file mode 100644 index b4736d6..0000000 --- a/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: kamaji.clastix.io/v1alpha1 -kind: TenantControlPlane -metadata: - name: tenantkind -spec: - controlPlane: - deployment: - replicas: 2 - additionalMetadata: - annotations: - environment.clastix.io: test - tier.clastix.io: "0" - labels: - tenant.clastix.io: test - kind.clastix.io: deployment - service: - additionalMetadata: - annotations: - environment.clastix.io: test - tier.clastix.io: "0" - labels: - tenant.clastix.io: test - kind.clastix.io: service - serviceType: ClusterIP - ingress: - enabled: true - hostname: kamaji.local - ingressClassName: nginx - additionalMetadata: - annotations: - kubernetes.io/ingress.allow-http: "false" - nginx.ingress.kubernetes.io/secure-backends: "true" - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - kubernetes: - version: "v1.23.4" - kubelet: - cgroupfs: systemd - admissionControllers: - - LimitRanger - - ResourceQuota - networkProfile: - address: "172.18.0.2" - port: 6443 - domain: "clastix.labs" - serviceCidr: "10.96.0.0/16" - podCidr: "10.244.0.0/16" - dnsServiceIPs: - - "10.96.0.10" - # addons: - # coreDNS: - # # enabled: true - # kubeProxy: - # # enabled: false \ No newline at end of file diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind_nodeport.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind_nodeport.yaml deleted file mode 100644 index 50b6d30..0000000 --- a/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind_nodeport.yaml +++ /dev/null @@ -1,48 +0,0 @@ -apiVersion: kamaji.clastix.io/v1alpha1 -kind: TenantControlPlane -metadata: - name: tenant1 -spec: - controlPlane: - deployment: - replicas: 2 - additionalMetadata: - annotations: - environment.clastix.io: test - tier.clastix.io: "0" - labels: - tenant.clastix.io: test - kind.clastix.io: deployment - service: - additionalMetadata: - annotations: - environment.clastix.io: test - tier.clastix.io: "0" - labels: - tenant.clastix.io: test - kind.clastix.io: service - serviceType: NodePort - ingress: - enabled: false - kubernetes: - version: "v1.23.4" - kubelet: - cgroupfs: cgroupfs - admissionControllers: - - LimitRanger - - ResourceQuota - networkProfile: - address: "172.18.0.2" - port: 31443 - domain: "clastix.labs" - serviceCidr: "10.96.0.0/16" - podCidr: "10.244.0.0/16" - dnsServiceIPs: - - "10.96.0.10" - addons: - konnectivity: - proxyPort: 31132 - proxyHost: "172.18.0.2" - version: v0.0.31 - coreDNS: {} - kubeProxy: diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind_test.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind_test.yaml deleted file mode 100644 index 33ebde2..0000000 --- a/config/samples/kamaji_v1alpha1_tenantcontrolplane_kind_test.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: kamaji.clastix.io/v1alpha1 -kind: TenantControlPlane -metadata: - name: tenantkind -spec: - controlPlane: - deployment: - replicas: 2 - additionalMetadata: - annotations: - environment.clastix.io: test - tier.clastix.io: "0" - labels: - tenant.clastix.io: test - kind.clastix.io: deployment - service: - additionalMetadata: - annotations: - environment.clastix.io: test - tier.clastix.io: "0" - labels: - tenant.clastix.io: test - kind.clastix.io: service - serviceType: ClusterIP - ingress: - enabled: true - hostname: kamaji.local - ingressClassName: nginx - additionalMetadata: - annotations: - kubernetes.io/ingress.allow-http: "false" - nginx.ingress.kubernetes.io/secure-backends: "true" - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - kubernetes: - version: "v1.23.4" - kubelet: - cgroupfs: systemd - # admissionControllers: - # - LimitRanger - # - ResourceQuota - networkProfile: - address: "172.18.0.2" - port: 6443 - domain: "clastix.labs" - serviceCidr: "10.96.0.0/16" - podCidr: "10.244.0.0/16" - dnsServiceIPs: - - "10.96.0.10" - addons: - coreDNS: - enabled: false - # kubeProxy: - # enabled: false \ No newline at end of file diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane_lab.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane_lab.yaml deleted file mode 100644 index 9355b2b..0000000 --- a/config/samples/kamaji_v1alpha1_tenantcontrolplane_lab.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: kamaji.clastix.io/v1alpha1 -kind: TenantControlPlane -metadata: - name: tenant-00 - namespace: tenants -spec: - controlPlane: - deployment: - replicas: 2 - additionalMetadata: - annotations: - environment.clastix.io: tenant-00 - labels: - tenant.clastix.io: tenant-00 - kind.clastix.io: deployment - service: - additionalMetadata: - annotations: - environment.clastix.io: tenant-00 - labels: - tenant.clastix.io: tenant-00 - kind.clastix.io: service - serviceType: LoadBalancer - ingress: - enabled: false - kubernetes: - version: v1.23.1 - kubelet: - cgroupfs: systemd - admissionControllers: - - ResourceQuota - - LimitRanger - networkProfile: - address: 192.168.32.150 - port: 6443 - domain: clastix.labs - serviceCidr: 10.96.0.0/16 - podCidr: 10.36.0.0/16 - dnsServiceIPs: - - 10.96.0.10 diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane_microk8s.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane_microk8s.yaml deleted file mode 100644 index 658d7ec..0000000 --- a/config/samples/kamaji_v1alpha1_tenantcontrolplane_microk8s.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: kamaji.clastix.io/v1alpha1 -kind: TenantControlPlane -metadata: - name: test - namespace: default -spec: - controlPlane: - deployment: - replicas: 2 - additionalMetadata: - annotations: - environment.clastix.io: test - tier.clastix.io: "0" - labels: - tenant.clastix.io: test - kind.clastix.io: deployment - service: - additionalMetadata: - annotations: - environment.clastix.io: test - tier.clastix.io: "0" - labels: - tenant.clastix.io: test - kind.clastix.io: service - serviceType: ClusterIP - ingress: - enabled: false - hostname: kamaji.local - ingressClassName: nginx - additionalMetadata: - annotations: - kubernetes.io/ingress.allow-http: "false" - nginx.ingress.kubernetes.io/secure-backends: "true" - nginx.ingress.kubernetes.io/ssl-passthrough: "true" - kubernetes: - version: "v1.23.1" - kubelet: - cgroupfs: systemd - admissionControllers: - - ResourceQuota - - LimitRanger - networkProfile: - address: "192.168.1.47" - port: 6443 - domain: "clastix.labs" - serviceCidr: "10.152.0.0/16" - podCidr: "10.1.0.0/16" - dnsServiceIPs: - - "10.152.183.10" diff --git a/config/samples/kamaji_v1alpha1_tenantcontrolplane_microk8s_raspberrypi_nodeport.yaml b/config/samples/kamaji_v1alpha1_tenantcontrolplane_microk8s_raspberrypi_nodeport.yaml deleted file mode 100644 index a6685ee..0000000 --- a/config/samples/kamaji_v1alpha1_tenantcontrolplane_microk8s_raspberrypi_nodeport.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: kamaji.clastix.io/v1alpha1 -kind: TenantControlPlane -metadata: - name: tenant1 -spec: - controlPlane: - deployment: - replicas: 2 - additionalMetadata: - annotations: - environment.clastix.io: test - tier.clastix.io: "0" - labels: - tenant.clastix.io: test - kind.clastix.io: deployment - service: - additionalMetadata: - annotations: - environment.clastix.io: test - tier.clastix.io: "0" - labels: - tenant.clastix.io: test - kind.clastix.io: service - serviceType: NodePort - ingress: - enabled: false - kubernetes: - version: "v1.23.4" - kubelet: - cgroupfs: cgroupfs - admissionControllers: - - LimitRanger - - ResourceQuota - networkProfile: - address: "192.168.1.47" - port: 31443 - domain: "clastix.labs" - serviceCidr: "10.96.0.0/16" - podCidr: "10.244.0.0/16" - dnsServiceIPs: - - "10.96.0.10" diff --git a/controllers/resources.go b/controllers/resources.go index 779cdab..c0569a2 100644 --- a/controllers/resources.go +++ b/controllers/resources.go @@ -1,3 +1,5 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 package controllers import ( @@ -41,7 +43,7 @@ func GetResources(config GroupResourceBuilderConfiguration) []resources.Resource // GetDeleteableResources returns a list of resources that have to be deleted when tenant control planes are deleted // Currently there is only a default approach -// TODO: the idea of this function is to become a factory to return the group of deleteable resources according to the given configuration +// TODO: the idea of this function is to become a factory to return the group of deleteable resources according to the given configuration. func GetDeleteableResources(config GroupDeleteableResourceBuilderConfiguration) []resources.DeleteableResource { return getDefaultDeleteableResources(config) } @@ -52,11 +54,12 @@ func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.R resources = append(resources, getKubernetesCertificatesResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...) resources = append(resources, getKubeconfigResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...) resources = append(resources, getKubernetesStorageResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...) - resources = append(resources, getKonnectivityResources(config.client, config.tenantControlPlane)...) + resources = append(resources, getInternalKonnectivityResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...) resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...) resources = append(resources, getKubernetesIngressResources(config.client, config.tenantControlPlane)...) resources = append(resources, getKubeadmPhaseResources(config.client, config.log, config.tenantControlPlane)...) resources = append(resources, getKubeadmAddonResources(config.client, config.log, config.tenantControlPlane)...) + resources = append(resources, getExternalKonnectivityResources(config.client, config.log, config.tcpReconcilerConfig, config.tenantControlPlane)...) return resources } @@ -258,12 +261,42 @@ func getKubeadmAddonResources(c client.Client, log logr.Logger, tenantControlPla } } -func getKonnectivityResources(c client.Client, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { +func getExternalKonnectivityResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { return []resources.Resource{ - &konnectivity.EgressSelectorConfiguration{ + &konnectivity.ServiceAccountResource{ + Client: c, + Name: "konnectivity-sa", + }, + &konnectivity.ClusterRoleBindingResource{ + Client: c, + Name: "konnectivity-clusterrolebinding", + }, + &konnectivity.ServiceResource{ + Client: c, + Name: "konnectivity-service", + }, + &konnectivity.Agent{ + Client: c, + Name: "konnectivity-agent", + }, + } +} + +func getInternalKonnectivityResources(c client.Client, log logr.Logger, tcpReconcilerConfig TenantControlPlaneReconcilerConfig, tenantControlPlane kamajiv1alpha1.TenantControlPlane) []resources.Resource { + return []resources.Resource{ + &konnectivity.EgressSelectorConfigurationResource{ Client: c, Name: "konnectivity-egress-selector-configuration", }, + &konnectivity.CertificateResource{ + Client: c, + Log: log, + Name: "konnectivity-certificate", + }, + &konnectivity.KubeconfigResource{ + Client: c, + Name: "konnectivity-kubeconfig", + }, } } diff --git a/deploy/kind/Makefile b/deploy/kind/Makefile index 046d204..2e5a3ff 100644 --- a/deploy/kind/Makefile +++ b/deploy/kind/Makefile @@ -37,3 +37,6 @@ kamaji-kind-worker-push: kamaji-kind-worker-build kamaji-kind-worker-join: $(kind_path)/join-node.bash + +kamaji-kind-worker-join-through-konnectivity: + $(kind_path)/join-node-konnectivity.bash diff --git a/deploy/kind/join-node-konnectivity.bash b/deploy/kind/join-node-konnectivity.bash new file mode 100755 index 0000000..25387a6 --- /dev/null +++ b/deploy/kind/join-node-konnectivity.bash @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e + +# Constants +export DOCKER_IMAGE_NAME="clastix/kamaji-kind-worker" + +# Variables +export KUBERNETES_VERSION=${1:-latest} + +if [ -z $2 ] +then + MAPPING_PORT="" +else + MAPPING_PORT="-p ${2}:80" +fi + +export KONNECTIVITY_PROXY_HOST=${3:-konnectiviy.local} + +clear +echo "Welcome to join a new node through Konnectivity" + +echo -ne "\nChecking right kubeconfig\n" +kubectl cluster-info +echo "Are you pointing to the right tenant control plane? (Type return to continue)" +read + +JOIN_CMD="$(kubeadm --kubeconfig=/tmp/kubeconfig token create --print-join-command) --ignore-preflight-errors=SystemVerification" +echo "Deploying new node..." +KIND_IP=$(docker inspect kamaji-control-plane --format='{{.NetworkSettings.Networks.kind.IPAddress}}') +NODE=$(docker run -d --add-host $KONNECTIVITY_PROXY_HOST:$KIND_IP --privileged -v /lib/modules:/lib/modules:ro -v /var --net host $MAPPING_PORT $DOCKER_IMAGE_NAME:$KUBERNETES_VERSION) +sleep 10 +echo "Joining new node..." +docker exec -e JOIN_CMD="$JOIN_CMD" $NODE /bin/bash -c "$JOIN_CMD" diff --git a/deploy/kind/kind-kamaji.yaml b/deploy/kind/kind-kamaji.yaml index 9fdee75..376030f 100644 --- a/deploy/kind/kind-kamaji.yaml +++ b/deploy/kind/kind-kamaji.yaml @@ -23,10 +23,14 @@ nodes: - containerPort: 443 hostPort: 443 protocol: TCP + ## expose port 31132 of the node to port 31132 on the host for konnectivity + - containerPort: 31132 + hostPort: 31132 + protocol: TCP ## expose port 31443 of the node to port 31443 on the host - containerPort: 31443 hostPort: 31443 - protocol: TCP + protocol: TCP ## expose port 6443 of the node to port 8443 on the host - containerPort: 6443 hostPort: 8443 diff --git a/docs/kamaji-tenant-deployment-guide.md b/docs/kamaji-tenant-deployment-guide.md index c73bb16..da2900b 100644 --- a/docs/kamaji-tenant-deployment-guide.md +++ b/docs/kamaji-tenant-deployment-guide.md @@ -73,6 +73,23 @@ spec: EOF ``` +If workers are not reachable from tenant control plane, konnectivity can be enabled (it is by default): + +```yaml +... + addons: + konnectivity: + enabled: true + proxyHost: "172.18.0.2" + proxyPort: 31132 +... +``` + +`proxyHost` is the address where konnectivity proxy server will be running. Konnectivity works as sidecar container into the tenant control plane pod. If no value is specified, it will take tenant IP. + +`proxyPort` is the port where konnectivity proxy server will be running. (default `8132`) + + ```bash kubectl create namespace ${TENANT_NAMESPACE} kubectl apply -f ${TENANT_NAMESPACE}-${TENANT_NAME}-tcp.yaml diff --git a/docs/reference.md b/docs/reference.md index 59f2c79..a27ec13 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -102,3 +102,16 @@ addons: addons: kubeProxy: {} ``` + +### Konnectivity + +```yaml +addons: + konnectivity: + proxyPort: 31132 # mandatory + proxyHost: "172.18.0.2" + allowAddressAsExternalIP: false + serviceType: NodePort # mandatory + version: v0.0.31 + serverImage: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server + agentImage: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent diff --git a/go.mod b/go.mod index 5a0c7b0..e809a10 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( google.golang.org/grpc v1.43.0 k8s.io/api v0.23.5 k8s.io/apimachinery v0.23.5 + k8s.io/apiserver v0.23.5 k8s.io/client-go v0.23.5 k8s.io/cluster-bootstrap v0.0.0 k8s.io/component-base v0.23.5 diff --git a/helm/kamaji/crds/tenantcontrolplane.yaml b/helm/kamaji/crds/tenantcontrolplane.yaml index 6c7feaa..58c745b 100644 --- a/helm/kamaji/crds/tenantcontrolplane.yaml +++ b/helm/kamaji/crds/tenantcontrolplane.yaml @@ -1,3 +1,5 @@ + +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: @@ -12,630 +14,907 @@ spec: listKind: TenantControlPlaneList plural: tenantcontrolplanes shortNames: - - tcp + - tcp singular: tenantcontrolplane scope: Namespaced versions: - - additionalPrinterColumns: - - description: Kubernetes version - jsonPath: .spec.kubernetes.version - name: Version - type: string - - description: Kubernetes version - jsonPath: .status.kubernetesResources.version.status - name: Status - type: string - - description: Tenant Control Plane Endpoint (API server) - jsonPath: .status.controlPlaneEndpoint - name: Control-Plane-Endpoint - type: string - - description: Secret which contains admin kubeconfig - jsonPath: .status.kubeconfig.admin.secretName - name: Kubeconfig - type: string - - description: Age - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1alpha1 - schema: - openAPIV3Schema: - description: TenantControlPlane is the Schema for the tenantcontrolplanes - API. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation + - additionalPrinterColumns: + - description: Kubernetes version + jsonPath: .spec.kubernetes.version + name: Version + type: string + - description: Kubernetes version + jsonPath: .status.kubernetesResources.version.status + name: Status + type: string + - description: Tenant Control Plane Endpoint (API server) + jsonPath: .status.controlPlaneEndpoint + name: Control-Plane-Endpoint + type: string + - description: Secret which contains admin kubeconfig + jsonPath: .status.kubeconfig.admin.secretName + name: Kubeconfig + type: string + - description: Age + jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: TenantControlPlane is the Schema for the tenantcontrolplanes + API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: TenantControlPlaneSpec defines the desired state of TenantControlPlane. - properties: - addons: - default: - coreDNS: - enabled: true - kubeProxy: - enabled: true - description: Addons contain which addons are enabled - properties: - coreDNS: - default: - enabled: true - description: AddonSpec defines the spec for every addon. - properties: - enabled: - default: true - type: boolean - type: object - kubeProxy: - default: - enabled: true - description: AddonSpec defines the spec for every addon. - properties: - enabled: - default: true - type: boolean - type: object - type: object - controlPlane: - description: ControlPlane defines how the Tenant Control Plane Kubernetes - resources must be created in the Admin Cluster, such as the number - of Pod replicas, the Service resource, or the Ingress. - properties: - deployment: - description: Defining the options for the deployed Tenant Control - Plane as Deployment resource. - properties: - additionalMetadata: - description: AdditionalMetadata defines which additional metadata, - such as labels and annotations, must be attached to the - created resource. - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - replicas: - default: 2 - format: int32 - type: integer - type: object - ingress: - description: Defining the options for an Optional Ingress which - will expose API Server of the Tenant Control Plane - properties: - additionalMetadata: - description: AdditionalMetadata defines which additional metadata, - such as labels and annotations, must be attached to the - created resource. - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - enabled: - type: boolean - hostname: - description: Hostname is an optional field which will be used - as Ingress's Host. If it is not defined, Ingress's host - will be "..", where domain is - specified under NetworkProfileSpec - type: string - ingressClassName: - type: string - required: - - enabled - type: object - service: - description: Defining the options for the Tenant Control Plane - Service resource. - properties: - additionalMetadata: - description: AdditionalMetadata defines which additional metadata, - such as labels and annotations, must be attached to the - created resource. - properties: - annotations: - additionalProperties: - type: string - type: object - labels: - additionalProperties: - type: string - type: object - type: object - serviceType: - description: ServiceType allows specifying how to expose the - Tenant Control Plane. - enum: - - ClusterIP - - NodePort - - LoadBalancer - type: string - required: - - serviceType - type: object - required: - - service - type: object - kubernetes: - description: Kubernetes specification for tenant control plane - properties: - admissionControllers: - default: - - CertificateApproval - - CertificateSigning - - CertificateSubjectRestriction - - DefaultIngressClass - - DefaultStorageClass - - DefaultTolerationSeconds - - LimitRanger - - MutatingAdmissionWebhook - - NamespaceLifecycle - - PersistentVolumeClaimResize - - Priority - - ResourceQuota - - RuntimeClass - - ServiceAccount - - StorageObjectInUseProtection - - TaintNodesByCondition - - ValidatingAdmissionWebhook - description: 'List of enabled Admission Controllers for the Tenant - cluster. Full reference available here: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers' - items: + type: string + metadata: + type: object + spec: + description: TenantControlPlaneSpec defines the desired state of TenantControlPlane. + properties: + addons: + description: Addons contain which addons are enabled + properties: + coreDNS: + description: AddonSpec defines the spec for every addon. + type: object + konnectivity: + description: KonnectivitySpec defines the spec for Konnectivity. + properties: + agentImage: + default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-agent + description: AgentImage defines the container image for Konnectivity's + agent. + type: string + allowAddressAsExternalIP: + type: boolean + proxyHost: + description: Host of Konnectivity proxy server. + type: string + proxyPort: + description: Port of Konnectivity proxy server. + format: int32 + type: integer + serverImage: + default: us.gcr.io/k8s-artifacts-prod/kas-network-proxy/proxy-server + description: ServerImage defines the container image for Konnectivity's + server. + type: string + serviceType: + description: ServiceType allows specifying how to expose the + Konnectivity Proxy Server. enum: - - AlwaysAdmit - - AlwaysDeny - - AlwaysPullImages - - CertificateApproval - - CertificateSigning - - CertificateSubjectRestriction - - DefaultIngressClass - - DefaultStorageClass - - DefaultTolerationSeconds - - DenyEscalatingExec - - DenyExecOnPrivileged - - DenyServiceExternalIPs - - EventRateLimit - - ExtendedResourceToleration - - ImagePolicyWebhook - - LimitPodHardAntiAffinityTopology - - LimitRanger - - MutatingAdmissionWebhook - - NamespaceAutoProvision - - NamespaceExists - - NamespaceLifecycle - - NodeRestriction - - OwnerReferencesPermissionEnforcement - - PersistentVolumeClaimResize - - PersistentVolumeLabel - - PodNodeSelector - - PodSecurity - - PodSecurityPolicy - - PodTolerationRestriction - - Priority - - ResourceQuota - - RuntimeClass - - SecurityContextDeny - - ServiceAccount - - StorageObjectInUseProtection - - TaintNodesByCondition - - ValidatingAdmissionWebhook + - ClusterIP + - NodePort + - LoadBalancer type: string - type: array - kubelet: - properties: - cgroupfs: - description: CGroupFS defines the cgroup driver for Kubelet - https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/ - enum: - - systemd - - cgroupfs - type: string - type: object - version: - description: Kubernetes Version for the tenant control plane - type: string - required: - - kubelet - - version - type: object - networkProfile: - description: NetworkProfile specifies how the network is - properties: - address: - description: Address where API server of will be exposed. In case - of LoadBalancer Service, this can be empty in order to use the - exposed IP provided by the cloud controller manager. - type: string - allowAddressAsExternalIP: - description: AllowAddressAsExternalIP will include tenantControlPlane.Spec.NetworkProfile.Address - in the section of ExternalIPs of the Kubernetes Service (only - ClusterIP or NodePort) - type: boolean - dnsServiceIPs: - default: - - 10.96.0.10 - items: + version: + default: v0.0.31 + description: Version for Konnectivity server and agent. type: string - type: array - domain: - description: Domain of the tenant control plane - type: string - podCidr: - default: 10.244.0.0/16 - description: CIDR for Kubernetes Pods - type: string - port: - default: 6443 - description: Port where API server of will be exposed - format: int32 - type: integer - serviceCidr: - default: 10.96.0.0/16 - description: Kubernetes Service - type: string - required: - - domain - type: object - required: - - controlPlane - - kubernetes - type: object - status: - description: TenantControlPlaneStatus defines the observed state of TenantControlPlane. - properties: - addons: - description: Addons contains the status of the different Addons - properties: - coreDNS: - description: AddonStatus defines the observed state of an Addon. - properties: - enabled: - type: boolean - kubeadmConfigResourceVersion: - type: string - lastUpdate: - format: date-time - type: string - required: - - enabled - type: object - kubeProxy: - description: AddonStatus defines the observed state of an Addon. - properties: - enabled: - type: boolean - kubeadmConfigResourceVersion: - type: string - lastUpdate: - format: date-time - type: string - required: - - enabled - type: object - type: object - certificates: - description: Certificates contains information about the different - certificates that are necessary to run a kubernetes control plane - properties: - apiServer: - description: CertificatePrivateKeyPair defines the status. - properties: - lastUpdate: - format: date-time - type: string - secretName: - type: string - type: object - apiServerKubeletClient: - description: CertificatePrivateKeyPair defines the status. - properties: - lastUpdate: - format: date-time - type: string - secretName: - type: string - type: object - ca: - description: CertificatePrivateKeyPair defines the status. - properties: - lastUpdate: - format: date-time - type: string - secretName: - type: string - type: object - etcd: - description: ETCDAPIServerCertificate defines the observed state - of ETCD Certificate for API server. - properties: - apiServer: - description: ETCDAPIServerCertificate defines the observed - state of ETCD Certificate for API server. - properties: - lastUpdate: - format: date-time + required: + - proxyPort + - serviceType + type: object + kubeProxy: + description: AddonSpec defines the spec for every addon. + type: object + type: object + controlPlane: + description: ControlPlane defines how the Tenant Control Plane Kubernetes + resources must be created in the Admin Cluster, such as the number + of Pod replicas, the Service resource, or the Ingress. + properties: + deployment: + description: Defining the options for the deployed Tenant Control + Plane as Deployment resource. + properties: + additionalMetadata: + description: AdditionalMetadata defines which additional metadata, + such as labels and annotations, must be attached to the + created resource. + properties: + annotations: + additionalProperties: type: string - secretName: + type: object + labels: + additionalProperties: type: string - type: object - ca: - description: ETCDAPIServerCertificate defines the observed - state of ETCD Certificate for API server. - properties: - lastUpdate: - format: date-time + type: object + type: object + replicas: + default: 2 + format: int32 + type: integer + type: object + ingress: + description: Defining the options for an Optional Ingress which + will expose API Server of the Tenant Control Plane + properties: + additionalMetadata: + description: AdditionalMetadata defines which additional metadata, + such as labels and annotations, must be attached to the + created resource. + properties: + annotations: + additionalProperties: type: string - secretName: + type: object + labels: + additionalProperties: type: string - type: object - type: object - frontProxyCA: - description: CertificatePrivateKeyPair defines the status. - properties: - lastUpdate: - format: date-time - type: string - secretName: - type: string - type: object - frontProxyClient: - description: CertificatePrivateKeyPair defines the status. - properties: - lastUpdate: - format: date-time - type: string - secretName: - type: string - type: object - sa: - description: CertificatePrivateKeyPair defines the status. - properties: - lastUpdate: - format: date-time - type: string - secretName: - type: string - type: object - type: object - controlPlaneEndpoint: - description: ControlPlaneEndpoint contains the status of the kubernetes - control plane - type: string - kubeadmPhase: - description: KubeadmPhase contains the status of the kubeadm phases - action - properties: - bootstrapToken: - description: KubeadmPhasesStatus contains the status of of a kubeadm - phase action. - properties: - kubeadmConfigResourceVersion: - type: string - lastUpdate: - format: date-time - type: string - type: object - uploadConfigKubeadm: - description: KubeadmPhasesStatus contains the status of of a kubeadm - phase action. - properties: - kubeadmConfigResourceVersion: - type: string - lastUpdate: - format: date-time - type: string - type: object - uploadConfigKubelet: - description: KubeadmPhasesStatus contains the status of of a kubeadm - phase action. - properties: - kubeadmConfigResourceVersion: - type: string - lastUpdate: - format: date-time - type: string - type: object - required: - - bootstrapToken - - uploadConfigKubeadm - - uploadConfigKubelet - type: object - kubeadmconfig: - description: KubeadmConfig contains the status of the configuration - required by kubeadm - properties: - configmapName: + type: object + type: object + enabled: + type: boolean + hostname: + description: Hostname is an optional field which will be used + as Ingress's Host. If it is not defined, Ingress's host + will be "..", where domain is + specified under NetworkProfileSpec + type: string + ingressClassName: + type: string + required: + - enabled + type: object + service: + description: Defining the options for the Tenant Control Plane + Service resource. + properties: + additionalMetadata: + description: AdditionalMetadata defines which additional metadata, + such as labels and annotations, must be attached to the + created resource. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + serviceType: + description: ServiceType allows specifying how to expose the + Tenant Control Plane. + enum: + - ClusterIP + - NodePort + - LoadBalancer + type: string + required: + - serviceType + type: object + required: + - service + type: object + kubernetes: + description: Kubernetes specification for tenant control plane + properties: + admissionControllers: + default: + - CertificateApproval + - CertificateSigning + - CertificateSubjectRestriction + - DefaultIngressClass + - DefaultStorageClass + - DefaultTolerationSeconds + - LimitRanger + - MutatingAdmissionWebhook + - NamespaceLifecycle + - PersistentVolumeClaimResize + - Priority + - ResourceQuota + - RuntimeClass + - ServiceAccount + - StorageObjectInUseProtection + - TaintNodesByCondition + - ValidatingAdmissionWebhook + description: 'List of enabled Admission Controllers for the Tenant + cluster. Full reference available here: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers' + items: + enum: + - AlwaysAdmit + - AlwaysDeny + - AlwaysPullImages + - CertificateApproval + - CertificateSigning + - CertificateSubjectRestriction + - DefaultIngressClass + - DefaultStorageClass + - DefaultTolerationSeconds + - DenyEscalatingExec + - DenyExecOnPrivileged + - DenyServiceExternalIPs + - EventRateLimit + - ExtendedResourceToleration + - ImagePolicyWebhook + - LimitPodHardAntiAffinityTopology + - LimitRanger + - MutatingAdmissionWebhook + - NamespaceAutoProvision + - NamespaceExists + - NamespaceLifecycle + - NodeRestriction + - OwnerReferencesPermissionEnforcement + - PersistentVolumeClaimResize + - PersistentVolumeLabel + - PodNodeSelector + - PodSecurity + - PodSecurityPolicy + - PodTolerationRestriction + - Priority + - ResourceQuota + - RuntimeClass + - SecurityContextDeny + - ServiceAccount + - StorageObjectInUseProtection + - TaintNodesByCondition + - ValidatingAdmissionWebhook type: string - lastUpdate: - format: date-time + type: array + kubelet: + properties: + cgroupfs: + description: CGroupFS defines the cgroup driver for Kubelet + https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/ + enum: + - systemd + - cgroupfs + type: string + type: object + version: + description: Kubernetes Version for the tenant control plane + type: string + required: + - kubelet + - version + type: object + networkProfile: + description: NetworkProfile specifies how the network is + properties: + address: + description: Address where API server of will be exposed. In case + of LoadBalancer Service, this can be empty in order to use the + exposed IP provided by the cloud controller manager. + type: string + allowAddressAsExternalIP: + description: AllowAddressAsExternalIP will include tenantControlPlane.Spec.NetworkProfile.Address + in the section of ExternalIPs of the Kubernetes Service (only + ClusterIP or NodePort) + type: boolean + dnsServiceIPs: + default: + - 10.96.0.10 + items: type: string - resourceVersion: - type: string - required: - - resourceVersion - type: object - kubeconfig: - description: KubeConfig contains information about the kubenconfigs - that control plane pieces need - properties: - admin: - description: TenantControlPlaneKubeconfigsStatus contains information - about a the generated kubeconfig. - properties: - lastUpdate: - format: date-time - type: string - secretName: - type: string - type: object - controlerManager: - description: TenantControlPlaneKubeconfigsStatus contains information - about a the generated kubeconfig. - properties: - lastUpdate: - format: date-time - type: string - secretName: - type: string - type: object - scheduler: - description: TenantControlPlaneKubeconfigsStatus contains information - about a the generated kubeconfig. - properties: - lastUpdate: - format: date-time - type: string - secretName: - type: string - type: object - type: object - kubernetesResources: - description: Kubernetes contains information about the reconciliation - of the required Kubernetes resources deployed in the admin cluster - properties: - deployment: - description: KubernetesDeploymentStatus defines the status for - the Tenant Control Plane Deployment in the management cluster. - properties: - availableReplicas: - description: Total number of available pods (ready for at - least minReadySeconds) targeted by this deployment. - format: int32 - type: integer - collisionCount: - description: Count of hash collisions for the Deployment. - The Deployment controller uses this field as a collision - avoidance mechanism when it needs to create the name for - the newest ReplicaSet. - format: int32 - type: integer - conditions: - description: Represents the latest available observations - of a deployment's current state. - items: - description: DeploymentCondition describes the state of - a deployment at a certain point. - properties: - lastTransitionTime: - description: Last time the condition transitioned from - one status to another. - format: date-time - type: string - lastUpdateTime: - description: The last time this condition was updated. - format: date-time - type: string - message: - description: A human readable message indicating details - about the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - status: - description: Status of the condition, one of True, False, - Unknown. - type: string - type: - description: Type of deployment condition. - type: string - required: + type: array + domain: + description: Domain of the tenant control plane + type: string + podCidr: + default: 10.244.0.0/16 + description: CIDR for Kubernetes Pods + type: string + port: + default: 6443 + description: Port where API server of will be exposed + format: int32 + type: integer + serviceCidr: + default: 10.96.0.0/16 + description: Kubernetes Service + type: string + required: + - domain + type: object + required: + - controlPlane + - kubernetes + type: object + status: + description: TenantControlPlaneStatus defines the observed state of TenantControlPlane. + properties: + addons: + description: Addons contains the status of the different Addons + properties: + coreDNS: + description: AddonStatus defines the observed state of an Addon. + properties: + enabled: + type: boolean + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + required: + - enabled + type: object + konnectivity: + description: KonnectivityStatus defines the status of Konnectivity + as Addon. + properties: + agent: + properties: + lastUpdate: + description: Last time when k8s object was updated + format: date-time + type: string + name: + type: string + namespace: + type: string + resourceVersion: + description: Resource version of k8s object + type: string + type: object + certificate: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + resourceVersion: + type: string + secretName: + type: string + type: object + clusterrolebinding: + properties: + lastUpdate: + description: Last time when k8s object was updated + format: date-time + type: string + name: + type: string + namespace: + type: string + resourceVersion: + description: Resource version of k8s object + type: string + type: object + egressSelectorConfiguration: + type: string + enabled: + type: boolean + kubeconfig: + description: TenantControlPlaneKubeconfigsStatus contains + information about a the generated kubeconfig. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + sa: + properties: + lastUpdate: + description: Last time when k8s object was updated + format: date-time + type: string + name: + type: string + namespace: + type: string + resourceVersion: + description: Resource version of k8s object + type: string + type: object + service: + description: KubernetesServiceStatus defines the status for + the Tenant Control Plane Service in the management cluster. + properties: + conditions: + description: Current service state + items: + description: "Condition contains details for one aspect + of the current state of this API Resource. --- This + struct is intended for direct use as an array at the + field path .status.conditions. For example, type + FooStatus struct{ // Represents the observations + of a foo's current state. // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\" + \ // +patchMergeKey=type // +patchStrategy=merge + \ // +listType=map // +listMapKey=type Conditions + []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` + \n // other fields }" + properties: + lastTransitionTime: + description: lastTransitionTime is the last time + the condition transitioned from one status to + another. This should be when the underlying condition + changed. If that is not known, then using the + time when the API field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message + indicating details about the transition. This + may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, + if .metadata.generation is currently 12, but the + .status.conditions[x].observedGeneration is 9, + the condition is out of date with respect to the + current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier + indicating the reason for the condition's last + transition. Producers of specific condition types + may define expected values and meanings for this + field, and whether the values are considered a + guaranteed API. The value should be a CamelCase + string. This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, + False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in + foo.example.com/CamelCase. --- Many .condition.type + values are consistent across resources like Available, + but because arbitrary conditions can be useful + (see .node.status.conditions), the ability to + deconflict is important. The regex it matches + is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason - status - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + loadBalancer: + description: LoadBalancer contains the current status + of the load-balancer, if one is present. + properties: + ingress: + description: Ingress is a list containing ingress + points for the load-balancer. Traffic intended for + the service should be sent to these ingress points. + items: + description: 'LoadBalancerIngress represents the + status of a load-balancer ingress point: traffic + intended for the service should be sent to an + ingress point.' + properties: + hostname: + description: Hostname is set for load-balancer + ingress points that are DNS based (typically + AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress + points that are IP based (typically GCE or + OpenStack load-balancers) + type: string + ports: + description: Ports is a list of records of service + ports If used, every port defined in the service + should have an entry in it + items: + properties: + error: + description: 'Error is to record the problem + with the service port The format of + the error shall comply with the following + rules: - built-in error values shall + be specified in this file and those + shall use CamelCase names - cloud + provider specific error values must + have names that comply with the format + foo.example.com/CamelCase. --- The regex + it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of + the service port of which status is + recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol + of the service port of which status + is recorded here The supported values + are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array type: object - type: array - name: - description: The name of the Deployment for the given cluster. - type: string - namespace: - description: The namespace which the Deployment for the given - cluster is deployed. - type: string - observedGeneration: - description: The generation observed by the deployment controller. - format: int64 - type: integer - readyReplicas: - description: readyReplicas is the number of pods targeted - by this Deployment with a Ready Condition. - format: int32 - type: integer - replicas: - description: Total number of non-terminated pods targeted - by this deployment (their labels match the selector). - format: int32 - type: integer - unavailableReplicas: - description: Total number of unavailable pods targeted by - this deployment. This is the total number of pods that are - still required for the deployment to have 100% available - capacity. They may either be pods that are running but not - yet available or pods that still have not been created. - format: int32 - type: integer - updatedReplicas: - description: Total number of non-terminated pods targeted - by this deployment that have the desired template spec. - format: int32 - type: integer - required: + name: + description: The name of the Service for the given cluster. + type: string + namespace: + description: The namespace which the Service for the given + cluster is deployed. + type: string + port: + description: The port where the service is running + format: int32 + type: integer + required: - name - namespace - type: object - ingress: - description: KubernetesIngressStatus defines the status for the - Tenant Control Plane Ingress in the management cluster. - properties: - loadBalancer: - description: LoadBalancer contains the current status of the - load-balancer. + - port + type: object + required: + - enabled + type: object + kubeProxy: + description: AddonStatus defines the observed state of an Addon. + properties: + enabled: + type: boolean + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + required: + - enabled + type: object + type: object + certificates: + description: Certificates contains information about the different + certificates that are necessary to run a kubernetes control plane + properties: + apiServer: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + resourceVersion: + type: string + secretName: + type: string + type: object + apiServerKubeletClient: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + resourceVersion: + type: string + secretName: + type: string + type: object + ca: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + resourceVersion: + type: string + secretName: + type: string + type: object + etcd: + description: ETCDAPIServerCertificate defines the observed state + of ETCD Certificate for API server. + properties: + apiServer: + description: ETCDAPIServerCertificate defines the observed + state of ETCD Certificate for API server. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + ca: + description: ETCDAPIServerCertificate defines the observed + state of ETCD Certificate for API server. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + type: object + frontProxyCA: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + resourceVersion: + type: string + secretName: + type: string + type: object + frontProxyClient: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + resourceVersion: + type: string + secretName: + type: string + type: object + sa: + description: CertificatePrivateKeyPair defines the status. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + type: object + controlPlaneEndpoint: + description: ControlPlaneEndpoint contains the status of the kubernetes + control plane + type: string + kubeadmPhase: + description: KubeadmPhase contains the status of the kubeadm phases + action + properties: + bootstrapToken: + description: KubeadmPhasesStatus contains the status of of a kubeadm + phase action. + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + uploadConfigKubeadm: + description: KubeadmPhasesStatus contains the status of of a kubeadm + phase action. + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + uploadConfigKubelet: + description: KubeadmPhasesStatus contains the status of of a kubeadm + phase action. + properties: + kubeadmConfigResourceVersion: + type: string + lastUpdate: + format: date-time + type: string + type: object + required: + - bootstrapToken + - uploadConfigKubeadm + - uploadConfigKubelet + type: object + kubeadmconfig: + description: KubeadmConfig contains the status of the configuration + required by kubeadm + properties: + configmapName: + type: string + lastUpdate: + format: date-time + type: string + resourceVersion: + type: string + required: + - resourceVersion + type: object + kubeconfig: + description: KubeConfig contains information about the kubenconfigs + that control plane pieces need + properties: + admin: + description: TenantControlPlaneKubeconfigsStatus contains information + about a the generated kubeconfig. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + controlerManager: + description: TenantControlPlaneKubeconfigsStatus contains information + about a the generated kubeconfig. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + scheduler: + description: TenantControlPlaneKubeconfigsStatus contains information + about a the generated kubeconfig. + properties: + lastUpdate: + format: date-time + type: string + secretName: + type: string + type: object + type: object + kubernetesResources: + description: Kubernetes contains information about the reconciliation + of the required Kubernetes resources deployed in the admin cluster + properties: + deployment: + description: KubernetesDeploymentStatus defines the status for + the Tenant Control Plane Deployment in the management cluster. + properties: + availableReplicas: + description: Total number of available pods (ready for at + least minReadySeconds) targeted by this deployment. + format: int32 + type: integer + collisionCount: + description: Count of hash collisions for the Deployment. + The Deployment controller uses this field as a collision + avoidance mechanism when it needs to create the name for + the newest ReplicaSet. + format: int32 + type: integer + conditions: + description: Represents the latest available observations + of a deployment's current state. + items: + description: DeploymentCondition describes the state of + a deployment at a certain point. properties: - ingress: - description: Ingress is a list containing ingress points - for the load-balancer. Traffic intended for the service - should be sent to these ingress points. - items: - description: 'LoadBalancerIngress represents the status + lastTransitionTime: + description: Last time the condition transitioned from + one status to another. + format: date-time + type: string + lastUpdateTime: + description: The last time this condition was updated. + format: date-time + type: string + message: + description: A human readable message indicating details + about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + type: string + type: + description: Type of deployment condition. + type: string + required: + - status + - type + type: object + type: array + lastUpdate: + description: Last time when deployment was updated + format: date-time + type: string + name: + description: The name of the Deployment for the given cluster. + type: string + namespace: + description: The namespace which the Deployment for the given + cluster is deployed. + type: string + observedGeneration: + description: The generation observed by the deployment controller. + format: int64 + type: integer + readyReplicas: + description: readyReplicas is the number of pods targeted + by this Deployment with a Ready Condition. + format: int32 + type: integer + replicas: + description: Total number of non-terminated pods targeted + by this deployment (their labels match the selector). + format: int32 + type: integer + unavailableReplicas: + description: Total number of unavailable pods targeted by + this deployment. This is the total number of pods that are + still required for the deployment to have 100% available + capacity. They may either be pods that are running but not + yet available or pods that still have not been created. + format: int32 + type: integer + updatedReplicas: + description: Total number of non-terminated pods targeted + by this deployment that have the desired template spec. + format: int32 + type: integer + required: + - name + - namespace + type: object + ingress: + description: KubernetesIngressStatus defines the status for the + Tenant Control Plane Ingress in the management cluster. + properties: + loadBalancer: + description: LoadBalancer contains the current status of the + load-balancer. + properties: + ingress: + description: Ingress is a list containing ingress points + for the load-balancer. Traffic intended for the service + should be sent to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.' - properties: - hostname: - description: Hostname is set for load-balancer ingress - points that are DNS based (typically AWS load-balancers) - type: string - ip: - description: IP is set for load-balancer ingress - points that are IP based (typically GCE or OpenStack - load-balancers) - type: string - ports: - description: Ports is a list of records of service - ports If used, every port defined in the service - should have an entry in it - items: - properties: - error: - description: 'Error is to record the problem + properties: + hostname: + description: Hostname is set for load-balancer ingress + points that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress + points that are IP based (typically GCE or OpenStack + load-balancers) + type: string + ports: + description: Ports is a list of records of service + ports If used, every port defined in the service + should have an entry in it + items: + properties: + error: + description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified @@ -644,50 +923,50 @@ spec: must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - port: - description: Port is the port number of the - service port of which status is recorded - here - format: int32 - type: integer - protocol: - default: TCP - description: 'Protocol is the protocol of + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the + service port of which status is recorded + here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"' - type: string - required: - - port - - protocol - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: array - type: object - name: - description: The name of the Ingress for the given cluster. - type: string - namespace: - description: The namespace which the Ingress for the given - cluster is deployed. - type: string - required: - - name - - namespace - type: object - service: - description: KubernetesServiceStatus defines the status for the - Tenant Control Plane Service in the management cluster. - properties: - conditions: - description: Current service state - items: - description: "Condition contains details for one aspect + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + name: + description: The name of the Ingress for the given cluster. + type: string + namespace: + description: The namespace which the Ingress for the given + cluster is deployed. + type: string + required: + - name + - namespace + type: object + service: + description: KubernetesServiceStatus defines the status for the + Tenant Control Plane Service in the management cluster. + properties: + conditions: + description: Current service state + items: + description: "Condition contains details for one aspect of the current state of this API Resource. --- This struct is intended for direct use as an array at the field path .status.conditions. For example, type FooStatus struct{ @@ -698,101 +977,101 @@ spec: +listMapKey=type Conditions []metav1.Condition `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the - condition transitioned from one status to another. - This should be when the underlying condition changed. If - that is not known, then using the time when the API - field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating - details about the transition. This may be an empty - string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, - if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to - the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier - indicating the reason for the condition's last transition. - Producers of specific condition types may define expected - values and meanings for this field, and whether the - values are considered a guaranteed API. The value - should be a CamelCase string. This field may not be - empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, - Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across - resources like Available, but because arbitrary conditions - can be useful (see .node.status.conditions), the ability - to deconflict is important. The regex it matches is - (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - loadBalancer: - description: LoadBalancer contains the current status of the - load-balancer, if one is present. properties: - ingress: - description: Ingress is a list containing ingress points - for the load-balancer. Traffic intended for the service - should be sent to these ingress points. - items: - description: 'LoadBalancerIngress represents the status + lastTransitionTime: + description: lastTransitionTime is the last time the + condition transitioned from one status to another. + This should be when the underlying condition changed. If + that is not known, then using the time when the API + field changed is acceptable. + format: date-time + type: string + message: + description: message is a human readable message indicating + details about the transition. This may be an empty + string. + maxLength: 32768 + type: string + observedGeneration: + description: observedGeneration represents the .metadata.generation + that the condition was set based upon. For instance, + if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration + is 9, the condition is out of date with respect to + the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: reason contains a programmatic identifier + indicating the reason for the condition's last transition. + Producers of specific condition types may define expected + values and meanings for this field, and whether the + values are considered a guaranteed API. The value + should be a CamelCase string. This field may not be + empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + --- Many .condition.type values are consistent across + resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability + to deconflict is important. The regex it matches is + (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + loadBalancer: + description: LoadBalancer contains the current status of the + load-balancer, if one is present. + properties: + ingress: + description: Ingress is a list containing ingress points + for the load-balancer. Traffic intended for the service + should be sent to these ingress points. + items: + description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.' - properties: - hostname: - description: Hostname is set for load-balancer ingress - points that are DNS based (typically AWS load-balancers) - type: string - ip: - description: IP is set for load-balancer ingress - points that are IP based (typically GCE or OpenStack - load-balancers) - type: string - ports: - description: Ports is a list of records of service - ports If used, every port defined in the service - should have an entry in it - items: - properties: - error: - description: 'Error is to record the problem + properties: + hostname: + description: Hostname is set for load-balancer ingress + points that are DNS based (typically AWS load-balancers) + type: string + ip: + description: IP is set for load-balancer ingress + points that are IP based (typically GCE or OpenStack + load-balancers) + type: string + ports: + description: Ports is a list of records of service + ports If used, every port defined in the service + should have an entry in it + items: + properties: + error: + description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified @@ -801,119 +1080,119 @@ spec: must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - port: - description: Port is the port number of the - service port of which status is recorded - here - format: int32 - type: integer - protocol: - default: TCP - description: 'Protocol is the protocol of + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the + service port of which status is recorded + here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"' - type: string - required: - - port - - protocol - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: array - type: object - name: - description: The name of the Service for the given cluster. - type: string - namespace: - description: The namespace which the Service for the given - cluster is deployed. - type: string - port: - description: The port where the service is running - format: int32 - type: integer - required: + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic + type: object + type: array + type: object + name: + description: The name of the Service for the given cluster. + type: string + namespace: + description: The namespace which the Service for the given + cluster is deployed. + type: string + port: + description: The port where the service is running + format: int32 + type: integer + required: + - name + - namespace + - port + type: object + version: + description: KubernetesVersion contains the information regarding + the running Kubernetes version, and its upgrade status. + properties: + status: + default: Provisioning + description: Status returns the current status of the Kubernetes + version, such as its provisioning state, or completed upgrade. + enum: + - Provisioning + - Upgrading + - Ready + - NotReady + type: string + version: + description: Version is the running Kubernetes version of + the Tenant Control Plane. + type: string + required: + - status + type: object + type: object + storage: + description: Storage Status contains information about Kubernetes + storage system + properties: + etcd: + description: ETCDStatus defines the observed state of ETCDStatus. + properties: + role: + properties: + exists: + type: boolean + name: + type: string + permissions: + items: + properties: + key: + type: string + rangeEnd: + type: string + type: + type: integer + type: object + type: array + required: + - exists - name - - namespace - - port - type: object - version: - description: KubernetesVersion contains the information regarding - the running Kubernetes version, and its upgrade status. - properties: - status: - default: Provisioning - description: Status returns the current status of the Kubernetes - version, such as its provisioning state, or completed upgrade. - enum: - - Provisioning - - Upgrading - - Ready - - NotReady - type: string - version: - description: Version is the running Kubernetes version of - the Tenant Control Plane. - type: string - required: - - status - type: object - type: object - storage: - description: Storage Status contains information about Kubernetes - storage system - properties: - etcd: - description: ETCDStatus defines the observed state of ETCDStatus. - properties: - role: - properties: - exists: - type: boolean - name: + type: object + user: + properties: + exists: + type: boolean + name: + type: string + roles: + items: type: string - permissions: - items: - properties: - key: - type: string - rangeEnd: - type: string - type: - type: integer - type: object - type: array - required: - - exists - - name - type: object - user: - properties: - exists: - type: boolean - name: - type: string - roles: - items: - type: string - type: array - required: - - exists - - name - type: object - type: object - type: object - type: object - type: object - served: true - storage: true - subresources: - status: {} + type: array + required: + - exists + - name + type: object + type: object + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/internal/resources/ca_certificate.go b/internal/resources/ca_certificate.go index b5aa08a..b347b0b 100644 --- a/internal/resources/ca_certificate.go +++ b/internal/resources/ca_certificate.go @@ -29,7 +29,8 @@ type CACertificate struct { } func (r *CACertificate) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { - return tenantControlPlane.Status.Certificates.CA.SecretName != r.resource.GetName() + return tenantControlPlane.Status.Certificates.CA.SecretName != r.resource.GetName() || + tenantControlPlane.Status.Certificates.CA.ResourceVersion != r.resource.ResourceVersion } func (r *CACertificate) ShouldCleanup(plane *kamajiv1alpha1.TenantControlPlane) bool { @@ -74,6 +75,7 @@ func (r *CACertificate) GetName() string { func (r *CACertificate) UpdateTenantControlPlaneStatus(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { tenantControlPlane.Status.Certificates.CA.LastUpdate = metav1.Now() tenantControlPlane.Status.Certificates.CA.SecretName = r.resource.GetName() + tenantControlPlane.Status.Certificates.CA.ResourceVersion = r.resource.ResourceVersion return nil } diff --git a/internal/resources/k8s_deployment_resource.go b/internal/resources/k8s_deployment_resource.go index 9f834de..ab7033b 100644 --- a/internal/resources/k8s_deployment_resource.go +++ b/internal/resources/k8s_deployment_resource.go @@ -25,9 +25,17 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/resources/konnectivity" "github.com/clastix/kamaji/internal/utilities" ) +const ( + konnectivityEgressSelectorConfigurationPath = "/etc/kubernetes/konnectivity/configurations/egress-selector-configuration.yaml" + konnectivityServerName = "konnectivity-server" + konnectivityServerPath = "/run/konnectivity" + konnectivityUDSName = "konnectivity-uds" +) + type KubernetesDeploymentResource struct { resource *appsv1.Deployment Client client.Client @@ -49,7 +57,7 @@ func (r *KubernetesDeploymentResource) ShouldCleanup(plane *kamajiv1alpha1.Tenan } func (r *KubernetesDeploymentResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { - return tenantControlPlane.Spec.Addons.Konnectivity != nil, nil + return tenantControlPlane.Spec.Addons.Konnectivity == nil, nil } func (r *KubernetesDeploymentResource) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { @@ -548,6 +556,10 @@ func (r *KubernetesDeploymentResource) CreateOrUpdate(ctx context.Context, tenan }, } + if err := r.reconcileKonnectivity(&r.resource.Spec.Template.Spec, *tenantControlPlane); err != nil { + return err + } + return controllerutil.SetControllerReference(tenantControlPlane, r.resource, r.Client.Scheme()) }) } @@ -573,6 +585,7 @@ func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(ctx contex DeploymentStatus: r.resource.Status, Name: r.resource.GetName(), Namespace: r.resource.GetNamespace(), + LastUpdate: metav1.Now(), } return nil @@ -605,7 +618,7 @@ func (r *KubernetesDeploymentResource) isNotReady() bool { } func (r *KubernetesDeploymentResource) reconcileKonnectivity(podSpec *corev1.PodSpec, tenantControlPlane kamajiv1alpha1.TenantControlPlane) error { - if tenantControlPlane.Spec.Addons.Konnectivity != nil { + if tenantControlPlane.Spec.Addons.Konnectivity == nil { return nil } diff --git a/internal/resources/konnectivity/agent.go b/internal/resources/konnectivity/agent.go index 96e6234..2f4d1da 100644 --- a/internal/resources/konnectivity/agent.go +++ b/internal/resources/konnectivity/agent.go @@ -34,7 +34,7 @@ func (r *Agent) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *k } func (r *Agent) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { - return tenantControlPlane.Spec.Addons.Konnectivity != nil + return tenantControlPlane.Spec.Addons.Konnectivity == nil } func (r *Agent) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { diff --git a/internal/resources/konnectivity/certificate_resource.go b/internal/resources/konnectivity/certificate_resource.go index 4bb0f4d..f2cb7fc 100644 --- a/internal/resources/konnectivity/certificate_resource.go +++ b/internal/resources/konnectivity/certificate_resource.go @@ -42,7 +42,7 @@ func (r *CertificateResource) ShouldStatusBeUpdated(ctx context.Context, tenantC } func (r *CertificateResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { - return tenantControlPlane.Spec.Addons.Konnectivity != nil + return tenantControlPlane.Spec.Addons.Konnectivity == nil } func (r *CertificateResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { diff --git a/internal/resources/konnectivity/cluster_role_binding_resource.go b/internal/resources/konnectivity/cluster_role_binding_resource.go index a9feaf4..368ad65 100644 --- a/internal/resources/konnectivity/cluster_role_binding_resource.go +++ b/internal/resources/konnectivity/cluster_role_binding_resource.go @@ -29,7 +29,7 @@ func (r *ClusterRoleBindingResource) ShouldStatusBeUpdated(ctx context.Context, } func (r *ClusterRoleBindingResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { - return tenantControlPlane.Spec.Addons.Konnectivity != nil + return tenantControlPlane.Spec.Addons.Konnectivity == nil } func (r *ClusterRoleBindingResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { diff --git a/internal/resources/konnectivity/constants.go b/internal/resources/konnectivity/constants.go new file mode 100644 index 0000000..6da2dc7 --- /dev/null +++ b/internal/resources/konnectivity/constants.go @@ -0,0 +1,19 @@ +package konnectivity + +const ( + AgentName = "konnectivity-agent" + CertCommonName = "system:konnectivity-server" + + agentTokenName = "konnectivity-agent-token" + apiServerAPIVersion = "apiserver.k8s.io/v1beta1" + certExpirationDelayYears = 10 + certOrganization = "system:master" + defaultClusterName = "kubernetes" + defaultUDSName = "/run/konnectivity/konnectivity-server.socket" + egressSelectorConfigurationKind = "EgressSelectorConfiguration" + egressSelectorConfigurationName = "cluster" + konnectivityCertAndKeyBaseName = "konnectivity" + konnectivityKubeconfigFileName = "konnectivity-server.conf" + kubeconfigAPIVersion = "v1" + roleAuthDelegator = "system:auth-delegator" +) diff --git a/internal/resources/konnectivity/egress_selector_configuration_resource.go b/internal/resources/konnectivity/egress_selector_configuration_resource.go index 636c677..00ef707 100644 --- a/internal/resources/konnectivity/egress_selector_configuration_resource.go +++ b/internal/resources/konnectivity/egress_selector_configuration_resource.go @@ -35,7 +35,7 @@ func (r *EgressSelectorConfigurationResource) Define(ctx context.Context, tenant } func (r *EgressSelectorConfigurationResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { - return tenantControlPlane.Spec.Addons.Konnectivity != nil + return tenantControlPlane.Spec.Addons.Konnectivity == nil } func (r *EgressSelectorConfigurationResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { diff --git a/internal/resources/konnectivity/kubeconfig_resource.go b/internal/resources/konnectivity/kubeconfig_resource.go index 27a117c..89cd3e8 100644 --- a/internal/resources/konnectivity/kubeconfig_resource.go +++ b/internal/resources/konnectivity/kubeconfig_resource.go @@ -32,7 +32,7 @@ func (r *KubeconfigResource) ShouldStatusBeUpdated(ctx context.Context, tenantCo } func (r *KubeconfigResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { - return tenantControlPlane.Spec.Addons.Konnectivity != nil + return tenantControlPlane.Spec.Addons.Konnectivity == nil } func (r *KubeconfigResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { diff --git a/internal/resources/konnectivity/service_account_resource.go b/internal/resources/konnectivity/service_account_resource.go index 57f88d4..7a04aac 100644 --- a/internal/resources/konnectivity/service_account_resource.go +++ b/internal/resources/konnectivity/service_account_resource.go @@ -30,7 +30,7 @@ func (r *ServiceAccountResource) ShouldStatusBeUpdated(ctx context.Context, tena } func (r *ServiceAccountResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { - return tenantControlPlane.Spec.Addons.Konnectivity != nil + return tenantControlPlane.Spec.Addons.Konnectivity == nil } func (r *ServiceAccountResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { diff --git a/internal/resources/konnectivity/service_resource.go b/internal/resources/konnectivity/service_resource.go index 03221ca..646169a 100644 --- a/internal/resources/konnectivity/service_resource.go +++ b/internal/resources/konnectivity/service_resource.go @@ -10,7 +10,6 @@ import ( corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -28,6 +27,18 @@ type ServiceResource struct { } func (r *ServiceResource) ShouldStatusBeUpdated(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { + if tenantControlPlane.Status.Addons.Konnectivity.Service.Name != r.resource.GetName() { + return true + } + + if tenantControlPlane.Status.Addons.Konnectivity.Service.Namespace != r.resource.GetNamespace() { + return true + } + + if tenantControlPlane.Status.Addons.Konnectivity.Service.Port != r.resource.Spec.Ports[0].Port { + return true + } + if len(r.resource.Status.Conditions) != len(tenantControlPlane.Status.Addons.Konnectivity.Service.Conditions) { return true } @@ -60,7 +71,7 @@ func (r *ServiceResource) ShouldStatusBeUpdated(ctx context.Context, tenantContr } func (r *ServiceResource) ShouldCleanup(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { - return tenantControlPlane.Spec.Addons.Konnectivity != nil + return tenantControlPlane.Spec.Addons.Konnectivity == nil } func (r *ServiceResource) CleanUp(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (bool, error) { @@ -86,7 +97,7 @@ func (r *ServiceResource) UpdateTenantControlPlaneStatus(ctx context.Context, te return nil } - tenantControlPlane.Status.Addons.Konnectivity.Service = corev1.ServiceStatus{} + tenantControlPlane.Status.Addons.Konnectivity.Service = kamajiv1alpha1.KubernetesServiceStatus{} tenantControlPlane.Status.Addons.Konnectivity.Enabled = false return nil @@ -108,8 +119,7 @@ func (r *ServiceResource) CreateOrUpdate(ctx context.Context, tenantControlPlane } func (r *ServiceResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) func() error { - namespacedName := k8stypes.NamespacedName{Namespace: r.resource.GetNamespace(), Name: r.resource.GetName()} - address, _ := tenantControlPlane.GetAddress(ctx, r.Client, namespacedName, tenantControlPlane.Spec.Addons.Konnectivity.ProxyHost) + address, _ := tenantControlPlane.GetKonnectivityServerAddress(ctx, r.Client) if address == "" { address = tenantControlPlane.Spec.NetworkProfile.Address } @@ -140,10 +150,10 @@ func (r *ServiceResource) mutate(ctx context.Context, tenantControlPlane *kamaji case utilities.IsValidIP(address): isIP = true case !utilities.IsValidHostname(address): - return fmt.Errorf("%s is not a valid address for Konnectivity proxy server.", address) + return fmt.Errorf("%s is not a valid address for konnectivity proxy server", address) } - switch tenantControlPlane.Spec.ControlPlane.Service.ServiceType { + switch tenantControlPlane.Spec.Addons.Konnectivity.ServiceType { case kamajiv1alpha1.ServiceTypeLoadBalancer: r.resource.Spec.Type = corev1.ServiceTypeLoadBalancer diff --git a/internal/utilities/utilities.go b/internal/utilities/utilities.go index c91a7be..09a25c2 100644 --- a/internal/utilities/utilities.go +++ b/internal/utilities/utilities.go @@ -6,6 +6,8 @@ package utilities import ( "bytes" "fmt" + "net" + "regexp" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer/json" @@ -56,3 +58,24 @@ func EncondeToYaml(o runtime.Object) ([]byte, error) { return buf.Bytes(), err } + +// IsValidIP checks if the given argument is an IP. +func IsValidIP(ip string) bool { + return net.ParseIP(ip) != nil +} + +// IsValidHostname checks if the given argument is a valid hostname. +func IsValidHostname(hostname string) bool { + pattern := "^([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9])(\\.([a-z0-9]|[a-z0-9][a-z0-9-]{0,61}[a-z0-9]))*$" + + return validateRegex(pattern, hostname) +} + +func validateRegex(pattern string, value string) bool { + isFound, err := regexp.MatchString(pattern, value) + if err != nil { + return false + } + + return isFound +}