diff --git a/Makefile b/Makefile index 5a41f65..f410573 100644 --- a/Makefile +++ b/Makefile @@ -240,6 +240,12 @@ cert-manager: $(HELM) repo add jetstack https://charts.jetstack.io $(HELM) upgrade --install cert-manager jetstack/cert-manager --namespace certmanager-system --create-namespace --set "installCRDs=true" +gateway-api: + kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.4.0/standard-install.yaml +# Required for the TLSRoutes. Experimentals. + kubectl apply --server-side -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.4.0/experimental-install.yaml + kubectl wait --for=condition=Established crd/gateways.gateway.networking.k8s.io --timeout=60s + load: kind $(KIND) load docker-image --name kamaji ${CONTAINER_REPOSITORY}:${VERSION} @@ -249,8 +255,11 @@ load: kind env: kind $(KIND) create cluster --name kamaji +cleanup: kind + $(KIND) delete cluster --name kamaji + .PHONY: e2e -e2e: env build load helm ginkgo cert-manager ## Create a KinD cluster, install Kamaji on it and run the test suite. +e2e: env build load helm ginkgo cert-manager gateway-api ## Create a KinD cluster, install Kamaji on it and run the test suite. $(HELM) upgrade --debug --install kamaji-crds ./charts/kamaji-crds --create-namespace --namespace kamaji-system $(HELM) repo add clastix https://clastix.github.io/charts $(HELM) dependency build ./charts/kamaji diff --git a/api/v1alpha1/groupversion_info.go b/api/v1alpha1/groupversion_info.go index 5fcb860..e13ab05 100644 --- a/api/v1alpha1/groupversion_info.go +++ b/api/v1alpha1/groupversion_info.go @@ -4,7 +4,6 @@ // Package v1alpha1 contains API Schema definitions for the kamaji v1alpha1 API group // +kubebuilder:object:generate=true // +groupName=kamaji.clastix.io -//nolint package v1alpha1 import ( diff --git a/api/v1alpha1/indexer_gateway_listener.go b/api/v1alpha1/indexer_gateway_listener.go new file mode 100644 index 0000000..b9c607d --- /dev/null +++ b/api/v1alpha1/indexer_gateway_listener.go @@ -0,0 +1,47 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package v1alpha1 + +import ( + "context" + "fmt" + + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +const ( + GatewayListenerNameKey = "spec.listeners.name" +) + +type GatewayListener struct{} + +func (g *GatewayListener) Object() client.Object { + return &gatewayv1.Gateway{} +} + +func (g *GatewayListener) Field() string { + return GatewayListenerNameKey +} + +func (g *GatewayListener) ExtractValue() client.IndexerFunc { + return func(object client.Object) []string { + gateway := object.(*gatewayv1.Gateway) //nolint:forcetypeassert + + listenerNames := make([]string, 0, len(gateway.Spec.Listeners)) + for _, listener := range gateway.Spec.Listeners { + // Create a composite key: namespace/gatewayName/listenerName + // This allows us to look up gateways by listener name while ensuring uniqueness + key := fmt.Sprintf("%s/%s/%s", gateway.Namespace, gateway.Name, listener.Name) + listenerNames = append(listenerNames, key) + } + + return listenerNames + } +} + +func (g *GatewayListener) SetupWithManager(ctx context.Context, mgr controllerruntime.Manager) error { + return mgr.GetFieldIndexer().IndexField(ctx, g.Object(), g.Field(), g.ExtractValue()) +} diff --git a/api/v1alpha1/tenantcontrolplane_status.go b/api/v1alpha1/tenantcontrolplane_status.go index 12ea8ec..b5c5df7 100644 --- a/api/v1alpha1/tenantcontrolplane_status.go +++ b/api/v1alpha1/tenantcontrolplane_status.go @@ -8,6 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" ) // APIServerCertificatesStatus defines the observed state of ETCD Certificate for API server. @@ -187,6 +188,7 @@ type KubernetesStatus struct { Deployment KubernetesDeploymentStatus `json:"deployment,omitempty"` Service KubernetesServiceStatus `json:"service,omitempty"` Ingress *KubernetesIngressStatus `json:"ingress,omitempty"` + Gateway *KubernetesGatewayStatus `json:"gateway,omitempty"` } // +kubebuilder:validation:Enum=Unknown;Provisioning;CertificateAuthorityRotating;Upgrading;Migrating;Ready;NotReady;Sleeping;WriteLimited @@ -244,3 +246,25 @@ type KubernetesIngressStatus struct { // The namespace which the Ingress for the given cluster is deployed. Namespace string `json:"namespace"` } + +type GatewayAccessPoint struct { + Type *gatewayv1.AddressType `json:"type"` + Value string `json:"value"` + Port int32 `json:"port"` + URLs []string `json:"urls,omitempty"` +} + +// +k8s:deepcopy-gen=false +type RouteStatus = gatewayv1.RouteStatus + +// KubernetesGatewayStatus defines the status for the Tenant Control Plane Gateway in the management cluster. +type KubernetesGatewayStatus struct { + // The TLSRoute status as resported by the gateway controllers. + RouteStatus `json:",inline"` + + // Reference to the route created for this tenant. + RouteRef corev1.LocalObjectReference `json:"routeRef,omitempty"` + + // A list of valid access points that the route exposes. + AccessPoints []GatewayAccessPoint `json:"accessPoints,omitempty"` +} diff --git a/api/v1alpha1/tenantcontrolplane_types.go b/api/v1alpha1/tenantcontrolplane_types.go index 666f185..0a8037d 100644 --- a/api/v1alpha1/tenantcontrolplane_types.go +++ b/api/v1alpha1/tenantcontrolplane_types.go @@ -8,6 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" ) // NetworkProfileSpec defines the desired state of NetworkProfile. @@ -124,6 +125,7 @@ type AdditionalMetadata struct { // ControlPlane defines how the Tenant Control Plane Kubernetes resources must be created in the Admin Cluster, // such as the number of Pod replicas, the Service resource, or the Ingress. +// +kubebuilder:validation:XValidation:rule="!(has(self.ingress) && has(self.gateway))",message="using both ingress and gateway is not supported" type ControlPlane struct { // Defining the options for the deployed Tenant Control Plane as Deployment resource. Deployment DeploymentSpec `json:"deployment,omitempty"` @@ -131,6 +133,8 @@ type ControlPlane struct { Service ServiceSpec `json:"service"` // Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane Ingress *IngressSpec `json:"ingress,omitempty"` + // Defining the options for an Optional Gateway which will expose API Server of the Tenant Control Plane + Gateway *GatewaySpec `json:"gateway,omitempty"` } // IngressSpec defines the options for the ingress which will expose API Server of the Tenant Control Plane. @@ -142,6 +146,16 @@ type IngressSpec struct { Hostname string `json:"hostname,omitempty"` } +// GatewaySpec defines the options for the Gateway which will expose API Server of the Tenant Control Plane. +type GatewaySpec struct { + // AdditionalMetadata to add Labels and Annotations support. + AdditionalMetadata AdditionalMetadata `json:"additionalMetadata,omitempty"` + // GatewayParentRefs is the class of the Gateway resource to use. + GatewayParentRefs []gatewayv1.ParentReference `json:"parentRefs,omitempty"` + // Hostname is an optional field which will be used as a route hostname. + Hostname gatewayv1.Hostname `json:"hostname,omitempty"` +} + type ControlPlaneComponentsResources struct { APIServer *corev1.ResourceRequirements `json:"apiServer,omitempty"` ControllerManager *corev1.ResourceRequirements `json:"controllerManager,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index cb32d93..478b149 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -8,8 +8,9 @@ package v1alpha1 import ( - "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/gateway-api/apis/v1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -83,21 +84,21 @@ func (in *AdditionalVolumeMounts) DeepCopyInto(out *AdditionalVolumeMounts) { *out = *in if in.APIServer != nil { in, out := &in.APIServer, &out.APIServer - *out = make([]v1.VolumeMount, len(*in)) + *out = make([]corev1.VolumeMount, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } if in.ControllerManager != nil { in, out := &in.ControllerManager, &out.ControllerManager - *out = make([]v1.VolumeMount, len(*in)) + *out = make([]corev1.VolumeMount, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } if in.Scheduler != nil { in, out := &in.Scheduler, &out.Scheduler - *out = make([]v1.VolumeMount, len(*in)) + *out = make([]corev1.VolumeMount, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -360,6 +361,11 @@ func (in *ControlPlane) DeepCopyInto(out *ControlPlane) { *out = new(IngressSpec) (*in).DeepCopyInto(*out) } + if in.Gateway != nil { + in, out := &in.Gateway, &out.Gateway + *out = new(GatewaySpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControlPlane. @@ -377,22 +383,22 @@ func (in *ControlPlaneComponentsResources) DeepCopyInto(out *ControlPlaneCompone *out = *in if in.APIServer != nil { in, out := &in.APIServer, &out.APIServer - *out = new(v1.ResourceRequirements) + *out = new(corev1.ResourceRequirements) (*in).DeepCopyInto(*out) } if in.ControllerManager != nil { in, out := &in.ControllerManager, &out.ControllerManager - *out = new(v1.ResourceRequirements) + *out = new(corev1.ResourceRequirements) (*in).DeepCopyInto(*out) } if in.Scheduler != nil { in, out := &in.Scheduler, &out.Scheduler - *out = new(v1.ResourceRequirements) + *out = new(corev1.ResourceRequirements) (*in).DeepCopyInto(*out) } if in.Kine != nil { in, out := &in.Kine, &out.Kine - *out = new(v1.ResourceRequirements) + *out = new(corev1.ResourceRequirements) (*in).DeepCopyInto(*out) } } @@ -632,19 +638,19 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { in.Strategy.DeepCopyInto(&out.Strategy) if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations - *out = make([]v1.Toleration, len(*in)) + *out = make([]corev1.Toleration, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } if in.Affinity != nil { in, out := &in.Affinity, &out.Affinity - *out = new(v1.Affinity) + *out = new(corev1.Affinity) (*in).DeepCopyInto(*out) } if in.TopologySpreadConstraints != nil { in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints - *out = make([]v1.TopologySpreadConstraint, len(*in)) + *out = make([]corev1.TopologySpreadConstraint, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -663,21 +669,21 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { in.PodAdditionalMetadata.DeepCopyInto(&out.PodAdditionalMetadata) if in.AdditionalInitContainers != nil { in, out := &in.AdditionalInitContainers, &out.AdditionalInitContainers - *out = make([]v1.Container, len(*in)) + *out = make([]corev1.Container, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } if in.AdditionalContainers != nil { in, out := &in.AdditionalContainers, &out.AdditionalContainers - *out = make([]v1.Container, len(*in)) + *out = make([]corev1.Container, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } if in.AdditionalVolumes != nil { in, out := &in.AdditionalVolumes, &out.AdditionalVolumes - *out = make([]v1.Volume, len(*in)) + *out = make([]corev1.Volume, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -786,6 +792,69 @@ func (in ExtraArgs) DeepCopy() ExtraArgs { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewayAccessPoint) DeepCopyInto(out *GatewayAccessPoint) { + *out = *in + if in.Type != nil { + in, out := &in.Type, &out.Type + *out = new(v1.AddressType) + **out = **in + } + if in.URLs != nil { + in, out := &in.URLs, &out.URLs + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayAccessPoint. +func (in *GatewayAccessPoint) DeepCopy() *GatewayAccessPoint { + if in == nil { + return nil + } + out := new(GatewayAccessPoint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewayListener) DeepCopyInto(out *GatewayListener) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayListener. +func (in *GatewayListener) DeepCopy() *GatewayListener { + if in == nil { + return nil + } + out := new(GatewayListener) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GatewaySpec) DeepCopyInto(out *GatewaySpec) { + *out = *in + in.AdditionalMetadata.DeepCopyInto(&out.AdditionalMetadata) + if in.GatewayParentRefs != nil { + in, out := &in.GatewayParentRefs, &out.GatewayParentRefs + *out = make([]v1.ParentReference, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewaySpec. +func (in *GatewaySpec) DeepCopy() *GatewaySpec { + if in == nil { + return nil + } + out := new(GatewaySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ImageOverrideTrait) DeepCopyInto(out *ImageOverrideTrait) { *out = *in @@ -822,7 +891,7 @@ func (in *KonnectivityAgentSpec) DeepCopyInto(out *KonnectivityAgentSpec) { *out = *in if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations - *out = make([]v1.Toleration, len(*in)) + *out = make([]corev1.Toleration, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -880,7 +949,7 @@ func (in *KonnectivityServerSpec) DeepCopyInto(out *KonnectivityServerSpec) { *out = *in if in.Resources != nil { in, out := &in.Resources, &out.Resources - *out = new(v1.ResourceRequirements) + *out = new(corev1.ResourceRequirements) (*in).DeepCopyInto(*out) } if in.ExtraArgs != nil { @@ -1175,6 +1244,30 @@ func (in *KubernetesDeploymentStatus) DeepCopy() *KubernetesDeploymentStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubernetesGatewayStatus) DeepCopyInto(out *KubernetesGatewayStatus) { + *out = *in + in.RouteStatus.DeepCopyInto(&out.RouteStatus) + out.RouteRef = in.RouteRef + if in.AccessPoints != nil { + in, out := &in.AccessPoints, &out.AccessPoints + *out = make([]GatewayAccessPoint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesGatewayStatus. +func (in *KubernetesGatewayStatus) DeepCopy() *KubernetesGatewayStatus { + if in == nil { + return nil + } + out := new(KubernetesGatewayStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubernetesIngressStatus) DeepCopyInto(out *KubernetesIngressStatus) { *out = *in @@ -1239,6 +1332,11 @@ func (in *KubernetesStatus) DeepCopyInto(out *KubernetesStatus) { *out = new(KubernetesIngressStatus) (*in).DeepCopyInto(*out) } + if in.Gateway != nil { + in, out := &in.Gateway, &out.Gateway + *out = new(KubernetesGatewayStatus) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesStatus. diff --git a/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml b/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml index aec5250..9c3508b 100644 --- a/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml +++ b/charts/kamaji-crds/hack/kamaji.clastix.io_tenantcontrolplanes_spec.yaml @@ -6700,6 +6700,178 @@ versions: type: object type: array type: object + gateway: + description: Defining the options for an Optional Gateway which will expose API Server of the Tenant Control Plane + properties: + additionalMetadata: + description: AdditionalMetadata to add Labels and Annotations support. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + hostname: + description: Hostname is an optional field which will be used as a route hostname. + maxLength: 253 + minLength: 1 + pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + parentRefs: + description: GatewayParentRefs is the class of the Gateway resource to use. + items: + description: |- + ParentReference identifies an API object (usually a Gateway) that can be considered + a parent of this resource (usually a route). There are two kinds of parent resources + with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + This API may be extended in the future to support additional kinds of parent + resources. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + type: array + type: object ingress: description: Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane properties: @@ -6801,6 +6973,9 @@ versions: required: - service type: object + x-kubernetes-validations: + - message: using both ingress and gateway is not supported + rule: '!(has(self.ingress) && has(self.gateway))' dataStore: description: |- DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Tenant Control Plane. @@ -7551,6 +7726,383 @@ versions: - namespace - selector type: object + gateway: + description: KubernetesGatewayStatus defines the status for the Tenant Control Plane Gateway in the management cluster. + properties: + accessPoints: + description: A list of valid access points that the route exposes. + items: + properties: + port: + format: int32 + type: integer + type: + description: |- + AddressType defines how a network address is represented as a text string. + This may take two possible forms: + + * A predefined CamelCase string identifier (currently limited to `IPAddress` or `Hostname`) + * A domain-prefixed string identifier (like `acme.io/CustomAddressType`) + + Values `IPAddress` and `Hostname` have Extended support. + + The `NamedAddress` value has been deprecated in favor of implementation + specific domain-prefixed strings. + + All other values, including domain-prefixed values have Implementation-specific support, + which are used in implementation-specific behaviors. Support for additional + predefined CamelCase identifiers may be added in future releases. + maxLength: 253 + minLength: 1 + pattern: ^Hostname|IPAddress|NamedAddress|[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + urls: + items: + type: string + type: array + value: + type: string + required: + - port + - type + - value + type: object + type: array + parents: + description: |- + Parents is a list of parent resources (usually Gateways) that are + associated with the route, and the status of the route with respect to + each parent. When this route attaches to a parent, the controller that + manages the parent must add an entry to this list when the controller + first sees the route and should update the entry as appropriate when the + route or gateway is modified. + + Note that parent references that cannot be resolved by an implementation + of this API will not be added to this list. Implementations of this API + can only populate Route status for the Gateways/parent resources they are + responsible for. + + A maximum of 32 Gateways will be represented in this list. An empty list + means the route has not been attached to any Gateway. + + + Notes for implementors: + + While parents is not a listType `map`, this is due to the fact that the + list key is not scalar, and Kubernetes is unable to represent this. + + Parent status MUST be considered to be namespaced by the combination of + the parentRef and controllerName fields, and implementations should keep + the following rules in mind when updating this status: + + * Implementations MUST update only entries that have a matching value of + `controllerName` for that implementation. + * Implementations MUST NOT update entries with non-matching `controllerName` + fields. + * Implementations MUST treat each `parentRef`` in the Route separately and + update its status based on the relationship with that parent. + * Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. + + + items: + description: |- + RouteParentStatus describes the status of a route with respect to an + associated Parent. + properties: + conditions: + description: |- + Conditions describes the status of the route with respect to the Gateway. + Note that the route's availability is also subject to the Gateway's own + status conditions and listener status. + + If the Route's ParentRef specifies an existing Gateway that supports + Routes of this kind AND that Gateway's controller has sufficient access, + then that Gateway's controller MUST set the "Accepted" condition on the + Route, to indicate whether the route has been accepted or rejected by the + Gateway, and why. + + A Route MUST be considered "Accepted" if at least one of the Route's + rules is implemented by the Gateway. + + There are a number of cases where the "Accepted" condition may not be set + due to lack of controller visibility, that includes when: + + * The Route refers to a nonexistent parent. + * The Route is of a type that the controller does not support. + * The Route is in a namespace the controller does not have access to. + + + + Notes for implementors: + + Conditions are a listType `map`, which means that they function like a + map with a key of the `type` field _in the k8s apiserver_. + + This means that implementations must obey some rules when updating this + section. + + * Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. + * Implementations MUST NOT remove or reorder Conditions that they are not + directly responsible for. For example, if an implementation sees a Condition + with type `special.io/SomeField`, it MUST NOT remove, change or update that + Condition. + * Implementations MUST always _merge_ changes into Conditions of the same Type, + rather than creating more than one Condition of the same Type. + * Implementations MUST always update the `observedGeneration` field of the + Condition to the `metadata.generation` of the Gateway at the time of update creation. + * If the `observedGeneration` of a Condition is _greater than_ the value the + implementation knows about, then it MUST NOT perform the update on that Condition, + but must wait for a future reconciliation and status update. (The assumption is that + the implementation's copy of the object is stale and an update will be re-triggered + if relevant.) + + + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + controllerName: + description: |- + ControllerName is a domain/path string that indicates the name of the + controller that wrote this status. This corresponds with the + controllerName field on GatewayClass. + + Example: "example.net/gateway-controller". + + The format of this field is DOMAIN "/" PATH, where DOMAIN and PATH are + valid Kubernetes names + (https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names). + + Controllers MUST populate this field when writing status. Controllers should ensure that + entries to status populated with their ControllerName are cleaned up when they are no + longer necessary. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + parentRef: + description: |- + ParentRef corresponds with a ParentRef in the spec that this + RouteParentStatus struct describes the status of. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + required: + - conditions + - controllerName + - parentRef + type: object + maxItems: 32 + type: array + x-kubernetes-list-type: atomic + routeRef: + description: Reference to the route created for this tenant. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - parents + type: object ingress: description: KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster. properties: diff --git a/charts/kamaji/controller-gen/clusterrole.yaml b/charts/kamaji/controller-gen/clusterrole.yaml index f68894e..23c75f3 100644 --- a/charts/kamaji/controller-gen/clusterrole.yaml +++ b/charts/kamaji/controller-gen/clusterrole.yaml @@ -42,6 +42,28 @@ - patch - update - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gateways + verbs: + - get + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - grpcroutes + - httproutes + - tlsroutes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - kamaji.clastix.io resources: diff --git a/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml b/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml index 647e4c2..18c1e86 100644 --- a/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml +++ b/charts/kamaji/crds/kamaji.clastix.io_tenantcontrolplanes.yaml @@ -6708,6 +6708,178 @@ spec: type: object type: array type: object + gateway: + description: Defining the options for an Optional Gateway which will expose API Server of the Tenant Control Plane + properties: + additionalMetadata: + description: AdditionalMetadata to add Labels and Annotations support. + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + hostname: + description: Hostname is an optional field which will be used as a route hostname. + maxLength: 253 + minLength: 1 + pattern: ^(\*\.)?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + parentRefs: + description: GatewayParentRefs is the class of the Gateway resource to use. + items: + description: |- + ParentReference identifies an API object (usually a Gateway) that can be considered + a parent of this resource (usually a route). There are two kinds of parent resources + with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + This API may be extended in the future to support additional kinds of parent + resources. + + The API object must be valid in the cluster; the Group and Kind must + be registered in the cluster for this reference to be valid. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + type: array + type: object ingress: description: Defining the options for an Optional Ingress which will expose API Server of the Tenant Control Plane properties: @@ -6809,6 +6981,9 @@ spec: required: - service type: object + x-kubernetes-validations: + - message: using both ingress and gateway is not supported + rule: '!(has(self.ingress) && has(self.gateway))' dataStore: description: |- DataStore specifies the DataStore that should be used to store the Kubernetes data for the given Tenant Control Plane. @@ -7559,6 +7734,383 @@ spec: - namespace - selector type: object + gateway: + description: KubernetesGatewayStatus defines the status for the Tenant Control Plane Gateway in the management cluster. + properties: + accessPoints: + description: A list of valid access points that the route exposes. + items: + properties: + port: + format: int32 + type: integer + type: + description: |- + AddressType defines how a network address is represented as a text string. + This may take two possible forms: + + * A predefined CamelCase string identifier (currently limited to `IPAddress` or `Hostname`) + * A domain-prefixed string identifier (like `acme.io/CustomAddressType`) + + Values `IPAddress` and `Hostname` have Extended support. + + The `NamedAddress` value has been deprecated in favor of implementation + specific domain-prefixed strings. + + All other values, including domain-prefixed values have Implementation-specific support, + which are used in implementation-specific behaviors. Support for additional + predefined CamelCase identifiers may be added in future releases. + maxLength: 253 + minLength: 1 + pattern: ^Hostname|IPAddress|NamedAddress|[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + urls: + items: + type: string + type: array + value: + type: string + required: + - port + - type + - value + type: object + type: array + parents: + description: |- + Parents is a list of parent resources (usually Gateways) that are + associated with the route, and the status of the route with respect to + each parent. When this route attaches to a parent, the controller that + manages the parent must add an entry to this list when the controller + first sees the route and should update the entry as appropriate when the + route or gateway is modified. + + Note that parent references that cannot be resolved by an implementation + of this API will not be added to this list. Implementations of this API + can only populate Route status for the Gateways/parent resources they are + responsible for. + + A maximum of 32 Gateways will be represented in this list. An empty list + means the route has not been attached to any Gateway. + + + Notes for implementors: + + While parents is not a listType `map`, this is due to the fact that the + list key is not scalar, and Kubernetes is unable to represent this. + + Parent status MUST be considered to be namespaced by the combination of + the parentRef and controllerName fields, and implementations should keep + the following rules in mind when updating this status: + + * Implementations MUST update only entries that have a matching value of + `controllerName` for that implementation. + * Implementations MUST NOT update entries with non-matching `controllerName` + fields. + * Implementations MUST treat each `parentRef`` in the Route separately and + update its status based on the relationship with that parent. + * Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. + + + items: + description: |- + RouteParentStatus describes the status of a route with respect to an + associated Parent. + properties: + conditions: + description: |- + Conditions describes the status of the route with respect to the Gateway. + Note that the route's availability is also subject to the Gateway's own + status conditions and listener status. + + If the Route's ParentRef specifies an existing Gateway that supports + Routes of this kind AND that Gateway's controller has sufficient access, + then that Gateway's controller MUST set the "Accepted" condition on the + Route, to indicate whether the route has been accepted or rejected by the + Gateway, and why. + + A Route MUST be considered "Accepted" if at least one of the Route's + rules is implemented by the Gateway. + + There are a number of cases where the "Accepted" condition may not be set + due to lack of controller visibility, that includes when: + + * The Route refers to a nonexistent parent. + * The Route is of a type that the controller does not support. + * The Route is in a namespace the controller does not have access to. + + + + Notes for implementors: + + Conditions are a listType `map`, which means that they function like a + map with a key of the `type` field _in the k8s apiserver_. + + This means that implementations must obey some rules when updating this + section. + + * Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. + * Implementations MUST NOT remove or reorder Conditions that they are not + directly responsible for. For example, if an implementation sees a Condition + with type `special.io/SomeField`, it MUST NOT remove, change or update that + Condition. + * Implementations MUST always _merge_ changes into Conditions of the same Type, + rather than creating more than one Condition of the same Type. + * Implementations MUST always update the `observedGeneration` field of the + Condition to the `metadata.generation` of the Gateway at the time of update creation. + * If the `observedGeneration` of a Condition is _greater than_ the value the + implementation knows about, then it MUST NOT perform the update on that Condition, + but must wait for a future reconciliation and status update. (The assumption is that + the implementation's copy of the object is stale and an update will be re-triggered + if relevant.) + + + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + maxItems: 8 + minItems: 1 + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + controllerName: + description: |- + ControllerName is a domain/path string that indicates the name of the + controller that wrote this status. This corresponds with the + controllerName field on GatewayClass. + + Example: "example.net/gateway-controller". + + The format of this field is DOMAIN "/" PATH, where DOMAIN and PATH are + valid Kubernetes names + (https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names). + + Controllers MUST populate this field when writing status. Controllers should ensure that + entries to status populated with their ControllerName are cleaned up when they are no + longer necessary. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[A-Za-z0-9\/\-._~%!$&'()*+,;=:]+$ + type: string + parentRef: + description: |- + ParentRef corresponds with a ParentRef in the spec that this + RouteParentStatus struct describes the status of. + properties: + group: + default: gateway.networking.k8s.io + description: |- + Group is the group of the referent. + When unspecified, "gateway.networking.k8s.io" is inferred. + To set the core API group (such as for a "Service" kind referent), + Group must be explicitly set to "" (empty string). + + Support: Core + maxLength: 253 + pattern: ^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + kind: + default: Gateway + description: |- + Kind is kind of the referent. + + There are two kinds of parent resources with "Core" support: + + * Gateway (Gateway conformance profile) + * Service (Mesh conformance profile, ClusterIP Services only) + + Support for other resources is Implementation-Specific. + maxLength: 63 + minLength: 1 + pattern: ^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$ + type: string + name: + description: |- + Name is the name of the referent. + + Support: Core + maxLength: 253 + minLength: 1 + type: string + namespace: + description: |- + Namespace is the namespace of the referent. When unspecified, this refers + to the local namespace of the Route. + + Note that there are specific rules for ParentRefs which cross namespace + boundaries. Cross-namespace references are only valid if they are explicitly + allowed by something in the namespace they are referring to. For example: + Gateway has the AllowedRoutes field, and ReferenceGrant provides a + generic way to enable any other kind of cross-namespace reference. + + + ParentRefs from a Route to a Service in the same namespace are "producer" + routes, which apply default routing rules to inbound connections from + any namespace to the Service. + + ParentRefs from a Route to a Service in a different namespace are + "consumer" routes, and these routing rules are only applied to outbound + connections originating from the same namespace as the Route, for which + the intended destination of the connections are a Service targeted as a + ParentRef of the Route. + + + Support: Core + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + port: + description: |- + Port is the network port this Route targets. It can be interpreted + differently based on the type of parent resource. + + When the parent resource is a Gateway, this targets all listeners + listening on the specified port that also support this kind of Route(and + select this Route). It's not recommended to set `Port` unless the + networking behaviors specified in a Route must apply to a specific port + as opposed to a listener(s) whose port(s) may be changed. When both Port + and SectionName are specified, the name and port of the selected listener + must match both specified values. + + + When the parent resource is a Service, this targets a specific port in the + Service spec. When both Port (experimental) and SectionName are specified, + the name and port of the selected port must match both specified values. + + + Implementations MAY choose to support other parent resources. + Implementations supporting other types of parent resources MUST clearly + document how/if Port is interpreted. + + For the purpose of status, an attachment is considered successful as + long as the parent resource accepts it partially. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment + from the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, + the Route MUST be considered detached from the Gateway. + + Support: Extended + format: int32 + maximum: 65535 + minimum: 1 + type: integer + sectionName: + description: |- + SectionName is the name of a section within the target resource. In the + following resources, SectionName is interpreted as the following: + + * Gateway: Listener name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + * Service: Port name. When both Port (experimental) and SectionName + are specified, the name and port of the selected listener must match + both specified values. + + Implementations MAY choose to support attaching Routes to other resources. + If that is the case, they MUST clearly document how SectionName is + interpreted. + + When unspecified (empty string), this will reference the entire resource. + For the purpose of status, an attachment is considered successful if at + least one section in the parent resource accepts it. For example, Gateway + listeners can restrict which Routes can attach to them by Route kind, + namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from + the referencing Route, the Route MUST be considered successfully + attached. If no Gateway listeners accept attachment from this Route, the + Route MUST be considered detached from the Gateway. + + Support: Core + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - name + type: object + required: + - conditions + - controllerName + - parentRef + type: object + maxItems: 32 + type: array + x-kubernetes-list-type: atomic + routeRef: + description: Reference to the route created for this tenant. + properties: + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + type: object + x-kubernetes-map-type: atomic + required: + - parents + type: object ingress: description: KubernetesIngressStatus defines the status for the Tenant Control Plane Ingress in the management cluster. properties: diff --git a/cmd/manager/cmd.go b/cmd/manager/cmd.go index c004a7e..0b77322 100644 --- a/cmd/manager/cmd.go +++ b/cmd/manager/cmd.go @@ -17,6 +17,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/discovery" "k8s.io/client-go/rest" "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" @@ -34,6 +35,7 @@ import ( "github.com/clastix/kamaji/internal" "github.com/clastix/kamaji/internal/builders/controlplane" datastoreutils "github.com/clastix/kamaji/internal/datastore/utils" + "github.com/clastix/kamaji/internal/utilities" "github.com/clastix/kamaji/internal/webhook" "github.com/clastix/kamaji/internal/webhook/handlers" "github.com/clastix/kamaji/internal/webhook/routes" @@ -146,6 +148,13 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { return err } + discoveryClient, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig()) + if err != nil { + setupLog.Error(err, "unable to create discovery client") + + return err + } + reconciler := &controllers.TenantControlPlaneReconciler{ Client: mgr.GetClient(), APIReader: mgr.GetAPIReader(), @@ -163,9 +172,10 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { KamajiService: managerServiceName, KamajiMigrateImage: migrateJobImage, MaxConcurrentReconciles: maxConcurrentReconciles, + DiscoveryClient: discoveryClient, } - if err = reconciler.SetupWithManager(mgr); err != nil { + if err = reconciler.SetupWithManager(ctx, mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Namespace") return err @@ -215,6 +225,15 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { return err } + // Only requires to look for the core api group. + if utilities.AreGatewayResourcesAvailable(ctx, mgr.GetClient(), discoveryClient) { + if err = (&kamajiv1alpha1.GatewayListener{}).SetupWithManager(ctx, mgr); err != nil { + setupLog.Error(err, "unable to create indexer", "indexer", "GatewayListener") + + return err + } + } + err = webhook.Register(mgr, map[routes.Route][]handlers.Handler{ routes.TenantControlPlaneMigrate{}: { handlers.Freeze{}, @@ -244,6 +263,10 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { }, handlers.TenantControlPlaneServiceCIDR{}, handlers.TenantControlPlaneLoadBalancerSourceRanges{}, + handlers.TenantControlPlaneGatewayValidation{ + Client: mgr.GetClient(), + DiscoveryClient: discoveryClient, + }, }, routes.TenantControlPlaneTelemetry{}: { handlers.TenantControlPlaneTelemetry{ diff --git a/cmd/root.go b/cmd/root.go index e24e1ee..3e28e1d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -10,6 +10,8 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" appsv1 "k8s.io/kubernetes/pkg/apis/apps/v1" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" ) @@ -22,6 +24,10 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(kamajiv1alpha1.AddToScheme(scheme)) utilruntime.Must(appsv1.RegisterDefaults(scheme)) + // NOTE: This will succeed even if Gateway API is not installed in the cluster. + // Only registers the go types. + utilruntime.Must(gatewayv1.Install(scheme)) + utilruntime.Must(gatewayv1alpha2.Install(scheme)) }, } } diff --git a/controllers/resources.go b/controllers/resources.go index f2f1346..31385f9 100644 --- a/controllers/resources.go +++ b/controllers/resources.go @@ -4,12 +4,14 @@ package controllers import ( + "context" "fmt" "time" "github.com/go-logr/logr" "github.com/google/uuid" k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/discovery" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -20,6 +22,7 @@ import ( "github.com/clastix/kamaji/internal/resources" ds "github.com/clastix/kamaji/internal/resources/datastore" "github.com/clastix/kamaji/internal/resources/konnectivity" + "github.com/clastix/kamaji/internal/utilities" ) type GroupResourceBuilderConfiguration struct { @@ -34,6 +37,7 @@ type GroupResourceBuilderConfiguration struct { KamajiServiceAccount string KamajiService string KamajiMigrateImage string + DiscoveryClient discovery.DiscoveryInterface } type GroupDeletableResourceBuilderConfiguration struct { @@ -48,8 +52,28 @@ type GroupDeletableResourceBuilderConfiguration struct { // GetResources returns a list of resources that will be used to provide tenant control planes // Currently there is only a default approach // TODO: the idea of this function is to become a factory to return the group of resources according to the given configuration. -func GetResources(config GroupResourceBuilderConfiguration) []resources.Resource { - return getDefaultResources(config) +func GetResources(ctx context.Context, config GroupResourceBuilderConfiguration) []resources.Resource { + resources := []resources.Resource{} + + resources = append(resources, getDataStoreMigratingResources(config.client, config.KamajiNamespace, config.KamajiMigrateImage, config.KamajiServiceAccount, config.KamajiService)...) + resources = append(resources, getUpgradeResources(config.client)...) + resources = append(resources, getKubernetesServiceResources(config.client)...) + resources = append(resources, getKubeadmConfigResources(config.client, getTmpDirectory(config.tcpReconcilerConfig.TmpBaseDirectory, config.tenantControlPlane), config.DataStore)...) + resources = append(resources, getKubernetesCertificatesResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...) + resources = append(resources, getKubeconfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...) + resources = append(resources, getKubernetesStorageResources(config.client, config.Connection, config.DataStore, config.ExpirationThreshold)...) + resources = append(resources, getKonnectivityServerRequirementsResources(config.client, config.ExpirationThreshold)...) + resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.DataStore)...) + resources = append(resources, getKonnectivityServerPatchResources(config.client)...) + resources = append(resources, getDataStoreMigratingCleanup(config.client, config.KamajiNamespace)...) + resources = append(resources, getKubernetesIngressResources(config.client)...) + + // Conditionally add Gateway resources + if utilities.AreGatewayResourcesAvailable(ctx, config.client, config.DiscoveryClient) { + resources = append(resources, getKubernetesGatewayResources(config.client)...) + } + + return resources } // GetDeletableResources returns a list of resources that have to be deleted when tenant control planes are deleted @@ -73,23 +97,6 @@ func GetDeletableResources(tcp *kamajiv1alpha1.TenantControlPlane, config GroupD return res } -func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.Resource { - resources := getDataStoreMigratingResources(config.client, config.KamajiNamespace, config.KamajiMigrateImage, config.KamajiServiceAccount, config.KamajiService) - resources = append(resources, getUpgradeResources(config.client)...) - resources = append(resources, getKubernetesServiceResources(config.client)...) - resources = append(resources, getKubeadmConfigResources(config.client, getTmpDirectory(config.tcpReconcilerConfig.TmpBaseDirectory, config.tenantControlPlane), config.DataStore)...) - resources = append(resources, getKubernetesCertificatesResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...) - resources = append(resources, getKubeconfigResources(config.client, config.tcpReconcilerConfig, config.tenantControlPlane)...) - resources = append(resources, getKubernetesStorageResources(config.client, config.Connection, config.DataStore, config.ExpirationThreshold)...) - resources = append(resources, getKonnectivityServerRequirementsResources(config.client, config.ExpirationThreshold)...) - resources = append(resources, getKubernetesDeploymentResources(config.client, config.tcpReconcilerConfig, config.DataStore)...) - resources = append(resources, getKonnectivityServerPatchResources(config.client)...) - resources = append(resources, getDataStoreMigratingCleanup(config.client, config.KamajiNamespace)...) - resources = append(resources, getKubernetesIngressResources(config.client)...) - - return resources -} - func getDataStoreMigratingCleanup(c client.Client, kamajiNamespace string) []resources.Resource { return []resources.Resource{ &ds.Migrate{ @@ -128,6 +135,14 @@ func getKubernetesServiceResources(c client.Client) []resources.Resource { } } +func getKubernetesGatewayResources(c client.Client) []resources.Resource { + return []resources.Resource{ + &resources.KubernetesGatewayResource{ + Client: c, + }, + } +} + func getKubeadmConfigResources(c client.Client, tmpDirectory string, dataStore kamajiv1alpha1.DataStore) []resources.Resource { var endpoints []string diff --git a/controllers/tenantcontrolplane_controller.go b/controllers/tenantcontrolplane_controller.go index 6baca1d..9e830d9 100644 --- a/controllers/tenantcontrolplane_controller.go +++ b/controllers/tenantcontrolplane_controller.go @@ -17,6 +17,7 @@ import ( networkingv1 "k8s.io/api/networking/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/discovery" "k8s.io/client-go/util/workqueue" "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" @@ -30,6 +31,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/controllers/finalizers" @@ -37,6 +40,7 @@ import ( "github.com/clastix/kamaji/internal/datastore" kamajierrors "github.com/clastix/kamaji/internal/errors" "github.com/clastix/kamaji/internal/resources" + "github.com/clastix/kamaji/internal/utilities" ) // TenantControlPlaneReconciler reconciles a TenantControlPlane object. @@ -51,6 +55,7 @@ type TenantControlPlaneReconciler struct { KamajiMigrateImage string MaxConcurrentReconciles int ReconcileTimeout time.Duration + DiscoveryClient discovery.DiscoveryInterface // CertificateChan is the channel used by the CertificateLifecycleController that is checking for // certificates and kubeconfig user certs validity: a generic event for the given TCP will be triggered // once the validity threshold for the given certificate is reached. @@ -76,6 +81,10 @@ type TenantControlPlaneReconcilerConfig struct { //+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;delete +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=httproutes,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=grpcroutes,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=tlsroutes,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gateways,verbs=get;list;watch func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := log.FromContext(ctx) @@ -184,8 +193,9 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R KamajiServiceAccount: r.KamajiServiceAccount, KamajiService: r.KamajiService, KamajiMigrateImage: r.KamajiMigrateImage, + DiscoveryClient: r.DiscoveryClient, } - registeredResources := GetResources(groupResourceBuilderConfiguration) + registeredResources := GetResources(ctx, groupResourceBuilderConfiguration) for _, resource := range registeredResources { result, err := resources.Handle(ctx, resource, tenantControlPlane) @@ -242,10 +252,10 @@ func (r *TenantControlPlaneReconciler) mutexSpec(obj client.Object) mutex.Spec { } // SetupWithManager sets up the controller with the Manager. -func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error { +func (r *TenantControlPlaneReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { r.clock = clock.RealClock{} - return ctrl.NewControllerManagedBy(mgr). + controllerBuilder := ctrl.NewControllerManagedBy(mgr). WatchesRawSource(source.Channel(r.CertificateChan, handler.Funcs{GenericFunc: func(_ context.Context, genericEvent event.TypedGenericEvent[client.Object], w workqueue.TypedRateLimitingInterface[reconcile.Request]) { w.AddRateLimited(ctrl.Request{ NamespacedName: k8stypes.NamespacedName{ @@ -295,7 +305,20 @@ func (r *TenantControlPlaneReconciler) SetupWithManager(mgr ctrl.Manager) error v, ok := labels["kamaji.clastix.io/component"] return ok && v == "migrate" - }))). + }))) + + // Conditionally add Gateway API ownership if available + if utilities.AreGatewayResourcesAvailable(ctx, r.Client, r.DiscoveryClient) { + controllerBuilder = controllerBuilder. + Owns(&gatewayv1.HTTPRoute{}). + Owns(&gatewayv1.GRPCRoute{}). + Owns(&gatewayv1alpha2.TLSRoute{}). + Watches(&gatewayv1.Gateway{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, object client.Object) []reconcile.Request { + return nil + })) + } + + return controllerBuilder. WithOptions(controller.Options{ MaxConcurrentReconciles: r.MaxConcurrentReconciles, }). diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index 0f73e74..13bbf51 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -28572,6 +28572,13 @@ such as the number of Pod replicas, the Service resource, or the Ingress. Defining the options for the deployed Tenant Control Plane as Deployment resource.
false + + gateway + object + + Defining the options for an Optional Gateway which will expose API Server of the Tenant Control Plane
+ + false ingress object @@ -41367,6 +41374,243 @@ merge patch.
+`TenantControlPlane.spec.controlPlane.gateway` + + +Defining the options for an Optional Gateway which will expose API Server of the Tenant Control Plane + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
additionalMetadataobject + AdditionalMetadata to add Labels and Annotations support.
+
false
hostnamestring + Hostname is an optional field which will be used as a route hostname.
+
false
parentRefs[]object + GatewayParentRefs is the class of the Gateway resource to use.
+
false
+ + +`TenantControlPlane.spec.controlPlane.gateway.additionalMetadata` + + +AdditionalMetadata to add Labels and Annotations support. + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
annotationsmap[string]string +
+
false
labelsmap[string]string +
+
false
+ + +`TenantControlPlane.spec.controlPlane.gateway.parentRefs[index]` + + +ParentReference identifies an API object (usually a Gateway) that can be considered +a parent of this resource (usually a route). There are two kinds of parent resources +with "Core" support: + +* Gateway (Gateway conformance profile) +* Service (Mesh conformance profile, ClusterIP Services only) + +This API may be extended in the future to support additional kinds of parent +resources. + +The API object must be valid in the cluster; the Group and Kind must +be registered in the cluster for this reference to be valid. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name is the name of the referent. + +Support: Core
+
true
groupstring + Group is the group of the referent. +When unspecified, "gateway.networking.k8s.io" is inferred. +To set the core API group (such as for a "Service" kind referent), +Group must be explicitly set to "" (empty string). + +Support: Core
+
+ Default: gateway.networking.k8s.io
+
false
kindstring + Kind is kind of the referent. + +There are two kinds of parent resources with "Core" support: + +* Gateway (Gateway conformance profile) +* Service (Mesh conformance profile, ClusterIP Services only) + +Support for other resources is Implementation-Specific.
+
+ Default: Gateway
+
false
namespacestring + Namespace is the namespace of the referent. When unspecified, this refers +to the local namespace of the Route. + +Note that there are specific rules for ParentRefs which cross namespace +boundaries. Cross-namespace references are only valid if they are explicitly +allowed by something in the namespace they are referring to. For example: +Gateway has the AllowedRoutes field, and ReferenceGrant provides a +generic way to enable any other kind of cross-namespace reference. + + +ParentRefs from a Route to a Service in the same namespace are "producer" +routes, which apply default routing rules to inbound connections from +any namespace to the Service. + +ParentRefs from a Route to a Service in a different namespace are +"consumer" routes, and these routing rules are only applied to outbound +connections originating from the same namespace as the Route, for which +the intended destination of the connections are a Service targeted as a +ParentRef of the Route. + + +Support: Core
+
false
portinteger + Port is the network port this Route targets. It can be interpreted +differently based on the type of parent resource. + +When the parent resource is a Gateway, this targets all listeners +listening on the specified port that also support this kind of Route(and +select this Route). It's not recommended to set `Port` unless the +networking behaviors specified in a Route must apply to a specific port +as opposed to a listener(s) whose port(s) may be changed. When both Port +and SectionName are specified, the name and port of the selected listener +must match both specified values. + + +When the parent resource is a Service, this targets a specific port in the +Service spec. When both Port (experimental) and SectionName are specified, +the name and port of the selected port must match both specified values. + + +Implementations MAY choose to support other parent resources. +Implementations supporting other types of parent resources MUST clearly +document how/if Port is interpreted. + +For the purpose of status, an attachment is considered successful as +long as the parent resource accepts it partially. For example, Gateway +listeners can restrict which Routes can attach to them by Route kind, +namespace, or hostname. If 1 of 2 Gateway listeners accept attachment +from the referencing Route, the Route MUST be considered successfully +attached. If no Gateway listeners accept attachment from this Route, +the Route MUST be considered detached from the Gateway. + +Support: Extended
+
+ Format: int32
+ Minimum: 1
+ Maximum: 65535
+
false
sectionNamestring + SectionName is the name of a section within the target resource. In the +following resources, SectionName is interpreted as the following: + +* Gateway: Listener name. When both Port (experimental) and SectionName +are specified, the name and port of the selected listener must match +both specified values. +* Service: Port name. When both Port (experimental) and SectionName +are specified, the name and port of the selected listener must match +both specified values. + +Implementations MAY choose to support attaching Routes to other resources. +If that is the case, they MUST clearly document how SectionName is +interpreted. + +When unspecified (empty string), this will reference the entire resource. +For the purpose of status, an attachment is considered successful if at +least one section in the parent resource accepts it. For example, Gateway +listeners can restrict which Routes can attach to them by Route kind, +namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from +the referencing Route, the Route MUST be considered successfully +attached. If no Gateway listeners accept attachment from this Route, the +Route MUST be considered detached from the Gateway. + +Support: Core
+
false
+ + `TenantControlPlane.spec.controlPlane.ingress` @@ -43595,6 +43839,13 @@ Kubernetes contains information about the reconciliation of the required Kuberne KubernetesDeploymentStatus defines the status for the Tenant Control Plane Deployment in the management cluster.
false + + gateway + object + + KubernetesGatewayStatus defines the status for the Tenant Control Plane Gateway in the management cluster.
+ + false ingress object @@ -43818,6 +44069,505 @@ DeploymentCondition describes the state of a deployment at a certain point. +`TenantControlPlane.status.kubernetesResources.gateway` + + +KubernetesGatewayStatus defines the status for the Tenant Control Plane Gateway in the management cluster. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
parents[]object + Parents is a list of parent resources (usually Gateways) that are +associated with the route, and the status of the route with respect to +each parent. When this route attaches to a parent, the controller that +manages the parent must add an entry to this list when the controller +first sees the route and should update the entry as appropriate when the +route or gateway is modified. + +Note that parent references that cannot be resolved by an implementation +of this API will not be added to this list. Implementations of this API +can only populate Route status for the Gateways/parent resources they are +responsible for. + +A maximum of 32 Gateways will be represented in this list. An empty list +means the route has not been attached to any Gateway. + + +Notes for implementors: + +While parents is not a listType `map`, this is due to the fact that the +list key is not scalar, and Kubernetes is unable to represent this. + +Parent status MUST be considered to be namespaced by the combination of +the parentRef and controllerName fields, and implementations should keep +the following rules in mind when updating this status: + +* Implementations MUST update only entries that have a matching value of + `controllerName` for that implementation. +* Implementations MUST NOT update entries with non-matching `controllerName` + fields. +* Implementations MUST treat each `parentRef`` in the Route separately and + update its status based on the relationship with that parent. +* Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. + +
+
true
accessPoints[]object + A list of valid access points that the route exposes.
+
false
routeRefobject + Reference to the route created for this tenant.
+
false
+ + +`TenantControlPlane.status.kubernetesResources.gateway.parents[index]` + + +RouteParentStatus describes the status of a route with respect to an +associated Parent. + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
conditions[]object + Conditions describes the status of the route with respect to the Gateway. +Note that the route's availability is also subject to the Gateway's own +status conditions and listener status. + +If the Route's ParentRef specifies an existing Gateway that supports +Routes of this kind AND that Gateway's controller has sufficient access, +then that Gateway's controller MUST set the "Accepted" condition on the +Route, to indicate whether the route has been accepted or rejected by the +Gateway, and why. + +A Route MUST be considered "Accepted" if at least one of the Route's +rules is implemented by the Gateway. + +There are a number of cases where the "Accepted" condition may not be set +due to lack of controller visibility, that includes when: + +* The Route refers to a nonexistent parent. +* The Route is of a type that the controller does not support. +* The Route is in a namespace the controller does not have access to. + + + +Notes for implementors: + +Conditions are a listType `map`, which means that they function like a +map with a key of the `type` field _in the k8s apiserver_. + +This means that implementations must obey some rules when updating this +section. + +* Implementations MUST perform a read-modify-write cycle on this field + before modifying it. That is, when modifying this field, implementations + must be confident they have fetched the most recent version of this field, + and ensure that changes they make are on that recent version. +* Implementations MUST NOT remove or reorder Conditions that they are not + directly responsible for. For example, if an implementation sees a Condition + with type `special.io/SomeField`, it MUST NOT remove, change or update that + Condition. +* Implementations MUST always _merge_ changes into Conditions of the same Type, + rather than creating more than one Condition of the same Type. +* Implementations MUST always update the `observedGeneration` field of the + Condition to the `metadata.generation` of the Gateway at the time of update creation. +* If the `observedGeneration` of a Condition is _greater than_ the value the + implementation knows about, then it MUST NOT perform the update on that Condition, + but must wait for a future reconciliation and status update. (The assumption is that + the implementation's copy of the object is stale and an update will be re-triggered + if relevant.) + +
+
true
controllerNamestring + ControllerName is a domain/path string that indicates the name of the +controller that wrote this status. This corresponds with the +controllerName field on GatewayClass. + +Example: "example.net/gateway-controller". + +The format of this field is DOMAIN "/" PATH, where DOMAIN and PATH are +valid Kubernetes names +(https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names). + +Controllers MUST populate this field when writing status. Controllers should ensure that +entries to status populated with their ControllerName are cleaned up when they are no +longer necessary.
+
true
parentRefobject + ParentRef corresponds with a ParentRef in the spec that this +RouteParentStatus struct describes the status of.
+
true
+ + +`TenantControlPlane.status.kubernetesResources.gateway.parents[index].conditions[index]` + + +Condition contains details for one aspect of the current state of this API Resource. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
lastTransitionTimestring + lastTransitionTime is the last time the condition transitioned from one status to another. +This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.
+
+ Format: date-time
+
true
messagestring + message is a human readable message indicating details about the transition. +This may be an empty string.
+
true
reasonstring + reason contains a programmatic identifier indicating the reason for the condition's last transition. +Producers of specific condition types may define expected values and meanings for this field, +and whether the values are considered a guaranteed API. +The value should be a CamelCase string. +This field may not be empty.
+
true
statusenum + status of the condition, one of True, False, Unknown.
+
+ Enum: True, False, Unknown
+
true
typestring + type of condition in CamelCase or in foo.example.com/CamelCase.
+
true
observedGenerationinteger + observedGeneration represents the .metadata.generation that the condition was set based upon. +For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date +with respect to the current state of the instance.
+
+ Format: int64
+ Minimum: 0
+
false
+ + +`TenantControlPlane.status.kubernetesResources.gateway.parents[index].parentRef` + + +ParentRef corresponds with a ParentRef in the spec that this +RouteParentStatus struct describes the status of. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name is the name of the referent. + +Support: Core
+
true
groupstring + Group is the group of the referent. +When unspecified, "gateway.networking.k8s.io" is inferred. +To set the core API group (such as for a "Service" kind referent), +Group must be explicitly set to "" (empty string). + +Support: Core
+
+ Default: gateway.networking.k8s.io
+
false
kindstring + Kind is kind of the referent. + +There are two kinds of parent resources with "Core" support: + +* Gateway (Gateway conformance profile) +* Service (Mesh conformance profile, ClusterIP Services only) + +Support for other resources is Implementation-Specific.
+
+ Default: Gateway
+
false
namespacestring + Namespace is the namespace of the referent. When unspecified, this refers +to the local namespace of the Route. + +Note that there are specific rules for ParentRefs which cross namespace +boundaries. Cross-namespace references are only valid if they are explicitly +allowed by something in the namespace they are referring to. For example: +Gateway has the AllowedRoutes field, and ReferenceGrant provides a +generic way to enable any other kind of cross-namespace reference. + + +ParentRefs from a Route to a Service in the same namespace are "producer" +routes, which apply default routing rules to inbound connections from +any namespace to the Service. + +ParentRefs from a Route to a Service in a different namespace are +"consumer" routes, and these routing rules are only applied to outbound +connections originating from the same namespace as the Route, for which +the intended destination of the connections are a Service targeted as a +ParentRef of the Route. + + +Support: Core
+
false
portinteger + Port is the network port this Route targets. It can be interpreted +differently based on the type of parent resource. + +When the parent resource is a Gateway, this targets all listeners +listening on the specified port that also support this kind of Route(and +select this Route). It's not recommended to set `Port` unless the +networking behaviors specified in a Route must apply to a specific port +as opposed to a listener(s) whose port(s) may be changed. When both Port +and SectionName are specified, the name and port of the selected listener +must match both specified values. + + +When the parent resource is a Service, this targets a specific port in the +Service spec. When both Port (experimental) and SectionName are specified, +the name and port of the selected port must match both specified values. + + +Implementations MAY choose to support other parent resources. +Implementations supporting other types of parent resources MUST clearly +document how/if Port is interpreted. + +For the purpose of status, an attachment is considered successful as +long as the parent resource accepts it partially. For example, Gateway +listeners can restrict which Routes can attach to them by Route kind, +namespace, or hostname. If 1 of 2 Gateway listeners accept attachment +from the referencing Route, the Route MUST be considered successfully +attached. If no Gateway listeners accept attachment from this Route, +the Route MUST be considered detached from the Gateway. + +Support: Extended
+
+ Format: int32
+ Minimum: 1
+ Maximum: 65535
+
false
sectionNamestring + SectionName is the name of a section within the target resource. In the +following resources, SectionName is interpreted as the following: + +* Gateway: Listener name. When both Port (experimental) and SectionName +are specified, the name and port of the selected listener must match +both specified values. +* Service: Port name. When both Port (experimental) and SectionName +are specified, the name and port of the selected listener must match +both specified values. + +Implementations MAY choose to support attaching Routes to other resources. +If that is the case, they MUST clearly document how SectionName is +interpreted. + +When unspecified (empty string), this will reference the entire resource. +For the purpose of status, an attachment is considered successful if at +least one section in the parent resource accepts it. For example, Gateway +listeners can restrict which Routes can attach to them by Route kind, +namespace, or hostname. If 1 of 2 Gateway listeners accept attachment from +the referencing Route, the Route MUST be considered successfully +attached. If no Gateway listeners accept attachment from this Route, the +Route MUST be considered detached from the Gateway. + +Support: Core
+
false
+ + +`TenantControlPlane.status.kubernetesResources.gateway.accessPoints[index]` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
portinteger +
+
+ Format: int32
+
true
typestring + AddressType defines how a network address is represented as a text string. +This may take two possible forms: + +* A predefined CamelCase string identifier (currently limited to `IPAddress` or `Hostname`) +* A domain-prefixed string identifier (like `acme.io/CustomAddressType`) + +Values `IPAddress` and `Hostname` have Extended support. + +The `NamedAddress` value has been deprecated in favor of implementation +specific domain-prefixed strings. + +All other values, including domain-prefixed values have Implementation-specific support, +which are used in implementation-specific behaviors. Support for additional +predefined CamelCase identifiers may be added in future releases.
+
true
valuestring +
+
true
urls[]string +
+
false
+ + +`TenantControlPlane.status.kubernetesResources.gateway.routeRef` + + +Reference to the route created for this tenant. + + + + + + + + + + + + + + + + +
NameTypeDescriptionRequired
namestring + Name of the referent. +This field is effectively required, but due to backwards compatibility is +allowed to be empty. Instances of this type with an empty value here are +almost certainly wrong. +More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+
+ Default:
+
false
+ + `TenantControlPlane.status.kubernetesResources.ingress` diff --git a/e2e/suite_test.go b/e2e/suite_test.go index 8e99892..7346b91 100644 --- a/e2e/suite_test.go +++ b/e2e/suite_test.go @@ -15,6 +15,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" ) @@ -56,6 +58,12 @@ var _ = BeforeSuite(func() { err = kamajiv1alpha1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = gatewayv1.Install(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = gatewayv1alpha2.Install(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + //+kubebuilder:scaffold:scheme k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) diff --git a/e2e/tcp_gateway_ready_test.go b/e2e/tcp_gateway_ready_test.go new file mode 100644 index 0000000..5cf3ff9 --- /dev/null +++ b/e2e/tcp_gateway_ready_test.go @@ -0,0 +1,102 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + pointer "k8s.io/utils/ptr" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +var _ = Describe("Deploy a TenantControlPlane with Gateway API", func() { + var tcp *kamajiv1alpha1.TenantControlPlane + + JustBeforeEach(func() { + tcp = &kamajiv1alpha1.TenantControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tcp-gateway", + Namespace: "default", + }, + Spec: kamajiv1alpha1.TenantControlPlaneSpec{ + ControlPlane: kamajiv1alpha1.ControlPlane{ + Deployment: kamajiv1alpha1.DeploymentSpec{ + Replicas: pointer.To(int32(1)), + }, + Service: kamajiv1alpha1.ServiceSpec{ + ServiceType: "ClusterIP", + }, + Gateway: &kamajiv1alpha1.GatewaySpec{ + Hostname: gatewayv1.Hostname("tcp-gateway.example.com"), + AdditionalMetadata: kamajiv1alpha1.AdditionalMetadata{ + Labels: map[string]string{ + "test.kamaji.io/gateway": "true", + }, + Annotations: map[string]string{ + "test.kamaji.io/created-by": "e2e-test", + }, + }, + GatewayParentRefs: []gatewayv1.ParentReference{ + { + Name: "test-gateway", + }, + }, + }, + }, + NetworkProfile: kamajiv1alpha1.NetworkProfileSpec{ + Address: "172.18.0.3", + }, + Kubernetes: kamajiv1alpha1.KubernetesSpec{ + Version: "v1.23.6", + Kubelet: kamajiv1alpha1.KubeletSpec{ + CGroupFS: "cgroupfs", + }, + AdmissionControllers: kamajiv1alpha1.AdmissionControllers{ + "LimitRanger", + "ResourceQuota", + }, + }, + Addons: kamajiv1alpha1.AddonsSpec{}, + }, + } + Expect(k8sClient.Create(context.Background(), tcp)).NotTo(HaveOccurred()) + }) + + JustAfterEach(func() { + Expect(k8sClient.Delete(context.Background(), tcp)).Should(Succeed()) + + // Wait for the object to be completely deleted + Eventually(func() bool { + err := k8sClient.Get(context.Background(), types.NamespacedName{ + Name: tcp.Name, + Namespace: tcp.Namespace, + }, &kamajiv1alpha1.TenantControlPlane{}) + + return err != nil // Returns true when object is not found (deleted) + }).WithTimeout(time.Minute).Should(BeTrue()) + }) + + It("Should be Ready", func() { + StatusMustEqualTo(tcp, kamajiv1alpha1.VersionReady) + }) + + It("Should create TLSRoute resource", func() { + Eventually(func() error { + route := &gatewayv1alpha2.TLSRoute{} + // TODO: Check ownership. + return k8sClient.Get(context.Background(), types.NamespacedName{ + Name: tcp.Name, + Namespace: tcp.Namespace, + }, route) + }).WithTimeout(time.Minute).Should(Succeed()) + }) +}) diff --git a/go.mod b/go.mod index 588f37d..b9ffdbc 100644 --- a/go.mod +++ b/go.mod @@ -35,8 +35,9 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/kubelet v0.0.0 k8s.io/kubernetes v1.34.2 - k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 + k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d sigs.k8s.io/controller-runtime v0.22.4 + sigs.k8s.io/gateway-api v1.4.0 ) require ( @@ -67,8 +68,8 @@ require ( github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/ebitengine/purego v0.8.4 // indirect - github.com/emicklei/go-restful/v3 v3.12.2 // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/evanphx/json-patch v5.7.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect @@ -77,9 +78,9 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/jsonpointer v0.21.2 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.1 // indirect github.com/go-pg/zerochecker v0.2.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect @@ -103,7 +104,7 @@ require ( github.com/lithammer/dedent v1.1.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/mailru/easyjson v0.9.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.1.0 // indirect github.com/moby/patternmatcher v0.6.0 // indirect @@ -126,7 +127,7 @@ require ( github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect - github.com/prometheus/procfs v0.16.1 // indirect + github.com/prometheus/procfs v0.17.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/shirou/gopsutil/v4 v4.25.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect @@ -150,12 +151,12 @@ require ( go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect - go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 // indirect - go.opentelemetry.io/otel/metric v1.35.0 // indirect - go.opentelemetry.io/otel/sdk v1.34.0 // indirect - go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/sdk v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect go.opentelemetry.io/proto/otlp v1.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect @@ -170,13 +171,13 @@ require ( golang.org/x/sys v0.37.0 // indirect golang.org/x/term v0.36.0 // indirect golang.org/x/text v0.30.0 // indirect - golang.org/x/time v0.9.0 // indirect + golang.org/x/time v0.12.0 // indirect golang.org/x/tools v0.37.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/grpc v1.72.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1 // indirect + google.golang.org/grpc v1.75.1 // indirect google.golang.org/protobuf v1.36.8 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect @@ -190,12 +191,12 @@ require ( k8s.io/cri-api v0.34.0 // indirect k8s.io/cri-client v0.0.0 // indirect k8s.io/kms v0.34.0 // indirect - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect + k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3 // indirect k8s.io/kube-proxy v0.0.0 // indirect k8s.io/system-validators v1.10.2 // indirect mellium.im/sasl v0.3.1 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/kustomize/api v0.20.1 // indirect sigs.k8s.io/kustomize/kyaml v0.20.1 // indirect sigs.k8s.io/randfill v1.0.0 // indirect diff --git a/go.sum b/go.sum index f3916fb..29cdefd 100644 --- a/go.sum +++ b/go.sum @@ -49,7 +49,6 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -68,10 +67,10 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= -github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= -github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -100,14 +99,12 @@ github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/jsonpointer v0.21.2 h1:AqQaNADVwq/VnkCmQg6ogE+M3FOsKTytwges0JdwVuA= +github.com/go-openapi/jsonpointer v0.21.2/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= +github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= github.com/go-pg/pg/v10 v10.15.0 h1:6DQwbaxJz/e4wvgzbxBkBLiL/Uuk87MGgHhkURtzx24= github.com/go-pg/pg/v10 v10.15.0/go.mod h1:FIn/x04hahOf9ywQ1p68rXqaDVbTRLYlu4MQR0lhoB8= github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU= @@ -196,7 +193,6 @@ github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCy github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -213,8 +209,8 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= @@ -284,8 +280,8 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= -github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= -github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= +github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -389,22 +385,22 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.6 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0 h1:tgJ0uaNS4c98WRNUEx5U3aDlrDOI5Rs+1Vifcw4DJ8U= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0/go.mod h1:U7HYyW0zt/a9x5J1Kjs+r1f/d4ZHnYFclhYY2+YbeoE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4= go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4= go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= @@ -468,8 +464,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -483,20 +479,22 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0= gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= -google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= -google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= -google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 h1:FiusG7LWj+4byqhbvmB+Q93B/mOxJLN2DTozDuZm4EU= +google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:kXqgZtrWaf6qS3jZOCnCH7WYfrvFjkC51bM8fz3RsCA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1 h1:pmJpJEvT846VzausCQ5d7KreSROcDqmO388w5YbnltA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1/go.mod h1:GmFNa4BdJZ2a8G+wCe9Bg3wwThLrJun751XstdJt5Og= +google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= +google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -543,8 +541,8 @@ k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kms v0.34.0 h1:u+/rcxQ3Jr7gC9AY5nXuEnBcGEB7ZOIJ9cdLdyHyEjQ= k8s.io/kms v0.34.0/go.mod h1:s1CFkLG7w9eaTYvctOxosx88fl4spqmixnNpys0JAtM= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3 h1:liMHz39T5dJO1aOKHLvwaCjDbf07wVh6yaUlTpunnkE= +k8s.io/kube-openapi v0.0.0-20250814151709-d7b6acb124c3/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= k8s.io/kube-proxy v0.34.0 h1:gU7MVbJHiXyPX8bXnod4bANtSC7rZSKkkLmM8gUqwT4= k8s.io/kube-proxy v0.34.0/go.mod h1:tfwI8dCKm5Q0r+aVIbrq/aC36Kk936w2LZu8/rvJzWI= k8s.io/kubelet v0.34.0 h1:1nZt1Q6Kfx7xCaTS9vnqR9sjZDxf3cRSQkAFCczULmc= @@ -553,16 +551,18 @@ k8s.io/kubernetes v1.34.2 h1:WQdDvYJazkmkwSncgNwGvVtaCt4TYXIU3wSMRgvp3MI= k8s.io/kubernetes v1.34.2/go.mod h1:m6pZk6a179pRo2wsTiCPORJ86iOEQmfIzUvtyEF8BwA= k8s.io/system-validators v1.10.2 h1:7rC7VdrQCaM55E08Pw3I1v1Op9ObLxdKAu5Ff5hIPwY= k8s.io/system-validators v1.10.2/go.mod h1:awfSS706v9R12VC7u7K89FKfqVy44G+E0L1A0FX9Wmw= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d h1:wAhiDyZ4Tdtt7e46e9M5ZSAJ/MnPGPs+Ki1gHw4w1R0= +k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.22.4 h1:GEjV7KV3TY8e+tJ2LCTxUTanW4z/FmNB7l327UfMq9A= sigs.k8s.io/controller-runtime v0.22.4/go.mod h1:+QX1XUpTXN4mLoblf4tqr5CQcyHPAki2HLXqQMY6vh8= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/gateway-api v1.4.0 h1:ZwlNM6zOHq0h3WUX2gfByPs2yAEsy/EenYJB78jpQfQ= +sigs.k8s.io/gateway-api v1.4.0/go.mod h1:AR5RSqciWP98OPckEjOjh2XJhAe2Na4LHyXD2FUY7Qk= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/kustomize/api v0.20.1 h1:iWP1Ydh3/lmldBnH/S5RXgT98vWYMaTUL1ADcr+Sv7I= sigs.k8s.io/kustomize/api v0.20.1/go.mod h1:t6hUFxO+Ph0VxIk1sKp1WS0dOjbPCtLJ4p8aADLwqjM= sigs.k8s.io/kustomize/kyaml v0.20.1 h1:PCMnA2mrVbRP3NIB6v9kYCAc38uvFLVs8j/CD567A78= diff --git a/internal/resources/k8s_gateway_resource.go b/internal/resources/k8s_gateway_resource.go new file mode 100644 index 0000000..7def916 --- /dev/null +++ b/internal/resources/k8s_gateway_resource.go @@ -0,0 +1,384 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "context" + "fmt" + "net/url" + + "github.com/prometheus/client_golang/prometheus" + v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/utilities" +) + +type KubernetesGatewayResource struct { + resource *gatewayv1alpha2.TLSRoute + Client client.Client +} + +func (r *KubernetesGatewayResource) GetHistogram() prometheus.Histogram { + gatewayCollector = LazyLoadHistogramFromResource(gatewayCollector, r) + + return gatewayCollector +} + +func (r *KubernetesGatewayResource) ShouldStatusBeUpdated(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) bool { + switch { + case tcp.Spec.ControlPlane.Gateway == nil && tcp.Status.Kubernetes.Gateway == nil: + return false + case tcp.Spec.ControlPlane.Gateway != nil && tcp.Status.Kubernetes.Gateway == nil: + return true + case tcp.Spec.ControlPlane.Gateway == nil && tcp.Status.Kubernetes.Gateway != nil: + return true + // Could be an alias for default here since the other cases are covered. + case tcp.Spec.ControlPlane.Gateway != nil && tcp.Status.Kubernetes.Gateway != nil: + return r.gatewayStatusNeedsUpdate(tcp) + } + + return false +} + +// gatewayStatusNeedsUpdate compares the current gateway resource status with the stored status. +func (r *KubernetesGatewayResource) gatewayStatusNeedsUpdate(tcp *kamajiv1alpha1.TenantControlPlane) bool { + currentStatus := tcp.Status.Kubernetes.Gateway + + // Check if route reference has changed + if currentStatus.RouteRef.Name != r.resource.Name { + return true + } + + // Compare RouteStatus - check if number of parents changed + if len(currentStatus.RouteStatus.Parents) != len(r.resource.Status.RouteStatus.Parents) { + return true + } + + // Compare individual parent statuses + // NOTE: Multiple Parent References are assumed. + for i, currentParent := range currentStatus.RouteStatus.Parents { + if i >= len(r.resource.Status.RouteStatus.Parents) { + return true + } + + resourceParent := r.resource.Status.RouteStatus.Parents[i] + + // Compare parent references + if currentParent.ParentRef.Name != resourceParent.ParentRef.Name || + (currentParent.ParentRef.Namespace == nil) != (resourceParent.ParentRef.Namespace == nil) || + (currentParent.ParentRef.Namespace != nil && resourceParent.ParentRef.Namespace != nil && + *currentParent.ParentRef.Namespace != *resourceParent.ParentRef.Namespace) || + (currentParent.ParentRef.SectionName == nil) != (resourceParent.ParentRef.SectionName == nil) || + (currentParent.ParentRef.SectionName != nil && resourceParent.ParentRef.SectionName != nil && + *currentParent.ParentRef.SectionName != *resourceParent.ParentRef.SectionName) { + return true + } + + if len(currentParent.Conditions) != len(resourceParent.Conditions) { + return true + } + + // Compare each condition + for j, currentCondition := range currentParent.Conditions { + if j >= len(resourceParent.Conditions) { + return true + } + + resourceCondition := resourceParent.Conditions[j] + + if currentCondition.Type != resourceCondition.Type || + currentCondition.Status != resourceCondition.Status || + currentCondition.Reason != resourceCondition.Reason || + currentCondition.Message != resourceCondition.Message || + !currentCondition.LastTransitionTime.Equal(&resourceCondition.LastTransitionTime) { + return true + } + } + } + + // Since access points are derived from route status and gateway conditions, + // and we've already compared the route status above, we can assume that + // if the route status hasn't changed, the access points calculation + // will produce the same result. This avoids the need for complex + // gateway fetching in the status comparison. + // + // If there are edge cases where gateway state changes but route status doesn't, + // those will be caught in the next reconciliation cycle anyway. + return false +} + +func (r *KubernetesGatewayResource) ShouldCleanup(tcp *kamajiv1alpha1.TenantControlPlane) bool { + return tcp.Spec.ControlPlane.Gateway == nil && tcp.Status.Kubernetes.Gateway != nil +} + +func (r *KubernetesGatewayResource) CleanUp(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) (bool, error) { + logger := log.FromContext(ctx, "resource", r.GetName()) + + route := gatewayv1alpha2.TLSRoute{} + if err := r.Client.Get(ctx, client.ObjectKey{ + Namespace: r.resource.GetNamespace(), + Name: r.resource.GetName(), + }, &route); err != nil { + if !k8serrors.IsNotFound(err) { + logger.Error(err, "failed to get TLSRoute before cleanup") + + return false, err + } + + return false, nil + } + + if !metav1.IsControlledBy(&route, tcp) { + logger.Info("skipping cleanup: route is not managed by Kamaji", "name", route.Name, "namespace", route.Namespace) + + return false, nil + } + + if err := r.Client.Delete(ctx, &route); err != nil { + if !k8serrors.IsNotFound(err) { + // TODO: Is that an error? Wanted to delete the resource anyways. + logger.Error(err, "cannot cleanup tcp route") + + return false, err + } + + return false, nil + } + + logger.V(1).Info("tcp route cleaned up successfully") + + return true, nil +} + +// fetchGatewayByListener uses the indexer to efficiently find a gateway with a specific listener. +// This avoids the need to iterate through all listeners in a gateway. +func (r *KubernetesGatewayResource) fetchGatewayByListener(ctx context.Context, ref gatewayv1.ParentReference) (*gatewayv1.Gateway, error) { + if ref.Namespace == nil { + return nil, fmt.Errorf("missing namespace") + } + if ref.SectionName == nil { + return nil, fmt.Errorf("missing sectionName") + } + + // Build the composite key that matches our indexer format: namespace/gatewayName/listenerName + listenerKey := fmt.Sprintf("%s/%s/%s", *ref.Namespace, ref.Name, *ref.SectionName) + + // Query gateways using the indexer + gatewayList := &gatewayv1.GatewayList{} + if err := r.Client.List(ctx, gatewayList, client.MatchingFieldsSelector{ + Selector: fields.OneTermEqualSelector(kamajiv1alpha1.GatewayListenerNameKey, listenerKey), + }); err != nil { + return nil, fmt.Errorf("failed to list gateways by listener: %w", err) + } + + if len(gatewayList.Items) == 0 { + return nil, fmt.Errorf("no gateway found with listener '%s'", *ref.SectionName) + } + + // Since we're using a composite key with namespace/name/listener, we should get exactly one result + if len(gatewayList.Items) > 1 { + return nil, fmt.Errorf("found multiple gateways with listener '%s', expected exactly one", *ref.SectionName) + } + + return &gatewayList.Items[0], nil +} + +func FindMatchingListener(listeners []gatewayv1.Listener, ref gatewayv1.ParentReference) (gatewayv1.Listener, error) { + if ref.SectionName == nil { + return gatewayv1.Listener{}, fmt.Errorf("missing sectionName") + } + name := *ref.SectionName + for _, listener := range listeners { + if listener.Name == name { + return listener, nil + } + } + + // TODO: Handle the cases according to the spec: + // - When both Port (experimental) and SectionName are + // specified, the name and port of the selected listener + // must match both specified values. + // - When unspecified (empty string) this will reference + // the entire resource [...] an attachment is considered + // successful if at least one section in the parent resource accepts it + + return gatewayv1.Listener{}, fmt.Errorf("could not find listener '%s'", name) +} + +func (r *KubernetesGatewayResource) UpdateTenantControlPlaneStatus(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) error { + logger := log.FromContext(ctx, "resource", r.GetName()) + + // Clean up status if Gateway routes are no longer configured + if tcp.Spec.ControlPlane.Gateway == nil { + tcp.Status.Kubernetes.Gateway = nil + + return nil + } + + tcp.Status.Kubernetes.Gateway = &kamajiv1alpha1.KubernetesGatewayStatus{ + RouteStatus: r.resource.Status.RouteStatus, + RouteRef: v1.LocalObjectReference{ + Name: r.resource.Name, + }, + } + + routeStatuses := tcp.Status.Kubernetes.Gateway.RouteStatus + + // TODO: Investigate the implications of having multiple parents / hostnames + // TODO: Use condition to report? + if len(routeStatuses.Parents) == 0 { + return fmt.Errorf("no gateway attached to the route") + } + if len(routeStatuses.Parents) > 1 { + return fmt.Errorf("too many gateway attached to the route") + } + if len(r.resource.Spec.Hostnames) == 0 { + return fmt.Errorf("no hostname in the route") + } + if len(r.resource.Spec.Hostnames) > 1 { + return fmt.Errorf("too many hostnames in the route") + } + + logger.V(1).Info("updating TenantControlPlane status for Gateway routes") + accessPoints := []kamajiv1alpha1.GatewayAccessPoint{} + for _, routeStatus := range routeStatuses.Parents { + routeAccepted := meta.IsStatusConditionTrue( + routeStatus.Conditions, + string(gatewayv1.RouteConditionAccepted), + ) + if !routeAccepted { + continue + } + + // Use the indexer to efficiently find the gateway with the specific listener + gateway, err := r.fetchGatewayByListener(ctx, routeStatus.ParentRef) + if err != nil { + return fmt.Errorf("could not fetch gateway with listener '%v': %w", + routeStatus.ParentRef.SectionName, err) + } + gatewayProgrammed := meta.IsStatusConditionTrue( + gateway.Status.Conditions, + string(gatewayv1.GatewayConditionProgrammed), + ) + if !gatewayProgrammed { + continue + } + + // Since we fetched the gateway using the indexer, we know the listener exists + // but we still need to get its details from the gateway spec + listener, err := FindMatchingListener( + gateway.Spec.Listeners, routeStatus.ParentRef, + ) + if err != nil { + return fmt.Errorf("failed to match listener: %w", err) + } + + for _, hostname := range r.resource.Spec.Hostnames { + rawURL := fmt.Sprintf("https://%s:%d", hostname, listener.Port) + url, err := url.Parse(rawURL) + if err != nil { + return fmt.Errorf("invalid url: %w", err) + } + + hostnameAddressType := gatewayv1.HostnameAddressType + accessPoints = append(accessPoints, kamajiv1alpha1.GatewayAccessPoint{ + Type: &hostnameAddressType, + Value: url.String(), + Port: listener.Port, + }) + } + } + tcp.Status.Kubernetes.Gateway.AccessPoints = accessPoints + + return nil +} + +func (r *KubernetesGatewayResource) Define(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) error { + r.resource = &gatewayv1alpha2.TLSRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: tcp.GetName(), + Namespace: tcp.GetNamespace(), + }, + } + + return nil +} + +func (r *KubernetesGatewayResource) mutate(tcp *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { + return func() error { + labels := utilities.MergeMaps( + r.resource.GetLabels(), + utilities.KamajiLabels(tcp.GetName(), r.GetName()), + tcp.Spec.ControlPlane.Gateway.AdditionalMetadata.Labels, + ) + r.resource.SetLabels(labels) + + annotations := utilities.MergeMaps( + r.resource.GetAnnotations(), + tcp.Spec.ControlPlane.Gateway.AdditionalMetadata.Annotations) + r.resource.SetAnnotations(annotations) + + if tcp.Spec.ControlPlane.Gateway.GatewayParentRefs != nil { + r.resource.Spec.ParentRefs = tcp.Spec.ControlPlane.Gateway.GatewayParentRefs + } + + serviceName := gatewayv1alpha2.ObjectName(tcp.Status.Kubernetes.Service.Name) + servicePort := tcp.Status.Kubernetes.Service.Port + + if serviceName == "" || servicePort == 0 { + return fmt.Errorf("service not ready, cannot create TLSRoute") + } + + rule := gatewayv1alpha2.TLSRouteRule{ + BackendRefs: []gatewayv1alpha2.BackendRef{ + { + BackendObjectReference: gatewayv1alpha2.BackendObjectReference{ + Name: serviceName, + Port: &servicePort, + }, + }, + }, + } + + r.resource.Spec.Hostnames = []gatewayv1.Hostname{tcp.Spec.ControlPlane.Gateway.Hostname} + r.resource.Spec.Rules = []gatewayv1alpha2.TLSRouteRule{rule} + + return controllerutil.SetControllerReference(tcp, r.resource, r.Client.Scheme()) + } +} + +func (r *KubernetesGatewayResource) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + logger := log.FromContext(ctx, "resource", r.GetName()) + + if tenantControlPlane.Spec.ControlPlane.Gateway == nil { + return controllerutil.OperationResultNone, nil + } + + if len(tenantControlPlane.Spec.ControlPlane.Gateway.Hostname) == 0 { + return controllerutil.OperationResultNone, fmt.Errorf("missing hostname to expose the Tenant Control Plane using a Gateway resource") + } + + logger.V(1).Info("creating or updating resource gateway routes") + + result, err := utilities.CreateOrUpdateWithConflict(ctx, r.Client, r.resource, r.mutate(tenantControlPlane)) + if err != nil { + return result, err + } + + return result, nil +} + +func (r *KubernetesGatewayResource) GetName() string { + return "gateway_routes" +} diff --git a/internal/resources/k8s_gateway_resource_test.go b/internal/resources/k8s_gateway_resource_test.go new file mode 100644 index 0000000..514def8 --- /dev/null +++ b/internal/resources/k8s_gateway_resource_test.go @@ -0,0 +1,238 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources_test + +import ( + "context" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/resources" +) + +func TestGatewayResource(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Gateway Resource Suite") +} + +var runtimeScheme *runtime.Scheme + +var _ = BeforeSuite(func() { + runtimeScheme = runtime.NewScheme() + Expect(scheme.AddToScheme(runtimeScheme)).To(Succeed()) + Expect(kamajiv1alpha1.AddToScheme(runtimeScheme)).To(Succeed()) + Expect(gatewayv1alpha2.Install(runtimeScheme)).To(Succeed()) +}) + +var _ = Describe("KubernetesGatewayResource", func() { + var ( + tcp *kamajiv1alpha1.TenantControlPlane + resource *resources.KubernetesGatewayResource + ctx context.Context + ) + + BeforeEach(func() { + ctx = context.Background() + + fakeClient := fake.NewClientBuilder(). + WithScheme(runtimeScheme). + Build() + + resource = &resources.KubernetesGatewayResource{ + Client: fakeClient, + } + + tcp = &kamajiv1alpha1.TenantControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-tcp", + Namespace: "default", + }, + Spec: kamajiv1alpha1.TenantControlPlaneSpec{ + ControlPlane: kamajiv1alpha1.ControlPlane{ + Gateway: &kamajiv1alpha1.GatewaySpec{ + Hostname: gatewayv1alpha2.Hostname("test.example.com"), + AdditionalMetadata: kamajiv1alpha1.AdditionalMetadata{ + Labels: map[string]string{ + "test-label": "test-value", + }, + }, + GatewayParentRefs: []gatewayv1alpha2.ParentReference{ + { + Name: "test-gateway", + }, + }, + }, + }, + }, + Status: kamajiv1alpha1.TenantControlPlaneStatus{ + Kubernetes: kamajiv1alpha1.KubernetesStatus{ + Service: kamajiv1alpha1.KubernetesServiceStatus{ + Name: "test-service", + Port: 6443, + }, + }, + }, + } + }) + + Context("When GatewayRoutes is configured", func() { + It("should not cleanup", func() { + Expect(resource.ShouldCleanup(tcp)).To(BeFalse()) + }) + + It("should define route resources", func() { + err := resource.Define(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should require status update when GatewayRoutes is configured but status is nil", func() { + tcp.Status.Kubernetes.Gateway = nil + shouldUpdate := resource.ShouldStatusBeUpdated(ctx, tcp) + Expect(shouldUpdate).To(BeTrue()) + }) + }) + + Context("When GatewayRoutes is not configured", func() { + BeforeEach(func() { + tcp.Spec.ControlPlane.Gateway = nil + tcp.Status.Kubernetes.Gateway = &kamajiv1alpha1.KubernetesGatewayStatus{ + AccessPoints: nil, + } + }) + + It("should cleanup", func() { + Expect(resource.ShouldCleanup(tcp)).To(BeTrue()) + }) + + It("should not require status update when both spec and status are nil", func() { + tcp.Status.Kubernetes.Gateway = nil + shouldUpdate := resource.ShouldStatusBeUpdated(ctx, tcp) + Expect(shouldUpdate).To(BeFalse()) + }) + }) + + Context("When hostname is missing", func() { + BeforeEach(func() { + tcp.Spec.ControlPlane.Gateway.Hostname = "" + }) + + It("should fail to create or update", func() { + err := resource.Define(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + _, err = resource.CreateOrUpdate(ctx, tcp) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("missing hostname")) + }) + }) + + Context("When service is not ready", func() { + BeforeEach(func() { + tcp.Status.Kubernetes.Service.Name = "" + tcp.Status.Kubernetes.Service.Port = 0 + }) + + It("should fail to create or update", func() { + err := resource.Define(ctx, tcp) + Expect(err).NotTo(HaveOccurred()) + + _, err = resource.CreateOrUpdate(ctx, tcp) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("service not ready")) + }) + }) + + It("should return correct resource name", func() { + Expect(resource.GetName()).To(Equal("gateway_routes")) + }) + + Describe("findMatchingListener", func() { + var ( + listeners []gatewayv1.Listener + ref gatewayv1.ParentReference + ) + + BeforeEach(func() { + listeners = []gatewayv1.Listener{ + { + Name: "first", + Port: gatewayv1.PortNumber(443), + }, + { + Name: "middle", + Port: gatewayv1.PortNumber(6443), + }, + { + Name: "last", + Port: gatewayv1.PortNumber(80), + }, + } + ref = gatewayv1.ParentReference{ + Name: "test-gateway", + } + }) + + It("should return an error when sectionName is nil", func() { + listener, err := resources.FindMatchingListener(listeners, ref) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("missing sectionName")) + Expect(listener).To(Equal(gatewayv1.Listener{})) + }) + It("should return an error when sectionName is an empty string", func() { + sectionName := gatewayv1.SectionName("") + ref.SectionName = §ionName + + listener, err := resources.FindMatchingListener(listeners, ref) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("could not find listener ''")) + Expect(listener).To(Equal(gatewayv1.Listener{})) + }) + + It("should return the matching listener when sectionName points to an existing listener", func() { + sectionName := gatewayv1.SectionName("middle") + ref.SectionName = §ionName + + listener, err := resources.FindMatchingListener(listeners, ref) + Expect(err).NotTo(HaveOccurred()) + Expect(listener.Port).To(Equal(gatewayv1.PortNumber(6443))) + }) + + It("should return an error when sectionName points to a non-existent listener", func() { + sectionName := gatewayv1.SectionName("non-existent") + ref.SectionName = §ionName + + listener, err := resources.FindMatchingListener(listeners, ref) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("could not find listener 'non-existent'")) + Expect(listener).To(Equal(gatewayv1.Listener{})) + }) + + It("should return the first listener", func() { + sectionName := gatewayv1.SectionName("first") + ref.SectionName = §ionName + + listener, err := resources.FindMatchingListener(listeners, ref) + Expect(err).NotTo(HaveOccurred()) + Expect(listener.Port).To(Equal(gatewayv1.PortNumber(443))) + }) + + It("should return the last listener when matching by name", func() { + sectionName := gatewayv1.SectionName("last") + ref.SectionName = §ionName + + listener, err := resources.FindMatchingListener(listeners, ref) + Expect(err).NotTo(HaveOccurred()) + Expect(listener.Port).To(Equal(gatewayv1.PortNumber(80))) + }) + }) +}) diff --git a/internal/resources/kubeadm_config.go b/internal/resources/kubeadm_config.go index 0ff7d6f..832f889 100644 --- a/internal/resources/kubeadm_config.go +++ b/internal/resources/kubeadm_config.go @@ -77,14 +77,6 @@ func (r *KubeadmConfigResource) UpdateTenantControlPlaneStatus(_ context.Context return nil } -func (r *KubeadmConfigResource) getControlPlaneEndpoint(ingress *kamajiv1alpha1.IngressSpec, address string, port int32) string { - if ingress != nil && len(ingress.Hostname) > 0 { - address, port = utilities.GetControlPlaneAddressAndPortFromHostname(ingress.Hostname, port) - } - - return net.JoinHostPort(address, strconv.FormatInt(int64(port), 10)) -} - func (r *KubeadmConfigResource) mutate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) controllerutil.MutateFn { return func() error { logger := log.FromContext(ctx, "resource", r.GetName()) @@ -98,12 +90,27 @@ func (r *KubeadmConfigResource) mutate(ctx context.Context, tenantControlPlane * r.resource.SetLabels(utilities.MergeMaps(r.resource.GetLabels(), utilities.KamajiLabels(tenantControlPlane.GetName(), r.GetName()))) + endpoint := net.JoinHostPort(address, strconv.FormatInt(int64(port), 10)) + spec := tenantControlPlane.Spec.ControlPlane + if spec.Gateway != nil { + if len(spec.Gateway.Hostname) > 0 { + gaddr, gport := utilities.GetControlPlaneAddressAndPortFromHostname(string(spec.Gateway.Hostname), port) + endpoint = net.JoinHostPort(gaddr, strconv.FormatInt(int64(gport), 10)) + } + } + if spec.Ingress != nil { + if len(spec.Ingress.Hostname) > 0 { + iaddr, iport := utilities.GetControlPlaneAddressAndPortFromHostname(spec.Ingress.Hostname, port) + endpoint = net.JoinHostPort(iaddr, strconv.FormatInt(int64(iport), 10)) + } + } + params := kubeadm.Parameters{ TenantControlPlaneAddress: address, TenantControlPlanePort: port, TenantControlPlaneName: tenantControlPlane.GetName(), TenantControlPlaneNamespace: tenantControlPlane.GetNamespace(), - TenantControlPlaneEndpoint: r.getControlPlaneEndpoint(tenantControlPlane.Spec.ControlPlane.Ingress, address, port), + TenantControlPlaneEndpoint: endpoint, TenantControlPlaneCertSANs: tenantControlPlane.Spec.NetworkProfile.CertSANs, TenantControlPlaneClusterDomain: tenantControlPlane.Spec.NetworkProfile.ClusterDomain, TenantControlPlanePodCIDR: tenantControlPlane.Spec.NetworkProfile.PodCIDR, diff --git a/internal/resources/metrics.go b/internal/resources/metrics.go index 86cb989..6076ca5 100644 --- a/internal/resources/metrics.go +++ b/internal/resources/metrics.go @@ -16,6 +16,7 @@ var ( frontproxycaCollector prometheus.Histogram deploymentCollector prometheus.Histogram ingressCollector prometheus.Histogram + gatewayCollector prometheus.Histogram serviceCollector prometheus.Histogram kubeadmconfigCollector prometheus.Histogram kubeadmupgradeCollector prometheus.Histogram diff --git a/internal/utilities/gateway_discovery.go b/internal/utilities/gateway_discovery.go new file mode 100644 index 0000000..628c443 --- /dev/null +++ b/internal/utilities/gateway_discovery.go @@ -0,0 +1,109 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package utilities + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" +) + +// AreGatewayResourcesAvailable checks if Gateway API is available in the cluster through a discovery Client +// with fallback to client-based check. +func AreGatewayResourcesAvailable(ctx context.Context, c client.Client, discoveryClient discovery.DiscoveryInterface) bool { + if discoveryClient == nil { + return IsGatewayAPIAvailableViaClient(ctx, c) + } + + available, err := GatewayAPIResourcesAvailable(ctx, discoveryClient) + if err != nil { + return false + } + + return available +} + +// NOTE: These functions are extremely similar, maybe they can be merged and accept a GVK. +// Explicit for now. +// GatewayAPIResourcesAvailable checks if Gateway API is available in the cluster. +func GatewayAPIResourcesAvailable(ctx context.Context, discoveryClient discovery.DiscoveryInterface) (bool, error) { + gatewayAPIGroup := gatewayv1.GroupName + + serverGroups, err := discoveryClient.ServerGroups() + if err != nil { + return false, err + } + + for _, group := range serverGroups.Groups { + if group.Name == gatewayAPIGroup { + return true, nil + } + } + + return false, nil +} + +// TLSRouteAPIAvailable checks specifically for TLSRoute resource availability. +func TLSRouteAPIAvailable(ctx context.Context, discoveryClient discovery.DiscoveryInterface) (bool, error) { + gv := gatewayv1alpha2.SchemeGroupVersion + + resourceList, err := discoveryClient.ServerResourcesForGroupVersion(gv.String()) + if err != nil { + return false, err + } + + for _, resource := range resourceList.APIResources { + if resource.Kind == "TLSRoute" { + return true, nil + } + } + + return false, nil +} + +// IsTLSRouteAvailable checks if TLSRoute is available with fallback to client-based check. +func IsTLSRouteAvailable(ctx context.Context, c client.Client, discoveryClient discovery.DiscoveryInterface) bool { + if discoveryClient == nil { + return IsTLSRouteAvailableViaClient(ctx, c) + } + + available, err := TLSRouteAPIAvailable(ctx, discoveryClient) + if err != nil { + return false + } + + return available +} + +// IsTLSRouteAvailableViaClient uses client to check TLSRoute availability. +func IsTLSRouteAvailableViaClient(ctx context.Context, c client.Client) bool { + // Try to check if TLSRoute GVK can be resolved + gvk := schema.GroupVersionKind{ + Group: gatewayv1alpha2.GroupName, + Version: "v1alpha2", + Kind: "TLSRoute", + } + + restMapper := c.RESTMapper() + _, err := restMapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + if meta.IsNoMatchError(err) { + return false + } + // Other errors might be transient, assume available + return true + } + + return true +} + +// IsGatewayAPIAvailableViaClient uses client to check Gateway API availability. +func IsGatewayAPIAvailableViaClient(ctx context.Context, c client.Client) bool { + return IsTLSRouteAvailableViaClient(ctx, c) +} diff --git a/internal/webhook/handlers/tcp_gateway_validate.go b/internal/webhook/handlers/tcp_gateway_validate.go new file mode 100644 index 0000000..4efcbcd --- /dev/null +++ b/internal/webhook/handlers/tcp_gateway_validate.go @@ -0,0 +1,77 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package handlers + +import ( + "context" + "fmt" + + "gomodules.xyz/jsonpatch/v2" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/discovery" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/utilities" + "github.com/clastix/kamaji/internal/webhook/utils" +) + +type TenantControlPlaneGatewayValidation struct { + Client client.Client + DiscoveryClient discovery.DiscoveryInterface +} + +func (t TenantControlPlaneGatewayValidation) OnCreate(object runtime.Object) AdmissionResponse { + return func(ctx context.Context, _ admission.Request) ([]jsonpatch.JsonPatchOperation, error) { + tcp, ok := object.(*kamajiv1alpha1.TenantControlPlane) + if !ok { + return nil, fmt.Errorf("cannot cast object to TenantControlPlane") + } + + if tcp.Spec.ControlPlane.Gateway != nil { + // NOTE: Do we actually want to deny here if Gateway API is not available or a warning? + // Seems sensible to deny to avoid anything. + if err := t.validateGatewayAPIAvailability(ctx); err != nil { + return nil, err + } + } + + return nil, nil + } +} + +func (t TenantControlPlaneGatewayValidation) OnUpdate(object runtime.Object, _ runtime.Object) AdmissionResponse { + return func(ctx context.Context, _ admission.Request) ([]jsonpatch.JsonPatchOperation, error) { + tcp, ok := object.(*kamajiv1alpha1.TenantControlPlane) + if !ok { + return nil, fmt.Errorf("cannot cast object to TenantControlPlane") + } + + if tcp.Spec.ControlPlane.Gateway != nil { + if err := t.validateGatewayAPIAvailability(ctx); err != nil { + return nil, err + } + } + + return nil, nil + } +} + +func (t TenantControlPlaneGatewayValidation) OnDelete(object runtime.Object) AdmissionResponse { + return utils.NilOp() +} + +func (t TenantControlPlaneGatewayValidation) validateGatewayAPIAvailability(ctx context.Context) error { + if !utilities.AreGatewayResourcesAvailable(ctx, t.Client, t.DiscoveryClient) { + return fmt.Errorf("the Gateway API is not available in this cluster, cannot use gatewayRoute configuration") + } + + // Additional check for TLSRoute specifically + if !utilities.IsTLSRouteAvailable(ctx, t.Client, t.DiscoveryClient) { + return fmt.Errorf("TLSRoute resource is not available in this cluster") + } + + return nil +} diff --git a/internal/webhook/handlers/tcp_gateway_validate_test.go b/internal/webhook/handlers/tcp_gateway_validate_test.go new file mode 100644 index 0000000..d5abe84 --- /dev/null +++ b/internal/webhook/handlers/tcp_gateway_validate_test.go @@ -0,0 +1,253 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package handlers_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/discovery" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/webhook/handlers" +) + +// Mock discovery client for testing. +type mockDiscoveryClient struct { + discovery.DiscoveryInterface + serverGroups *metav1.APIGroupList + serverGroupsError error + serverResources map[string]*metav1.APIResourceList + serverResourcesError map[string]error +} + +func (m *mockDiscoveryClient) ServerGroups() (*metav1.APIGroupList, error) { + return m.serverGroups, m.serverGroupsError +} + +func (m *mockDiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (*metav1.APIResourceList, error) { + if err, exists := m.serverResourcesError[groupVersion]; exists { + return nil, err + } + if resources, exists := m.serverResources[groupVersion]; exists { + return resources, nil + } + + return &metav1.APIResourceList{}, nil +} + +var _ = Describe("TCP Gateway Validation Webhook", func() { + var ( + ctx context.Context + handler handlers.TenantControlPlaneGatewayValidation + tcp *kamajiv1alpha1.TenantControlPlane + mockClient client.Client + mockDiscovery *mockDiscoveryClient + ) + + BeforeEach(func() { + ctx = context.Background() + mockClient = nil + mockDiscovery = &mockDiscoveryClient{ + serverResources: make(map[string]*metav1.APIResourceList), + serverResourcesError: make(map[string]error), + } + + handler = handlers.TenantControlPlaneGatewayValidation{ + Client: mockClient, + DiscoveryClient: mockDiscovery, + } + + tcp = &kamajiv1alpha1.TenantControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-tcp", + Namespace: "default", + }, + Spec: kamajiv1alpha1.TenantControlPlaneSpec{}, + } + }) + + Context("when TenantControlPlane has no Gateway configuration", func() { + It("should allow creation without Gateway APIs", func() { + mockDiscovery.serverGroups = &metav1.APIGroupList{ + Groups: []metav1.APIGroup{}, + } + + _, err := handler.OnCreate(tcp)(ctx, admission.Request{}) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should allow creation with Gateway APIs available", func() { + mockDiscovery.serverGroups = &metav1.APIGroupList{ + Groups: []metav1.APIGroup{ + {Name: "gateway.networking.k8s.io"}, + }, + } + + _, err := handler.OnCreate(tcp)(ctx, admission.Request{}) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("when TenantControlPlane has Gateway configuration", func() { + BeforeEach(func() { + tcp.Spec.ControlPlane.Gateway = &kamajiv1alpha1.GatewaySpec{ + Hostname: gatewayv1.Hostname("api.example.com"), + } + }) + + Context("and Gateway APIs are available", func() { + BeforeEach(func() { + mockDiscovery.serverGroups = &metav1.APIGroupList{ + Groups: []metav1.APIGroup{ + {Name: "gateway.networking.k8s.io"}, + }, + } + mockDiscovery.serverResources["gateway.networking.k8s.io/v1alpha2"] = &metav1.APIResourceList{ + APIResources: []metav1.APIResource{ + {Kind: "TLSRoute"}, + }, + } + }) + + It("should allow creation", func() { + _, err := handler.OnCreate(tcp)(ctx, admission.Request{}) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should allow updates", func() { + oldTCP := tcp.DeepCopy() + _, err := handler.OnUpdate(tcp, oldTCP)(ctx, admission.Request{}) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("and Gateway APIs are not available", func() { + BeforeEach(func() { + mockDiscovery.serverGroups = &metav1.APIGroupList{ + Groups: []metav1.APIGroup{}, + } + }) + + It("should deny creation with clear error message", func() { + _, err := handler.OnCreate(tcp)(ctx, admission.Request{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Gateway API is not available in this cluster")) + }) + + It("should deny updates with clear error message", func() { + oldTCP := tcp.DeepCopy() + _, err := handler.OnUpdate(tcp, oldTCP)(ctx, admission.Request{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Gateway API is not available in this cluster")) + }) + }) + + Context("and Gateway API group exists but TLSRoute is not available", func() { + BeforeEach(func() { + mockDiscovery.serverGroups = &metav1.APIGroupList{ + Groups: []metav1.APIGroup{ + {Name: "gateway.networking.k8s.io"}, + }, + } + mockDiscovery.serverResources["gateway.networking.k8s.io/v1alpha2"] = &metav1.APIResourceList{ + APIResources: []metav1.APIResource{ + {Kind: "Gateway"}, + {Kind: "HTTPRoute"}, + }, + } + }) + + It("should deny creation when TLSRoute is missing", func() { + _, err := handler.OnCreate(tcp)(ctx, admission.Request{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("TLSRoute resource is not available")) + }) + }) + }) + + Context("when Gateway configuration is added in update", func() { + It("should validate Gateway APIs when adding Gateway configuration", func() { + oldTCP := tcp.DeepCopy() + + tcp.Spec.ControlPlane.Gateway = &kamajiv1alpha1.GatewaySpec{ + Hostname: gatewayv1.Hostname("api.example.com"), + } + + mockDiscovery.serverGroups = &metav1.APIGroupList{ + Groups: []metav1.APIGroup{}, + } + + _, err := handler.OnUpdate(tcp, oldTCP)(ctx, admission.Request{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Gateway API is not available")) + }) + + It("should allow removing Gateway configuration", func() { + // Start with Gateway configuration + oldTCP := tcp.DeepCopy() + oldTCP.Spec.ControlPlane.Gateway = &kamajiv1alpha1.GatewaySpec{ + Hostname: gatewayv1.Hostname("api.example.com"), + } + + mockDiscovery.serverGroups = &metav1.APIGroupList{ + Groups: []metav1.APIGroup{}, + } + + _, err := handler.OnUpdate(tcp, oldTCP)(ctx, admission.Request{}) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("OnDelete operations", func() { + It("should always allow delete operations", func() { + tcp.Spec.ControlPlane.Gateway = &kamajiv1alpha1.GatewaySpec{ + Hostname: gatewayv1.Hostname("api.example.com"), + } + + mockDiscovery.serverGroups = &metav1.APIGroupList{ + Groups: []metav1.APIGroup{}, + } + + admissionResponse := handler.OnDelete(tcp) + _, err := admissionResponse(ctx, admission.Request{}) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("with different Gateway API versions", func() { + BeforeEach(func() { + tcp.Spec.ControlPlane.Gateway = &kamajiv1alpha1.GatewaySpec{ + Hostname: gatewayv1.Hostname("api.example.com"), + } + mockDiscovery.serverGroups = &metav1.APIGroupList{ + Groups: []metav1.APIGroup{ + {Name: "gateway.networking.k8s.io"}, + }, + } + }) + + It("should work with v1alpha2 TLSRoute", func() { + mockDiscovery.serverResources["gateway.networking.k8s.io/v1alpha2"] = &metav1.APIResourceList{ + APIResources: []metav1.APIResource{ + {Kind: "TLSRoute"}, + }, + } + + _, err := handler.OnCreate(tcp)(ctx, admission.Request{}) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should handle missing version gracefully", func() { + _, err := handler.OnCreate(tcp)(ctx, admission.Request{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("TLSRoute resource is not available")) + }) + }) +})