mirror of
				https://github.com/optim-enterprises-bv/kubernetes.git
				synced 2025-11-04 04:08:16 +00:00 
			
		
		
		
	Merge pull request #127326 from stlaz/ctb_new_signer
trustbundles: add a new kube-apiserver-serving signer
This commit is contained in:
		@@ -23,14 +23,21 @@ import (
 | 
			
		||||
	"context"
 | 
			
		||||
	"fmt"
 | 
			
		||||
 | 
			
		||||
	certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1"
 | 
			
		||||
	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 | 
			
		||||
	utilfeature "k8s.io/apiserver/pkg/util/feature"
 | 
			
		||||
	"k8s.io/client-go/kubernetes"
 | 
			
		||||
	"k8s.io/component-base/featuregate"
 | 
			
		||||
	"k8s.io/controller-manager/controller"
 | 
			
		||||
	"k8s.io/klog/v2"
 | 
			
		||||
	"k8s.io/kubernetes/cmd/kube-controller-manager/names"
 | 
			
		||||
	"k8s.io/kubernetes/pkg/controller/certificates/approver"
 | 
			
		||||
	"k8s.io/kubernetes/pkg/controller/certificates/cleaner"
 | 
			
		||||
	ctbpublisher "k8s.io/kubernetes/pkg/controller/certificates/clustertrustbundlepublisher"
 | 
			
		||||
	"k8s.io/kubernetes/pkg/controller/certificates/rootcacertpublisher"
 | 
			
		||||
	"k8s.io/kubernetes/pkg/controller/certificates/signer"
 | 
			
		||||
	csrsigningconfig "k8s.io/kubernetes/pkg/controller/certificates/signer/config"
 | 
			
		||||
	"k8s.io/kubernetes/pkg/features"
 | 
			
		||||
)
 | 
			
		||||
 | 
			
		||||
func newCertificateSigningRequestSigningControllerDescriptor() *ControllerDescriptor {
 | 
			
		||||
@@ -200,16 +207,9 @@ func newRootCACertificatePublisherControllerDescriptor() *ControllerDescriptor {
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func startRootCACertificatePublisherController(ctx context.Context, controllerContext ControllerContext, controllerName string) (controller.Interface, bool, error) {
 | 
			
		||||
	var (
 | 
			
		||||
		rootCA []byte
 | 
			
		||||
		err    error
 | 
			
		||||
	)
 | 
			
		||||
	if controllerContext.ComponentConfig.SAController.RootCAFile != "" {
 | 
			
		||||
		if rootCA, err = readCA(controllerContext.ComponentConfig.SAController.RootCAFile); err != nil {
 | 
			
		||||
			return nil, true, fmt.Errorf("error parsing root-ca-file at %s: %v", controllerContext.ComponentConfig.SAController.RootCAFile, err)
 | 
			
		||||
		}
 | 
			
		||||
	} else {
 | 
			
		||||
		rootCA = controllerContext.ClientBuilder.ConfigOrDie("root-ca-cert-publisher").CAData
 | 
			
		||||
	rootCA, err := getKubeAPIServerCAFileContents(controllerContext)
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		return nil, true, err
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	sac, err := rootcacertpublisher.NewPublisher(
 | 
			
		||||
@@ -224,3 +224,77 @@ func startRootCACertificatePublisherController(ctx context.Context, controllerCo
 | 
			
		||||
	go sac.Run(ctx, 1)
 | 
			
		||||
	return nil, true, nil
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func newKubeAPIServerSignerClusterTrustBundledPublisherDescriptor() *ControllerDescriptor {
 | 
			
		||||
	return &ControllerDescriptor{
 | 
			
		||||
		name:                 names.KubeAPIServerClusterTrustBundlePublisherController,
 | 
			
		||||
		initFunc:             newKubeAPIServerSignerClusterTrustBundledPublisherController,
 | 
			
		||||
		requiredFeatureGates: []featuregate.Feature{features.ClusterTrustBundle},
 | 
			
		||||
	}
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func newKubeAPIServerSignerClusterTrustBundledPublisherController(ctx context.Context, controllerContext ControllerContext, controllerName string) (controller.Interface, bool, error) {
 | 
			
		||||
	rootCA, err := getKubeAPIServerCAFileContents(controllerContext)
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		return nil, false, err
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	if len(rootCA) == 0 || !utilfeature.DefaultFeatureGate.Enabled(features.ClusterTrustBundle) {
 | 
			
		||||
		return nil, false, nil
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	apiserverSignerClient := controllerContext.ClientBuilder.ClientOrDie("kube-apiserver-serving-clustertrustbundle-publisher")
 | 
			
		||||
	ctbAvailable, err := clusterTrustBundlesAvailable(apiserverSignerClient)
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		return nil, false, fmt.Errorf("discovery failed for ClusterTrustBundle: %w", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	if !ctbAvailable {
 | 
			
		||||
		return nil, false, nil
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	servingSigners, err := dynamiccertificates.NewStaticCAContent("kube-apiserver-serving", rootCA)
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		return nil, false, fmt.Errorf("failed to create a static CA content provider for the kube-apiserver-serving signer: %w", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	ctbPublisher, err := ctbpublisher.NewClusterTrustBundlePublisher(
 | 
			
		||||
		"kubernetes.io/kube-apiserver-serving",
 | 
			
		||||
		servingSigners,
 | 
			
		||||
		apiserverSignerClient,
 | 
			
		||||
	)
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		return nil, false, fmt.Errorf("error creating kube-apiserver-serving signer certificates publisher: %w", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	go ctbPublisher.Run(ctx)
 | 
			
		||||
	return nil, true, nil
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func clusterTrustBundlesAvailable(client kubernetes.Interface) (bool, error) {
 | 
			
		||||
	resList, err := client.Discovery().ServerResourcesForGroupVersion(certificatesv1alpha1.SchemeGroupVersion.String())
 | 
			
		||||
 | 
			
		||||
	if resList != nil {
 | 
			
		||||
		// even in case of an error above there might be a partial list for APIs that
 | 
			
		||||
		// were already successfully discovered
 | 
			
		||||
		for _, r := range resList.APIResources {
 | 
			
		||||
			if r.Name == "clustertrustbundles" {
 | 
			
		||||
				return true, nil
 | 
			
		||||
			}
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
	return false, err
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func getKubeAPIServerCAFileContents(controllerContext ControllerContext) ([]byte, error) {
 | 
			
		||||
	if controllerContext.ComponentConfig.SAController.RootCAFile == "" {
 | 
			
		||||
		return controllerContext.ClientBuilder.ConfigOrDie("root-ca-cert-publisher").CAData, nil
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	rootCA, err := readCA(controllerContext.ComponentConfig.SAController.RootCAFile)
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		return nil, fmt.Errorf("error parsing root-ca-file at %s: %w", controllerContext.ComponentConfig.SAController.RootCAFile, err)
 | 
			
		||||
	}
 | 
			
		||||
	return rootCA, nil
 | 
			
		||||
 | 
			
		||||
}
 | 
			
		||||
 
 | 
			
		||||
@@ -570,6 +570,7 @@ func NewControllerDescriptors() map[string]*ControllerDescriptor {
 | 
			
		||||
	register(newVolumeAttributesClassProtectionControllerDescriptor())
 | 
			
		||||
	register(newTTLAfterFinishedControllerDescriptor())
 | 
			
		||||
	register(newRootCACertificatePublisherControllerDescriptor())
 | 
			
		||||
	register(newKubeAPIServerSignerClusterTrustBundledPublisherDescriptor())
 | 
			
		||||
	register(newEphemeralVolumeControllerDescriptor())
 | 
			
		||||
 | 
			
		||||
	// feature gated
 | 
			
		||||
 
 | 
			
		||||
@@ -89,6 +89,7 @@ func TestControllerNamesDeclaration(t *testing.T) {
 | 
			
		||||
		names.VolumeAttributesClassProtectionController,
 | 
			
		||||
		names.TTLAfterFinishedController,
 | 
			
		||||
		names.RootCACertificatePublisherController,
 | 
			
		||||
		names.KubeAPIServerClusterTrustBundlePublisherController,
 | 
			
		||||
		names.EphemeralVolumeController,
 | 
			
		||||
		names.StorageVersionGarbageCollectorController,
 | 
			
		||||
		names.ResourceClaimController,
 | 
			
		||||
 
 | 
			
		||||
@@ -42,48 +42,49 @@ package names
 | 
			
		||||
//     3.2. when defined flag's help mentions a controller name
 | 
			
		||||
//  4. defining a new service account for a new controller (old controllers may have inconsistent service accounts to stay backwards compatible)
 | 
			
		||||
const (
 | 
			
		||||
	ServiceAccountTokenController                = "serviceaccount-token-controller"
 | 
			
		||||
	EndpointsController                          = "endpoints-controller"
 | 
			
		||||
	EndpointSliceController                      = "endpointslice-controller"
 | 
			
		||||
	EndpointSliceMirroringController             = "endpointslice-mirroring-controller"
 | 
			
		||||
	ReplicationControllerController              = "replicationcontroller-controller"
 | 
			
		||||
	PodGarbageCollectorController                = "pod-garbage-collector-controller"
 | 
			
		||||
	ResourceQuotaController                      = "resourcequota-controller"
 | 
			
		||||
	NamespaceController                          = "namespace-controller"
 | 
			
		||||
	ServiceAccountController                     = "serviceaccount-controller"
 | 
			
		||||
	GarbageCollectorController                   = "garbage-collector-controller"
 | 
			
		||||
	DaemonSetController                          = "daemonset-controller"
 | 
			
		||||
	JobController                                = "job-controller"
 | 
			
		||||
	DeploymentController                         = "deployment-controller"
 | 
			
		||||
	ReplicaSetController                         = "replicaset-controller"
 | 
			
		||||
	HorizontalPodAutoscalerController            = "horizontal-pod-autoscaler-controller"
 | 
			
		||||
	DisruptionController                         = "disruption-controller"
 | 
			
		||||
	StatefulSetController                        = "statefulset-controller"
 | 
			
		||||
	CronJobController                            = "cronjob-controller"
 | 
			
		||||
	CertificateSigningRequestSigningController   = "certificatesigningrequest-signing-controller"
 | 
			
		||||
	CertificateSigningRequestApprovingController = "certificatesigningrequest-approving-controller"
 | 
			
		||||
	CertificateSigningRequestCleanerController   = "certificatesigningrequest-cleaner-controller"
 | 
			
		||||
	TTLController                                = "ttl-controller"
 | 
			
		||||
	BootstrapSignerController                    = "bootstrap-signer-controller"
 | 
			
		||||
	TokenCleanerController                       = "token-cleaner-controller"
 | 
			
		||||
	NodeIpamController                           = "node-ipam-controller"
 | 
			
		||||
	NodeLifecycleController                      = "node-lifecycle-controller"
 | 
			
		||||
	TaintEvictionController                      = "taint-eviction-controller"
 | 
			
		||||
	PersistentVolumeBinderController             = "persistentvolume-binder-controller"
 | 
			
		||||
	PersistentVolumeAttachDetachController       = "persistentvolume-attach-detach-controller"
 | 
			
		||||
	PersistentVolumeExpanderController           = "persistentvolume-expander-controller"
 | 
			
		||||
	ClusterRoleAggregationController             = "clusterrole-aggregation-controller"
 | 
			
		||||
	PersistentVolumeClaimProtectionController    = "persistentvolumeclaim-protection-controller"
 | 
			
		||||
	PersistentVolumeProtectionController         = "persistentvolume-protection-controller"
 | 
			
		||||
	TTLAfterFinishedController                   = "ttl-after-finished-controller"
 | 
			
		||||
	RootCACertificatePublisherController         = "root-ca-certificate-publisher-controller"
 | 
			
		||||
	EphemeralVolumeController                    = "ephemeral-volume-controller"
 | 
			
		||||
	StorageVersionGarbageCollectorController     = "storageversion-garbage-collector-controller"
 | 
			
		||||
	ResourceClaimController                      = "resourceclaim-controller"
 | 
			
		||||
	LegacyServiceAccountTokenCleanerController   = "legacy-serviceaccount-token-cleaner-controller"
 | 
			
		||||
	ValidatingAdmissionPolicyStatusController    = "validatingadmissionpolicy-status-controller"
 | 
			
		||||
	VolumeAttributesClassProtectionController    = "volumeattributesclass-protection-controller"
 | 
			
		||||
	ServiceCIDRController                        = "service-cidr-controller"
 | 
			
		||||
	StorageVersionMigratorController             = "storage-version-migrator-controller"
 | 
			
		||||
	SELinuxWarningController                     = "selinux-warning-controller"
 | 
			
		||||
	ServiceAccountTokenController                      = "serviceaccount-token-controller"
 | 
			
		||||
	EndpointsController                                = "endpoints-controller"
 | 
			
		||||
	EndpointSliceController                            = "endpointslice-controller"
 | 
			
		||||
	EndpointSliceMirroringController                   = "endpointslice-mirroring-controller"
 | 
			
		||||
	ReplicationControllerController                    = "replicationcontroller-controller"
 | 
			
		||||
	PodGarbageCollectorController                      = "pod-garbage-collector-controller"
 | 
			
		||||
	ResourceQuotaController                            = "resourcequota-controller"
 | 
			
		||||
	NamespaceController                                = "namespace-controller"
 | 
			
		||||
	ServiceAccountController                           = "serviceaccount-controller"
 | 
			
		||||
	GarbageCollectorController                         = "garbage-collector-controller"
 | 
			
		||||
	DaemonSetController                                = "daemonset-controller"
 | 
			
		||||
	JobController                                      = "job-controller"
 | 
			
		||||
	DeploymentController                               = "deployment-controller"
 | 
			
		||||
	ReplicaSetController                               = "replicaset-controller"
 | 
			
		||||
	HorizontalPodAutoscalerController                  = "horizontal-pod-autoscaler-controller"
 | 
			
		||||
	DisruptionController                               = "disruption-controller"
 | 
			
		||||
	StatefulSetController                              = "statefulset-controller"
 | 
			
		||||
	CronJobController                                  = "cronjob-controller"
 | 
			
		||||
	CertificateSigningRequestSigningController         = "certificatesigningrequest-signing-controller"
 | 
			
		||||
	CertificateSigningRequestApprovingController       = "certificatesigningrequest-approving-controller"
 | 
			
		||||
	CertificateSigningRequestCleanerController         = "certificatesigningrequest-cleaner-controller"
 | 
			
		||||
	TTLController                                      = "ttl-controller"
 | 
			
		||||
	BootstrapSignerController                          = "bootstrap-signer-controller"
 | 
			
		||||
	TokenCleanerController                             = "token-cleaner-controller"
 | 
			
		||||
	NodeIpamController                                 = "node-ipam-controller"
 | 
			
		||||
	NodeLifecycleController                            = "node-lifecycle-controller"
 | 
			
		||||
	TaintEvictionController                            = "taint-eviction-controller"
 | 
			
		||||
	PersistentVolumeBinderController                   = "persistentvolume-binder-controller"
 | 
			
		||||
	PersistentVolumeAttachDetachController             = "persistentvolume-attach-detach-controller"
 | 
			
		||||
	PersistentVolumeExpanderController                 = "persistentvolume-expander-controller"
 | 
			
		||||
	ClusterRoleAggregationController                   = "clusterrole-aggregation-controller"
 | 
			
		||||
	PersistentVolumeClaimProtectionController          = "persistentvolumeclaim-protection-controller"
 | 
			
		||||
	PersistentVolumeProtectionController               = "persistentvolume-protection-controller"
 | 
			
		||||
	TTLAfterFinishedController                         = "ttl-after-finished-controller"
 | 
			
		||||
	RootCACertificatePublisherController               = "root-ca-certificate-publisher-controller"
 | 
			
		||||
	KubeAPIServerClusterTrustBundlePublisherController = "kube-apiserver-serving-clustertrustbundle-publisher-controller"
 | 
			
		||||
	EphemeralVolumeController                          = "ephemeral-volume-controller"
 | 
			
		||||
	StorageVersionGarbageCollectorController           = "storageversion-garbage-collector-controller"
 | 
			
		||||
	ResourceClaimController                            = "resourceclaim-controller"
 | 
			
		||||
	LegacyServiceAccountTokenCleanerController         = "legacy-serviceaccount-token-cleaner-controller"
 | 
			
		||||
	ValidatingAdmissionPolicyStatusController          = "validatingadmissionpolicy-status-controller"
 | 
			
		||||
	VolumeAttributesClassProtectionController          = "volumeattributesclass-protection-controller"
 | 
			
		||||
	ServiceCIDRController                              = "service-cidr-controller"
 | 
			
		||||
	StorageVersionMigratorController                   = "storage-version-migrator-controller"
 | 
			
		||||
	SELinuxWarningController                           = "selinux-warning-controller"
 | 
			
		||||
)
 | 
			
		||||
 
 | 
			
		||||
@@ -0,0 +1,78 @@
 | 
			
		||||
/*
 | 
			
		||||
Copyright 2024 The Kubernetes Authors.
 | 
			
		||||
 | 
			
		||||
Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
you may not use this file except in compliance with the License.
 | 
			
		||||
You may obtain a copy of the License at
 | 
			
		||||
 | 
			
		||||
    http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 | 
			
		||||
Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
See the License for the specific language governing permissions and
 | 
			
		||||
limitations under the License.
 | 
			
		||||
*/
 | 
			
		||||
 | 
			
		||||
package clustertrustbundlepublisher
 | 
			
		||||
 | 
			
		||||
import (
 | 
			
		||||
	"errors"
 | 
			
		||||
	"strconv"
 | 
			
		||||
	"sync"
 | 
			
		||||
	"time"
 | 
			
		||||
 | 
			
		||||
	apierrors "k8s.io/apimachinery/pkg/api/errors"
 | 
			
		||||
	"k8s.io/component-base/metrics"
 | 
			
		||||
	"k8s.io/component-base/metrics/legacyregistry"
 | 
			
		||||
)
 | 
			
		||||
 | 
			
		||||
// clustertrustbundlePublisher - subsystem name used by clustertrustbundle_publisher
 | 
			
		||||
const (
 | 
			
		||||
	clustertrustbundlePublisher = "clustertrustbundle_publisher"
 | 
			
		||||
)
 | 
			
		||||
 | 
			
		||||
var (
 | 
			
		||||
	syncCounter = metrics.NewCounterVec(
 | 
			
		||||
		&metrics.CounterOpts{
 | 
			
		||||
			Subsystem:      clustertrustbundlePublisher,
 | 
			
		||||
			Name:           "sync_total",
 | 
			
		||||
			Help:           "Number of syncs that occurred in cluster trust bundle publisher.",
 | 
			
		||||
			StabilityLevel: metrics.ALPHA,
 | 
			
		||||
		},
 | 
			
		||||
		[]string{"code"},
 | 
			
		||||
	)
 | 
			
		||||
	syncLatency = metrics.NewHistogramVec(
 | 
			
		||||
		&metrics.HistogramOpts{
 | 
			
		||||
			Subsystem:      clustertrustbundlePublisher,
 | 
			
		||||
			Name:           "sync_duration_seconds",
 | 
			
		||||
			Help:           "The time it took to sync a cluster trust bundle.",
 | 
			
		||||
			Buckets:        metrics.ExponentialBuckets(0.001, 2, 15),
 | 
			
		||||
			StabilityLevel: metrics.ALPHA,
 | 
			
		||||
		},
 | 
			
		||||
		[]string{"code"},
 | 
			
		||||
	)
 | 
			
		||||
)
 | 
			
		||||
 | 
			
		||||
func recordMetrics(start time.Time, err error) {
 | 
			
		||||
	code := "500"
 | 
			
		||||
	if err == nil {
 | 
			
		||||
		code = "200"
 | 
			
		||||
	} else {
 | 
			
		||||
		var statusErr apierrors.APIStatus
 | 
			
		||||
		if errors.As(err, &statusErr) && statusErr.Status().Code != 0 {
 | 
			
		||||
			code = strconv.Itoa(int(statusErr.Status().Code))
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
	syncLatency.WithLabelValues(code).Observe(time.Since(start).Seconds())
 | 
			
		||||
	syncCounter.WithLabelValues(code).Inc()
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
var once sync.Once
 | 
			
		||||
 | 
			
		||||
func registerMetrics() {
 | 
			
		||||
	once.Do(func() {
 | 
			
		||||
		legacyregistry.MustRegister(syncCounter)
 | 
			
		||||
		legacyregistry.MustRegister(syncLatency)
 | 
			
		||||
	})
 | 
			
		||||
}
 | 
			
		||||
@@ -0,0 +1,110 @@
 | 
			
		||||
/*
 | 
			
		||||
Copyright 2024 The Kubernetes Authors.
 | 
			
		||||
 | 
			
		||||
Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
you may not use this file except in compliance with the License.
 | 
			
		||||
You may obtain a copy of the License at
 | 
			
		||||
 | 
			
		||||
    http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 | 
			
		||||
Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
See the License for the specific language governing permissions and
 | 
			
		||||
limitations under the License.
 | 
			
		||||
*/
 | 
			
		||||
 | 
			
		||||
package clustertrustbundlepublisher
 | 
			
		||||
 | 
			
		||||
import (
 | 
			
		||||
	"errors"
 | 
			
		||||
	"fmt"
 | 
			
		||||
	"strings"
 | 
			
		||||
	"testing"
 | 
			
		||||
	"time"
 | 
			
		||||
 | 
			
		||||
	certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1"
 | 
			
		||||
	apierrors "k8s.io/apimachinery/pkg/api/errors"
 | 
			
		||||
	"k8s.io/component-base/metrics/legacyregistry"
 | 
			
		||||
	"k8s.io/component-base/metrics/testutil"
 | 
			
		||||
)
 | 
			
		||||
 | 
			
		||||
func TestSyncCounter(t *testing.T) {
 | 
			
		||||
	testCases := []struct {
 | 
			
		||||
		desc    string
 | 
			
		||||
		err     error
 | 
			
		||||
		metrics []string
 | 
			
		||||
		want    string
 | 
			
		||||
	}{
 | 
			
		||||
		{
 | 
			
		||||
			desc: "nil error",
 | 
			
		||||
			err:  nil,
 | 
			
		||||
			metrics: []string{
 | 
			
		||||
				"clustertrustbundle_publisher_sync_total",
 | 
			
		||||
			},
 | 
			
		||||
			want: `
 | 
			
		||||
# HELP clustertrustbundle_publisher_sync_total [ALPHA] Number of syncs that occurred in cluster trust bundle publisher.
 | 
			
		||||
# TYPE clustertrustbundle_publisher_sync_total counter
 | 
			
		||||
clustertrustbundle_publisher_sync_total{code="200"} 1
 | 
			
		||||
				`,
 | 
			
		||||
		},
 | 
			
		||||
		{
 | 
			
		||||
			desc: "kube api error",
 | 
			
		||||
			err:  apierrors.NewNotFound(certificatesv1alpha1.Resource("clustertrustbundle"), "test.test:testSigner:something"),
 | 
			
		||||
			metrics: []string{
 | 
			
		||||
				"clustertrustbundle_publisher_sync_total",
 | 
			
		||||
			},
 | 
			
		||||
			want: `
 | 
			
		||||
# HELP clustertrustbundle_publisher_sync_total [ALPHA] Number of syncs that occurred in cluster trust bundle publisher.
 | 
			
		||||
# TYPE clustertrustbundle_publisher_sync_total counter
 | 
			
		||||
clustertrustbundle_publisher_sync_total{code="404"} 1
 | 
			
		||||
				`,
 | 
			
		||||
		},
 | 
			
		||||
		{
 | 
			
		||||
			desc: "nested kube api error",
 | 
			
		||||
			err:  fmt.Errorf("oh noes: %w", apierrors.NewBadRequest("bad request!")),
 | 
			
		||||
			metrics: []string{
 | 
			
		||||
				"clustertrustbundle_publisher_sync_total",
 | 
			
		||||
			},
 | 
			
		||||
			want: `
 | 
			
		||||
# HELP clustertrustbundle_publisher_sync_total [ALPHA] Number of syncs that occurred in cluster trust bundle publisher.
 | 
			
		||||
# TYPE clustertrustbundle_publisher_sync_total counter
 | 
			
		||||
clustertrustbundle_publisher_sync_total{code="400"} 1
 | 
			
		||||
				`,
 | 
			
		||||
		},
 | 
			
		||||
		{
 | 
			
		||||
			desc: "kube api error without code",
 | 
			
		||||
			err:  &apierrors.StatusError{},
 | 
			
		||||
			metrics: []string{
 | 
			
		||||
				"clustertrustbundle_publisher_sync_total",
 | 
			
		||||
			},
 | 
			
		||||
			want: `
 | 
			
		||||
# HELP clustertrustbundle_publisher_sync_total [ALPHA] Number of syncs that occurred in cluster trust bundle publisher.
 | 
			
		||||
# TYPE clustertrustbundle_publisher_sync_total counter
 | 
			
		||||
clustertrustbundle_publisher_sync_total{code="500"} 1
 | 
			
		||||
				`,
 | 
			
		||||
		},
 | 
			
		||||
		{
 | 
			
		||||
			desc: "general error",
 | 
			
		||||
			err:  errors.New("test"),
 | 
			
		||||
			metrics: []string{
 | 
			
		||||
				"clustertrustbundle_publisher_sync_total",
 | 
			
		||||
			},
 | 
			
		||||
			want: `
 | 
			
		||||
# HELP clustertrustbundle_publisher_sync_total [ALPHA] Number of syncs that occurred in cluster trust bundle publisher.
 | 
			
		||||
# TYPE clustertrustbundle_publisher_sync_total counter
 | 
			
		||||
clustertrustbundle_publisher_sync_total{code="500"} 1
 | 
			
		||||
				`,
 | 
			
		||||
		},
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	for _, tc := range testCases {
 | 
			
		||||
		t.Run(tc.desc, func(t *testing.T) {
 | 
			
		||||
			recordMetrics(time.Now(), tc.err)
 | 
			
		||||
			defer syncCounter.Reset()
 | 
			
		||||
			if err := testutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(tc.want), tc.metrics...); err != nil {
 | 
			
		||||
				t.Fatal(err)
 | 
			
		||||
			}
 | 
			
		||||
		})
 | 
			
		||||
	}
 | 
			
		||||
}
 | 
			
		||||
@@ -0,0 +1,229 @@
 | 
			
		||||
/*
 | 
			
		||||
Copyright 2024 The Kubernetes Authors.
 | 
			
		||||
 | 
			
		||||
Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
you may not use this file except in compliance with the License.
 | 
			
		||||
You may obtain a copy of the License at
 | 
			
		||||
 | 
			
		||||
    http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 | 
			
		||||
Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
See the License for the specific language governing permissions and
 | 
			
		||||
limitations under the License.
 | 
			
		||||
*/
 | 
			
		||||
 | 
			
		||||
package clustertrustbundlepublisher
 | 
			
		||||
 | 
			
		||||
import (
 | 
			
		||||
	"context"
 | 
			
		||||
	"crypto/sha256"
 | 
			
		||||
	"fmt"
 | 
			
		||||
	"strings"
 | 
			
		||||
	"time"
 | 
			
		||||
 | 
			
		||||
	certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1"
 | 
			
		||||
	apierrors "k8s.io/apimachinery/pkg/api/errors"
 | 
			
		||||
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 | 
			
		||||
	"k8s.io/apimachinery/pkg/fields"
 | 
			
		||||
	"k8s.io/apimachinery/pkg/labels"
 | 
			
		||||
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 | 
			
		||||
	"k8s.io/apimachinery/pkg/util/wait"
 | 
			
		||||
	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 | 
			
		||||
	certinformers "k8s.io/client-go/informers/certificates/v1alpha1"
 | 
			
		||||
	clientset "k8s.io/client-go/kubernetes"
 | 
			
		||||
	certlisters "k8s.io/client-go/listers/certificates/v1alpha1"
 | 
			
		||||
	"k8s.io/client-go/tools/cache"
 | 
			
		||||
	"k8s.io/client-go/util/workqueue"
 | 
			
		||||
	"k8s.io/klog/v2"
 | 
			
		||||
)
 | 
			
		||||
 | 
			
		||||
func init() {
 | 
			
		||||
	registerMetrics()
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
type ClusterTrustBundlePublisher struct {
 | 
			
		||||
	signerName string
 | 
			
		||||
	ca         dynamiccertificates.CAContentProvider
 | 
			
		||||
 | 
			
		||||
	client clientset.Interface
 | 
			
		||||
 | 
			
		||||
	ctbInformer     cache.SharedIndexInformer
 | 
			
		||||
	ctbLister       certlisters.ClusterTrustBundleLister
 | 
			
		||||
	ctbListerSynced cache.InformerSynced
 | 
			
		||||
 | 
			
		||||
	queue workqueue.TypedRateLimitingInterface[string]
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
type caContentListener func()
 | 
			
		||||
 | 
			
		||||
func (f caContentListener) Enqueue() {
 | 
			
		||||
	f()
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
// NewClusterTrustBundlePublisher creates and maintains a cluster trust bundle object
 | 
			
		||||
// for a signer named `signerName`. The cluster trust bundle object contains the
 | 
			
		||||
// CA from the `caProvider` in its .spec.TrustBundle.
 | 
			
		||||
func NewClusterTrustBundlePublisher(
 | 
			
		||||
	signerName string,
 | 
			
		||||
	caProvider dynamiccertificates.CAContentProvider,
 | 
			
		||||
	kubeClient clientset.Interface,
 | 
			
		||||
) (*ClusterTrustBundlePublisher, error) {
 | 
			
		||||
	if len(signerName) == 0 {
 | 
			
		||||
		return nil, fmt.Errorf("signerName cannot be empty")
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	p := &ClusterTrustBundlePublisher{
 | 
			
		||||
		signerName: signerName,
 | 
			
		||||
		ca:         caProvider,
 | 
			
		||||
		client:     kubeClient,
 | 
			
		||||
 | 
			
		||||
		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
 | 
			
		||||
			workqueue.DefaultTypedControllerRateLimiter[string](),
 | 
			
		||||
			workqueue.TypedRateLimitingQueueConfig[string]{
 | 
			
		||||
				Name: "ca_cert_publisher_cluster_trust_bundles",
 | 
			
		||||
			},
 | 
			
		||||
		),
 | 
			
		||||
	}
 | 
			
		||||
	p.ctbInformer = setupSignerNameFilteredCTBInformer(p.client, p.signerName)
 | 
			
		||||
	p.ctbLister = certlisters.NewClusterTrustBundleLister(p.ctbInformer.GetIndexer())
 | 
			
		||||
	p.ctbListerSynced = p.ctbInformer.HasSynced
 | 
			
		||||
 | 
			
		||||
	_, err := p.ctbInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 | 
			
		||||
		AddFunc: func(obj interface{}) {
 | 
			
		||||
			p.queue.Add("")
 | 
			
		||||
		},
 | 
			
		||||
		UpdateFunc: func(_, _ interface{}) {
 | 
			
		||||
			p.queue.Add("")
 | 
			
		||||
		},
 | 
			
		||||
		DeleteFunc: func(_ interface{}) {
 | 
			
		||||
			p.queue.Add("")
 | 
			
		||||
		},
 | 
			
		||||
	})
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		return nil, fmt.Errorf("failed to register ClusterTrustBundle event handler: %w", err)
 | 
			
		||||
	}
 | 
			
		||||
	p.ca.AddListener(p.caContentChangedListener())
 | 
			
		||||
 | 
			
		||||
	return p, nil
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func (p *ClusterTrustBundlePublisher) caContentChangedListener() dynamiccertificates.Listener {
 | 
			
		||||
	return caContentListener(func() {
 | 
			
		||||
		p.queue.Add("")
 | 
			
		||||
	})
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func (p *ClusterTrustBundlePublisher) Run(ctx context.Context) {
 | 
			
		||||
	defer utilruntime.HandleCrash()
 | 
			
		||||
	defer p.queue.ShutDown()
 | 
			
		||||
 | 
			
		||||
	logger := klog.FromContext(ctx)
 | 
			
		||||
	logger.Info("Starting ClusterTrustBundle CA cert publisher controller")
 | 
			
		||||
	defer logger.Info("Shutting down ClusterTrustBundle CA cert publisher controller")
 | 
			
		||||
 | 
			
		||||
	go p.ctbInformer.Run(ctx.Done())
 | 
			
		||||
 | 
			
		||||
	if !cache.WaitForNamedCacheSync("cluster trust bundle", ctx.Done(), p.ctbListerSynced) {
 | 
			
		||||
		return
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	// init the signer syncer
 | 
			
		||||
	p.queue.Add("")
 | 
			
		||||
	go wait.UntilWithContext(ctx, p.runWorker(), time.Second)
 | 
			
		||||
 | 
			
		||||
	<-ctx.Done()
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func (p *ClusterTrustBundlePublisher) runWorker() func(context.Context) {
 | 
			
		||||
	return func(ctx context.Context) {
 | 
			
		||||
		for p.processNextWorkItem(ctx) {
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
// processNextWorkItem deals with one key off the queue. It returns false when
 | 
			
		||||
// it's time to quit.
 | 
			
		||||
func (p *ClusterTrustBundlePublisher) processNextWorkItem(ctx context.Context) bool {
 | 
			
		||||
	key, quit := p.queue.Get()
 | 
			
		||||
	if quit {
 | 
			
		||||
		return false
 | 
			
		||||
	}
 | 
			
		||||
	defer p.queue.Done(key)
 | 
			
		||||
 | 
			
		||||
	if err := p.syncClusterTrustBundle(ctx); err != nil {
 | 
			
		||||
		utilruntime.HandleError(fmt.Errorf("syncing %q failed: %w", key, err))
 | 
			
		||||
		p.queue.AddRateLimited(key)
 | 
			
		||||
		return true
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	p.queue.Forget(key)
 | 
			
		||||
	return true
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func (p *ClusterTrustBundlePublisher) syncClusterTrustBundle(ctx context.Context) (err error) {
 | 
			
		||||
	startTime := time.Now()
 | 
			
		||||
	defer func() {
 | 
			
		||||
		recordMetrics(startTime, err)
 | 
			
		||||
		klog.FromContext(ctx).V(4).Info("Finished syncing ClusterTrustBundle", "signerName", p.signerName, "elapsedTime", time.Since(startTime))
 | 
			
		||||
	}()
 | 
			
		||||
 | 
			
		||||
	caBundle := string(p.ca.CurrentCABundleContent())
 | 
			
		||||
	bundleName := constructBundleName(p.signerName, []byte(caBundle))
 | 
			
		||||
 | 
			
		||||
	bundle, err := p.ctbLister.Get(bundleName)
 | 
			
		||||
	if apierrors.IsNotFound(err) {
 | 
			
		||||
		_, err = p.client.CertificatesV1alpha1().ClusterTrustBundles().Create(ctx, &certificatesv1alpha1.ClusterTrustBundle{
 | 
			
		||||
			ObjectMeta: metav1.ObjectMeta{
 | 
			
		||||
				Name: bundleName,
 | 
			
		||||
			},
 | 
			
		||||
			Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
 | 
			
		||||
				SignerName:  p.signerName,
 | 
			
		||||
				TrustBundle: caBundle,
 | 
			
		||||
			},
 | 
			
		||||
		}, metav1.CreateOptions{})
 | 
			
		||||
	} else if err == nil && bundle.Spec.TrustBundle != caBundle {
 | 
			
		||||
		bundle = bundle.DeepCopy()
 | 
			
		||||
		bundle.Spec.TrustBundle = caBundle
 | 
			
		||||
		_, err = p.client.CertificatesV1alpha1().ClusterTrustBundles().Update(ctx, bundle, metav1.UpdateOptions{})
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		return err
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	signerTrustBundles, err := p.ctbLister.List(labels.Everything())
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		return err
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	// keep the deletion error to be returned in the end in order to retrigger the reconciliation loop
 | 
			
		||||
	var deletionError error
 | 
			
		||||
	for _, bundleObject := range signerTrustBundles {
 | 
			
		||||
		if bundleObject.Name == bundleName {
 | 
			
		||||
			continue
 | 
			
		||||
		}
 | 
			
		||||
 | 
			
		||||
		if err := p.client.CertificatesV1alpha1().ClusterTrustBundles().Delete(ctx, bundleObject.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
 | 
			
		||||
			klog.FromContext(ctx).Error(err, "failed to remove a cluster trust bundle", "bundleName", bundleObject.Name)
 | 
			
		||||
			deletionError = err
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	return deletionError
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func setupSignerNameFilteredCTBInformer(client clientset.Interface, signerName string) cache.SharedIndexInformer {
 | 
			
		||||
	return certinformers.NewFilteredClusterTrustBundleInformer(client, 0, cache.Indexers{},
 | 
			
		||||
		func(options *metav1.ListOptions) {
 | 
			
		||||
			options.FieldSelector = fields.OneTermEqualSelector("spec.signerName", signerName).String()
 | 
			
		||||
		})
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func constructBundleName(signerName string, bundleBytes []byte) string {
 | 
			
		||||
	namePrefix := strings.ReplaceAll(signerName, "/", ":") + ":"
 | 
			
		||||
	bundleHash := sha256.Sum256(bundleBytes)
 | 
			
		||||
	return fmt.Sprintf("%s%x", namePrefix, bundleHash[:12])
 | 
			
		||||
 | 
			
		||||
}
 | 
			
		||||
@@ -0,0 +1,362 @@
 | 
			
		||||
/*
 | 
			
		||||
Copyright 2024 The Kubernetes Authors.
 | 
			
		||||
 | 
			
		||||
Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
you may not use this file except in compliance with the License.
 | 
			
		||||
You may obtain a copy of the License at
 | 
			
		||||
 | 
			
		||||
    http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 | 
			
		||||
Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
See the License for the specific language governing permissions and
 | 
			
		||||
limitations under the License.
 | 
			
		||||
*/
 | 
			
		||||
 | 
			
		||||
package clustertrustbundlepublisher
 | 
			
		||||
 | 
			
		||||
import (
 | 
			
		||||
	"crypto/ecdsa"
 | 
			
		||||
	"crypto/elliptic"
 | 
			
		||||
	cryptorand "crypto/rand"
 | 
			
		||||
	"testing"
 | 
			
		||||
 | 
			
		||||
	certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1"
 | 
			
		||||
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 | 
			
		||||
	"k8s.io/apimachinery/pkg/runtime"
 | 
			
		||||
	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 | 
			
		||||
	"k8s.io/client-go/kubernetes/fake"
 | 
			
		||||
	clienttesting "k8s.io/client-go/testing"
 | 
			
		||||
	"k8s.io/client-go/tools/cache"
 | 
			
		||||
	certutil "k8s.io/client-go/util/cert"
 | 
			
		||||
	"k8s.io/kubernetes/test/utils/ktesting"
 | 
			
		||||
)
 | 
			
		||||
 | 
			
		||||
const testSignerName = "test.test/testSigner"
 | 
			
		||||
 | 
			
		||||
func TestCTBPublisherSync(t *testing.T) {
 | 
			
		||||
	checkCreatedTestSignerBundle := func(t *testing.T, actions []clienttesting.Action) {
 | 
			
		||||
		filteredActions := filterOutListWatch(actions)
 | 
			
		||||
		if len(filteredActions) != 1 {
 | 
			
		||||
			t.Fatalf("expected 1 action, got %v", filteredActions)
 | 
			
		||||
		}
 | 
			
		||||
 | 
			
		||||
		createAction := expectAction[clienttesting.CreateAction](t, filteredActions[0], "create")
 | 
			
		||||
 | 
			
		||||
		ctb, ok := createAction.GetObject().(*certificatesv1alpha1.ClusterTrustBundle)
 | 
			
		||||
		if !ok {
 | 
			
		||||
			t.Fatalf("expected ClusterTrustBundle create, got %v", createAction.GetObject())
 | 
			
		||||
		}
 | 
			
		||||
 | 
			
		||||
		if ctb.Spec.SignerName != testSignerName {
 | 
			
		||||
			t.Fatalf("expected signer name %q, got %q", testSignerName, ctb.Spec.SignerName)
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	checkUpdatedTestSignerBundle := func(expectedCABytes []byte) func(t *testing.T, actions []clienttesting.Action) {
 | 
			
		||||
		return func(t *testing.T, actions []clienttesting.Action) {
 | 
			
		||||
			filteredActions := filterOutListWatch(actions)
 | 
			
		||||
			if len(filteredActions) != 1 {
 | 
			
		||||
				t.Fatalf("expected 1 action, got %v", filteredActions)
 | 
			
		||||
			}
 | 
			
		||||
 | 
			
		||||
			updateAction := expectAction[clienttesting.UpdateAction](t, filteredActions[0], "update")
 | 
			
		||||
 | 
			
		||||
			ctb, ok := updateAction.GetObject().(*certificatesv1alpha1.ClusterTrustBundle)
 | 
			
		||||
			if !ok {
 | 
			
		||||
				t.Fatalf("expected ClusterTrustBundle update, got %v", updateAction.GetObject())
 | 
			
		||||
			}
 | 
			
		||||
 | 
			
		||||
			if ctb.Spec.SignerName != testSignerName {
 | 
			
		||||
				t.Fatalf("expected signer name %q, got %q", testSignerName, ctb.Spec.SignerName)
 | 
			
		||||
			}
 | 
			
		||||
 | 
			
		||||
			if ctb.Spec.TrustBundle != string(expectedCABytes) {
 | 
			
		||||
				t.Fatalf("expected trust bundle payload:\n%s\n, got %q", expectedCABytes, ctb.Spec.TrustBundle)
 | 
			
		||||
			}
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	checkDeletedTestSignerBundle := func(name string) func(t *testing.T, actions []clienttesting.Action) {
 | 
			
		||||
		return func(t *testing.T, actions []clienttesting.Action) {
 | 
			
		||||
			filteredActions := filterOutListWatch(actions)
 | 
			
		||||
			if len(filteredActions) != 1 {
 | 
			
		||||
				t.Fatalf("expected 1 action, got %v", filteredActions)
 | 
			
		||||
			}
 | 
			
		||||
 | 
			
		||||
			deleteAction := expectAction[clienttesting.DeleteAction](t, filteredActions[0], "delete")
 | 
			
		||||
 | 
			
		||||
			if actionName := deleteAction.GetName(); actionName != name {
 | 
			
		||||
				t.Fatalf("expected deleted CTB name %q, got %q", name, actionName)
 | 
			
		||||
			}
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	testCAProvider := testingCABundlleProvider(t)
 | 
			
		||||
	testBundleName := constructBundleName(testSignerName, testCAProvider.CurrentCABundleContent())
 | 
			
		||||
 | 
			
		||||
	for _, tt := range []struct {
 | 
			
		||||
		name          string
 | 
			
		||||
		existingCTBs  []runtime.Object
 | 
			
		||||
		expectActions func(t *testing.T, actions []clienttesting.Action)
 | 
			
		||||
		wantErr       bool
 | 
			
		||||
	}{
 | 
			
		||||
		{
 | 
			
		||||
			name:          "no CTBs exist",
 | 
			
		||||
			expectActions: checkCreatedTestSignerBundle,
 | 
			
		||||
		},
 | 
			
		||||
		{
 | 
			
		||||
			name: "no CTBs for the current signer exist",
 | 
			
		||||
			existingCTBs: []runtime.Object{
 | 
			
		||||
				&certificatesv1alpha1.ClusterTrustBundle{
 | 
			
		||||
					ObjectMeta: metav1.ObjectMeta{
 | 
			
		||||
						Name: "nosigner",
 | 
			
		||||
					},
 | 
			
		||||
					Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
 | 
			
		||||
						TrustBundle: "somedatahere",
 | 
			
		||||
					},
 | 
			
		||||
				},
 | 
			
		||||
				&certificatesv1alpha1.ClusterTrustBundle{
 | 
			
		||||
					ObjectMeta: metav1.ObjectMeta{
 | 
			
		||||
						Name: "signer:one",
 | 
			
		||||
					},
 | 
			
		||||
					Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
 | 
			
		||||
						SignerName:  "signer",
 | 
			
		||||
						TrustBundle: "signerdata",
 | 
			
		||||
					},
 | 
			
		||||
				},
 | 
			
		||||
			},
 | 
			
		||||
			expectActions: checkCreatedTestSignerBundle,
 | 
			
		||||
		},
 | 
			
		||||
		{
 | 
			
		||||
			name: "CTB for the signer exists with different content",
 | 
			
		||||
			existingCTBs: []runtime.Object{
 | 
			
		||||
				&certificatesv1alpha1.ClusterTrustBundle{
 | 
			
		||||
					ObjectMeta: metav1.ObjectMeta{
 | 
			
		||||
						Name: testBundleName,
 | 
			
		||||
					},
 | 
			
		||||
					Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
 | 
			
		||||
						SignerName:  testSignerName,
 | 
			
		||||
						TrustBundle: "olddata",
 | 
			
		||||
					},
 | 
			
		||||
				},
 | 
			
		||||
			},
 | 
			
		||||
			expectActions: checkUpdatedTestSignerBundle(testCAProvider.CurrentCABundleContent()),
 | 
			
		||||
		},
 | 
			
		||||
		{
 | 
			
		||||
			name: "multiple CTBs for the signer",
 | 
			
		||||
			existingCTBs: []runtime.Object{
 | 
			
		||||
				&certificatesv1alpha1.ClusterTrustBundle{
 | 
			
		||||
					ObjectMeta: metav1.ObjectMeta{
 | 
			
		||||
						Name: testBundleName,
 | 
			
		||||
					},
 | 
			
		||||
					Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
 | 
			
		||||
						SignerName:  testSignerName,
 | 
			
		||||
						TrustBundle: string(testCAProvider.CurrentCABundleContent()),
 | 
			
		||||
					},
 | 
			
		||||
				},
 | 
			
		||||
				&certificatesv1alpha1.ClusterTrustBundle{
 | 
			
		||||
					ObjectMeta: metav1.ObjectMeta{
 | 
			
		||||
						Name: "test.test/testSigner:name2",
 | 
			
		||||
					},
 | 
			
		||||
					Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
 | 
			
		||||
						SignerName:  testSignerName,
 | 
			
		||||
						TrustBundle: string(testCAProvider.CurrentCABundleContent()),
 | 
			
		||||
					},
 | 
			
		||||
				},
 | 
			
		||||
			},
 | 
			
		||||
			expectActions: checkDeletedTestSignerBundle("test.test/testSigner:name2"),
 | 
			
		||||
		},
 | 
			
		||||
		{
 | 
			
		||||
			name: "multiple CTBs for the signer - the one with the proper name needs changing",
 | 
			
		||||
			existingCTBs: []runtime.Object{
 | 
			
		||||
				&certificatesv1alpha1.ClusterTrustBundle{
 | 
			
		||||
					ObjectMeta: metav1.ObjectMeta{
 | 
			
		||||
						Name: testBundleName,
 | 
			
		||||
					},
 | 
			
		||||
					Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
 | 
			
		||||
						SignerName:  testSignerName,
 | 
			
		||||
						TrustBundle: "olddata",
 | 
			
		||||
					},
 | 
			
		||||
				},
 | 
			
		||||
				&certificatesv1alpha1.ClusterTrustBundle{
 | 
			
		||||
					ObjectMeta: metav1.ObjectMeta{
 | 
			
		||||
						Name: "test.test/testSigner:name2",
 | 
			
		||||
					},
 | 
			
		||||
					Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
 | 
			
		||||
						SignerName:  testSignerName,
 | 
			
		||||
						TrustBundle: string(testCAProvider.CurrentCABundleContent()),
 | 
			
		||||
					},
 | 
			
		||||
				},
 | 
			
		||||
			},
 | 
			
		||||
			expectActions: func(t *testing.T, actions []clienttesting.Action) {
 | 
			
		||||
				filteredActions := filterOutListWatch(actions)
 | 
			
		||||
				if len(filteredActions) != 2 {
 | 
			
		||||
					t.Fatalf("expected 2 actions, got %v", filteredActions)
 | 
			
		||||
				}
 | 
			
		||||
				checkUpdatedTestSignerBundle(testCAProvider.CurrentCABundleContent())(t, filteredActions[:1])
 | 
			
		||||
				checkDeletedTestSignerBundle("test.test/testSigner:name2")(t, filteredActions[1:])
 | 
			
		||||
			},
 | 
			
		||||
		},
 | 
			
		||||
		{
 | 
			
		||||
			name: "another CTB with a different name exists for the signer",
 | 
			
		||||
			existingCTBs: []runtime.Object{
 | 
			
		||||
				&certificatesv1alpha1.ClusterTrustBundle{
 | 
			
		||||
					ObjectMeta: metav1.ObjectMeta{
 | 
			
		||||
						Name: "test.test/testSigner:preexisting",
 | 
			
		||||
					},
 | 
			
		||||
					Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
 | 
			
		||||
						SignerName:  testSignerName,
 | 
			
		||||
						TrustBundle: string(testCAProvider.CurrentCABundleContent()),
 | 
			
		||||
					},
 | 
			
		||||
				},
 | 
			
		||||
			},
 | 
			
		||||
			expectActions: func(t *testing.T, actions []clienttesting.Action) {
 | 
			
		||||
				filteredActions := filterOutListWatch(actions)
 | 
			
		||||
				if len(filteredActions) != 2 {
 | 
			
		||||
					t.Fatalf("expected 2 actions, got %v", filteredActions)
 | 
			
		||||
				}
 | 
			
		||||
				checkCreatedTestSignerBundle(t, filteredActions[:1])
 | 
			
		||||
				checkDeletedTestSignerBundle("test.test/testSigner:preexisting")(t, filteredActions[1:])
 | 
			
		||||
			},
 | 
			
		||||
		},
 | 
			
		||||
		{
 | 
			
		||||
			name: "CTB at the correct state - noop",
 | 
			
		||||
			existingCTBs: []runtime.Object{
 | 
			
		||||
				&certificatesv1alpha1.ClusterTrustBundle{
 | 
			
		||||
					ObjectMeta: metav1.ObjectMeta{
 | 
			
		||||
						Name: "nosigner",
 | 
			
		||||
					},
 | 
			
		||||
					Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
 | 
			
		||||
						TrustBundle: "somedatahere",
 | 
			
		||||
					},
 | 
			
		||||
				},
 | 
			
		||||
				&certificatesv1alpha1.ClusterTrustBundle{
 | 
			
		||||
					ObjectMeta: metav1.ObjectMeta{
 | 
			
		||||
						Name: "signer:one",
 | 
			
		||||
					},
 | 
			
		||||
					Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
 | 
			
		||||
						SignerName:  "signer",
 | 
			
		||||
						TrustBundle: "signerdata",
 | 
			
		||||
					},
 | 
			
		||||
				},
 | 
			
		||||
				&certificatesv1alpha1.ClusterTrustBundle{
 | 
			
		||||
					ObjectMeta: metav1.ObjectMeta{
 | 
			
		||||
						Name: testBundleName,
 | 
			
		||||
					},
 | 
			
		||||
					Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
 | 
			
		||||
						SignerName:  testSignerName,
 | 
			
		||||
						TrustBundle: string(testCAProvider.CurrentCABundleContent()),
 | 
			
		||||
					},
 | 
			
		||||
				},
 | 
			
		||||
			},
 | 
			
		||||
			expectActions: func(t *testing.T, actions []clienttesting.Action) {
 | 
			
		||||
				actions = filterOutListWatch(actions)
 | 
			
		||||
				if len(actions) != 0 {
 | 
			
		||||
					t.Fatalf("expected no actions, got %v", actions)
 | 
			
		||||
				}
 | 
			
		||||
			},
 | 
			
		||||
		},
 | 
			
		||||
	} {
 | 
			
		||||
		t.Run(tt.name, func(t *testing.T) {
 | 
			
		||||
			testCtx := ktesting.Init(t)
 | 
			
		||||
 | 
			
		||||
			fakeClient := fakeKubeClientSetWithCTBList(t, testSignerName, tt.existingCTBs...)
 | 
			
		||||
 | 
			
		||||
			p, err := NewClusterTrustBundlePublisher(testSignerName, testCAProvider, fakeClient)
 | 
			
		||||
			if err != nil {
 | 
			
		||||
				t.Fatalf("failed to set up a new cluster trust bundle publisher: %v", err)
 | 
			
		||||
			}
 | 
			
		||||
 | 
			
		||||
			go p.ctbInformer.Run(testCtx.Done())
 | 
			
		||||
			if !cache.WaitForCacheSync(testCtx.Done(), p.ctbInformer.HasSynced) {
 | 
			
		||||
				t.Fatal("timed out waiting for informer to sync")
 | 
			
		||||
			}
 | 
			
		||||
 | 
			
		||||
			if err := p.syncClusterTrustBundle(testCtx); (err != nil) != tt.wantErr {
 | 
			
		||||
				t.Errorf("syncClusterTrustBundle() error = %v, wantErr %v", err, tt.wantErr)
 | 
			
		||||
			}
 | 
			
		||||
 | 
			
		||||
			tt.expectActions(t, fakeClient.Actions())
 | 
			
		||||
		})
 | 
			
		||||
	}
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func fakeKubeClientSetWithCTBList(t *testing.T, signerName string, ctbs ...runtime.Object) *fake.Clientset {
 | 
			
		||||
	fakeClient := fake.NewSimpleClientset(ctbs...)
 | 
			
		||||
	fakeClient.PrependReactor("list", "clustertrustbundles", func(action clienttesting.Action) (handled bool, ret runtime.Object, err error) {
 | 
			
		||||
		listAction, ok := action.(clienttesting.ListAction)
 | 
			
		||||
		if !ok {
 | 
			
		||||
			t.Fatalf("expected list action, got %v", action)
 | 
			
		||||
		}
 | 
			
		||||
 | 
			
		||||
		// fakeClient does not implement field selector, we have to do it manually
 | 
			
		||||
		listRestrictions := listAction.GetListRestrictions()
 | 
			
		||||
		if listRestrictions.Fields == nil || listRestrictions.Fields.String() != ("spec.signerName="+signerName) {
 | 
			
		||||
			return false, nil, nil
 | 
			
		||||
		}
 | 
			
		||||
 | 
			
		||||
		retList := &certificatesv1alpha1.ClusterTrustBundleList{}
 | 
			
		||||
		for _, ctb := range ctbs {
 | 
			
		||||
			ctbObj, ok := ctb.(*certificatesv1alpha1.ClusterTrustBundle)
 | 
			
		||||
			if !ok {
 | 
			
		||||
				continue
 | 
			
		||||
			}
 | 
			
		||||
			if ctbObj.Spec.SignerName == testSignerName {
 | 
			
		||||
				retList.Items = append(retList.Items, *ctbObj)
 | 
			
		||||
			}
 | 
			
		||||
		}
 | 
			
		||||
 | 
			
		||||
		return true, retList, nil
 | 
			
		||||
	})
 | 
			
		||||
 | 
			
		||||
	return fakeClient
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func expectAction[A clienttesting.Action](t *testing.T, action clienttesting.Action, verb string) A {
 | 
			
		||||
	if action.GetVerb() != verb {
 | 
			
		||||
		t.Fatalf("expected action with verb %q, got %q", verb, action.GetVerb())
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	retAction, ok := action.(A)
 | 
			
		||||
	if !ok {
 | 
			
		||||
		t.Fatalf("expected %T action, got %v", *new(A), action)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	return retAction
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func filterOutListWatch(actions []clienttesting.Action) []clienttesting.Action {
 | 
			
		||||
	var filtered []clienttesting.Action
 | 
			
		||||
	for _, a := range actions {
 | 
			
		||||
		if a.Matches("list", "clustertrustbundles") || a.Matches("watch", "clustertrustbundles") {
 | 
			
		||||
			continue
 | 
			
		||||
		}
 | 
			
		||||
		filtered = append(filtered, a)
 | 
			
		||||
	}
 | 
			
		||||
	return filtered
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func testingCABundlleProvider(t *testing.T) dynamiccertificates.CAContentProvider {
 | 
			
		||||
	key, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		t.Fatalf("failed to create a private key: %v", err)
 | 
			
		||||
	}
 | 
			
		||||
	caCert, err := certutil.NewSelfSignedCACert(certutil.Config{CommonName: "test-ca"}, key)
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		t.Fatalf("failed to create a self-signed CA cert: %v", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	caPEM, err := certutil.EncodeCertificates(caCert)
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		t.Fatalf("failed to PEM-encode cert: %v", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	caProvider, err := dynamiccertificates.NewStaticCAContent("testca", caPEM)
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		t.Fatalf("failed to create a static CA provider: %v", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	return caProvider
 | 
			
		||||
}
 | 
			
		||||
@@ -453,6 +453,18 @@ func buildControllerRoles() ([]rbacv1.ClusterRole, []rbacv1.ClusterRoleBinding)
 | 
			
		||||
			eventsRule(),
 | 
			
		||||
		},
 | 
			
		||||
	})
 | 
			
		||||
 | 
			
		||||
	if utilfeature.DefaultFeatureGate.Enabled(features.ClusterTrustBundle) {
 | 
			
		||||
		addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{
 | 
			
		||||
			ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "kube-apiserver-serving-clustertrustbundle-publisher"},
 | 
			
		||||
			Rules: []rbacv1.PolicyRule{
 | 
			
		||||
				rbacv1helpers.NewRule("attest").Groups(certificatesGroup).Resources("signers").Names("kubernetes.io/kube-apiserver-serving").RuleOrDie(),
 | 
			
		||||
				rbacv1helpers.NewRule("create", "update", "delete", "list", "watch").Groups(certificatesGroup).Resources("clustertrustbundles").RuleOrDie(),
 | 
			
		||||
				eventsRule(),
 | 
			
		||||
			},
 | 
			
		||||
		})
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	addControllerRole(&controllerRoles, &controllerRoleBindings, rbacv1.ClusterRole{
 | 
			
		||||
		ObjectMeta: metav1.ObjectMeta{Name: saRolePrefix + "validatingadmissionpolicy-status-controller"},
 | 
			
		||||
		Rules: []rbacv1.PolicyRule{
 | 
			
		||||
 
 | 
			
		||||
							
								
								
									
										257
									
								
								test/integration/clustertrustbundles/apiserversigner_test.go
									
									
									
									
									
										Normal file
									
								
							
							
						
						
									
										257
									
								
								test/integration/clustertrustbundles/apiserversigner_test.go
									
									
									
									
									
										Normal file
									
								
							@@ -0,0 +1,257 @@
 | 
			
		||||
/*
 | 
			
		||||
Copyright 2024 The Kubernetes Authors.
 | 
			
		||||
 | 
			
		||||
Licensed under the Apache License, Version 2.0 (the "License");
 | 
			
		||||
you may not use this file except in compliance with the License.
 | 
			
		||||
You may obtain a copy of the License at
 | 
			
		||||
 | 
			
		||||
    http://www.apache.org/licenses/LICENSE-2.0
 | 
			
		||||
 | 
			
		||||
Unless required by applicable law or agreed to in writing, software
 | 
			
		||||
distributed under the License is distributed on an "AS IS" BASIS,
 | 
			
		||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | 
			
		||||
See the License for the specific language governing permissions and
 | 
			
		||||
limitations under the License.
 | 
			
		||||
*/
 | 
			
		||||
 | 
			
		||||
package clustertrustbundles
 | 
			
		||||
 | 
			
		||||
import (
 | 
			
		||||
	"context"
 | 
			
		||||
	"crypto/x509"
 | 
			
		||||
	"crypto/x509/pkix"
 | 
			
		||||
	"encoding/pem"
 | 
			
		||||
	"fmt"
 | 
			
		||||
	"math/big"
 | 
			
		||||
	"path/filepath"
 | 
			
		||||
	"regexp"
 | 
			
		||||
	"strconv"
 | 
			
		||||
	"strings"
 | 
			
		||||
	"testing"
 | 
			
		||||
	"time"
 | 
			
		||||
 | 
			
		||||
	"k8s.io/api/certificates/v1alpha1"
 | 
			
		||||
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 | 
			
		||||
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme"
 | 
			
		||||
	"k8s.io/apimachinery/pkg/runtime/schema"
 | 
			
		||||
	"k8s.io/apimachinery/pkg/util/wait"
 | 
			
		||||
	clientset "k8s.io/client-go/kubernetes"
 | 
			
		||||
	"k8s.io/client-go/rest"
 | 
			
		||||
	"k8s.io/client-go/tools/clientcmd"
 | 
			
		||||
	certutil "k8s.io/client-go/util/cert"
 | 
			
		||||
	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
 | 
			
		||||
	kubecontrollermanagertesting "k8s.io/kubernetes/cmd/kube-controller-manager/app/testing"
 | 
			
		||||
	"k8s.io/kubernetes/test/integration/framework"
 | 
			
		||||
	"k8s.io/kubernetes/test/utils/ktesting"
 | 
			
		||||
	"k8s.io/kubernetes/test/utils/kubeconfig"
 | 
			
		||||
)
 | 
			
		||||
 | 
			
		||||
func TestClusterTrustBundlesPublisherController(t *testing.T) {
 | 
			
		||||
	// KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE allows for APIs pending removal to not block tests
 | 
			
		||||
	// TODO: Remove this line once certificates v1alpha1 types to be removed in 1.32 are fully removed
 | 
			
		||||
	t.Setenv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE", "true")
 | 
			
		||||
	ctx := ktesting.Init(t)
 | 
			
		||||
 | 
			
		||||
	certBytes := mustMakeCertificate(t, &x509.Certificate{
 | 
			
		||||
		SerialNumber: big.NewInt(0),
 | 
			
		||||
		Subject: pkix.Name{
 | 
			
		||||
			CommonName: "testsigner-kas",
 | 
			
		||||
		},
 | 
			
		||||
		IsCA:                  true,
 | 
			
		||||
		BasicConstraintsValid: true,
 | 
			
		||||
	})
 | 
			
		||||
 | 
			
		||||
	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certBytes})
 | 
			
		||||
 | 
			
		||||
	tmpDir := t.TempDir()
 | 
			
		||||
	cacertPath := filepath.Join(tmpDir, "kube-apiserver-serving.crt")
 | 
			
		||||
	if err := certutil.WriteCert(cacertPath, certPEM); err != nil {
 | 
			
		||||
		t.Fatalf("failed to write the CA cert into a file: %v", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	apiServerFlags := []string{
 | 
			
		||||
		"--disable-admission-plugins", "ServiceAccount",
 | 
			
		||||
		"--authorization-mode=RBAC",
 | 
			
		||||
		"--feature-gates", "ClusterTrustBundle=true",
 | 
			
		||||
	}
 | 
			
		||||
	storageConfig := framework.SharedEtcd()
 | 
			
		||||
	server := kubeapiservertesting.StartTestServerOrDie(t, nil, apiServerFlags, storageConfig)
 | 
			
		||||
	defer server.TearDownFn()
 | 
			
		||||
 | 
			
		||||
	kubeConfigFile := createKubeConfigFileForRestConfig(t, server.ClientConfig)
 | 
			
		||||
 | 
			
		||||
	kcm := kubecontrollermanagertesting.StartTestServerOrDie(ctx, []string{
 | 
			
		||||
		"--kubeconfig=" + kubeConfigFile,
 | 
			
		||||
		"--controllers=kube-apiserver-serving-clustertrustbundle-publisher-controller", // these are the only controllers needed for this test
 | 
			
		||||
		"--use-service-account-credentials=true",                                       // exercise RBAC of kube-apiserver-serving-clustertrustbundle-publisher controller
 | 
			
		||||
		"--leader-elect=false",                                                         // KCM leader election calls os.Exit when it ends, so it is easier to just turn it off altogether
 | 
			
		||||
		"--root-ca-file=" + cacertPath,
 | 
			
		||||
		"--feature-gates=ClusterTrustBundle=true",
 | 
			
		||||
	})
 | 
			
		||||
	defer kcm.TearDownFn()
 | 
			
		||||
 | 
			
		||||
	// setup finished, tests follow
 | 
			
		||||
	clientSet, err := clientset.NewForConfig(server.ClientConfig)
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		t.Fatalf("error in create clientset: %v", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	unrelatedSigner := mustMakeCertificate(t, &x509.Certificate{
 | 
			
		||||
		SerialNumber: big.NewInt(0),
 | 
			
		||||
		Subject: pkix.Name{
 | 
			
		||||
			CommonName: "testsigner-kas",
 | 
			
		||||
		},
 | 
			
		||||
		IsCA:                  true,
 | 
			
		||||
		BasicConstraintsValid: true,
 | 
			
		||||
	})
 | 
			
		||||
	unrelatedPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: unrelatedSigner})
 | 
			
		||||
	// set up a signer that's completely unrelated to the controller to check
 | 
			
		||||
	// it's not anyhow handled by it
 | 
			
		||||
	unrelatedCTB, err := clientSet.CertificatesV1alpha1().ClusterTrustBundles().Create(ctx,
 | 
			
		||||
		&v1alpha1.ClusterTrustBundle{
 | 
			
		||||
			ObjectMeta: metav1.ObjectMeta{
 | 
			
		||||
				Name: "test.test:unrelated:0",
 | 
			
		||||
			},
 | 
			
		||||
			Spec: v1alpha1.ClusterTrustBundleSpec{
 | 
			
		||||
				SignerName:  "test.test/unrelated",
 | 
			
		||||
				TrustBundle: string(unrelatedPEM),
 | 
			
		||||
			},
 | 
			
		||||
		},
 | 
			
		||||
		metav1.CreateOptions{})
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		t.Fatalf("failed to set up an unrelated signer CTB: %v", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	t.Log("check that the controller creates a single buundle with expected PEM content")
 | 
			
		||||
	waitUntilSingleKASSignerCTB(ctx, t, clientSet, certPEM)
 | 
			
		||||
 | 
			
		||||
	t.Log("check that the controller deletes any additional bundles for the same signer")
 | 
			
		||||
	if _, err := clientSet.CertificatesV1alpha1().ClusterTrustBundles().Create(ctx, &v1alpha1.ClusterTrustBundle{
 | 
			
		||||
		ObjectMeta: metav1.ObjectMeta{
 | 
			
		||||
			Name: "kubernetes.io:kube-apiserver-serving:testname",
 | 
			
		||||
		},
 | 
			
		||||
		Spec: v1alpha1.ClusterTrustBundleSpec{
 | 
			
		||||
			SignerName:  "kubernetes.io/kube-apiserver-serving",
 | 
			
		||||
			TrustBundle: string(certPEM),
 | 
			
		||||
		},
 | 
			
		||||
	}, metav1.CreateOptions{}); err != nil {
 | 
			
		||||
		t.Fatalf("failed to create an additional cluster trust bundle: %v", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	waitUntilSingleKASSignerCTB(ctx, t, clientSet, certPEM)
 | 
			
		||||
 | 
			
		||||
	t.Log("check that the controller reconciles the bundle back to its original state if changed")
 | 
			
		||||
	differentSigner := mustMakeCertificate(t, &x509.Certificate{
 | 
			
		||||
		SerialNumber: big.NewInt(0),
 | 
			
		||||
		Subject: pkix.Name{
 | 
			
		||||
			CommonName: "testsigner-kas-different",
 | 
			
		||||
		},
 | 
			
		||||
		IsCA:                  true,
 | 
			
		||||
		BasicConstraintsValid: true,
 | 
			
		||||
	})
 | 
			
		||||
	differentSignerPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: differentSigner})
 | 
			
		||||
 | 
			
		||||
	ctbList, err := clientSet.CertificatesV1alpha1().ClusterTrustBundles().List(ctx, metav1.ListOptions{
 | 
			
		||||
		FieldSelector: "spec.signerName=kubernetes.io/kube-apiserver-serving",
 | 
			
		||||
	})
 | 
			
		||||
	if err != nil || len(ctbList.Items) != 1 {
 | 
			
		||||
		t.Fatalf("failed to retrieve CTB list containing the single CTB for the KAS serving signer: %v", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	ctbToUpdate := ctbList.Items[0].DeepCopy()
 | 
			
		||||
	ctbToUpdate.Spec.TrustBundle = string(differentSignerPEM)
 | 
			
		||||
 | 
			
		||||
	if _, err = clientSet.CertificatesV1alpha1().ClusterTrustBundles().Update(ctx, ctbToUpdate, metav1.UpdateOptions{}); err != nil {
 | 
			
		||||
		t.Fatalf("failed to update ctb with new PEM bundle: %v", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	waitUntilSingleKASSignerCTB(ctx, t, clientSet, certPEM)
 | 
			
		||||
 | 
			
		||||
	unrelatedCTB, err = clientSet.CertificatesV1alpha1().ClusterTrustBundles().Get(ctx, unrelatedCTB.Name, metav1.GetOptions{})
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		t.Fatalf("failed to get the unrelated CTB back: %v", err)
 | 
			
		||||
	}
 | 
			
		||||
	if unrelatedCTB.Spec.TrustBundle != string(unrelatedPEM) {
 | 
			
		||||
		t.Fatalf("the PEM content changed for the unrelated CTB:\n%s\n", unrelatedCTB.Spec.TrustBundle)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	totalSynncs := getTotalSyncMetric(ctx, t, server.ClientConfig, "clustertrustbundle_publisher_sync_total")
 | 
			
		||||
	if totalSynncs <= 0 {
 | 
			
		||||
		t.Fatalf("expected non-zero total syncs: %d", totalSynncs)
 | 
			
		||||
	}
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func waitUntilSingleKASSignerCTB(ctx context.Context, t *testing.T, clientSet *clientset.Clientset, caPEM []byte) {
 | 
			
		||||
	err := wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, 30*time.Second, true, func(ctx context.Context) (done bool, err error) {
 | 
			
		||||
		ctbList, err := clientSet.CertificatesV1alpha1().ClusterTrustBundles().List(ctx, metav1.ListOptions{
 | 
			
		||||
			FieldSelector: "spec.signerName=kubernetes.io/kube-apiserver-serving",
 | 
			
		||||
		})
 | 
			
		||||
 | 
			
		||||
		if err != nil {
 | 
			
		||||
			t.Logf("failed to list kube-apiserver-signer trust bundles: %v", err)
 | 
			
		||||
			return false, nil
 | 
			
		||||
		}
 | 
			
		||||
 | 
			
		||||
		if len(ctbList.Items) != 1 {
 | 
			
		||||
			t.Logf("expected a single CTB, got %v", ctbList.Items)
 | 
			
		||||
			return false, nil
 | 
			
		||||
		}
 | 
			
		||||
 | 
			
		||||
		if ctbList.Items[0].Spec.TrustBundle != string(caPEM) {
 | 
			
		||||
			t.Logf("CTB trustBundles are different")
 | 
			
		||||
			return false, nil
 | 
			
		||||
		}
 | 
			
		||||
		return true, nil
 | 
			
		||||
	})
 | 
			
		||||
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		t.Fatalf("there has always been a wrong number of trust bundles: %v", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func createKubeConfigFileForRestConfig(t *testing.T, restConfig *rest.Config) string {
 | 
			
		||||
	t.Helper()
 | 
			
		||||
 | 
			
		||||
	clientConfig := kubeconfig.CreateKubeConfig(restConfig)
 | 
			
		||||
 | 
			
		||||
	kubeConfigFile := filepath.Join(t.TempDir(), "kubeconfig.yaml")
 | 
			
		||||
	if err := clientcmd.WriteToFile(*clientConfig, kubeConfigFile); err != nil {
 | 
			
		||||
		t.Fatal(err)
 | 
			
		||||
	}
 | 
			
		||||
	return kubeConfigFile
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
func getTotalSyncMetric(ctx context.Context, t *testing.T, clientConfig *rest.Config, metric string) int {
 | 
			
		||||
	t.Helper()
 | 
			
		||||
 | 
			
		||||
	copyConfig := rest.CopyConfig(clientConfig)
 | 
			
		||||
	copyConfig.GroupVersion = &schema.GroupVersion{}
 | 
			
		||||
	copyConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
 | 
			
		||||
	rc, err := rest.RESTClientFor(copyConfig)
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		t.Fatalf("Failed to create REST client: %v", err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	body, err := rc.Get().AbsPath("/metrics").DoRaw(ctx)
 | 
			
		||||
	if err != nil {
 | 
			
		||||
		t.Fatal(err)
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	metricRegex := regexp.MustCompile(fmt.Sprintf(`%s{.*} (\d+)`, metric))
 | 
			
		||||
	for _, line := range strings.Split(string(body), "\n") {
 | 
			
		||||
		if strings.HasPrefix(line, metric) {
 | 
			
		||||
			matches := metricRegex.FindStringSubmatch(line)
 | 
			
		||||
			if len(matches) == 2 {
 | 
			
		||||
				metricValue, err := strconv.Atoi(matches[1])
 | 
			
		||||
				if err != nil {
 | 
			
		||||
					t.Fatalf("Failed to convert metric value to integer: %v", err)
 | 
			
		||||
				}
 | 
			
		||||
				return metricValue
 | 
			
		||||
			}
 | 
			
		||||
		}
 | 
			
		||||
	}
 | 
			
		||||
 | 
			
		||||
	t.Fatalf("metric %q not seen in body:\n%s\n", metric, string(body))
 | 
			
		||||
	return 0
 | 
			
		||||
}
 | 
			
		||||
		Reference in New Issue
	
	Block a user