Merge pull request #128499 from stlaz/ctb_betav1

ClusterTrustBundles - move to beta
This commit is contained in:
Kubernetes Prow Robot
2025-03-11 12:47:45 -07:00
committed by GitHub
56 changed files with 6231 additions and 347 deletions

View File

@@ -23,7 +23,7 @@ import (
"testing"
"time"
certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1"
certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/component-base/metrics/legacyregistry"
"k8s.io/component-base/metrics/testutil"
@@ -50,7 +50,7 @@ clustertrustbundle_publisher_sync_total{code="200"} 1
},
{
desc: "kube api error",
err: apierrors.NewNotFound(certificatesv1alpha1.Resource("clustertrustbundle"), "test.test:testSigner:something"),
err: apierrors.NewNotFound(certificatesv1beta1.Resource("clustertrustbundle"), "test.test:testSigner:something"),
metrics: []string{
"clustertrustbundle_publisher_sync_total",
},

View File

@@ -24,6 +24,7 @@ import (
"time"
certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1"
certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
@@ -31,9 +32,11 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
certinformers "k8s.io/client-go/informers/certificates/v1alpha1"
certalpha1informers "k8s.io/client-go/informers/certificates/v1alpha1"
certbeta1informers "k8s.io/client-go/informers/certificates/v1beta1"
clientset "k8s.io/client-go/kubernetes"
certlisters "k8s.io/client-go/listers/certificates/v1alpha1"
certalphav1listers "k8s.io/client-go/listers/certificates/v1alpha1"
certbetav1listers "k8s.io/client-go/listers/certificates/v1beta1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog/v2"
@@ -43,41 +46,196 @@ func init() {
registerMetrics()
}
type ClusterTrustBundlePublisher struct {
type PublisherRunner interface {
Run(context.Context)
}
type ClusterTrustBundlePublisher[T clusterTrustBundle] struct {
signerName string
ca dynamiccertificates.CAContentProvider
client clientset.Interface
client clusterTrustBundlesClient[T]
ctbInformer cache.SharedIndexInformer
ctbLister certlisters.ClusterTrustBundleLister
ctbLister clusterTrustBundlesLister[T]
ctbListerSynced cache.InformerSynced
handlers clusterTrustBundleHandlers[T]
queue workqueue.TypedRateLimitingInterface[string]
}
// clusterTrustBundle is a type constraint grouping all APIs versions of ClusterTrustBundles
type clusterTrustBundle interface {
certificatesv1alpha1.ClusterTrustBundle | certificatesv1beta1.ClusterTrustBundle
}
// clusterTrustBundlesClient is an API-version independent client for the ClusterTrustBundles API
type clusterTrustBundlesClient[T clusterTrustBundle] interface {
Create(context.Context, *T, metav1.CreateOptions) (*T, error)
Update(context.Context, *T, metav1.UpdateOptions) (*T, error)
Delete(context.Context, string, metav1.DeleteOptions) error
}
// clusterTrustBundlesLister is an API-version independent lister for the ClusterTrustBundles API
type clusterTrustBundlesLister[T clusterTrustBundle] interface {
Get(string) (*T, error)
List(labels.Selector) ([]*T, error)
}
type clusterTrustBundleHandlers[T clusterTrustBundle] interface {
createClusterTrustBundle(bundleName, signerName, trustBundle string) *T
updateWithTrustBundle(ctbObject *T, newBundle string) *T
containsTrustBundle(ctbObject *T, bundle string) bool
getName(ctbObject *T) string
}
var _ clusterTrustBundleHandlers[certificatesv1beta1.ClusterTrustBundle] = &betaHandlers{}
var _ clusterTrustBundleHandlers[certificatesv1alpha1.ClusterTrustBundle] = &alphaHandlers{}
// betaHandlers groups the `clusterTrustBundleHandlers` for the v1beta1 API of
// clusterTrustBundles
type betaHandlers struct{}
func (w *betaHandlers) createClusterTrustBundle(bundleName, signerName, trustBundle string) *certificatesv1beta1.ClusterTrustBundle {
return &certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: bundleName,
},
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
SignerName: signerName,
TrustBundle: trustBundle,
},
}
}
func (w *betaHandlers) updateWithTrustBundle(ctbObject *certificatesv1beta1.ClusterTrustBundle, newBundle string) *certificatesv1beta1.ClusterTrustBundle {
newObj := ctbObject.DeepCopy()
newObj.Spec.TrustBundle = newBundle
return newObj
}
func (w *betaHandlers) containsTrustBundle(ctbObject *certificatesv1beta1.ClusterTrustBundle, bundle string) bool {
return ctbObject.Spec.TrustBundle == bundle
}
func (w *betaHandlers) getName(ctbObject *certificatesv1beta1.ClusterTrustBundle) string {
return ctbObject.Name
}
// alphaHandlers groups the `clusterTrustBundleHandlers` for the v1alpha1 API of
// clusterTrustBundles
type alphaHandlers struct{}
func (w *alphaHandlers) createClusterTrustBundle(bundleName, signerName, trustBundle string) *certificatesv1alpha1.ClusterTrustBundle {
return &certificatesv1alpha1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: bundleName,
},
Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
SignerName: signerName,
TrustBundle: trustBundle,
},
}
}
func (w *alphaHandlers) updateWithTrustBundle(ctbObject *certificatesv1alpha1.ClusterTrustBundle, newBundle string) *certificatesv1alpha1.ClusterTrustBundle {
newObj := ctbObject.DeepCopy()
newObj.Spec.TrustBundle = newBundle
return newObj
}
func (w *alphaHandlers) containsTrustBundle(ctbObject *certificatesv1alpha1.ClusterTrustBundle, bundle string) bool {
return ctbObject.Spec.TrustBundle == bundle
}
func (w *alphaHandlers) getName(ctbObject *certificatesv1alpha1.ClusterTrustBundle) string {
return ctbObject.Name
}
type caContentListener func()
func (f caContentListener) Enqueue() {
f()
}
// NewClusterTrustBundlePublisher creates and maintains a cluster trust bundle object
// for a signer named `signerName`. The cluster trust bundle object contains the
// CA from the `caProvider` in its .spec.TrustBundle.
func NewClusterTrustBundlePublisher(
// NewBetaClusterTrustBundlePublisher sets up a ClusterTrustBundlePublisher for the
// v1beta1 API
func NewBetaClusterTrustBundlePublisher(
signerName string,
caProvider dynamiccertificates.CAContentProvider,
kubeClient clientset.Interface,
) (*ClusterTrustBundlePublisher, error) {
) (
PublisherRunner,
error,
) {
ctbInformer := certbeta1informers.NewFilteredClusterTrustBundleInformer(kubeClient, 0, cache.Indexers{},
func(options *metav1.ListOptions) {
options.FieldSelector = fields.OneTermEqualSelector("spec.signerName", signerName).String()
})
return newClusterTrustBundlePublisher(
signerName,
caProvider,
kubeClient.CertificatesV1beta1().ClusterTrustBundles(),
ctbInformer,
certbetav1listers.NewClusterTrustBundleLister(ctbInformer.GetIndexer()),
&betaHandlers{},
)
}
// NewAlphaClusterTrustBundlePublisher sets up a ClusterTrustBundlePublisher for the
// v1alpha1 API
func NewAlphaClusterTrustBundlePublisher(
signerName string,
caProvider dynamiccertificates.CAContentProvider,
kubeClient clientset.Interface,
) (
PublisherRunner,
error,
) {
ctbInformer := certalpha1informers.NewFilteredClusterTrustBundleInformer(kubeClient, 0, cache.Indexers{},
func(options *metav1.ListOptions) {
options.FieldSelector = fields.OneTermEqualSelector("spec.signerName", signerName).String()
})
return newClusterTrustBundlePublisher(
signerName,
caProvider,
kubeClient.CertificatesV1alpha1().ClusterTrustBundles(),
ctbInformer,
certalphav1listers.NewClusterTrustBundleLister(ctbInformer.GetIndexer()),
&alphaHandlers{},
)
}
// NewClusterTrustBundlePublisher creates and maintains a cluster trust bundle object
// for a signer named `signerName`. The cluster trust bundle object contains the
// CA from the `caProvider` in its .spec.TrustBundle.
func newClusterTrustBundlePublisher[T clusterTrustBundle](
signerName string,
caProvider dynamiccertificates.CAContentProvider,
bundleClient clusterTrustBundlesClient[T],
ctbInformer cache.SharedIndexInformer,
ctbLister clusterTrustBundlesLister[T],
handlers clusterTrustBundleHandlers[T],
) (PublisherRunner, error) {
if len(signerName) == 0 {
return nil, fmt.Errorf("signerName cannot be empty")
}
p := &ClusterTrustBundlePublisher{
p := &ClusterTrustBundlePublisher[T]{
signerName: signerName,
ca: caProvider,
client: kubeClient,
client: bundleClient,
ctbInformer: ctbInformer,
ctbLister: ctbLister,
ctbListerSynced: ctbInformer.HasSynced,
handlers: handlers,
queue: workqueue.NewTypedRateLimitingQueueWithConfig(
workqueue.DefaultTypedControllerRateLimiter[string](),
@@ -86,9 +244,6 @@ func NewClusterTrustBundlePublisher(
},
),
}
p.ctbInformer = setupSignerNameFilteredCTBInformer(p.client, p.signerName)
p.ctbLister = certlisters.NewClusterTrustBundleLister(p.ctbInformer.GetIndexer())
p.ctbListerSynced = p.ctbInformer.HasSynced
_, err := p.ctbInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
@@ -109,13 +264,13 @@ func NewClusterTrustBundlePublisher(
return p, nil
}
func (p *ClusterTrustBundlePublisher) caContentChangedListener() dynamiccertificates.Listener {
func (p *ClusterTrustBundlePublisher[T]) caContentChangedListener() dynamiccertificates.Listener {
return caContentListener(func() {
p.queue.Add("")
})
}
func (p *ClusterTrustBundlePublisher) Run(ctx context.Context) {
func (p *ClusterTrustBundlePublisher[T]) Run(ctx context.Context) {
defer utilruntime.HandleCrash()
defer p.queue.ShutDown()
@@ -136,7 +291,7 @@ func (p *ClusterTrustBundlePublisher) Run(ctx context.Context) {
<-ctx.Done()
}
func (p *ClusterTrustBundlePublisher) runWorker() func(context.Context) {
func (p *ClusterTrustBundlePublisher[T]) runWorker() func(context.Context) {
return func(ctx context.Context) {
for p.processNextWorkItem(ctx) {
}
@@ -145,7 +300,7 @@ func (p *ClusterTrustBundlePublisher) runWorker() func(context.Context) {
// processNextWorkItem deals with one key off the queue. It returns false when
// it's time to quit.
func (p *ClusterTrustBundlePublisher) processNextWorkItem(ctx context.Context) bool {
func (p *ClusterTrustBundlePublisher[T]) processNextWorkItem(ctx context.Context) bool {
key, quit := p.queue.Get()
if quit {
return false
@@ -162,7 +317,7 @@ func (p *ClusterTrustBundlePublisher) processNextWorkItem(ctx context.Context) b
return true
}
func (p *ClusterTrustBundlePublisher) syncClusterTrustBundle(ctx context.Context) (err error) {
func (p *ClusterTrustBundlePublisher[T]) syncClusterTrustBundle(ctx context.Context) (err error) {
startTime := time.Now()
defer func() {
recordMetrics(startTime, err)
@@ -174,19 +329,10 @@ func (p *ClusterTrustBundlePublisher) syncClusterTrustBundle(ctx context.Context
bundle, err := p.ctbLister.Get(bundleName)
if apierrors.IsNotFound(err) {
_, err = p.client.CertificatesV1alpha1().ClusterTrustBundles().Create(ctx, &certificatesv1alpha1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: bundleName,
},
Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
SignerName: p.signerName,
TrustBundle: caBundle,
},
}, metav1.CreateOptions{})
} else if err == nil && bundle.Spec.TrustBundle != caBundle {
bundle = bundle.DeepCopy()
bundle.Spec.TrustBundle = caBundle
_, err = p.client.CertificatesV1alpha1().ClusterTrustBundles().Update(ctx, bundle, metav1.UpdateOptions{})
_, err = p.client.Create(ctx, p.handlers.createClusterTrustBundle(bundleName, p.signerName, caBundle), metav1.CreateOptions{})
} else if err == nil && !p.handlers.containsTrustBundle(bundle, caBundle) {
updatedBundle := p.handlers.updateWithTrustBundle(bundle, caBundle)
_, err = p.client.Update(ctx, updatedBundle, metav1.UpdateOptions{})
}
if err != nil {
@@ -201,12 +347,13 @@ func (p *ClusterTrustBundlePublisher) syncClusterTrustBundle(ctx context.Context
// keep the deletion error to be returned in the end in order to retrigger the reconciliation loop
var deletionError error
for _, bundleObject := range signerTrustBundles {
if bundleObject.Name == bundleName {
if p.handlers.getName(bundleObject) == bundleName {
continue
}
if err := p.client.CertificatesV1alpha1().ClusterTrustBundles().Delete(ctx, bundleObject.Name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
klog.FromContext(ctx).Error(err, "failed to remove a cluster trust bundle", "bundleName", bundleObject.Name)
deleteName := p.handlers.getName(bundleObject)
if err := p.client.Delete(ctx, deleteName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
klog.FromContext(ctx).Error(err, "failed to remove a cluster trust bundle", "bundleName", deleteName)
deletionError = err
}
}
@@ -214,13 +361,6 @@ func (p *ClusterTrustBundlePublisher) syncClusterTrustBundle(ctx context.Context
return deletionError
}
func setupSignerNameFilteredCTBInformer(client clientset.Interface, signerName string) cache.SharedIndexInformer {
return certinformers.NewFilteredClusterTrustBundleInformer(client, 0, cache.Indexers{},
func(options *metav1.ListOptions) {
options.FieldSelector = fields.OneTermEqualSelector("spec.signerName", signerName).String()
})
}
func constructBundleName(signerName string, bundleBytes []byte) string {
namePrefix := strings.ReplaceAll(signerName, "/", ":") + ":"
bundleHash := sha256.Sum256(bundleBytes)

View File

@@ -22,7 +22,7 @@ import (
cryptorand "crypto/rand"
"testing"
certificatesv1alpha1 "k8s.io/api/certificates/v1alpha1"
certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
@@ -44,7 +44,7 @@ func TestCTBPublisherSync(t *testing.T) {
createAction := expectAction[clienttesting.CreateAction](t, filteredActions[0], "create")
ctb, ok := createAction.GetObject().(*certificatesv1alpha1.ClusterTrustBundle)
ctb, ok := createAction.GetObject().(*certificatesv1beta1.ClusterTrustBundle)
if !ok {
t.Fatalf("expected ClusterTrustBundle create, got %v", createAction.GetObject())
}
@@ -63,7 +63,7 @@ func TestCTBPublisherSync(t *testing.T) {
updateAction := expectAction[clienttesting.UpdateAction](t, filteredActions[0], "update")
ctb, ok := updateAction.GetObject().(*certificatesv1alpha1.ClusterTrustBundle)
ctb, ok := updateAction.GetObject().(*certificatesv1beta1.ClusterTrustBundle)
if !ok {
t.Fatalf("expected ClusterTrustBundle update, got %v", updateAction.GetObject())
}
@@ -109,19 +109,19 @@ func TestCTBPublisherSync(t *testing.T) {
{
name: "no CTBs for the current signer exist",
existingCTBs: []runtime.Object{
&certificatesv1alpha1.ClusterTrustBundle{
&certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: "nosigner",
},
Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
TrustBundle: "somedatahere",
},
},
&certificatesv1alpha1.ClusterTrustBundle{
&certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: "signer:one",
},
Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
SignerName: "signer",
TrustBundle: "signerdata",
},
@@ -132,11 +132,11 @@ func TestCTBPublisherSync(t *testing.T) {
{
name: "CTB for the signer exists with different content",
existingCTBs: []runtime.Object{
&certificatesv1alpha1.ClusterTrustBundle{
&certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: testBundleName,
},
Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
SignerName: testSignerName,
TrustBundle: "olddata",
},
@@ -147,20 +147,20 @@ func TestCTBPublisherSync(t *testing.T) {
{
name: "multiple CTBs for the signer",
existingCTBs: []runtime.Object{
&certificatesv1alpha1.ClusterTrustBundle{
&certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: testBundleName,
},
Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
SignerName: testSignerName,
TrustBundle: string(testCAProvider.CurrentCABundleContent()),
},
},
&certificatesv1alpha1.ClusterTrustBundle{
&certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: "test.test/testSigner:name2",
},
Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
SignerName: testSignerName,
TrustBundle: string(testCAProvider.CurrentCABundleContent()),
},
@@ -171,20 +171,20 @@ func TestCTBPublisherSync(t *testing.T) {
{
name: "multiple CTBs for the signer - the one with the proper name needs changing",
existingCTBs: []runtime.Object{
&certificatesv1alpha1.ClusterTrustBundle{
&certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: testBundleName,
},
Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
SignerName: testSignerName,
TrustBundle: "olddata",
},
},
&certificatesv1alpha1.ClusterTrustBundle{
&certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: "test.test/testSigner:name2",
},
Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
SignerName: testSignerName,
TrustBundle: string(testCAProvider.CurrentCABundleContent()),
},
@@ -202,11 +202,11 @@ func TestCTBPublisherSync(t *testing.T) {
{
name: "another CTB with a different name exists for the signer",
existingCTBs: []runtime.Object{
&certificatesv1alpha1.ClusterTrustBundle{
&certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: "test.test/testSigner:preexisting",
},
Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
SignerName: testSignerName,
TrustBundle: string(testCAProvider.CurrentCABundleContent()),
},
@@ -224,28 +224,28 @@ func TestCTBPublisherSync(t *testing.T) {
{
name: "CTB at the correct state - noop",
existingCTBs: []runtime.Object{
&certificatesv1alpha1.ClusterTrustBundle{
&certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: "nosigner",
},
Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
TrustBundle: "somedatahere",
},
},
&certificatesv1alpha1.ClusterTrustBundle{
&certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: "signer:one",
},
Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
SignerName: "signer",
TrustBundle: "signerdata",
},
},
&certificatesv1alpha1.ClusterTrustBundle{
&certificatesv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{
Name: testBundleName,
},
Spec: certificatesv1alpha1.ClusterTrustBundleSpec{
Spec: certificatesv1beta1.ClusterTrustBundleSpec{
SignerName: testSignerName,
TrustBundle: string(testCAProvider.CurrentCABundleContent()),
},
@@ -264,17 +264,22 @@ func TestCTBPublisherSync(t *testing.T) {
fakeClient := fakeKubeClientSetWithCTBList(t, testSignerName, tt.existingCTBs...)
p, err := NewClusterTrustBundlePublisher(testSignerName, testCAProvider, fakeClient)
p, err := NewBetaClusterTrustBundlePublisher(testSignerName, testCAProvider, fakeClient)
if err != nil {
t.Fatalf("failed to set up a new cluster trust bundle publisher: %v", err)
}
go p.ctbInformer.Run(testCtx.Done())
if !cache.WaitForCacheSync(testCtx.Done(), p.ctbInformer.HasSynced) {
controller, ok := p.(*ClusterTrustBundlePublisher[certificatesv1beta1.ClusterTrustBundle])
if !ok {
t.Fatalf("failed to assert the controller for the beta API")
}
go controller.ctbInformer.Run(testCtx.Done())
if !cache.WaitForCacheSync(testCtx.Done(), controller.ctbInformer.HasSynced) {
t.Fatal("timed out waiting for informer to sync")
}
if err := p.syncClusterTrustBundle(testCtx); (err != nil) != tt.wantErr {
if err := controller.syncClusterTrustBundle(testCtx); (err != nil) != tt.wantErr {
t.Errorf("syncClusterTrustBundle() error = %v, wantErr %v", err, tt.wantErr)
}
@@ -297,9 +302,9 @@ func fakeKubeClientSetWithCTBList(t *testing.T, signerName string, ctbs ...runti
return false, nil, nil
}
retList := &certificatesv1alpha1.ClusterTrustBundleList{}
retList := &certificatesv1beta1.ClusterTrustBundleList{}
for _, ctb := range ctbs {
ctbObj, ok := ctb.(*certificatesv1alpha1.ClusterTrustBundle)
ctbObj, ok := ctb.(*certificatesv1beta1.ClusterTrustBundle)
if !ok {
continue
}