github-personal/kamaji
1
0
Fork 0
mirror of https://github.com/outbackdingo/kamaji.git synced 2026-01-27 10:19:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: pausing reconciliation of controlled objects (#874)

Browse Source 
* feat: pausing reconciliation of controlled objects

Objects such as TenantControlPlane and Secret can be annotated with
kamaji.clastix.io/paused to prevent controllers from processing them.

This will stop reconciling objects for debugging or other purposes.
Annotation value is irrelevant, just the key presence is evaluated.

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

* docs: pausing reconciliation of controlled objects

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

* chore(logs): typo for deleted resources

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

---------

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>
This commit is contained in:
Dario Tranchitella
2025-07-16 10:44:48 +02:00
committed by GitHub
parent 0ab8843418
commit e366dc3959
15 changed files with 149 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
api/v1alpha1/tenantcontrolplane_const.go Normal file
View File

@@ -0,0 +1,10 @@
// Copyright 2022 Clastix Labs
// SPDX-License-Identifier: Apache-2.0
package v1alpha1
const (
// PausedReconciliationAnnotation is an annotation that can be applied to
// Tenant Control Plane objects to prevent the controller from processing such a resource.
PausedReconciliationAnnotation = "kamaji.clastix.io/paused"
)

22
controllers/certificate_lifecycle_controller.go
View File

@@ -24,6 +24,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/reconcile"
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
"github.com/clastix/kamaji/controllers/utils"
"github.com/clastix/kamaji/internal/constants"
"github.com/clastix/kamaji/internal/crypto"
"github.com/clastix/kamaji/internal/utilities"
@@ -41,19 +42,25 @@ func (s *CertificateLifecycle) Reconcile(ctx context.Context, request reconcile.
logger.Info("starting CertificateLifecycle handling")
secret := corev1.Secret{}
err := s.client.Get(ctx, request.NamespacedName, &secret)
if k8serrors.IsNotFound(err) {
logger.Info("resource have been deleted, skipping")
var secret corev1.Secret
if err := s.client.Get(ctx, request.NamespacedName, &secret); err != nil {
if k8serrors.IsNotFound(err) {
logger.Info("resource may have been deleted, skipping")
return reconcile.Result{}, nil
}
return reconcile.Result{}, nil
}
if err != nil {
logger.Error(err, "cannot retrieve the required resource")
return reconcile.Result{}, err
}
if utils.IsPaused(&secret) {
logger.Info("paused reconciliation, no further actions")
return reconcile.Result{}, nil
}
checkType, ok := secret.GetLabels()[constants.ControllerLabelResource]
if !ok {
logger.Info("missing controller label, shouldn't happen")
@@ -62,6 +69,7 @@ func (s *CertificateLifecycle) Reconcile(ctx context.Context, request reconcile.
}
var crt *x509.Certificate
var err error
switch checkType {
case "x509":

8
controllers/datastore_controller.go
View File

@@ -43,7 +43,7 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (r
var ds kamajiv1alpha1.DataStore
if err := r.Client.Get(ctx, request.NamespacedName, &ds); err != nil {
if k8serrors.IsNotFound(err) {
logger.Info("resource have been deleted, skipping")
logger.Info("resource may have been deleted, skipping")
return reconcile.Result{}, nil
}
@@ -53,6 +53,12 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (r
return reconcile.Result{}, err
}
if utils.IsPaused(&ds) {
logger.Info("paused reconciliation, no further actions")
return reconcile.Result{}, nil
}
var tcpList kamajiv1alpha1.TenantControlPlaneList
updateErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {

8
controllers/soot/controllers/coredns.go
View File

@@ -7,6 +7,7 @@ import (
"context"
"github.com/go-logr/logr"
"github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
@@ -23,6 +24,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
"github.com/clastix/kamaji/controllers/utils"
"github.com/clastix/kamaji/internal/kubeadm"
"github.com/clastix/kamaji/internal/resources"
@@ -39,7 +41,11 @@ type CoreDNS struct {
func (c *CoreDNS) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
tcp, err := c.GetTenantControlPlaneFunc()
if err != nil {
c.Logger.Error(err, "cannot retrieve TenantControlPlane")
if errors.Is(err, sooterrors.ErrPausedReconciliation) {
c.Logger.Info(err.Error())
return reconcile.Result{}, nil
}
return reconcile.Result{}, err
}

10
controllers/soot/controllers/errors/paused_reconciliation.go Normal file
View File

@@ -0,0 +1,10 @@
// Copyright 2022 Clastix Labs
// SPDX-License-Identifier: Apache-2.0
package errors
import (
"github.com/pkg/errors"
)
var ErrPausedReconciliation = errors.New("paused reconciliation, no further actions")

8
controllers/soot/controllers/konnectivity.go
View File

@@ -7,6 +7,7 @@ import (
"context"
"github.com/go-logr/logr"
"github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/rbac/v1"
@@ -25,6 +26,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/source"
"github.com/clastix/kamaji/controllers"
sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
"github.com/clastix/kamaji/controllers/utils"
"github.com/clastix/kamaji/internal/resources"
"github.com/clastix/kamaji/internal/resources/konnectivity"
@@ -40,6 +42,12 @@ type KonnectivityAgent struct {
func (k *KonnectivityAgent) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
tcp, err := k.GetTenantControlPlaneFunc()
if err != nil {
if errors.Is(err, sooterrors.ErrPausedReconciliation) {
k.Logger.Info(err.Error())
return reconcile.Result{}, nil
}
k.Logger.Error(err, "cannot retrieve TenantControlPlane")
return reconcile.Result{}, err

8
controllers/soot/controllers/kubeadm_phase.go
View File

@@ -7,6 +7,7 @@ import (
"context"
"github.com/go-logr/logr"
"github.com/pkg/errors"
"k8s.io/utils/ptr"
controllerruntime "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -19,6 +20,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
"github.com/clastix/kamaji/controllers/utils"
"github.com/clastix/kamaji/internal/resources"
)
@@ -34,6 +36,12 @@ type KubeadmPhase struct {
func (k *KubeadmPhase) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
tcp, err := k.GetTenantControlPlaneFunc()
if err != nil {
if errors.Is(err, sooterrors.ErrPausedReconciliation) {
k.logger.Info(err.Error())
return reconcile.Result{}, nil
}
return reconcile.Result{}, err
}

8
controllers/soot/controllers/kubeproxy.go
View File

@@ -7,6 +7,7 @@ import (
"context"
"github.com/go-logr/logr"
"github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
@@ -23,6 +24,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
"github.com/clastix/kamaji/controllers/utils"
"github.com/clastix/kamaji/internal/kubeadm"
"github.com/clastix/kamaji/internal/resources"
@@ -39,6 +41,12 @@ type KubeProxy struct {
func (k *KubeProxy) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
tcp, err := k.GetTenantControlPlaneFunc()
if err != nil {
if errors.Is(err, sooterrors.ErrPausedReconciliation) {
k.Logger.Info(err.Error())
return reconcile.Result{}, nil
}
k.Logger.Error(err, "cannot retrieve TenantControlPlane")
return reconcile.Result{}, err

12
controllers/soot/controllers/migrate.go
View File

@@ -9,8 +9,9 @@ import (
"time"
"github.com/go-logr/logr"
"github.com/pkg/errors"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
pointer "k8s.io/utils/ptr"
controllerruntime "sigs.k8s.io/controller-runtime"
@@ -25,6 +26,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/source"
"github.com/clastix/kamaji/api/v1alpha1"
sooterrors "github.com/clastix/kamaji/controllers/soot/controllers/errors"
"github.com/clastix/kamaji/controllers/utils"
"github.com/clastix/kamaji/internal/utilities"
)
@@ -42,6 +44,12 @@ type Migrate struct {
func (m *Migrate) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
tcp, err := m.GetTenantControlPlaneFunc()
if err != nil {
if errors.Is(err, sooterrors.ErrPausedReconciliation) {
m.Logger.Info(err.Error())
return reconcile.Result{}, nil
}
return reconcile.Result{}, err
}
// Cannot detect the status of the TenantControlPlane, enqueuing back
@@ -67,7 +75,7 @@ func (m *Migrate) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile
func (m *Migrate) cleanup(ctx context.Context) error {
if err := m.Client.Delete(ctx, m.object()); err != nil {
if errors.IsNotFound(err) {
if apierrors.IsNotFound(err) {
return nil
}

5
controllers/soot/manager.go
View File

@@ -29,6 +29,7 @@ import (
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
"github.com/clastix/kamaji/controllers/finalizers"
"github.com/clastix/kamaji/controllers/soot/controllers"
"github.com/clastix/kamaji/controllers/soot/controllers/errors"
"github.com/clastix/kamaji/controllers/utils"
"github.com/clastix/kamaji/internal/resources"
"github.com/clastix/kamaji/internal/utilities"
@@ -69,6 +70,10 @@ func (m *Manager) retrieveTenantControlPlane(ctx context.Context, request reconc
return nil, err
}
if utils.IsPaused(tcp) {
return nil, errors.ErrPausedReconciliation
}
return tcp, nil
}
}

8
controllers/tenantcontrolplane_controller.go
View File

@@ -85,7 +85,7 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
tenantControlPlane, err := r.getTenantControlPlane(ctx, req.NamespacedName)()
if k8serrors.IsNotFound(err) {
log.Info("resource have been deleted, skipping")
log.Info("resource may have been deleted, skipping")
return reconcile.Result{}, nil
}
@@ -95,6 +95,12 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
return reconcile.Result{}, err
}
if utils.IsPaused(tenantControlPlane) {
log.Info("paused reconciliation, no further actions")
return ctrl.Result{}, nil
}
releaser, err := mutex.Acquire(r.mutexSpec(tenantControlPlane))
if err != nil {
switch {

19
controllers/utils/is_paused.go Normal file
View File

@@ -0,0 +1,19 @@
// Copyright 2022 Clastix Labs
// SPDX-License-Identifier: Apache-2.0
package utils
import (
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/clastix/kamaji/api/v1alpha1"
)
func IsPaused(obj client.Object) bool {
if obj.GetAnnotations() == nil {
return false
}
_, paused := obj.GetAnnotations()[v1alpha1.PausedReconciliationAnnotation]
return paused
}

33
docs/content/guides/pausing.md Normal file
View File

@@ -0,0 +1,33 @@
# Pausing Reconciliations
Kamaji follows the Kubernetes Operator pattern, which includes implementing a reconciliation loop.
This loop continuously reacts to events such as creation, updates, and deletions of resources.
To temporarily disable reconciliation for a resource, you can use the following annotation:
> `kamaji.clastix.io/paused`
!!! info "Annotation value"
The annotation key is sufficient on its own: no value is required.
Its mere presence disables controller reconciliations.
## Pausing `TenantControlPlane` reconciliations
When you add the `kamaji.clastix.io/paused` annotation to a TenantControlPlane object,
Kamaji will halt all reconciliation processes for that object.
This affects **all controllers**, including:
- The primary controller responsible for provisioning resources in the management cluster
- Secondary (soot) controllers responsible for bootstrapping the control plane, deploying addons, and managing any additional resources handled by Kamaji.
## Pausing Secret rotation
Kamaji automatically generates and manages several `Secret` resources, such as:
- `x509` certificates
- `kubeconfig` credentials
These secrets are automatically rotated by Kamaji's built-in **Certificate Lifecycle** feature.
To temporarily disable secret rotation for these resources,
apply the `kamaji.clastix.io/paused` annotation to the corresponding object.

2
docs/content/llms.txt
View File

@@ -94,7 +94,7 @@ Clastix Labs no longer provides release artifacts following its own semantic ver
- [Releases and Versions](https://github.com/clastix/kamaji/blob/master/docs/content/reference/versioning.md): Kamaji versions are available in different types of release artifacts
- [API Reference](https://github.com/clastix/kamaji/blob/master/docs/content/reference/api.md): Kamaji Custom Resources full API documentation
## GitHub
## GitHubK
- [Readme](https://github.com/clastix/kamaji/blob/master/README.md): GitHub Readme file
- [License](https://github.com/clastix/kamaji/blob/master/LICENSE): Apache 2.0 license

1
docs/mkdocs.yml
View File

@@ -73,6 +73,7 @@ nav:
- guides/alternative-datastore.md
- guides/backup-and-restore.md
- guides/certs-lifecycle.md
- guides/pausing.md
- guides/datastore-migration.md
- guides/gitops.md
- guides/console.md