diff --git a/cmd/manager/cmd.go b/cmd/manager/cmd.go index 52152e6..7758a17 100644 --- a/cmd/manager/cmd.go +++ b/cmd/manager/cmd.go @@ -113,7 +113,6 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { TriggerChan: tcpChannel, KamajiNamespace: managerNamespace, KamajiServiceAccount: managerServiceAccountName, - WebhookCABundle: webhookCABundle, KamajiService: managerServiceName, KamajiMigrateImage: migrateJobImage, } @@ -147,7 +146,11 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command { return err } - if err = (&soot.Manager{}).SetupWithManager(mgr); err != nil { + if err = (&soot.Manager{ + MigrateCABundle: webhookCABundle, + MigrateServiceName: managerServiceName, + MigrateServiceNamespace: managerServiceName, + }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to set up soot manager") return err diff --git a/controllers/resources.go b/controllers/resources.go index 3c3db37..6e4073f 100644 --- a/controllers/resources.go +++ b/controllers/resources.go @@ -30,7 +30,6 @@ type GroupResourceBuilderConfiguration struct { KamajiNamespace string KamajiServiceAccount string KamajiService string - CABundle []byte KamajiMigrateImage string } @@ -66,7 +65,7 @@ func GetDeletableResources(tcp *kamajiv1alpha1.TenantControlPlane, config GroupD } func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.Resource { - resources := getDataStoreMigratingResources(config.client, config.KamajiNamespace, config.KamajiMigrateImage, config.KamajiServiceAccount, config.KamajiService, config.CABundle) + resources := getDataStoreMigratingResources(config.client, config.KamajiNamespace, config.KamajiMigrateImage, config.KamajiServiceAccount, config.KamajiService) resources = append(resources, getUpgradeResources(config.client)...) resources = append(resources, getKubernetesServiceResources(config.client)...) resources = append(resources, getKubeadmConfigResources(config.client, getTmpDirectory(config.tcpReconcilerConfig.TmpBaseDirectory, config.tenantControlPlane), config.DataStore)...) @@ -94,7 +93,7 @@ func getDataStoreMigratingCleanup(c client.Client, kamajiNamespace string) []res } } -func getDataStoreMigratingResources(c client.Client, kamajiNamespace, kamajiServiceAccount, migrateImage string, kamajiService string, caBundle []byte) []resources.Resource { +func getDataStoreMigratingResources(c client.Client, kamajiNamespace, migrateImage string, kamajiServiceAccount, kamajiService string) []resources.Resource { return []resources.Resource{ &ds.Migrate{ Client: c, @@ -102,7 +101,6 @@ func getDataStoreMigratingResources(c client.Client, kamajiNamespace, kamajiServ KamajiNamespace: kamajiNamespace, KamajiServiceAccount: kamajiServiceAccount, KamajiServiceName: kamajiService, - CABundle: caBundle, }, } } diff --git a/controllers/soot/controllers/migrate.go b/controllers/soot/controllers/migrate.go new file mode 100644 index 0000000..3daf034 --- /dev/null +++ b/controllers/soot/controllers/migrate.go @@ -0,0 +1,153 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package controllers + +import ( + "context" + "fmt" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/controllers/soot/helpers" + "github.com/clastix/kamaji/internal/utilities" +) + +type Migrate struct { + client client.Client + + GetTenantControlPlaneFunc helpers.TenantControlPlaneRetrievalFn + WebhookNamespace string + WebhookServiceName string + WebhookCABundle []byte + TriggerChannel chan event.GenericEvent +} + +func (m *Migrate) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { + tcp, err := m.GetTenantControlPlaneFunc() + if err != nil { + return reconcile.Result{}, err + } + // Cannot detect the status of the TenantControlPlane, enqueuing back + if tcp.Status.Kubernetes.Version.Status == nil { + return reconcile.Result{Requeue: true}, nil + } + + switch *tcp.Status.Kubernetes.Version.Status { + case v1alpha1.VersionMigrating: + err = m.createOrUpdate(ctx) + case v1alpha1.VersionReady: + err = m.cleanup(ctx) + } + + if err != nil { + log.FromContext(ctx).Error(err, "migrate reconciliation failed") + + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +func (m *Migrate) cleanup(ctx context.Context) error { + if err := m.client.Delete(ctx, m.object()); err != nil { + if errors.IsNotFound(err) { + return nil + } + + return fmt.Errorf("unable to clean-up ValidationWebhook required for migration: %w", err) + } + + return nil +} + +func (m *Migrate) createOrUpdate(ctx context.Context) error { + obj := m.object() + + _, err := utilities.CreateOrUpdateWithConflict(ctx, m.client, obj, func() error { + obj.Webhooks = []admissionregistrationv1.ValidatingWebhook{ + { + Name: "migrate.kamaji.clastix.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + URL: pointer.String(fmt.Sprintf("https://%s.%s.svc:443/migrate", m.WebhookServiceName, m.WebhookNamespace)), + CABundle: m.WebhookCABundle, + }, + Rules: []admissionregistrationv1.RuleWithOperations{ + { + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll}, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{"*"}, + APIVersions: []string{"*"}, + Resources: []string{"*"}, + Scope: func(v admissionregistrationv1.ScopeType) *admissionregistrationv1.ScopeType { + return &v + }(admissionregistrationv1.AllScopes), + }, + }, + }, + FailurePolicy: func(v admissionregistrationv1.FailurePolicyType) *admissionregistrationv1.FailurePolicyType { + return &v + }(admissionregistrationv1.Fail), + MatchPolicy: func(v admissionregistrationv1.MatchPolicyType) *admissionregistrationv1.MatchPolicyType { + return &v + }(admissionregistrationv1.Equivalent), + NamespaceSelector: &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "kubernetes.io/metadata.name", + Operator: metav1.LabelSelectorOpNotIn, + Values: []string{ + "kube-system", + }, + }, + }, + }, + SideEffects: func(v admissionregistrationv1.SideEffectClass) *admissionregistrationv1.SideEffectClass { + return &v + }(admissionregistrationv1.SideEffectClassNoneOnDryRun), + TimeoutSeconds: nil, + AdmissionReviewVersions: []string{"v1"}, + }, + } + + return nil + }) + + return err +} + +func (m *Migrate) SetupWithManager(mgr manager.Manager) error { + m.client = mgr.GetClient() + + return controllerruntime.NewControllerManagedBy(mgr). + WithLogger(mgr.GetLogger().WithName("migrate")). + For(&admissionregistrationv1.ValidatingWebhookConfiguration{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool { + vwc := m.object() + + return object.GetName() == vwc.GetName() + }))). + Watches(&source.Channel{Source: m.TriggerChannel}, &handler.EnqueueRequestForObject{}). + Complete(m) +} + +func (m *Migrate) object() *admissionregistrationv1.ValidatingWebhookConfiguration { + return &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kamaji-freeze", + }, + } +} diff --git a/controllers/soot/helpers/tcp_retrieval.go b/controllers/soot/helpers/tcp_retrieval.go new file mode 100644 index 0000000..ba5aff7 --- /dev/null +++ b/controllers/soot/helpers/tcp_retrieval.go @@ -0,0 +1,10 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package helpers + +import ( + kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" +) + +type TenantControlPlaneRetrievalFn func() (*kamajiv1alpha1.TenantControlPlane, error) diff --git a/controllers/soot/manager.go b/controllers/soot/manager.go index f8a22db..807b577 100644 --- a/controllers/soot/manager.go +++ b/controllers/soot/manager.go @@ -19,9 +19,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/clastix/kamaji/controllers/soot/helpers" - kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/controllers/soot/controllers" + "github.com/clastix/kamaji/controllers/soot/helpers" "github.com/clastix/kamaji/internal/utilities" ) @@ -35,6 +35,10 @@ type sootMap map[string]sootItem type Manager struct { client client.Client sootMap sootMap + + MigrateCABundle []byte + MigrateServiceName string + MigrateServiceNamespace string } // retrieveTenantControlPlane is the function used to let an underlying controller of the soot manager @@ -143,6 +147,18 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res } ch = make(chan event.GenericEvent) + // + // Register all the controllers of the soot here: + // + if err = (&controllers.Migrate{ + WebhookNamespace: m.MigrateServiceName, + WebhookServiceName: m.MigrateServiceNamespace, + WebhookCABundle: m.MigrateCABundle, + GetTenantControlPlaneFunc: m.retrieveTenantControlPlane(tcpCtx, request), + TriggerChannel: ch, + }).SetupWithManager(mgr); err != nil { + return reconcile.Result{}, err + } // Starting the manager go func() { if err = mgr.Start(tcpCtx); err != nil { diff --git a/controllers/tenantcontrolplane_controller.go b/controllers/tenantcontrolplane_controller.go index 9c7dbf6..41a21fa 100644 --- a/controllers/tenantcontrolplane_controller.go +++ b/controllers/tenantcontrolplane_controller.go @@ -44,7 +44,6 @@ type TenantControlPlaneReconciler struct { TriggerChan TenantControlPlaneChannel KamajiNamespace string KamajiServiceAccount string - WebhookCABundle []byte KamajiService string KamajiMigrateImage string } @@ -135,7 +134,6 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R KamajiNamespace: r.KamajiNamespace, KamajiServiceAccount: r.KamajiServiceAccount, KamajiService: r.KamajiService, - CABundle: r.WebhookCABundle, KamajiMigrateImage: r.KamajiMigrateImage, } registeredResources := GetResources(groupResourceBuilderConfiguration) diff --git a/internal/resources/datastore/datastore_migrate.go b/internal/resources/datastore/datastore_migrate.go index ff04be3..db38b51 100644 --- a/internal/resources/datastore/datastore_migrate.go +++ b/internal/resources/datastore/datastore_migrate.go @@ -7,13 +7,11 @@ import ( "context" "fmt" - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" @@ -27,14 +25,12 @@ type Migrate struct { KamajiNamespace string KamajiServiceAccount string KamajiServiceName string - CABundle []byte ShouldCleanUp bool MigrateImage string actualDatastore *kamajiv1alpha1.DataStore desiredDatastore *kamajiv1alpha1.DataStore job *batchv1.Job - webhook *admissionregistrationv1.ValidatingWebhookConfiguration inProgress bool } @@ -51,13 +47,6 @@ func (d *Migrate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1 }, } - d.webhook = &admissionregistrationv1.ValidatingWebhookConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kamaji-freeze", - Namespace: "kube-system", - }, - } - if d.ShouldCleanUp { return nil } @@ -85,31 +74,17 @@ func (d *Migrate) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool { return d.ShouldCleanUp } -func (d *Migrate) CleanUp(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) (bool, error) { - // Deleting migrate Job in the admin cluster - var jobErr, webhookErr error - - if err := d.Client.Get(ctx, types.NamespacedName{Name: d.job.GetName(), Namespace: d.job.GetNamespace()}, d.job); err == nil { - jobErr = d.Client.Delete(ctx, d.job) - } - // Deleting webhook deployed in the Tenant cluster - tcpClient, err := utilities.GetTenantClient(ctx, d.Client, tcp) +func (d *Migrate) CleanUp(ctx context.Context, _ *kamajiv1alpha1.TenantControlPlane) (bool, error) { + err := d.Client.Get(ctx, types.NamespacedName{Name: d.job.GetName(), Namespace: d.job.GetNamespace()}, d.job) if err != nil { - return false, fmt.Errorf("unable to create TenantControlPlane client: %w", err) + if errors.IsNotFound(err) { + return false, nil + } + + return false, err } - if err = tcpClient.Get(ctx, types.NamespacedName{Name: d.webhook.GetName(), Namespace: d.webhook.GetNamespace()}, d.webhook); err == nil { - jobErr = tcpClient.Delete(ctx, d.webhook) - } - - switch { - case jobErr != nil: - return false, jobErr - case webhookErr != nil: - return false, webhookErr - default: - return false, nil - } + return false, d.Client.Delete(ctx, d.job) } func (d *Migrate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { @@ -121,12 +96,7 @@ func (d *Migrate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamaji return controllerutil.OperationResultNone, nil } - tcpClient, err := utilities.GetTenantClient(ctx, d.Client, tenantControlPlane) - if err != nil { - return controllerutil.OperationResultNone, fmt.Errorf("unable to create TenantControlPlane client: %w", err) - } - - jobRessult, err := utilities.CreateOrUpdateWithConflict(ctx, d.Client, d.job, func() error { + res, err := utilities.CreateOrUpdateWithConflict(ctx, d.Client, d.job, func() error { d.job.SetLabels(map[string]string{ "tcp.kamaji.clastix.io/name": tenantControlPlane.GetName(), "tcp.kamaji.clastix.io/namespace": tenantControlPlane.GetNamespace(), @@ -151,62 +121,10 @@ func (d *Migrate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamaji return nil }) if err != nil { - return jobRessult, fmt.Errorf("unable to launch migrate job: %w", err) + return res, fmt.Errorf("unable to launch migrate job: %w", err) } - webhookResult, err := utilities.CreateOrUpdateWithConflict(ctx, tcpClient, d.webhook, func() error { - d.webhook.Webhooks = []admissionregistrationv1.ValidatingWebhook{ - { - Name: "migrate.kamaji.clastix.io", - ClientConfig: admissionregistrationv1.WebhookClientConfig{ - URL: pointer.String(fmt.Sprintf("https://%s.%s.svc:443/migrate", d.KamajiServiceName, d.KamajiNamespace)), - CABundle: d.CABundle, - }, - Rules: []admissionregistrationv1.RuleWithOperations{ - { - Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll}, - Rule: admissionregistrationv1.Rule{ - APIGroups: []string{"*"}, - APIVersions: []string{"*"}, - Resources: []string{"*"}, - Scope: func(v admissionregistrationv1.ScopeType) *admissionregistrationv1.ScopeType { - return &v - }(admissionregistrationv1.AllScopes), - }, - }, - }, - FailurePolicy: func(v admissionregistrationv1.FailurePolicyType) *admissionregistrationv1.FailurePolicyType { - return &v - }(admissionregistrationv1.Fail), - MatchPolicy: func(v admissionregistrationv1.MatchPolicyType) *admissionregistrationv1.MatchPolicyType { - return &v - }(admissionregistrationv1.Equivalent), - NamespaceSelector: &metav1.LabelSelector{ - MatchExpressions: []metav1.LabelSelectorRequirement{ - { - Key: "kubernetes.io/metadata.name", - Operator: metav1.LabelSelectorOpNotIn, - Values: []string{ - "kube-system", - }, - }, - }, - }, - SideEffects: func(v admissionregistrationv1.SideEffectClass) *admissionregistrationv1.SideEffectClass { - return &v - }(admissionregistrationv1.SideEffectClassNoneOnDryRun), - TimeoutSeconds: nil, - AdmissionReviewVersions: []string{"v1"}, - }, - } - - return nil - }) - if err != nil { - return webhookResult, fmt.Errorf("unable to create webhook on TenantControlPlane: %w", err) - } - - switch jobRessult { + switch res { case controllerutil.OperationResultNone: if len(d.job.Status.Conditions) == 0 { break @@ -221,7 +139,7 @@ func (d *Migrate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamaji case controllerutil.OperationResultCreated: break default: - return "", fmt.Errorf("unexpected status %s from the migration job", jobRessult) + return "", fmt.Errorf("unexpected status %s from the migration job", res) } d.inProgress = true