github-personal/kamaji
1
0
Fork 0
mirror of https://github.com/outbackdingo/kamaji.git synced 2026-01-27 10:19:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: moving migrate webhook handling from tcp to soot manager

Browse Source
This commit is contained in:
Dario Tranchitella
2022-12-08 16:50:07 +01:00
parent 1ec257a729
commit 28c47d9d13
7 changed files with 200 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/manager/cmd.go
View File

@@ -113,7 +113,6 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
TriggerChan: tcpChannel,
KamajiNamespace: managerNamespace,
KamajiServiceAccount: managerServiceAccountName,
WebhookCABundle: webhookCABundle,
KamajiService: managerServiceName,
KamajiMigrateImage: migrateJobImage,
}
@@ -147,7 +146,11 @@ func NewCmd(scheme *runtime.Scheme) *cobra.Command {
return err
}
if err = (&soot.Manager{}).SetupWithManager(mgr); err != nil {
if err = (&soot.Manager{
MigrateCABundle: webhookCABundle,
MigrateServiceName: managerServiceName,
MigrateServiceNamespace: managerServiceName,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to set up soot manager")
return err

6
controllers/resources.go
View File

@@ -30,7 +30,6 @@ type GroupResourceBuilderConfiguration struct {
KamajiNamespace string
KamajiServiceAccount string
KamajiService string
CABundle []byte
KamajiMigrateImage string
}
@@ -66,7 +65,7 @@ func GetDeletableResources(tcp *kamajiv1alpha1.TenantControlPlane, config GroupD
}
func getDefaultResources(config GroupResourceBuilderConfiguration) []resources.Resource {
resources := getDataStoreMigratingResources(config.client, config.KamajiNamespace, config.KamajiMigrateImage, config.KamajiServiceAccount, config.KamajiService, config.CABundle)
resources := getDataStoreMigratingResources(config.client, config.KamajiNamespace, config.KamajiMigrateImage, config.KamajiServiceAccount, config.KamajiService)
resources = append(resources, getUpgradeResources(config.client)...)
resources = append(resources, getKubernetesServiceResources(config.client)...)
resources = append(resources, getKubeadmConfigResources(config.client, getTmpDirectory(config.tcpReconcilerConfig.TmpBaseDirectory, config.tenantControlPlane), config.DataStore)...)
@@ -94,7 +93,7 @@ func getDataStoreMigratingCleanup(c client.Client, kamajiNamespace string) []res
}
}
func getDataStoreMigratingResources(c client.Client, kamajiNamespace, kamajiServiceAccount, migrateImage string, kamajiService string, caBundle []byte) []resources.Resource {
func getDataStoreMigratingResources(c client.Client, kamajiNamespace, migrateImage string, kamajiServiceAccount, kamajiService string) []resources.Resource {
return []resources.Resource{
&ds.Migrate{
Client: c,
@@ -102,7 +101,6 @@ func getDataStoreMigratingResources(c client.Client, kamajiNamespace, kamajiServ
KamajiNamespace: kamajiNamespace,
KamajiServiceAccount: kamajiServiceAccount,
KamajiServiceName: kamajiService,
CABundle: caBundle,
},
}
}

153
controllers/soot/controllers/migrate.go Normal file
View File

@@ -0,0 +1,153 @@
// Copyright 2022 Clastix Labs
// SPDX-License-Identifier: Apache-2.0
package controllers
import (
"context"
"fmt"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/pointer"
controllerruntime "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/source"
"github.com/clastix/kamaji/api/v1alpha1"
"github.com/clastix/kamaji/controllers/soot/helpers"
"github.com/clastix/kamaji/internal/utilities"
)
type Migrate struct {
client client.Client
GetTenantControlPlaneFunc helpers.TenantControlPlaneRetrievalFn
WebhookNamespace string
WebhookServiceName string
WebhookCABundle []byte
TriggerChannel chan event.GenericEvent
}
func (m *Migrate) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) {
tcp, err := m.GetTenantControlPlaneFunc()
if err != nil {
return reconcile.Result{}, err
}
// Cannot detect the status of the TenantControlPlane, enqueuing back
if tcp.Status.Kubernetes.Version.Status == nil {
return reconcile.Result{Requeue: true}, nil
}
switch *tcp.Status.Kubernetes.Version.Status {
case v1alpha1.VersionMigrating:
err = m.createOrUpdate(ctx)
case v1alpha1.VersionReady:
err = m.cleanup(ctx)
}
if err != nil {
log.FromContext(ctx).Error(err, "migrate reconciliation failed")
return reconcile.Result{}, err
}
return reconcile.Result{}, nil
}
func (m *Migrate) cleanup(ctx context.Context) error {
if err := m.client.Delete(ctx, m.object()); err != nil {
if errors.IsNotFound(err) {
return nil
}
return fmt.Errorf("unable to clean-up ValidationWebhook required for migration: %w", err)
}
return nil
}
func (m *Migrate) createOrUpdate(ctx context.Context) error {
obj := m.object()
_, err := utilities.CreateOrUpdateWithConflict(ctx, m.client, obj, func() error {
obj.Webhooks = []admissionregistrationv1.ValidatingWebhook{
{
Name: "migrate.kamaji.clastix.io",
ClientConfig: admissionregistrationv1.WebhookClientConfig{
URL: pointer.String(fmt.Sprintf("https://%s.%s.svc:443/migrate", m.WebhookServiceName, m.WebhookNamespace)),
CABundle: m.WebhookCABundle,
},
Rules: []admissionregistrationv1.RuleWithOperations{
{
Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll},
Rule: admissionregistrationv1.Rule{
APIGroups: []string{"*"},
APIVersions: []string{"*"},
Resources: []string{"*"},
Scope: func(v admissionregistrationv1.ScopeType) *admissionregistrationv1.ScopeType {
return &v
}(admissionregistrationv1.AllScopes),
},
},
},
FailurePolicy: func(v admissionregistrationv1.FailurePolicyType) *admissionregistrationv1.FailurePolicyType {
return &v
}(admissionregistrationv1.Fail),
MatchPolicy: func(v admissionregistrationv1.MatchPolicyType) *admissionregistrationv1.MatchPolicyType {
return &v
}(admissionregistrationv1.Equivalent),
NamespaceSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "kubernetes.io/metadata.name",
Operator: metav1.LabelSelectorOpNotIn,
Values: []string{
"kube-system",
},
},
},
},
SideEffects: func(v admissionregistrationv1.SideEffectClass) *admissionregistrationv1.SideEffectClass {
return &v
}(admissionregistrationv1.SideEffectClassNoneOnDryRun),
TimeoutSeconds: nil,
AdmissionReviewVersions: []string{"v1"},
},
}
return nil
})
return err
}
func (m *Migrate) SetupWithManager(mgr manager.Manager) error {
m.client = mgr.GetClient()
return controllerruntime.NewControllerManagedBy(mgr).
WithLogger(mgr.GetLogger().WithName("migrate")).
For(&admissionregistrationv1.ValidatingWebhookConfiguration{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(object client.Object) bool {
vwc := m.object()
return object.GetName() == vwc.GetName()
}))).
Watches(&source.Channel{Source: m.TriggerChannel}, &handler.EnqueueRequestForObject{}).
Complete(m)
}
func (m *Migrate) object() *admissionregistrationv1.ValidatingWebhookConfiguration {
return &admissionregistrationv1.ValidatingWebhookConfiguration{
ObjectMeta: metav1.ObjectMeta{
Name: "kamaji-freeze",
},
}
}

10
controllers/soot/helpers/tcp_retrieval.go Normal file
View File

@@ -0,0 +1,10 @@
// Copyright 2022 Clastix Labs
// SPDX-License-Identifier: Apache-2.0
package helpers
import (
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
)
type TenantControlPlaneRetrievalFn func() (*kamajiv1alpha1.TenantControlPlane, error)

20
controllers/soot/manager.go
View File

@@ -19,9 +19,9 @@ import (
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"github.com/clastix/kamaji/controllers/soot/helpers"
kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
"github.com/clastix/kamaji/controllers/soot/controllers"
"github.com/clastix/kamaji/controllers/soot/helpers"
"github.com/clastix/kamaji/internal/utilities"
)
@@ -35,6 +35,10 @@ type sootMap map[string]sootItem
type Manager struct {
client client.Client
sootMap sootMap
MigrateCABundle []byte
MigrateServiceName string
MigrateServiceNamespace string
}
// retrieveTenantControlPlane is the function used to let an underlying controller of the soot manager
@@ -143,6 +147,18 @@ func (m *Manager) Reconcile(ctx context.Context, request reconcile.Request) (res
}
ch = make(chan event.GenericEvent)
//
// Register all the controllers of the soot here:
//
if err = (&controllers.Migrate{
WebhookNamespace: m.MigrateServiceName,
WebhookServiceName: m.MigrateServiceNamespace,
WebhookCABundle: m.MigrateCABundle,
GetTenantControlPlaneFunc: m.retrieveTenantControlPlane(tcpCtx, request),
TriggerChannel: ch,
}).SetupWithManager(mgr); err != nil {
return reconcile.Result{}, err
}
// Starting the manager
go func() {
if err = mgr.Start(tcpCtx); err != nil {

2
controllers/tenantcontrolplane_controller.go
View File

@@ -44,7 +44,6 @@ type TenantControlPlaneReconciler struct {
TriggerChan TenantControlPlaneChannel
KamajiNamespace string
KamajiServiceAccount string
WebhookCABundle []byte
KamajiService string
KamajiMigrateImage string
}
@@ -135,7 +134,6 @@ func (r *TenantControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.R
KamajiNamespace: r.KamajiNamespace,
KamajiServiceAccount: r.KamajiServiceAccount,
KamajiService: r.KamajiService,
CABundle: r.WebhookCABundle,
KamajiMigrateImage: r.KamajiMigrateImage,
}
registeredResources := GetResources(groupResourceBuilderConfiguration)

106
internal/resources/datastore/datastore_migrate.go
View File

@@ -7,13 +7,11 @@ import (
"context"
"fmt"
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/log"
@@ -27,14 +25,12 @@ type Migrate struct {
KamajiNamespace string
KamajiServiceAccount string
KamajiServiceName string
CABundle []byte
ShouldCleanUp bool
MigrateImage string
actualDatastore *kamajiv1alpha1.DataStore
desiredDatastore *kamajiv1alpha1.DataStore
job *batchv1.Job
webhook *admissionregistrationv1.ValidatingWebhookConfiguration
inProgress bool
}
@@ -51,13 +47,6 @@ func (d *Migrate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1
},
}
d.webhook = &admissionregistrationv1.ValidatingWebhookConfiguration{
ObjectMeta: metav1.ObjectMeta{
Name: "kamaji-freeze",
Namespace: "kube-system",
},
}
if d.ShouldCleanUp {
return nil
}
@@ -85,31 +74,17 @@ func (d *Migrate) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
return d.ShouldCleanUp
}
func (d *Migrate) CleanUp(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) (bool, error) {
// Deleting migrate Job in the admin cluster
var jobErr, webhookErr error
if err := d.Client.Get(ctx, types.NamespacedName{Name: d.job.GetName(), Namespace: d.job.GetNamespace()}, d.job); err == nil {
jobErr = d.Client.Delete(ctx, d.job)
}
// Deleting webhook deployed in the Tenant cluster
tcpClient, err := utilities.GetTenantClient(ctx, d.Client, tcp)
func (d *Migrate) CleanUp(ctx context.Context, _ *kamajiv1alpha1.TenantControlPlane) (bool, error) {
err := d.Client.Get(ctx, types.NamespacedName{Name: d.job.GetName(), Namespace: d.job.GetNamespace()}, d.job)
if err != nil {
return false, fmt.Errorf("unable to create TenantControlPlane client: %w", err)
if errors.IsNotFound(err) {
return false, nil
}
return false, err
}
if err = tcpClient.Get(ctx, types.NamespacedName{Name: d.webhook.GetName(), Namespace: d.webhook.GetNamespace()}, d.webhook); err == nil {
jobErr = tcpClient.Delete(ctx, d.webhook)
}
switch {
case jobErr != nil:
return false, jobErr
case webhookErr != nil:
return false, webhookErr
default:
return false, nil
}
return false, d.Client.Delete(ctx, d.job)
}
func (d *Migrate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
@@ -121,12 +96,7 @@ func (d *Migrate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamaji
return controllerutil.OperationResultNone, nil
}
tcpClient, err := utilities.GetTenantClient(ctx, d.Client, tenantControlPlane)
if err != nil {
return controllerutil.OperationResultNone, fmt.Errorf("unable to create TenantControlPlane client: %w", err)
}
jobRessult, err := utilities.CreateOrUpdateWithConflict(ctx, d.Client, d.job, func() error {
res, err := utilities.CreateOrUpdateWithConflict(ctx, d.Client, d.job, func() error {
d.job.SetLabels(map[string]string{
"tcp.kamaji.clastix.io/name": tenantControlPlane.GetName(),
"tcp.kamaji.clastix.io/namespace": tenantControlPlane.GetNamespace(),
@@ -151,62 +121,10 @@ func (d *Migrate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamaji
return nil
})
if err != nil {
return jobRessult, fmt.Errorf("unable to launch migrate job: %w", err)
return res, fmt.Errorf("unable to launch migrate job: %w", err)
}
webhookResult, err := utilities.CreateOrUpdateWithConflict(ctx, tcpClient, d.webhook, func() error {
d.webhook.Webhooks = []admissionregistrationv1.ValidatingWebhook{
{
Name: "migrate.kamaji.clastix.io",
ClientConfig: admissionregistrationv1.WebhookClientConfig{
URL: pointer.String(fmt.Sprintf("https://%s.%s.svc:443/migrate", d.KamajiServiceName, d.KamajiNamespace)),
CABundle: d.CABundle,
},
Rules: []admissionregistrationv1.RuleWithOperations{
{
Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.OperationAll},
Rule: admissionregistrationv1.Rule{
APIGroups: []string{"*"},
APIVersions: []string{"*"},
Resources: []string{"*"},
Scope: func(v admissionregistrationv1.ScopeType) *admissionregistrationv1.ScopeType {
return &v
}(admissionregistrationv1.AllScopes),
},
},
},
FailurePolicy: func(v admissionregistrationv1.FailurePolicyType) *admissionregistrationv1.FailurePolicyType {
return &v
}(admissionregistrationv1.Fail),
MatchPolicy: func(v admissionregistrationv1.MatchPolicyType) *admissionregistrationv1.MatchPolicyType {
return &v
}(admissionregistrationv1.Equivalent),
NamespaceSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "kubernetes.io/metadata.name",
Operator: metav1.LabelSelectorOpNotIn,
Values: []string{
"kube-system",
},
},
},
},
SideEffects: func(v admissionregistrationv1.SideEffectClass) *admissionregistrationv1.SideEffectClass {
return &v
}(admissionregistrationv1.SideEffectClassNoneOnDryRun),
TimeoutSeconds: nil,
AdmissionReviewVersions: []string{"v1"},
},
}
return nil
})
if err != nil {
return webhookResult, fmt.Errorf("unable to create webhook on TenantControlPlane: %w", err)
}
switch jobRessult {
switch res {
case controllerutil.OperationResultNone:
if len(d.job.Status.Conditions) == 0 {
break
@@ -221,7 +139,7 @@ func (d *Migrate) CreateOrUpdate(ctx context.Context, tenantControlPlane *kamaji
case controllerutil.OperationResultCreated:
break
default:
return "", fmt.Errorf("unexpected status %s from the migration job", jobRessult)
return "", fmt.Errorf("unexpected status %s from the migration job", res)
}
d.inProgress = true