feat(metrics): exposing resource handlers time bucket (#836)

* refactor: static names and avoiding clash

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

* feat(metrics): exposing resource handlers time bucket

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

---------

Signed-off-by: Dario Tranchitella <dario@tranchitella.eu>

internal/resources/addons/coredns.go

@@ -8,6 +8,7 @@ import (
 	"context"
 
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -37,6 +38,12 @@ type CoreDNS struct {
 	serviceAccount     *corev1.ServiceAccount
 }
 
+func (c *CoreDNS) GetHistogram() prometheus.Histogram {
+	coreDNSCollector = resources.LazyLoadHistogramFromResource(coreDNSCollector, c)
+
+	return coreDNSCollector
+}
+
 func (c *CoreDNS) Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error {
 	c.deployment = &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{

internal/resources/addons/kube_proxy.go

@@ -8,6 +8,7 @@ import (
 	"context"
 
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -39,6 +40,12 @@ type KubeProxy struct {
 	daemonSet          *appsv1.DaemonSet
 }
 
+func (k *KubeProxy) GetHistogram() prometheus.Histogram {
+	kubeProxyCollector = resources.LazyLoadHistogramFromResource(kubeProxyCollector, k)
+
+	return kubeProxyCollector
+}
+
 func (k *KubeProxy) Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error {
 	k.clusterRoleBinding = &rbacv1.ClusterRoleBinding{
 		ObjectMeta: metav1.ObjectMeta{

internal/resources/addons/metrics.go

@@ -0,0 +1,13 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package addons
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	kubeProxyCollector prometheus.Histogram
+	coreDNSCollector   prometheus.Histogram
+)

internal/resources/api_server_certificate.go

@@ -8,6 +8,7 @@ import (
 	"crypto/x509"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8stypes "k8s.io/apimachinery/pkg/types"
@@ -31,6 +32,12 @@ type APIServerCertificate struct {
 	TmpDirectory string
 }
 
+func (r *APIServerCertificate) GetHistogram() prometheus.Histogram {
+	apiservercertificateCollector = LazyLoadHistogramFromResource(apiservercertificateCollector, r)
+
+	return apiservercertificateCollector
+}
+
 func (r *APIServerCertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Certificates.APIServer.Checksum != utilities.GetObjectChecksum(r.resource)
 }

internal/resources/api_server_kubelet_client_certificate.go

@@ -8,6 +8,7 @@ import (
 	"crypto/x509"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8stypes "k8s.io/apimachinery/pkg/types"
@@ -30,6 +31,12 @@ type APIServerKubeletClientCertificate struct {
 	TmpDirectory string
 }
 
+func (r *APIServerKubeletClientCertificate) GetHistogram() prometheus.Histogram {
+	clientcertificateCollector = LazyLoadHistogramFromResource(clientcertificateCollector, r)
+
+	return clientcertificateCollector
+}
+
 func (r *APIServerKubeletClientCertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Certificates.APIServerKubeletClient.Checksum != utilities.GetObjectChecksum(r.resource)
 }

internal/resources/ca_certificate.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
@@ -30,6 +31,12 @@ type CACertificate struct {
 	TmpDirectory string
 }
 
+func (r *CACertificate) GetHistogram() prometheus.Histogram {
+	certificateauthorityCollector = LazyLoadHistogramFromResource(certificateauthorityCollector, r)
+
+	return certificateauthorityCollector
+}
+
 func (r *CACertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return r.isRotatingCA || tenantControlPlane.Status.Certificates.CA.SecretName != r.resource.GetName() ||
 		tenantControlPlane.Status.Certificates.CA.Checksum != utilities.GetObjectChecksum(r.resource)

internal/resources/datastore/datastore_certificate.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -18,6 +19,7 @@ import (
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/internal/constants"
 	"github.com/clastix/kamaji/internal/crypto"
+	"github.com/clastix/kamaji/internal/resources"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
@@ -28,6 +30,12 @@ type Certificate struct {
 	DataStore kamajiv1alpha1.DataStore
 }
 
+func (r *Certificate) GetHistogram() prometheus.Histogram {
+	certificateCollector = resources.LazyLoadHistogramFromResource(certificateCollector, r)
+
+	return certificateCollector
+}
+
 func (r *Certificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Storage.Certificate.Checksum != utilities.GetObjectChecksum(r.resource)
 }

internal/resources/datastore/datastore_migrate.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -36,6 +37,12 @@ type Migrate struct {
 	inProgress bool
 }
 
+func (d *Migrate) GetHistogram() prometheus.Histogram {
+	migrateCollector = resources.LazyLoadHistogramFromResource(migrateCollector, d)
+
+	return migrateCollector
+}
+
 func (d *Migrate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	if len(tenantControlPlane.Status.Storage.DataStoreName) == 0 {
 		return nil

internal/resources/datastore/datastore_multitenancy.go

@@ -7,29 +7,37 @@ import (
 	"context"
 
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/resources"
 )
 
 type MultiTenancy struct {
 	DataStore kamajiv1alpha1.DataStore
 }
 
-func (m MultiTenancy) Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error {
+func (m *MultiTenancy) GetHistogram() prometheus.Histogram {
+	multiTenancyCollector = resources.LazyLoadHistogramFromResource(multiTenancyCollector, m)
+
+	return multiTenancyCollector
+}
+
+func (m *MultiTenancy) Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error {
 	return nil
 }
 
-func (m MultiTenancy) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
+func (m *MultiTenancy) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool {
 	return false
 }
 
-func (m MultiTenancy) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
+func (m *MultiTenancy) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) {
 	return false, nil
 }
 
-func (m MultiTenancy) CreateOrUpdate(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+func (m *MultiTenancy) CreateOrUpdate(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
 	// If the NATS Datastore is already used by a Tenant Control Plane
 	// and a new one is reclaiming it, we need to stop it, since it's not allowed.
 	// TODO(prometherion): remove this after multi-tenancy is implemented for NATS.
@@ -49,14 +57,14 @@ func (m MultiTenancy) CreateOrUpdate(_ context.Context, tcp *kamajiv1alpha1.Tena
 	}
 }
 
-func (m MultiTenancy) GetName() string {
+func (m *MultiTenancy) GetName() string {
 	return "ds.multitenancy"
 }
 
-func (m MultiTenancy) ShouldStatusBeUpdated(context.Context, *kamajiv1alpha1.TenantControlPlane) bool {
+func (m *MultiTenancy) ShouldStatusBeUpdated(context.Context, *kamajiv1alpha1.TenantControlPlane) bool {
 	return false
 }
 
-func (m MultiTenancy) UpdateTenantControlPlaneStatus(context.Context, *kamajiv1alpha1.TenantControlPlane) error {
+func (m *MultiTenancy) UpdateTenantControlPlaneStatus(context.Context, *kamajiv1alpha1.TenantControlPlane) error {
 	return nil
 }

internal/resources/datastore/datastore_setup.go

@@ -7,6 +7,7 @@ import (
 	"context"
 
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -18,6 +19,7 @@ import (
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/controllers/finalizers"
 	"github.com/clastix/kamaji/internal/datastore"
+	"github.com/clastix/kamaji/internal/resources"
 	"github.com/clastix/kamaji/internal/resources/utils"
 )
 
@@ -34,6 +36,12 @@ type Setup struct {
 	DataStore  kamajiv1alpha1.DataStore
 }
 
+func (r *Setup) GetHistogram() prometheus.Histogram {
+	setupCollector = resources.LazyLoadHistogramFromResource(setupCollector, r)
+
+	return setupCollector
+}
+
 func (r *Setup) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Storage.Driver != string(r.DataStore.Spec.Driver) ||
 		tenantControlPlane.Status.Storage.Setup.Checksum != tenantControlPlane.Status.Storage.Config.Checksum ||

internal/resources/datastore/datastore_storage_config.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	kubeerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -22,6 +23,7 @@ import (
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/controllers/finalizers"
+	"github.com/clastix/kamaji/internal/resources"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
@@ -32,6 +34,12 @@ type Config struct {
 	DataStore  kamajiv1alpha1.DataStore
 }
 
+func (r *Config) GetHistogram() prometheus.Histogram {
+	storageCollector = resources.LazyLoadHistogramFromResource(storageCollector, r)
+
+	return storageCollector
+}
+
 func (r *Config) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Storage.Config.Checksum != utilities.GetObjectChecksum(r.resource) ||
 		tenantControlPlane.Status.Storage.DataStoreName != r.DataStore.GetName()

internal/resources/datastore/metrics.go

@@ -0,0 +1,16 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package datastore
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	certificateCollector  prometheus.Histogram
+	migrateCollector      prometheus.Histogram
+	multiTenancyCollector prometheus.Histogram
+	setupCollector        prometheus.Histogram
+	storageCollector      prometheus.Histogram
+)

internal/resources/front-proxy-client-certificate.go

@@ -8,6 +8,7 @@ import (
 	"crypto/x509"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8stypes "k8s.io/apimachinery/pkg/types"
@@ -30,6 +31,12 @@ type FrontProxyClientCertificate struct {
 	TmpDirectory string
 }
 
+func (r *FrontProxyClientCertificate) GetHistogram() prometheus.Histogram {
+	frontproxycertificateCollector = LazyLoadHistogramFromResource(frontproxycertificateCollector, r)
+
+	return frontproxycertificateCollector
+}
+
 func (r *FrontProxyClientCertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Certificates.FrontProxyClient.Checksum != utilities.GetObjectChecksum(r.resource)
 }

internal/resources/front_proxy_ca_certificate.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
@@ -27,6 +28,12 @@ type FrontProxyCACertificate struct {
 	TmpDirectory string
 }
 
+func (r *FrontProxyCACertificate) GetHistogram() prometheus.Histogram {
+	frontproxycaCollector = LazyLoadHistogramFromResource(frontproxycaCollector, r)
+
+	return frontproxycaCollector
+}
+
 func (r *FrontProxyCACertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Certificates.FrontProxyCA.Checksum != utilities.GetObjectChecksum(r.resource)
 }

internal/resources/k8s_deployment_resource.go

@@ -6,6 +6,7 @@ package resources
 import (
 	"context"
 
+	"github.com/prometheus/client_golang/prometheus"
 	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
@@ -21,10 +22,15 @@ type KubernetesDeploymentResource struct {
 	resource           *appsv1.Deployment
 	Client             client.Client
 	DataStore          kamajiv1alpha1.DataStore
-	Name               string
 	KineContainerImage string
 }
 
+func (r *KubernetesDeploymentResource) GetHistogram() prometheus.Histogram {
+	deploymentCollector = LazyLoadHistogramFromResource(deploymentCollector, r)
+
+	return deploymentCollector
+}
+
 func (r *KubernetesDeploymentResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return r.resource.Status.String() == tenantControlPlane.Status.Kubernetes.Deployment.DeploymentStatus.String()
 }
@@ -49,8 +55,6 @@ func (r *KubernetesDeploymentResource) Define(_ context.Context, tenantControlPl
 		},
 	}
 
-	r.Name = "deployment"
-
 	return nil
 }
 
@@ -71,7 +75,7 @@ func (r *KubernetesDeploymentResource) CreateOrUpdate(ctx context.Context, tenan
 }
 
 func (r *KubernetesDeploymentResource) GetName() string {
-	return r.Name
+	return "deployment"
 }
 
 func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {

internal/resources/k8s_ingress_resource.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
 	networkingv1 "k8s.io/api/networking/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -22,7 +23,12 @@ import (
 type KubernetesIngressResource struct {
 	resource *networkingv1.Ingress
 	Client   client.Client
-	Name     string
+}
+
+func (r *KubernetesIngressResource) GetHistogram() prometheus.Histogram {
+	ingressCollector = LazyLoadHistogramFromResource(ingressCollector, r)
+
+	return ingressCollector
 }
 
 func (r *KubernetesIngressResource) ShouldStatusBeUpdated(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) bool {
@@ -140,8 +146,6 @@ func (r *KubernetesIngressResource) Define(_ context.Context, tenantControlPlane
 		},
 	}
 
-	r.Name = "ingress"
-
 	return nil
 }
 
@@ -211,5 +215,5 @@ func (r *KubernetesIngressResource) CreateOrUpdate(ctx context.Context, tenantCo
 }
 
 func (r *KubernetesIngressResource) GetName() string {
-	return r.Name
+	return "ingress"
 }

internal/resources/k8s_service_resource.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"strconv"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -26,6 +27,12 @@ type KubernetesServiceResource struct {
 	Client   client.Client
 }
 
+func (r *KubernetesServiceResource) GetHistogram() prometheus.Histogram {
+	serviceCollector = LazyLoadHistogramFromResource(serviceCollector, r)
+
+	return serviceCollector
+}
+
 func (r *KubernetesServiceResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Kubernetes.Service.Name != r.resource.GetName() ||
 		tenantControlPlane.Status.Kubernetes.Service.Namespace != r.resource.GetNamespace() ||

internal/resources/konnectivity/agent.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -19,6 +20,7 @@ import (
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/internal/constants"
+	"github.com/clastix/kamaji/internal/resources"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
@@ -28,6 +30,12 @@ type Agent struct {
 	tenantClient client.Client
 }
 
+func (r *Agent) GetHistogram() prometheus.Histogram {
+	agentCollector = resources.LazyLoadHistogramFromResource(agentCollector, r)
+
+	return agentCollector
+}
+
 func (r *Agent) ShouldStatusBeUpdated(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) bool {
 	return tcp.Spec.Addons.Konnectivity == nil && (tcp.Status.Addons.Konnectivity.Agent.Namespace != "" || tcp.Status.Addons.Konnectivity.Agent.Name != "") ||
 		tcp.Spec.Addons.Konnectivity != nil && (tcp.Status.Addons.Konnectivity.Agent.Namespace != r.resource.Namespace || tcp.Status.Addons.Konnectivity.Agent.Name != r.resource.Name)

internal/resources/konnectivity/certificate_resource.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -21,6 +22,7 @@ import (
 	"github.com/clastix/kamaji/internal/constants"
 	"github.com/clastix/kamaji/internal/crypto"
 	"github.com/clastix/kamaji/internal/kubeadm"
+	"github.com/clastix/kamaji/internal/resources"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
@@ -29,6 +31,12 @@ type CertificateResource struct {
 	Client   client.Client
 }
 
+func (r *CertificateResource) GetHistogram() prometheus.Histogram {
+	certificateCollector = resources.LazyLoadHistogramFromResource(certificateCollector, r)
+
+	return certificateCollector
+}
+
 func (r *CertificateResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Addons.Konnectivity.Certificate.Checksum != utilities.GetObjectChecksum(r.resource)
 }

internal/resources/konnectivity/cluster_role_binding_resource.go

@@ -6,6 +6,7 @@ package konnectivity
 import (
 	"context"
 
+	"github.com/prometheus/client_golang/prometheus"
 	rbacv1 "k8s.io/api/rbac/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -15,6 +16,7 @@ import (
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/internal/constants"
+	"github.com/clastix/kamaji/internal/resources"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
@@ -25,6 +27,12 @@ type ClusterRoleBindingResource struct {
 	tenantClient client.Client
 }
 
+func (r *ClusterRoleBindingResource) GetHistogram() prometheus.Histogram {
+	clusterrolebindingCollector = resources.LazyLoadHistogramFromResource(clusterrolebindingCollector, r)
+
+	return clusterrolebindingCollector
+}
+
 func (r *ClusterRoleBindingResource) ShouldStatusBeUpdated(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) bool {
 	return tcp.Spec.Addons.Konnectivity == nil && tcp.Status.Addons.Konnectivity.ClusterRoleBinding.Name != "" ||
 		tcp.Spec.Addons.Konnectivity != nil && (tcp.Status.Addons.Konnectivity.ClusterRoleBinding.Name == "" ||

internal/resources/konnectivity/deployment_resource.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
 	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -15,6 +16,7 @@ import (
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	builder "github.com/clastix/kamaji/internal/builders/controlplane"
+	"github.com/clastix/kamaji/internal/resources"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
@@ -25,6 +27,12 @@ type KubernetesDeploymentResource struct {
 	Client  client.Client
 }
 
+func (r *KubernetesDeploymentResource) GetHistogram() prometheus.Histogram {
+	deploymentCollector = resources.LazyLoadHistogramFromResource(deploymentCollector, r)
+
+	return deploymentCollector
+}
+
 func (r *KubernetesDeploymentResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	switch {
 	case tenantControlPlane.Spec.Addons.Konnectivity == nil && tenantControlPlane.Status.Addons.Konnectivity.Enabled,

internal/resources/konnectivity/egress_selector_configuration_resource.go

@@ -6,6 +6,7 @@ package konnectivity
 import (
 	"context"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -16,6 +17,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/resources"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
@@ -24,6 +26,12 @@ type EgressSelectorConfigurationResource struct {
 	Client   client.Client
 }
 
+func (r *EgressSelectorConfigurationResource) GetHistogram() prometheus.Histogram {
+	egressCollector = resources.LazyLoadHistogramFromResource(egressCollector, r)
+
+	return egressCollector
+}
+
 func (r *EgressSelectorConfigurationResource) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	r.resource = &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{

internal/resources/konnectivity/kubeconfig_resource.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -20,6 +21,7 @@ import (
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/internal/constants"
+	"github.com/clastix/kamaji/internal/resources"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
@@ -28,6 +30,12 @@ type KubeconfigResource struct {
 	Client   client.Client
 }
 
+func (r *KubeconfigResource) GetHistogram() prometheus.Histogram {
+	kubeconfigCollector = resources.LazyLoadHistogramFromResource(kubeconfigCollector, r)
+
+	return kubeconfigCollector
+}
+
 func (r *KubeconfigResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig.Checksum != utilities.GetObjectChecksum(r.resource)
 }

internal/resources/konnectivity/metrics.go

@@ -0,0 +1,19 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package konnectivity
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	agentCollector              prometheus.Histogram
+	certificateCollector        prometheus.Histogram
+	clusterrolebindingCollector prometheus.Histogram
+	deploymentCollector         prometheus.Histogram
+	egressCollector             prometheus.Histogram
+	kubeconfigCollector         prometheus.Histogram
+	serviceaccountCollector     prometheus.Histogram
+	serviceCollector            prometheus.Histogram
+)

internal/resources/konnectivity/service_account_resource.go

@@ -6,6 +6,7 @@ package konnectivity
 import (
 	"context"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -15,6 +16,7 @@ import (
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
 	"github.com/clastix/kamaji/internal/constants"
+	"github.com/clastix/kamaji/internal/resources"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
@@ -25,6 +27,12 @@ type ServiceAccountResource struct {
 	tenantClient client.Client
 }
 
+func (r *ServiceAccountResource) GetHistogram() prometheus.Histogram {
+	serviceaccountCollector = resources.LazyLoadHistogramFromResource(serviceaccountCollector, r)
+
+	return serviceaccountCollector
+}
+
 func (r *ServiceAccountResource) ShouldStatusBeUpdated(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) bool {
 	return tcp.Spec.Addons.Konnectivity == nil && len(tcp.Status.Addons.Konnectivity.ServiceAccount.Name) > 0 && len(tcp.Status.Addons.Konnectivity.ServiceAccount.Namespace) > 0 ||
 		tcp.Spec.Addons.Konnectivity != nil && tcp.Status.Addons.Konnectivity.ServiceAccount.Name == "" && tcp.Status.Addons.Konnectivity.ServiceAccount.Namespace == ""

internal/resources/konnectivity/service_resource.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -15,6 +16,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1"
+	"github.com/clastix/kamaji/internal/resources"
 	"github.com/clastix/kamaji/internal/utilities"
 )
 
@@ -23,6 +25,12 @@ type ServiceResource struct {
 	Client   client.Client
 }
 
+func (r *ServiceResource) GetHistogram() prometheus.Histogram {
+	serviceCollector = resources.LazyLoadHistogramFromResource(serviceCollector, r)
+
+	return serviceCollector
+}
+
 func (r *ServiceResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	if tenantControlPlane.Spec.Addons.Konnectivity == nil &&
 		tenantControlPlane.Status.Addons.Konnectivity.Service.Port == 0 &&

internal/resources/kubeadm_config.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"strconv"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -27,6 +28,12 @@ type KubeadmConfigResource struct {
 	TmpDirectory string
 }
 
+func (r *KubeadmConfigResource) GetHistogram() prometheus.Histogram {
+	kubeadmconfigCollector = LazyLoadHistogramFromResource(kubeadmconfigCollector, r)
+
+	return kubeadmconfigCollector
+}
+
 func (r *KubeadmConfigResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.KubeadmConfig.Checksum != utilities.GetObjectChecksum(r.resource)
 }

internal/resources/kubeadm_phases.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -45,6 +46,29 @@ type KubeadmPhase struct {
 	checksum string
 }
 
+func (r *KubeadmPhase) GetHistogram() prometheus.Histogram {
+	switch r.Phase {
+	case PhaseUploadConfigKubeadm:
+		kubeadmphaseUploadConfigKubeadmCollector = LazyLoadHistogramFromResource(kubeadmphaseUploadConfigKubeadmCollector, r)
+
+		return kubeadmphaseUploadConfigKubeadmCollector
+	case PhaseUploadConfigKubelet:
+		kubeadmphaseUploadConfigKubeletCollector = LazyLoadHistogramFromResource(kubeadmphaseUploadConfigKubeletCollector, r)
+
+		return kubeadmphaseUploadConfigKubeletCollector
+	case PhaseBootstrapToken:
+		kubeadmphaseBootstrapTokenCollector = LazyLoadHistogramFromResource(kubeadmphaseBootstrapTokenCollector, r)
+
+		return kubeadmphaseBootstrapTokenCollector
+	case PhaseClusterAdminRBAC:
+		kubeadmphaseClusterAdminRBACCollector = LazyLoadHistogramFromResource(kubeadmphaseClusterAdminRBACCollector, r)
+
+		return kubeadmphaseClusterAdminRBACCollector
+	default:
+		panic("should not happen")
+	}
+}
+
 func (r *KubeadmPhase) GetWatchedObject() client.Object {
 	switch r.Phase {
 	case PhaseUploadConfigKubeadm:

internal/resources/kubeadm_upgrade.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -26,6 +27,12 @@ type KubernetesUpgrade struct {
 	inProgress bool
 }
 
+func (k *KubernetesUpgrade) GetHistogram() prometheus.Histogram {
+	kubeadmupgradeCollector = LazyLoadHistogramFromResource(kubeadmupgradeCollector, k)
+
+	return kubeadmupgradeCollector
+}
+
 func (k *KubernetesUpgrade) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error {
 	k.upgrade = upgrade.Upgrade{
 		Before: upgrade.ClusterState{

internal/resources/kubeconfig.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8stypes "k8s.io/apimachinery/pkg/types"
@@ -39,6 +40,12 @@ type KubeconfigResource struct {
 	TmpDirectory       string
 }
 
+func (r *KubeconfigResource) GetHistogram() prometheus.Histogram {
+	kubeconfigCollector = LazyLoadHistogramFromResource(kubeconfigCollector, r)
+
+	return kubeconfigCollector
+}
+
 func (r *KubeconfigResource) ShouldStatusBeUpdated(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) bool {
 	// an update is required only in case of missing status checksum, or name:
 	// this data is required by the following resource handlers.

internal/resources/metrics.go

@@ -0,0 +1,52 @@
+// Copyright 2022 Clastix Labs
+// SPDX-License-Identifier: Apache-2.0
+
+package resources
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+var (
+	apiservercertificateCollector      prometheus.Histogram
+	clientcertificateCollector         prometheus.Histogram
+	certificateauthorityCollector      prometheus.Histogram
+	frontproxycertificateCollector     prometheus.Histogram
+	frontproxycaCollector              prometheus.Histogram
+	deploymentCollector                prometheus.Histogram
+	ingressCollector                   prometheus.Histogram
+	serviceCollector                   prometheus.Histogram
+	kubeadmconfigCollector             prometheus.Histogram
+	kubeadmupgradeCollector            prometheus.Histogram
+	kubeconfigCollector                prometheus.Histogram
+	serviceaccountcertificateCollector prometheus.Histogram
+
+	kubeadmphaseUploadConfigKubeadmCollector prometheus.Histogram
+	kubeadmphaseUploadConfigKubeletCollector prometheus.Histogram
+	kubeadmphaseBootstrapTokenCollector      prometheus.Histogram
+	kubeadmphaseClusterAdminRBACCollector    prometheus.Histogram
+)
+
+func LazyLoadHistogramFromResource(collector prometheus.Histogram, resource Resource) prometheus.Histogram {
+	n := resource.GetName()
+
+	if collector == nil {
+		c := prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: "kamaji",
+			Subsystem: "handler",
+			Name:      n + "_time_seconds",
+			Help:      "Bucket time requested for the given handler to complete its handling.",
+			Buckets: []float64{
+				0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
+				1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60,
+			},
+		})
+
+		metrics.Registry.MustRegister(c)
+
+		return c
+	}
+
+	return collector
+}

internal/resources/resource.go

@@ -5,7 +5,9 @@ package resources
 
 import (
 	"context"
+	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	k8stypes "k8s.io/apimachinery/pkg/types"
 	clientset "k8s.io/client-go/kubernetes"
@@ -20,7 +22,13 @@ const (
 	OperationResultEnqueueBack controllerutil.OperationResult = "enqueueBack"
 )
 
+type ResourceMetric interface {
+	GetHistogram() prometheus.Histogram
+}
+
 type Resource interface {
+	ResourceMetric
+
 	Define(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) error
 	ShouldCleanup(tcp *kamajiv1alpha1.TenantControlPlane) bool
 	CleanUp(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) (bool, error)
@@ -59,6 +67,11 @@ type HandlerConfig struct {
 
 // Handle handles the given resource and returns a boolean to say if the tenantControlPlane has been modified.
 func Handle(ctx context.Context, resource Resource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) {
+	startTime := time.Now()
+	defer func() {
+		resource.GetHistogram().Observe(time.Since(startTime).Seconds())
+	}()
+
 	if err := resource.Define(ctx, tenantControlPlane); err != nil {
 		return "", err
 	}

internal/resources/sa_certificate.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/prometheus/client_golang/prometheus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
@@ -28,6 +29,12 @@ type SACertificate struct {
 	TmpDirectory string
 }
 
+func (r *SACertificate) GetHistogram() prometheus.Histogram {
+	serviceaccountcertificateCollector = LazyLoadHistogramFromResource(serviceaccountcertificateCollector, r)
+
+	return serviceaccountcertificateCollector
+}
+
 func (r *SACertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool {
 	return tenantControlPlane.Status.Certificates.SA.SecretName != r.resource.GetName() ||
 		tenantControlPlane.Status.Certificates.SA.Checksum != utilities.GetObjectChecksum(r.resource)