From eeb12c232b04a69bfbe31ab2e1df083d69346a4f Mon Sep 17 00:00:00 2001 From: Dario Tranchitella Date: Tue, 10 Jun 2025 17:28:10 +0200 Subject: [PATCH] feat(metrics): exposing resource handlers time bucket (#836) * refactor: static names and avoiding clash Signed-off-by: Dario Tranchitella * feat(metrics): exposing resource handlers time bucket Signed-off-by: Dario Tranchitella --------- Signed-off-by: Dario Tranchitella --- internal/resources/addons/coredns.go | 7 +++ internal/resources/addons/kube_proxy.go | 7 +++ internal/resources/addons/metrics.go | 13 +++++ internal/resources/api_server_certificate.go | 7 +++ .../api_server_kubelet_client_certificate.go | 7 +++ internal/resources/ca_certificate.go | 7 +++ .../datastore/datastore_certificate.go | 8 +++ .../resources/datastore/datastore_migrate.go | 7 +++ .../datastore/datastore_multitenancy.go | 22 +++++--- .../resources/datastore/datastore_setup.go | 8 +++ .../datastore/datastore_storage_config.go | 8 +++ internal/resources/datastore/metrics.go | 16 ++++++ .../front-proxy-client-certificate.go | 7 +++ .../resources/front_proxy_ca_certificate.go | 7 +++ internal/resources/k8s_deployment_resource.go | 12 +++-- internal/resources/k8s_ingress_resource.go | 12 +++-- internal/resources/k8s_service_resource.go | 7 +++ internal/resources/konnectivity/agent.go | 8 +++ .../konnectivity/certificate_resource.go | 8 +++ .../cluster_role_binding_resource.go | 8 +++ .../konnectivity/deployment_resource.go | 8 +++ .../egress_selector_configuration_resource.go | 8 +++ .../konnectivity/kubeconfig_resource.go | 8 +++ internal/resources/konnectivity/metrics.go | 19 +++++++ .../konnectivity/service_account_resource.go | 8 +++ .../konnectivity/service_resource.go | 8 +++ internal/resources/kubeadm_config.go | 7 +++ internal/resources/kubeadm_phases.go | 24 +++++++++ internal/resources/kubeadm_upgrade.go | 7 +++ internal/resources/kubeconfig.go | 7 +++ internal/resources/metrics.go | 52 +++++++++++++++++++ internal/resources/resource.go | 13 +++++ internal/resources/sa_certificate.go | 7 +++ 33 files changed, 347 insertions(+), 15 deletions(-) create mode 100644 internal/resources/addons/metrics.go create mode 100644 internal/resources/datastore/metrics.go create mode 100644 internal/resources/konnectivity/metrics.go create mode 100644 internal/resources/metrics.go diff --git a/internal/resources/addons/coredns.go b/internal/resources/addons/coredns.go index 8fcc9dd..bf0d71b 100644 --- a/internal/resources/addons/coredns.go +++ b/internal/resources/addons/coredns.go @@ -8,6 +8,7 @@ import ( "context" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -37,6 +38,12 @@ type CoreDNS struct { serviceAccount *corev1.ServiceAccount } +func (c *CoreDNS) GetHistogram() prometheus.Histogram { + coreDNSCollector = resources.LazyLoadHistogramFromResource(coreDNSCollector, c) + + return coreDNSCollector +} + func (c *CoreDNS) Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error { c.deployment = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/resources/addons/kube_proxy.go b/internal/resources/addons/kube_proxy.go index 8640135..51bab92 100644 --- a/internal/resources/addons/kube_proxy.go +++ b/internal/resources/addons/kube_proxy.go @@ -8,6 +8,7 @@ import ( "context" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -39,6 +40,12 @@ type KubeProxy struct { daemonSet *appsv1.DaemonSet } +func (k *KubeProxy) GetHistogram() prometheus.Histogram { + kubeProxyCollector = resources.LazyLoadHistogramFromResource(kubeProxyCollector, k) + + return kubeProxyCollector +} + func (k *KubeProxy) Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error { k.clusterRoleBinding = &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/resources/addons/metrics.go b/internal/resources/addons/metrics.go new file mode 100644 index 0000000..c67e23b --- /dev/null +++ b/internal/resources/addons/metrics.go @@ -0,0 +1,13 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package addons + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + kubeProxyCollector prometheus.Histogram + coreDNSCollector prometheus.Histogram +) diff --git a/internal/resources/api_server_certificate.go b/internal/resources/api_server_certificate.go index 436cdf9..8a43da3 100644 --- a/internal/resources/api_server_certificate.go +++ b/internal/resources/api_server_certificate.go @@ -8,6 +8,7 @@ import ( "crypto/x509" "fmt" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" @@ -31,6 +32,12 @@ type APIServerCertificate struct { TmpDirectory string } +func (r *APIServerCertificate) GetHistogram() prometheus.Histogram { + apiservercertificateCollector = LazyLoadHistogramFromResource(apiservercertificateCollector, r) + + return apiservercertificateCollector +} + func (r *APIServerCertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return tenantControlPlane.Status.Certificates.APIServer.Checksum != utilities.GetObjectChecksum(r.resource) } diff --git a/internal/resources/api_server_kubelet_client_certificate.go b/internal/resources/api_server_kubelet_client_certificate.go index 85b4d42..5353358 100644 --- a/internal/resources/api_server_kubelet_client_certificate.go +++ b/internal/resources/api_server_kubelet_client_certificate.go @@ -8,6 +8,7 @@ import ( "crypto/x509" "fmt" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" @@ -30,6 +31,12 @@ type APIServerKubeletClientCertificate struct { TmpDirectory string } +func (r *APIServerKubeletClientCertificate) GetHistogram() prometheus.Histogram { + clientcertificateCollector = LazyLoadHistogramFromResource(clientcertificateCollector, r) + + return clientcertificateCollector +} + func (r *APIServerKubeletClientCertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return tenantControlPlane.Status.Certificates.APIServerKubeletClient.Checksum != utilities.GetObjectChecksum(r.resource) } diff --git a/internal/resources/ca_certificate.go b/internal/resources/ca_certificate.go index 5425b0b..d7381f0 100644 --- a/internal/resources/ca_certificate.go +++ b/internal/resources/ca_certificate.go @@ -8,6 +8,7 @@ import ( "context" "fmt" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" @@ -30,6 +31,12 @@ type CACertificate struct { TmpDirectory string } +func (r *CACertificate) GetHistogram() prometheus.Histogram { + certificateauthorityCollector = LazyLoadHistogramFromResource(certificateauthorityCollector, r) + + return certificateauthorityCollector +} + func (r *CACertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return r.isRotatingCA || tenantControlPlane.Status.Certificates.CA.SecretName != r.resource.GetName() || tenantControlPlane.Status.Certificates.CA.Checksum != utilities.GetObjectChecksum(r.resource) diff --git a/internal/resources/datastore/datastore_certificate.go b/internal/resources/datastore/datastore_certificate.go index dea45ae..3c7bd74 100644 --- a/internal/resources/datastore/datastore_certificate.go +++ b/internal/resources/datastore/datastore_certificate.go @@ -8,6 +8,7 @@ import ( "context" "fmt" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -18,6 +19,7 @@ import ( kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/internal/constants" "github.com/clastix/kamaji/internal/crypto" + "github.com/clastix/kamaji/internal/resources" "github.com/clastix/kamaji/internal/utilities" ) @@ -28,6 +30,12 @@ type Certificate struct { DataStore kamajiv1alpha1.DataStore } +func (r *Certificate) GetHistogram() prometheus.Histogram { + certificateCollector = resources.LazyLoadHistogramFromResource(certificateCollector, r) + + return certificateCollector +} + func (r *Certificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return tenantControlPlane.Status.Storage.Certificate.Checksum != utilities.GetObjectChecksum(r.resource) } diff --git a/internal/resources/datastore/datastore_migrate.go b/internal/resources/datastore/datastore_migrate.go index 6fea1ae..81ef538 100644 --- a/internal/resources/datastore/datastore_migrate.go +++ b/internal/resources/datastore/datastore_migrate.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/prometheus/client_golang/prometheus" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -36,6 +37,12 @@ type Migrate struct { inProgress bool } +func (d *Migrate) GetHistogram() prometheus.Histogram { + migrateCollector = resources.LazyLoadHistogramFromResource(migrateCollector, d) + + return migrateCollector +} + func (d *Migrate) Define(ctx context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { if len(tenantControlPlane.Status.Storage.DataStoreName) == 0 { return nil diff --git a/internal/resources/datastore/datastore_multitenancy.go b/internal/resources/datastore/datastore_multitenancy.go index ce551e6..26499dd 100644 --- a/internal/resources/datastore/datastore_multitenancy.go +++ b/internal/resources/datastore/datastore_multitenancy.go @@ -7,29 +7,37 @@ import ( "context" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/resources" ) type MultiTenancy struct { DataStore kamajiv1alpha1.DataStore } -func (m MultiTenancy) Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error { +func (m *MultiTenancy) GetHistogram() prometheus.Histogram { + multiTenancyCollector = resources.LazyLoadHistogramFromResource(multiTenancyCollector, m) + + return multiTenancyCollector +} + +func (m *MultiTenancy) Define(context.Context, *kamajiv1alpha1.TenantControlPlane) error { return nil } -func (m MultiTenancy) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool { +func (m *MultiTenancy) ShouldCleanup(*kamajiv1alpha1.TenantControlPlane) bool { return false } -func (m MultiTenancy) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) { +func (m *MultiTenancy) CleanUp(context.Context, *kamajiv1alpha1.TenantControlPlane) (bool, error) { return false, nil } -func (m MultiTenancy) CreateOrUpdate(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { +func (m *MultiTenancy) CreateOrUpdate(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { // If the NATS Datastore is already used by a Tenant Control Plane // and a new one is reclaiming it, we need to stop it, since it's not allowed. // TODO(prometherion): remove this after multi-tenancy is implemented for NATS. @@ -49,14 +57,14 @@ func (m MultiTenancy) CreateOrUpdate(_ context.Context, tcp *kamajiv1alpha1.Tena } } -func (m MultiTenancy) GetName() string { +func (m *MultiTenancy) GetName() string { return "ds.multitenancy" } -func (m MultiTenancy) ShouldStatusBeUpdated(context.Context, *kamajiv1alpha1.TenantControlPlane) bool { +func (m *MultiTenancy) ShouldStatusBeUpdated(context.Context, *kamajiv1alpha1.TenantControlPlane) bool { return false } -func (m MultiTenancy) UpdateTenantControlPlaneStatus(context.Context, *kamajiv1alpha1.TenantControlPlane) error { +func (m *MultiTenancy) UpdateTenantControlPlaneStatus(context.Context, *kamajiv1alpha1.TenantControlPlane) error { return nil } diff --git a/internal/resources/datastore/datastore_setup.go b/internal/resources/datastore/datastore_setup.go index abde4c6..ac61a4e 100644 --- a/internal/resources/datastore/datastore_setup.go +++ b/internal/resources/datastore/datastore_setup.go @@ -7,6 +7,7 @@ import ( "context" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -18,6 +19,7 @@ import ( kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/controllers/finalizers" "github.com/clastix/kamaji/internal/datastore" + "github.com/clastix/kamaji/internal/resources" "github.com/clastix/kamaji/internal/resources/utils" ) @@ -34,6 +36,12 @@ type Setup struct { DataStore kamajiv1alpha1.DataStore } +func (r *Setup) GetHistogram() prometheus.Histogram { + setupCollector = resources.LazyLoadHistogramFromResource(setupCollector, r) + + return setupCollector +} + func (r *Setup) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return tenantControlPlane.Status.Storage.Driver != string(r.DataStore.Spec.Driver) || tenantControlPlane.Status.Storage.Setup.Checksum != tenantControlPlane.Status.Storage.Config.Checksum || diff --git a/internal/resources/datastore/datastore_storage_config.go b/internal/resources/datastore/datastore_storage_config.go index 7d03420..3065d51 100644 --- a/internal/resources/datastore/datastore_storage_config.go +++ b/internal/resources/datastore/datastore_storage_config.go @@ -10,6 +10,7 @@ import ( "github.com/google/uuid" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" kubeerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -22,6 +23,7 @@ import ( kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/controllers/finalizers" + "github.com/clastix/kamaji/internal/resources" "github.com/clastix/kamaji/internal/utilities" ) @@ -32,6 +34,12 @@ type Config struct { DataStore kamajiv1alpha1.DataStore } +func (r *Config) GetHistogram() prometheus.Histogram { + storageCollector = resources.LazyLoadHistogramFromResource(storageCollector, r) + + return storageCollector +} + func (r *Config) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return tenantControlPlane.Status.Storage.Config.Checksum != utilities.GetObjectChecksum(r.resource) || tenantControlPlane.Status.Storage.DataStoreName != r.DataStore.GetName() diff --git a/internal/resources/datastore/metrics.go b/internal/resources/datastore/metrics.go new file mode 100644 index 0000000..3ec315d --- /dev/null +++ b/internal/resources/datastore/metrics.go @@ -0,0 +1,16 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package datastore + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + certificateCollector prometheus.Histogram + migrateCollector prometheus.Histogram + multiTenancyCollector prometheus.Histogram + setupCollector prometheus.Histogram + storageCollector prometheus.Histogram +) diff --git a/internal/resources/front-proxy-client-certificate.go b/internal/resources/front-proxy-client-certificate.go index f5ed67c..d22c49c 100644 --- a/internal/resources/front-proxy-client-certificate.go +++ b/internal/resources/front-proxy-client-certificate.go @@ -8,6 +8,7 @@ import ( "crypto/x509" "fmt" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" @@ -30,6 +31,12 @@ type FrontProxyClientCertificate struct { TmpDirectory string } +func (r *FrontProxyClientCertificate) GetHistogram() prometheus.Histogram { + frontproxycertificateCollector = LazyLoadHistogramFromResource(frontproxycertificateCollector, r) + + return frontproxycertificateCollector +} + func (r *FrontProxyClientCertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return tenantControlPlane.Status.Certificates.FrontProxyClient.Checksum != utilities.GetObjectChecksum(r.resource) } diff --git a/internal/resources/front_proxy_ca_certificate.go b/internal/resources/front_proxy_ca_certificate.go index d410720..c919ddd 100644 --- a/internal/resources/front_proxy_ca_certificate.go +++ b/internal/resources/front_proxy_ca_certificate.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" @@ -27,6 +28,12 @@ type FrontProxyCACertificate struct { TmpDirectory string } +func (r *FrontProxyCACertificate) GetHistogram() prometheus.Histogram { + frontproxycaCollector = LazyLoadHistogramFromResource(frontproxycaCollector, r) + + return frontproxycaCollector +} + func (r *FrontProxyCACertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return tenantControlPlane.Status.Certificates.FrontProxyCA.Checksum != utilities.GetObjectChecksum(r.resource) } diff --git a/internal/resources/k8s_deployment_resource.go b/internal/resources/k8s_deployment_resource.go index caa9195..d8fabb1 100644 --- a/internal/resources/k8s_deployment_resource.go +++ b/internal/resources/k8s_deployment_resource.go @@ -6,6 +6,7 @@ package resources import ( "context" + "github.com/prometheus/client_golang/prometheus" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" @@ -21,10 +22,15 @@ type KubernetesDeploymentResource struct { resource *appsv1.Deployment Client client.Client DataStore kamajiv1alpha1.DataStore - Name string KineContainerImage string } +func (r *KubernetesDeploymentResource) GetHistogram() prometheus.Histogram { + deploymentCollector = LazyLoadHistogramFromResource(deploymentCollector, r) + + return deploymentCollector +} + func (r *KubernetesDeploymentResource) isStatusEqual(tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return r.resource.Status.String() == tenantControlPlane.Status.Kubernetes.Deployment.DeploymentStatus.String() } @@ -49,8 +55,6 @@ func (r *KubernetesDeploymentResource) Define(_ context.Context, tenantControlPl }, } - r.Name = "deployment" - return nil } @@ -71,7 +75,7 @@ func (r *KubernetesDeploymentResource) CreateOrUpdate(ctx context.Context, tenan } func (r *KubernetesDeploymentResource) GetName() string { - return r.Name + return "deployment" } func (r *KubernetesDeploymentResource) UpdateTenantControlPlaneStatus(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { diff --git a/internal/resources/k8s_ingress_resource.go b/internal/resources/k8s_ingress_resource.go index f2e014f..a008bfb 100644 --- a/internal/resources/k8s_ingress_resource.go +++ b/internal/resources/k8s_ingress_resource.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/prometheus/client_golang/prometheus" networkingv1 "k8s.io/api/networking/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -22,7 +23,12 @@ import ( type KubernetesIngressResource struct { resource *networkingv1.Ingress Client client.Client - Name string +} + +func (r *KubernetesIngressResource) GetHistogram() prometheus.Histogram { + ingressCollector = LazyLoadHistogramFromResource(ingressCollector, r) + + return ingressCollector } func (r *KubernetesIngressResource) ShouldStatusBeUpdated(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) bool { @@ -140,8 +146,6 @@ func (r *KubernetesIngressResource) Define(_ context.Context, tenantControlPlane }, } - r.Name = "ingress" - return nil } @@ -211,5 +215,5 @@ func (r *KubernetesIngressResource) CreateOrUpdate(ctx context.Context, tenantCo } func (r *KubernetesIngressResource) GetName() string { - return r.Name + return "ingress" } diff --git a/internal/resources/k8s_service_resource.go b/internal/resources/k8s_service_resource.go index 7e7f11f..b1665da 100644 --- a/internal/resources/k8s_service_resource.go +++ b/internal/resources/k8s_service_resource.go @@ -8,6 +8,7 @@ import ( "net" "strconv" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -26,6 +27,12 @@ type KubernetesServiceResource struct { Client client.Client } +func (r *KubernetesServiceResource) GetHistogram() prometheus.Histogram { + serviceCollector = LazyLoadHistogramFromResource(serviceCollector, r) + + return serviceCollector +} + func (r *KubernetesServiceResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return tenantControlPlane.Status.Kubernetes.Service.Name != r.resource.GetName() || tenantControlPlane.Status.Kubernetes.Service.Namespace != r.resource.GetNamespace() || diff --git a/internal/resources/konnectivity/agent.go b/internal/resources/konnectivity/agent.go index 092d652..2125b82 100644 --- a/internal/resources/konnectivity/agent.go +++ b/internal/resources/konnectivity/agent.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/prometheus/client_golang/prometheus" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -19,6 +20,7 @@ import ( kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/internal/constants" + "github.com/clastix/kamaji/internal/resources" "github.com/clastix/kamaji/internal/utilities" ) @@ -28,6 +30,12 @@ type Agent struct { tenantClient client.Client } +func (r *Agent) GetHistogram() prometheus.Histogram { + agentCollector = resources.LazyLoadHistogramFromResource(agentCollector, r) + + return agentCollector +} + func (r *Agent) ShouldStatusBeUpdated(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) bool { return tcp.Spec.Addons.Konnectivity == nil && (tcp.Status.Addons.Konnectivity.Agent.Namespace != "" || tcp.Status.Addons.Konnectivity.Agent.Name != "") || tcp.Spec.Addons.Konnectivity != nil && (tcp.Status.Addons.Konnectivity.Agent.Namespace != r.resource.Namespace || tcp.Status.Addons.Konnectivity.Agent.Name != r.resource.Name) diff --git a/internal/resources/konnectivity/certificate_resource.go b/internal/resources/konnectivity/certificate_resource.go index 7dfe784..05a8eda 100644 --- a/internal/resources/konnectivity/certificate_resource.go +++ b/internal/resources/konnectivity/certificate_resource.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,6 +22,7 @@ import ( "github.com/clastix/kamaji/internal/constants" "github.com/clastix/kamaji/internal/crypto" "github.com/clastix/kamaji/internal/kubeadm" + "github.com/clastix/kamaji/internal/resources" "github.com/clastix/kamaji/internal/utilities" ) @@ -29,6 +31,12 @@ type CertificateResource struct { Client client.Client } +func (r *CertificateResource) GetHistogram() prometheus.Histogram { + certificateCollector = resources.LazyLoadHistogramFromResource(certificateCollector, r) + + return certificateCollector +} + func (r *CertificateResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return tenantControlPlane.Status.Addons.Konnectivity.Certificate.Checksum != utilities.GetObjectChecksum(r.resource) } diff --git a/internal/resources/konnectivity/cluster_role_binding_resource.go b/internal/resources/konnectivity/cluster_role_binding_resource.go index dcf9f50..4f6b0ea 100644 --- a/internal/resources/konnectivity/cluster_role_binding_resource.go +++ b/internal/resources/konnectivity/cluster_role_binding_resource.go @@ -6,6 +6,7 @@ package konnectivity import ( "context" + "github.com/prometheus/client_golang/prometheus" rbacv1 "k8s.io/api/rbac/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -15,6 +16,7 @@ import ( kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/internal/constants" + "github.com/clastix/kamaji/internal/resources" "github.com/clastix/kamaji/internal/utilities" ) @@ -25,6 +27,12 @@ type ClusterRoleBindingResource struct { tenantClient client.Client } +func (r *ClusterRoleBindingResource) GetHistogram() prometheus.Histogram { + clusterrolebindingCollector = resources.LazyLoadHistogramFromResource(clusterrolebindingCollector, r) + + return clusterrolebindingCollector +} + func (r *ClusterRoleBindingResource) ShouldStatusBeUpdated(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) bool { return tcp.Spec.Addons.Konnectivity == nil && tcp.Status.Addons.Konnectivity.ClusterRoleBinding.Name != "" || tcp.Spec.Addons.Konnectivity != nil && (tcp.Status.Addons.Konnectivity.ClusterRoleBinding.Name == "" || diff --git a/internal/resources/konnectivity/deployment_resource.go b/internal/resources/konnectivity/deployment_resource.go index bbadd43..e95423f 100644 --- a/internal/resources/konnectivity/deployment_resource.go +++ b/internal/resources/konnectivity/deployment_resource.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/prometheus/client_golang/prometheus" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -15,6 +16,7 @@ import ( kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" builder "github.com/clastix/kamaji/internal/builders/controlplane" + "github.com/clastix/kamaji/internal/resources" "github.com/clastix/kamaji/internal/utilities" ) @@ -25,6 +27,12 @@ type KubernetesDeploymentResource struct { Client client.Client } +func (r *KubernetesDeploymentResource) GetHistogram() prometheus.Histogram { + deploymentCollector = resources.LazyLoadHistogramFromResource(deploymentCollector, r) + + return deploymentCollector +} + func (r *KubernetesDeploymentResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { switch { case tenantControlPlane.Spec.Addons.Konnectivity == nil && tenantControlPlane.Status.Addons.Konnectivity.Enabled, diff --git a/internal/resources/konnectivity/egress_selector_configuration_resource.go b/internal/resources/konnectivity/egress_selector_configuration_resource.go index 8b9dcc9..0d897a3 100644 --- a/internal/resources/konnectivity/egress_selector_configuration_resource.go +++ b/internal/resources/konnectivity/egress_selector_configuration_resource.go @@ -6,6 +6,7 @@ package konnectivity import ( "context" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -16,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/resources" "github.com/clastix/kamaji/internal/utilities" ) @@ -24,6 +26,12 @@ type EgressSelectorConfigurationResource struct { Client client.Client } +func (r *EgressSelectorConfigurationResource) GetHistogram() prometheus.Histogram { + egressCollector = resources.LazyLoadHistogramFromResource(egressCollector, r) + + return egressCollector +} + func (r *EgressSelectorConfigurationResource) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { r.resource = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ diff --git a/internal/resources/konnectivity/kubeconfig_resource.go b/internal/resources/konnectivity/kubeconfig_resource.go index c194799..14336f3 100644 --- a/internal/resources/konnectivity/kubeconfig_resource.go +++ b/internal/resources/konnectivity/kubeconfig_resource.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -20,6 +21,7 @@ import ( kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/internal/constants" + "github.com/clastix/kamaji/internal/resources" "github.com/clastix/kamaji/internal/utilities" ) @@ -28,6 +30,12 @@ type KubeconfigResource struct { Client client.Client } +func (r *KubeconfigResource) GetHistogram() prometheus.Histogram { + kubeconfigCollector = resources.LazyLoadHistogramFromResource(kubeconfigCollector, r) + + return kubeconfigCollector +} + func (r *KubeconfigResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return tenantControlPlane.Status.Addons.Konnectivity.Kubeconfig.Checksum != utilities.GetObjectChecksum(r.resource) } diff --git a/internal/resources/konnectivity/metrics.go b/internal/resources/konnectivity/metrics.go new file mode 100644 index 0000000..3bab93b --- /dev/null +++ b/internal/resources/konnectivity/metrics.go @@ -0,0 +1,19 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package konnectivity + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + agentCollector prometheus.Histogram + certificateCollector prometheus.Histogram + clusterrolebindingCollector prometheus.Histogram + deploymentCollector prometheus.Histogram + egressCollector prometheus.Histogram + kubeconfigCollector prometheus.Histogram + serviceaccountCollector prometheus.Histogram + serviceCollector prometheus.Histogram +) diff --git a/internal/resources/konnectivity/service_account_resource.go b/internal/resources/konnectivity/service_account_resource.go index 15b5963..9fa2ecc 100644 --- a/internal/resources/konnectivity/service_account_resource.go +++ b/internal/resources/konnectivity/service_account_resource.go @@ -6,6 +6,7 @@ package konnectivity import ( "context" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -15,6 +16,7 @@ import ( kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" "github.com/clastix/kamaji/internal/constants" + "github.com/clastix/kamaji/internal/resources" "github.com/clastix/kamaji/internal/utilities" ) @@ -25,6 +27,12 @@ type ServiceAccountResource struct { tenantClient client.Client } +func (r *ServiceAccountResource) GetHistogram() prometheus.Histogram { + serviceaccountCollector = resources.LazyLoadHistogramFromResource(serviceaccountCollector, r) + + return serviceaccountCollector +} + func (r *ServiceAccountResource) ShouldStatusBeUpdated(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) bool { return tcp.Spec.Addons.Konnectivity == nil && len(tcp.Status.Addons.Konnectivity.ServiceAccount.Name) > 0 && len(tcp.Status.Addons.Konnectivity.ServiceAccount.Namespace) > 0 || tcp.Spec.Addons.Konnectivity != nil && tcp.Status.Addons.Konnectivity.ServiceAccount.Name == "" && tcp.Status.Addons.Konnectivity.ServiceAccount.Namespace == "" diff --git a/internal/resources/konnectivity/service_resource.go b/internal/resources/konnectivity/service_resource.go index 390d171..8cc98c4 100644 --- a/internal/resources/konnectivity/service_resource.go +++ b/internal/resources/konnectivity/service_resource.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -15,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" kamajiv1alpha1 "github.com/clastix/kamaji/api/v1alpha1" + "github.com/clastix/kamaji/internal/resources" "github.com/clastix/kamaji/internal/utilities" ) @@ -23,6 +25,12 @@ type ServiceResource struct { Client client.Client } +func (r *ServiceResource) GetHistogram() prometheus.Histogram { + serviceCollector = resources.LazyLoadHistogramFromResource(serviceCollector, r) + + return serviceCollector +} + func (r *ServiceResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { if tenantControlPlane.Spec.Addons.Konnectivity == nil && tenantControlPlane.Status.Addons.Konnectivity.Service.Port == 0 && diff --git a/internal/resources/kubeadm_config.go b/internal/resources/kubeadm_config.go index ae4cfc0..c3b4c6f 100644 --- a/internal/resources/kubeadm_config.go +++ b/internal/resources/kubeadm_config.go @@ -8,6 +8,7 @@ import ( "net" "strconv" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -27,6 +28,12 @@ type KubeadmConfigResource struct { TmpDirectory string } +func (r *KubeadmConfigResource) GetHistogram() prometheus.Histogram { + kubeadmconfigCollector = LazyLoadHistogramFromResource(kubeadmconfigCollector, r) + + return kubeadmconfigCollector +} + func (r *KubeadmConfigResource) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return tenantControlPlane.Status.KubeadmConfig.Checksum != utilities.GetObjectChecksum(r.resource) } diff --git a/internal/resources/kubeadm_phases.go b/internal/resources/kubeadm_phases.go index bf64259..4bc3e35 100644 --- a/internal/resources/kubeadm_phases.go +++ b/internal/resources/kubeadm_phases.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -45,6 +46,29 @@ type KubeadmPhase struct { checksum string } +func (r *KubeadmPhase) GetHistogram() prometheus.Histogram { + switch r.Phase { + case PhaseUploadConfigKubeadm: + kubeadmphaseUploadConfigKubeadmCollector = LazyLoadHistogramFromResource(kubeadmphaseUploadConfigKubeadmCollector, r) + + return kubeadmphaseUploadConfigKubeadmCollector + case PhaseUploadConfigKubelet: + kubeadmphaseUploadConfigKubeletCollector = LazyLoadHistogramFromResource(kubeadmphaseUploadConfigKubeletCollector, r) + + return kubeadmphaseUploadConfigKubeletCollector + case PhaseBootstrapToken: + kubeadmphaseBootstrapTokenCollector = LazyLoadHistogramFromResource(kubeadmphaseBootstrapTokenCollector, r) + + return kubeadmphaseBootstrapTokenCollector + case PhaseClusterAdminRBAC: + kubeadmphaseClusterAdminRBACCollector = LazyLoadHistogramFromResource(kubeadmphaseClusterAdminRBACCollector, r) + + return kubeadmphaseClusterAdminRBACCollector + default: + panic("should not happen") + } +} + func (r *KubeadmPhase) GetWatchedObject() client.Object { switch r.Phase { case PhaseUploadConfigKubeadm: diff --git a/internal/resources/kubeadm_upgrade.go b/internal/resources/kubeadm_upgrade.go index bfc279c..e225302 100644 --- a/internal/resources/kubeadm_upgrade.go +++ b/internal/resources/kubeadm_upgrade.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/util/version" "k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade" "sigs.k8s.io/controller-runtime/pkg/client" @@ -26,6 +27,12 @@ type KubernetesUpgrade struct { inProgress bool } +func (k *KubernetesUpgrade) GetHistogram() prometheus.Histogram { + kubeadmupgradeCollector = LazyLoadHistogramFromResource(kubeadmupgradeCollector, k) + + return kubeadmupgradeCollector +} + func (k *KubernetesUpgrade) Define(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) error { k.upgrade = upgrade.Upgrade{ Before: upgrade.ClusterState{ diff --git a/internal/resources/kubeconfig.go b/internal/resources/kubeconfig.go index a87da7f..6f57406 100644 --- a/internal/resources/kubeconfig.go +++ b/internal/resources/kubeconfig.go @@ -8,6 +8,7 @@ import ( "fmt" "strings" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8stypes "k8s.io/apimachinery/pkg/types" @@ -39,6 +40,12 @@ type KubeconfigResource struct { TmpDirectory string } +func (r *KubeconfigResource) GetHistogram() prometheus.Histogram { + kubeconfigCollector = LazyLoadHistogramFromResource(kubeconfigCollector, r) + + return kubeconfigCollector +} + func (r *KubeconfigResource) ShouldStatusBeUpdated(_ context.Context, tcp *kamajiv1alpha1.TenantControlPlane) bool { // an update is required only in case of missing status checksum, or name: // this data is required by the following resource handlers. diff --git a/internal/resources/metrics.go b/internal/resources/metrics.go new file mode 100644 index 0000000..86cb989 --- /dev/null +++ b/internal/resources/metrics.go @@ -0,0 +1,52 @@ +// Copyright 2022 Clastix Labs +// SPDX-License-Identifier: Apache-2.0 + +package resources + +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + apiservercertificateCollector prometheus.Histogram + clientcertificateCollector prometheus.Histogram + certificateauthorityCollector prometheus.Histogram + frontproxycertificateCollector prometheus.Histogram + frontproxycaCollector prometheus.Histogram + deploymentCollector prometheus.Histogram + ingressCollector prometheus.Histogram + serviceCollector prometheus.Histogram + kubeadmconfigCollector prometheus.Histogram + kubeadmupgradeCollector prometheus.Histogram + kubeconfigCollector prometheus.Histogram + serviceaccountcertificateCollector prometheus.Histogram + + kubeadmphaseUploadConfigKubeadmCollector prometheus.Histogram + kubeadmphaseUploadConfigKubeletCollector prometheus.Histogram + kubeadmphaseBootstrapTokenCollector prometheus.Histogram + kubeadmphaseClusterAdminRBACCollector prometheus.Histogram +) + +func LazyLoadHistogramFromResource(collector prometheus.Histogram, resource Resource) prometheus.Histogram { + n := resource.GetName() + + if collector == nil { + c := prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "kamaji", + Subsystem: "handler", + Name: n + "_time_seconds", + Help: "Bucket time requested for the given handler to complete its handling.", + Buckets: []float64{ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, + 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60, + }, + }) + + metrics.Registry.MustRegister(c) + + return c + } + + return collector +} diff --git a/internal/resources/resource.go b/internal/resources/resource.go index b92ecb3..18ca59c 100644 --- a/internal/resources/resource.go +++ b/internal/resources/resource.go @@ -5,7 +5,9 @@ package resources import ( "context" + "time" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" k8stypes "k8s.io/apimachinery/pkg/types" clientset "k8s.io/client-go/kubernetes" @@ -20,7 +22,13 @@ const ( OperationResultEnqueueBack controllerutil.OperationResult = "enqueueBack" ) +type ResourceMetric interface { + GetHistogram() prometheus.Histogram +} + type Resource interface { + ResourceMetric + Define(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) error ShouldCleanup(tcp *kamajiv1alpha1.TenantControlPlane) bool CleanUp(ctx context.Context, tcp *kamajiv1alpha1.TenantControlPlane) (bool, error) @@ -59,6 +67,11 @@ type HandlerConfig struct { // Handle handles the given resource and returns a boolean to say if the tenantControlPlane has been modified. func Handle(ctx context.Context, resource Resource, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) (controllerutil.OperationResult, error) { + startTime := time.Now() + defer func() { + resource.GetHistogram().Observe(time.Since(startTime).Seconds()) + }() + if err := resource.Define(ctx, tenantControlPlane); err != nil { return "", err } diff --git a/internal/resources/sa_certificate.go b/internal/resources/sa_certificate.go index b53c7b0..8486f3b 100644 --- a/internal/resources/sa_certificate.go +++ b/internal/resources/sa_certificate.go @@ -7,6 +7,7 @@ import ( "context" "fmt" + "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants" @@ -28,6 +29,12 @@ type SACertificate struct { TmpDirectory string } +func (r *SACertificate) GetHistogram() prometheus.Histogram { + serviceaccountcertificateCollector = LazyLoadHistogramFromResource(serviceaccountcertificateCollector, r) + + return serviceaccountcertificateCollector +} + func (r *SACertificate) ShouldStatusBeUpdated(_ context.Context, tenantControlPlane *kamajiv1alpha1.TenantControlPlane) bool { return tenantControlPlane.Status.Certificates.SA.SecretName != r.resource.GetName() || tenantControlPlane.Status.Certificates.SA.Checksum != utilities.GetObjectChecksum(r.resource)