feat: prometheus metrics

Expose proxmox api calls metrics.

Signed-off-by: Serge Logvinov <serge.logvinov@sinextra.dev>

.golangci.yml

@@ -167,6 +167,7 @@ linters:
     - varnamelen # too annoying
     - wrapcheck
     - perfsprint
+    - exportloopref
 
     # temporarily disabled linters
     - copyloopvar

Dockerfile

@@ -1,7 +1,7 @@
 # syntax = docker/dockerfile:1.5
 ########################################
 
-FROM --platform=${BUILDPLATFORM} golang:1.22.6-alpine3.20 AS builder
+FROM --platform=${BUILDPLATFORM} golang:1.23.1-alpine3.20 AS builder
 RUN apk update && apk add --no-cache make
 ENV GO111MODULE on
 WORKDIR /src

Makefile

@@ -70,7 +70,7 @@ build: ## Build
 .PHONY: run
 run: build ## Run
 	./bin/proxmox-cloud-controller-manager-$(ARCH) --v=5 --kubeconfig=kubeconfig --cloud-config=proxmox-config.yaml --controllers=cloud-node,cloud-node-lifecycle \
-		--use-service-account-credentials --leader-elect=false --bind-address=127.0.0.1
+		--use-service-account-credentials --leader-elect=false --bind-address=127.0.0.1 --authorization-always-allow-paths=/healthz,/livez,/readyz,/metrics
 
 .PHONY: lint
 lint: ## Lint Code

charts/proxmox-cloud-controller-manager/Chart.yaml

@@ -14,7 +14,7 @@ maintainers:
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.3
+version: 0.2.4
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.

charts/proxmox-cloud-controller-manager/templates/deployment.yaml

@@ -67,6 +67,7 @@ spec:
             - --leader-elect-resource-name=cloud-controller-manager-proxmox
             - --use-service-account-credentials
             - --secure-port=10258
+            - --authorization-always-allow-paths=/healthz,/livez,/readyz,/metrics
           {{- with .Values.extraArgs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}

docs/metrics.md

@@ -0,0 +1,47 @@
+# Metrics documentation
+
+This document is a reflection of the current state of the exposed metrics of the Proxmox CCM.
+
+## Gather metrics
+
+By default, the Proxmox CCM exposes metrics on the `https://localhost:10258/metrics` endpoint.
+
+```yaml
+proxmox-cloud-controller-manager --authorization-always-allow-paths="/metrics" --secure-port=10258
+```
+
+### Helm chart values
+
+The following values can be set in the Helm chart to expose the metrics of the Talos CCM.
+
+```yaml
+podAnnotations:
+  prometheus.io/scrape: "true"
+  prometheus.io/scheme: "https"
+  prometheus.io/port: "10258"
+```
+
+## Metrics exposed by the CCM
+
+### Proxmox API calls
+
+|Metric name|Metric type|Labels/tags|
+|-----------|-----------|-----------|
+|proxmox_api_request_duration_seconds|Histogram|`request`=<api_request>|
+|proxmox_api_request_errors_total|Counter|`request`=<api_request>|
+
+Example output:
+
+```txt
+proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="0.1"} 13
+proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="0.25"} 172
+proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="0.5"} 199
+proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="1"} 210
+proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="2.5"} 210
+proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="5"} 210
+proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="10"} 210
+proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="30"} 210
+proxmox_api_request_duration_seconds_bucket{request="getVmInfo",le="+Inf"} 210
+proxmox_api_request_duration_seconds_sum{request="getVmInfo"} 39.698945394000006
+proxmox_api_request_duration_seconds_count{request="getVmInfo"} 210
+```

go.mod

@@ -1,6 +1,6 @@
 module github.com/sergelogvinov/proxmox-cloud-controller-manager
 
-go 1.22.6
+go 1.23.1
 
 require (
 	github.com/Telmate/proxmox-api-go v0.0.0-20240901160831-032717b543a5
@@ -8,18 +8,18 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.9.0
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/api v0.31.0
+	k8s.io/api v0.31.1
-	k8s.io/apimachinery v0.31.0
+	k8s.io/apimachinery v0.31.1
-	k8s.io/client-go v0.31.0
+	k8s.io/client-go v0.31.1
-	k8s.io/cloud-provider v0.31.0
+	k8s.io/cloud-provider v0.31.1
-	k8s.io/component-base v0.31.0
+	k8s.io/component-base v0.31.1
 	k8s.io/klog/v2 v2.130.1
 )
 
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
-	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
@@ -60,9 +60,9 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.20.0 // indirect
+	github.com/prometheus/client_golang v1.20.3 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/common v0.59.1 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
@@ -82,9 +82,9 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/crypto v0.27.0 // indirect
-	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
+	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
 	golang.org/x/net v0.29.0 // indirect
-	golang.org/x/oauth2 v0.22.0 // indirect
+	golang.org/x/oauth2 v0.23.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
 	golang.org/x/sys v0.25.0 // indirect
 	golang.org/x/term v0.24.0 // indirect
@@ -99,10 +99,10 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/apiserver v0.31.0 // indirect
+	k8s.io/apiserver v0.31.1 // indirect
-	k8s.io/component-helpers v0.31.0 // indirect
+	k8s.io/component-helpers v0.31.1 // indirect
-	k8s.io/controller-manager v0.31.0 // indirect
+	k8s.io/controller-manager v0.31.1 // indirect
-	k8s.io/kms v0.31.0 // indirect
+	k8s.io/kms v0.31.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect
 	k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect

go.sum

@@ -4,8 +4,8 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/Telmate/proxmox-api-go v0.0.0-20240901160831-032717b543a5 h1:piiGhiaoOLVnmzDD1xn18X+y5sGy5wb8JKeK5s1ITz8=
 github.com/Telmate/proxmox-api-go v0.0.0-20240901160831-032717b543a5/go.mod h1:Gu6n6vEn1hlyFUkjrvU+X1fdgaSXLoM9HKYYJqy1fsY=
-github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
+github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
-github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
+github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -131,12 +131,14 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.20.0 h1:jBzTZ7B099Rg24tny+qngoynol8LtVYlA2bqx3vEloI=
+github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4=
-github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
 github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
 github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
 github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
 github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0=
+github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
@@ -216,6 +218,8 @@ golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
 golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
 golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI=
 golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ=
+golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk=
+golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -226,6 +230,8 @@ golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
 golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
 golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA=
 golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
+golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -280,26 +286,26 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo=
+k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU=
-k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE=
+k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI=
-k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc=
+k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U=
-k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
-k8s.io/apiserver v0.31.0 h1:p+2dgJjy+bk+B1Csz+mc2wl5gHwvNkC9QJV+w55LVrY=
+k8s.io/apiserver v0.31.1 h1:Sars5ejQDCRBY5f7R3QFHdqN3s61nhkpaX8/k1iEw1c=
-k8s.io/apiserver v0.31.0/go.mod h1:KI9ox5Yu902iBnnyMmy7ajonhKnkeZYJhTZ/YI+WEMk=
+k8s.io/apiserver v0.31.1/go.mod h1:lzDhpeToamVZJmmFlaLwdYZwd7zB+WYRYIboqA1kGxM=
-k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8=
+k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0=
-k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU=
+k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg=
-k8s.io/cloud-provider v0.31.0 h1:qNOs78I2/7zQmyStfDtY2M7EdilUl9fCSYMcqBju/tA=
+k8s.io/cloud-provider v0.31.1 h1:40b6AgDizwm5eWratZbqubTHMob25VWr6NX2Ei5TwZA=
-k8s.io/cloud-provider v0.31.0/go.mod h1:QgUPqLoL6aXhLlrNg1U4IrJk/PvvxgeOnT2ixkgnqT0=
+k8s.io/cloud-provider v0.31.1/go.mod h1:xAdkE7fdZdu9rKLuOZUMBfagu7bM+bas3iPux/2nLGg=
-k8s.io/component-base v0.31.0 h1:/KIzGM5EvPNQcYgwq5NwoQBaOlVFrghoVGr8lG6vNRs=
+k8s.io/component-base v0.31.1 h1:UpOepcrX3rQ3ab5NB6g5iP0tvsgJWzxTyAo20sgYSy8=
-k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo=
+k8s.io/component-base v0.31.1/go.mod h1:WGeaw7t/kTsqpVTaCoVEtillbqAhF2/JgvO0LDOMa0w=
-k8s.io/component-helpers v0.31.0 h1:jyRUKA+GX+q19o81k4x94imjNICn+e6Gzi6T89va1/A=
+k8s.io/component-helpers v0.31.1 h1:5hZUf3747atdgtR3gPntrG35rC2CkK7rYq2KUraz6Os=
-k8s.io/component-helpers v0.31.0/go.mod h1:MrNIvT4iB7wXIseYSWfHUJB/aNUiFvbilp4qDfBQi6s=
+k8s.io/component-helpers v0.31.1/go.mod h1:ye0Gi8KzFNTfpIuzvVDtxJQMP/0Owkukf1vGf22Hl6U=
-k8s.io/controller-manager v0.31.0 h1:OmM0JfkzMvNXGbKIInj8SOrqIHLW4ymDGaNaa4KqyGc=
+k8s.io/controller-manager v0.31.1 h1:bwiy8y//EG5lJL2mdbOvZWrOgw2EXXIvwp95VYgoIis=
-k8s.io/controller-manager v0.31.0/go.mod h1:slaIzbI1ecqVphjSuHwlzJQ2pclSwtjUzTEQ6fYAB8A=
+k8s.io/controller-manager v0.31.1/go.mod h1:O440MSE6EI1AEVhB2Fc8FYqv6r8BHrSXjm5aj3886No=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
 k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
-k8s.io/kms v0.31.0 h1:KchILPfB1ZE+ka7223mpU5zeFNkmb45jl7RHnlImUaI=
+k8s.io/kms v0.31.1 h1:cGLyV3cIwb0ovpP/jtyIe2mEuQ/MkbhmeBF2IYCA9Io=
-k8s.io/kms v0.31.0/go.mod h1:OZKwl1fan3n3N5FFxnW5C4V3ygrah/3YXeJWS3O6+94=
+k8s.io/kms v0.31.1/go.mod h1:OZKwl1fan3n3N5FFxnW5C4V3ygrah/3YXeJWS3O6+94=
 k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUxmcUV/CtNU8QM7h1FLWQOo=
 k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA=
 k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA=

pkg/metrics/metrics.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package metrics collects metrics.
+package metrics
+
+import (
+	"time"
+)
+
+// MetricContext indicates the context for Talos client metrics.
+type MetricContext struct {
+	start      time.Time
+	attributes []string
+}
+
+// NewMetricContext creates a new MetricContext.
+func NewMetricContext(resource string) *MetricContext {
+	return &MetricContext{
+		start:      time.Now(),
+		attributes: []string{resource},
+	}
+}

pkg/metrics/metrics_api.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"time"
+
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+// CSIMetrics contains the metrics for Talos API calls.
+type CSIMetrics struct {
+	Duration *metrics.HistogramVec
+	Errors   *metrics.CounterVec
+}
+
+var apiMetrics = registerAPIMetrics()
+
+// ObserveRequest records the request latency and counts the errors.
+func (mc *MetricContext) ObserveRequest(err error) error {
+	apiMetrics.Duration.WithLabelValues(mc.attributes...).Observe(
+		time.Since(mc.start).Seconds())
+
+	if err != nil {
+		apiMetrics.Errors.WithLabelValues(mc.attributes...).Inc()
+	}
+
+	return err
+}
+
+func registerAPIMetrics() *CSIMetrics {
+	metrics := &CSIMetrics{
+		Duration: metrics.NewHistogramVec(
+			&metrics.HistogramOpts{
+				Name:    "proxmox_api_request_duration_seconds",
+				Help:    "Latency of an Proxmox API call",
+				Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 30},
+			}, []string{"request"}),
+		Errors: metrics.NewCounterVec(
+			&metrics.CounterOpts{
+				Name: "proxmox_api_request_errors_total",
+				Help: "Total number of errors for an Proxmox API call",
+			}, []string{"request"}),
+	}
+
+	legacyregistry.MustRegister(
+		metrics.Duration,
+		metrics.Errors,
+	)
+
+	return metrics
+}

pkg/proxmox/instances.go

@@ -24,6 +24,7 @@ import (
 	pxapi "github.com/Telmate/proxmox-api-go/proxmox"
 
 	"github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/cluster"
+	metrics "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/metrics"
 	provider "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/provider"
 
 	v1 "k8s.io/api/core/v1"
@@ -53,8 +54,8 @@ func (i *instances) InstanceExists(_ context.Context, node *v1.Node) (bool, erro
 		return true, nil
 	}
 
-	_, _, err := i.getInstance(node)
+	mc := metrics.NewMetricContext("getVmInfo")
-	if err != nil {
+	if _, _, err := i.getInstance(node); mc.ObserveRequest(err) != nil {
 		if err == cloudprovider.InstanceNotFound {
 			klog.V(4).InfoS("instances.InstanceExists() instance not found", "node", klog.KObj(node), "providerID", node.Spec.ProviderID)
 
@@ -92,8 +93,10 @@ func (i *instances) InstanceShutdown(_ context.Context, node *v1.Node) (bool, er
 		return false, nil
 	}
 
+	mc := metrics.NewMetricContext("getVmState")
+
 	vmState, err := px.GetVmState(vmr)
-	if err != nil {
+	if mc.ObserveRequest(err) != nil {
 		return false, err
 	}
 
@@ -121,8 +124,10 @@ func (i *instances) InstanceMetadata(_ context.Context, node *v1.Node) (*cloudpr
 		if providerID == "" {
 			klog.V(4).InfoS("instances.InstanceMetadata() empty providerID, trying find by Name", "node", klog.KObj(node))
 
+			mc := metrics.NewMetricContext("findVmByName")
+
 			vmRef, region, err = i.c.FindVMByName(node.Name)
-			if err != nil {
+			if mc.ObserveRequest(err) != nil {
 				return nil, fmt.Errorf("instances.InstanceMetadata() - failed to find instance by name %s: %v, skipped", node.Name, err)
 			}
 		} else if !strings.HasPrefix(node.Spec.ProviderID, provider.ProviderName) {
@@ -132,8 +137,10 @@ func (i *instances) InstanceMetadata(_ context.Context, node *v1.Node) (*cloudpr
 		}
 
 		if vmRef == nil {
+			mc := metrics.NewMetricContext("getVmInfo")
+
 			vmRef, region, err = i.getInstance(node)
-			if err != nil {
+			if mc.ObserveRequest(err) != nil {
 				return nil, err
 			}
 		}
@@ -176,8 +183,10 @@ func (i *instances) getInstance(node *v1.Node) (*pxapi.VmRef, string, error) {
 		return nil, "", fmt.Errorf("instances.getInstance() error: %v", err)
 	}
 
+	mc := metrics.NewMetricContext("getVmInfo")
+
 	vmInfo, err := px.GetVmInfo(vm)
-	if err != nil {
+	if mc.ObserveRequest(err) != nil {
 		if strings.Contains(err.Error(), "not found") {
 			return nil, "", cloudprovider.InstanceNotFound
 		}
@@ -200,8 +209,10 @@ func (i *instances) getInstanceType(vmRef *pxapi.VmRef, region string) (string,
 		return "", err
 	}
 
+	mc := metrics.NewMetricContext("getVmInfo")
+
 	vmInfo, err := px.GetVmInfo(vmRef)
-	if err != nil {
+	if mc.ObserveRequest(err) != nil {
 		return "", err
 	}