From 0a31716c17dd601fbe36025186de86e2d47e82cd Mon Sep 17 00:00:00 2001 From: Serge Logvinov Date: Tue, 11 Nov 2025 16:39:09 +0700 Subject: [PATCH] fix: handle inaccessible nodes Enhanced instance existence checks to handle inaccessible Proxmox nodes. Improved test cases for instance existence and metadata retrieval. Signed-off-by: Serge Logvinov --- .github/workflows/build-test.yaml | 2 +- Dockerfile | 2 +- README.md | 2 +- .../Chart.yaml | 2 +- .../README.md | 2 +- .../README.md.gotmpl | 2 +- docs/install.md | 2 +- go.mod | 42 +-- go.sum | 80 +++--- pkg/proxmox/instances.go | 120 +++++---- pkg/proxmox/instances_test.go | 246 +++++++++++++++--- pkg/proxmox/labels.go | 3 - pkg/proxmoxpool/errors.go | 3 + pkg/proxmoxpool/pool.go | 33 ++- test/cluster/cluster.go | 50 +++- 15 files changed, 426 insertions(+), 165 deletions(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 427e8da..41c7bc4 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -31,7 +31,7 @@ jobs: - name: Lint uses: golangci/golangci-lint-action@v8 with: - version: v2.6.0 + version: v2.6.1 args: --timeout=5m --config=.golangci.yml - name: Unit run: make unit diff --git a/Dockerfile b/Dockerfile index ec39a89..96f5d86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # syntax = docker/dockerfile:1.18 ######################################## -FROM --platform=${BUILDPLATFORM} golang:1.25.3-alpine AS builder +FROM --platform=${BUILDPLATFORM} golang:1.25.4-alpine AS builder RUN apk update && apk add --no-cache make ENV GO111MODULE=on WORKDIR /src diff --git a/README.md b/README.md index feb7980..018e57f 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ metadata: # Proxmox specific labels topology.proxmox.sinextra.dev/region: cluster-1 - topology.proxmox.sinextra.dev/node: pve-node-1 + topology.proxmox.sinextra.dev/zone: pve-node-1 topology.proxmox.sinextra.dev/ha-group: default name: worker-1 diff --git a/charts/proxmox-cloud-controller-manager/Chart.yaml b/charts/proxmox-cloud-controller-manager/Chart.yaml index b2269c9..bf67b01 100644 --- a/charts/proxmox-cloud-controller-manager/Chart.yaml +++ b/charts/proxmox-cloud-controller-manager/Chart.yaml @@ -16,7 +16,7 @@ maintainers: # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.18 +version: 0.2.19 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. diff --git a/charts/proxmox-cloud-controller-manager/README.md b/charts/proxmox-cloud-controller-manager/README.md index 7b15d38..2d7f469 100644 --- a/charts/proxmox-cloud-controller-manager/README.md +++ b/charts/proxmox-cloud-controller-manager/README.md @@ -30,7 +30,7 @@ You need to set `--cloud-provider=external` in the kubelet argument for all node ```shell # Create role CCM -pveum role add CCM -privs "VM.Audit" +pveum role add CCM -privs "VM.Audit Sys.Audit" # Create user and grant permissions pveum user add kubernetes@pve pveum aclmod / -user kubernetes@pve -role CCM diff --git a/charts/proxmox-cloud-controller-manager/README.md.gotmpl b/charts/proxmox-cloud-controller-manager/README.md.gotmpl index 6c4ae6d..167dde9 100644 --- a/charts/proxmox-cloud-controller-manager/README.md.gotmpl +++ b/charts/proxmox-cloud-controller-manager/README.md.gotmpl @@ -28,7 +28,7 @@ You need to set `--cloud-provider=external` in the kubelet argument for all node ```shell # Create role CCM -pveum role add CCM -privs "VM.Audit" +pveum role add CCM -privs "VM.Audit Sys.Audit" # Create user and grant permissions pveum user add kubernetes@pve pveum aclmod / -user kubernetes@pve -role CCM diff --git a/docs/install.md b/docs/install.md index ec63c95..2151fe2 100644 --- a/docs/install.md +++ b/docs/install.md @@ -48,7 +48,7 @@ Official [documentation](https://pve.proxmox.com/wiki/User_Management) ```shell # Create role CCM -pveum role add CCM -privs "VM.Audit" +pveum role add CCM -privs "VM.Audit Sys.Audit" # Create user and grant permissions pveum user add kubernetes@pve pveum aclmod / -user kubernetes@pve -role CCM diff --git a/go.mod b/go.mod index 21c4c31..17ad557 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/sergelogvinov/proxmox-cloud-controller-manager -go 1.25.3 +go 1.25.4 // replace github.com/sergelogvinov/go-proxmox => ../proxmox/go-proxmox @@ -9,9 +9,10 @@ require ( github.com/luthermonson/go-proxmox v0.2.4-0.20250923162601-ef332f9e265b github.com/pkg/errors v0.9.1 github.com/samber/lo v1.52.0 - github.com/sergelogvinov/go-proxmox v0.0.0-20251110010552-654365b267da + github.com/sergelogvinov/go-proxmox v0.0.0-20251111120129-70a3eea3125a github.com/spf13/pflag v1.0.10 github.com/stretchr/testify v1.11.1 + go.uber.org/multierr v1.11.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.34.1 k8s.io/apimachinery v0.34.1 @@ -70,8 +71,8 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.66.1 // indirect - github.com/prometheus/procfs v0.17.0 // indirect + github.com/prometheus/common v0.67.2 // indirect + github.com/prometheus/procfs v0.19.2 // indirect github.com/spf13/cobra v1.10.1 // indirect github.com/stoewer/go-strcase v1.3.1 // indirect github.com/x448/float16 v0.8.4 // indirect @@ -87,24 +88,23 @@ require ( go.opentelemetry.io/otel/metric v1.38.0 // indirect go.opentelemetry.io/otel/sdk v1.38.0 // indirect go.opentelemetry.io/otel/trace v1.38.0 // indirect - go.opentelemetry.io/proto/otlp v1.8.0 // indirect - go.uber.org/multierr v1.11.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/zap v1.27.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.42.0 // indirect - golang.org/x/exp v0.0.0-20250911091902-df9299821621 // indirect - golang.org/x/net v0.44.0 // indirect - golang.org/x/oauth2 v0.31.0 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.36.0 // indirect - golang.org/x/term v0.35.0 // indirect - golang.org/x/text v0.29.0 // indirect - golang.org/x/time v0.13.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250826171959-ef028d996bc1 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1 // indirect - google.golang.org/grpc v1.75.1 // indirect - google.golang.org/protobuf v1.36.9 // indirect + golang.org/x/crypto v0.43.0 // indirect + golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect + golang.org/x/net v0.46.0 // indirect + golang.org/x/oauth2 v0.33.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/term v0.36.0 // indirect + golang.org/x/text v0.30.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251103181224-f26f9409b101 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 // indirect + google.golang.org/grpc v1.76.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect @@ -113,8 +113,8 @@ require ( k8s.io/controller-manager v0.34.1 // indirect k8s.io/kms v0.34.1 // indirect k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect - k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d // indirect - sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0 // indirect + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect diff --git a/go.sum b/go.sum index 7754038..f4860be 100644 --- a/go.sum +++ b/go.sum @@ -154,17 +154,17 @@ github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= -github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= -github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= -github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= +github.com/prometheus/common v0.67.2 h1:PcBAckGFTIHt2+L3I33uNRTlKTplNzFctXcWhPyAEN8= +github.com/prometheus/common v0.67.2/go.mod h1:63W3KZb1JOKgcjlIr64WW/LvFGAqKPj0atm+knVGEko= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw= github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= -github.com/sergelogvinov/go-proxmox v0.0.0-20251110010552-654365b267da h1:uK/GNZyaU+b1o4Ax8TJ/c99dNtT1S5pM2nj91mj1S6Q= -github.com/sergelogvinov/go-proxmox v0.0.0-20251110010552-654365b267da/go.mod h1:vSTg/WC771SByc5087tu7uyGaXUv6fS8q3ak2X+xwqk= +github.com/sergelogvinov/go-proxmox v0.0.0-20251111120129-70a3eea3125a h1:R8ngi2YoMY3Ju/lFi6FfWA99fPxLy140Dg5YSmFSXVo= +github.com/sergelogvinov/go-proxmox v0.0.0-20251111120129-70a3eea3125a/go.mod h1:vSTg/WC771SByc5087tu7uyGaXUv6fS8q3ak2X+xwqk= github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af h1:Sp5TG9f7K39yfB+If0vjp97vuT74F72r8hfRpP8jLU0= github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= @@ -231,8 +231,8 @@ go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6 go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= -go.opentelemetry.io/proto/otlp v1.8.0 h1:fRAZQDcAFHySxpJ1TwlA1cJ4tvcrw7nXl9xWWC8N5CE= -go.opentelemetry.io/proto/otlp v1.8.0/go.mod h1:tIeYOeNBU4cvmPqpaji1P+KbB4Oloai8wN4rWzRrFF0= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -246,61 +246,61 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= -golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= -golang.org/x/exp v0.0.0-20250911091902-df9299821621 h1:2id6c1/gto0kaHYyrixvknJ8tUK/Qs5IsmBtrc+FtgU= -golang.org/x/exp v0.0.0-20250911091902-df9299821621/go.mod h1:TwQYMMnGpvZyc+JpB/UAuTNIsVJifOlSkrZkhcvpVUk= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= +golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= -golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= -golang.org/x/oauth2 v0.31.0 h1:8Fq0yVZLh4j4YA47vHKFTa9Ew5XIrCP8LC6UeNZnLxo= -golang.org/x/oauth2 v0.31.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo= +golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= -golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= -golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI= -golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= -golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20250826171959-ef028d996bc1 h1:APHvLLYBhtZvsbnpkfknDZ7NyH4z5+ub/I0u8L3Oz6g= -google.golang.org/genproto/googleapis/api v0.0.0-20250826171959-ef028d996bc1/go.mod h1:xUjFWUnWDpZ/C0Gu0qloASKFb6f8/QXiiXhSPFsD668= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1 h1:pmJpJEvT846VzausCQ5d7KreSROcDqmO388w5YbnltA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250826171959-ef028d996bc1/go.mod h1:GmFNa4BdJZ2a8G+wCe9Bg3wwThLrJun751XstdJt5Og= -google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= -google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= -google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= -google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +google.golang.org/genproto/googleapis/api v0.0.0-20251103181224-f26f9409b101 h1:vk5TfqZHNn0obhPIYeS+cxIFKFQgser/M2jnI+9c6MM= +google.golang.org/genproto/googleapis/api v0.0.0-20251103181224-f26f9409b101/go.mod h1:E17fc4PDhkr22dE3RgnH2hEubUaky6ZwW4VhANxyspg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101 h1:tRPGkdGHuewF4UisLzzHHr1spKw92qLM98nIzxbC0wY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251103181224-f26f9409b101/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= +google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -335,10 +335,10 @@ k8s.io/kms v0.34.1 h1:iCFOvewDPzWM9fMTfyIPO+4MeuZ0tcZbugxLNSHFG4w= k8s.io/kms v0.34.1/go.mod h1:s1CFkLG7w9eaTYvctOxosx88fl4spqmixnNpys0JAtM= k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= -k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d h1:wAhiDyZ4Tdtt7e46e9M5ZSAJ/MnPGPs+Ki1gHw4w1R0= -k8s.io/utils v0.0.0-20250820121507-0af2bda4dd1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0 h1:qPrZsv1cwQiFeieFlRqT627fVZ+tyfou/+S5S0H5ua0= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.33.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= diff --git a/pkg/proxmox/instances.go b/pkg/proxmox/instances.go index 8b5e0aa..bd0155c 100644 --- a/pkg/proxmox/instances.go +++ b/pkg/proxmox/instances.go @@ -25,6 +25,7 @@ import ( "strconv" "strings" + goproxmox "github.com/sergelogvinov/go-proxmox" providerconfig "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/config" metrics "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/metrics" provider "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/provider" @@ -112,12 +113,18 @@ func (i *instances) InstanceExists(ctx context.Context, node *v1.Node) (bool, er mc := metrics.NewMetricContext("getVmInfo") if _, err := i.getInstanceInfo(ctx, node); mc.ObserveRequest(err) != nil { - if err == cloudprovider.InstanceNotFound { + if errors.Is(err, cloudprovider.InstanceNotFound) { klog.V(4).InfoS("instances.InstanceExists() instance not found", "node", klog.KObj(node), "providerID", node.Spec.ProviderID) return false, nil } + if errors.Is(err, proxmoxpool.ErrNodeInaccessible) { + klog.V(4).InfoS("instances.InstanceExists() proxmox node inaccessible, cannot define instance status", "node", klog.KObj(node), "providerID", node.Spec.ProviderID) + + return true, nil + } + return false, err } @@ -193,34 +200,34 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud if mc.ObserveRequest(err) != nil { klog.ErrorS(err, "instances.InstanceMetadata() failed to get instance info", "node", klog.KObj(node)) - if err == proxmoxpool.ErrInstanceNotFound { + if errors.Is(err, cloudprovider.InstanceNotFound) { klog.V(4).InfoS("instances.InstanceMetadata() instance not found", "node", klog.KObj(node), "providerID", providerID) return &cloudprovider.InstanceMetadata{}, nil } + if errors.Is(err, proxmoxpool.ErrNodeInaccessible) { + klog.V(4).InfoS("instances.InstanceMetadata() proxmox node inaccessible, cannot get instance metadata", "node", klog.KObj(node), "providerID", providerID) + + return &cloudprovider.InstanceMetadata{}, nil + } + return nil, err } - additionalLabels := map[string]string{ + annotations := map[string]string{} + labels := map[string]string{ LabelTopologyRegion: info.Region, - LabelTopologyNode: info.Node, + LabelTopologyZone: info.Zone, } if providerID == "" { if i.provider == providerconfig.ProviderCapmox { providerID = provider.GetProviderIDFromUUID(info.UUID) + annotations[AnnotationProxmoxInstanceID] = fmt.Sprintf("%d", info.ID) } else { providerID = provider.GetProviderIDFromID(info.Region, info.ID) } - - annotations := map[string]string{ - AnnotationProxmoxInstanceID: fmt.Sprintf("%d", info.ID), - } - - if err := syncNodeAnnotations(ctx, i.c.kclient, node, annotations); err != nil { - klog.ErrorS(err, "error updating annotations for the node", "node", klog.KRef("", node.Name)) - } } metadata := &cloudprovider.InstanceMetadata{ @@ -229,7 +236,7 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud InstanceType: info.Type, Zone: info.Zone, Region: info.Region, - AdditionalLabels: additionalLabels, + AdditionalLabels: labels, } if i.zoneAsHAGroup { @@ -241,12 +248,20 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud } metadata.Zone = haGroup - additionalLabels[LabelTopologyHAGroup] = haGroup + labels[LabelTopologyHAGroup] = haGroup } - if len(additionalLabels) > 0 && !hasUninitializedTaint(node) { - if err := syncNodeLabels(i.c, node, additionalLabels); err != nil { - klog.ErrorS(err, "error updating labels for the node", "node", klog.KRef("", node.Name)) + if !hasUninitializedTaint(node) { + if len(labels) > 0 { + if err := syncNodeLabels(i.c, node, labels); err != nil { + klog.ErrorS(err, "error updating labels for the node", "node", klog.KRef("", node.Name)) + } + } + } + + if len(annotations) > 0 { + if err := syncNodeAnnotations(ctx, i.c.kclient, node, annotations); err != nil { + klog.ErrorS(err, "error updating annotations for the node", "node", klog.KRef("", node.Name)) } } @@ -265,28 +280,37 @@ func (i *instances) getInstanceInfo(ctx context.Context, node *v1.Node) (*instan ) providerID := node.Spec.ProviderID - if providerID == "" && node.Annotations[AnnotationProxmoxInstanceID] != "" { - region = node.Labels[LabelTopologyRegion] - if region == "" { - region = node.Labels[v1.LabelTopologyRegion] + + vmID, region, err = provider.ParseProviderID(providerID) + if err != nil { + if i.provider == providerconfig.ProviderDefault { + klog.V(4).InfoS("instances.getInstanceInfo() failed to parse providerID", "node", klog.KObj(node), "providerID", providerID) } - vmID, err = strconv.Atoi(node.Annotations[AnnotationProxmoxInstanceID]) - if err != nil { - return nil, fmt.Errorf("instances.getInstanceInfo() parse annotation error: %v", err) - } + // ProviderID parsing failed, probably cluster is running with kubernetes distribution + if node.Annotations[AnnotationProxmoxInstanceID] != "" { + region = node.Labels[LabelTopologyRegion] + if region == "" { + region = node.Labels[v1.LabelTopologyRegion] + } - if _, err := i.c.pxpool.GetProxmoxCluster(region); err == nil { - providerID = provider.GetProviderIDFromID(region, vmID) + vmID, err = strconv.Atoi(node.Annotations[AnnotationProxmoxInstanceID]) + if err != nil { + return nil, fmt.Errorf("instances.getInstanceInfo() parse annotation error: %v", err) + } - klog.V(4).InfoS("instances.getInstanceInfo() set providerID", "node", klog.KObj(node), "providerID", providerID) + if _, err := i.c.pxpool.GetProxmoxCluster(region); err == nil { + providerID = provider.GetProviderIDFromID(region, vmID) + + klog.V(4).InfoS("instances.getInstanceInfo() set providerID", "node", klog.KObj(node), "providerID", providerID) + } } } - if providerID == "" { - klog.V(4).InfoS("instances.getInstanceInfo() empty providerID, trying find node", "node", klog.KObj(node)) + if vmID == 0 || region == "" { + klog.V(4).InfoS("instances.getInstanceInfo() trying find node", "node", klog.KObj(node), "providerID", providerID) - mc := metrics.NewMetricContext("findVmByName") + mc := metrics.NewMetricContext("findVmByNode") vmID, region, err = i.c.pxpool.FindVMByNode(ctx, node) if mc.ObserveRequest(err) != nil { @@ -294,41 +318,25 @@ func (i *instances) getInstanceInfo(ctx context.Context, node *v1.Node) (*instan vmID, region, err = i.c.pxpool.FindVMByUUID(ctx, node.Status.NodeInfo.SystemUUID) if mc.ObserveRequest(err) != nil { - return nil, err - } - } - - if vmID == 0 { - return nil, cloudprovider.InstanceNotFound - } - - providerID = provider.GetProviderIDFromID(region, vmID) - } - - if vmID == 0 { - vmID, region, err = provider.ParseProviderID(providerID) - if err != nil { - if i.provider == providerconfig.ProviderDefault { - klog.V(4).InfoS("instances.getInstanceInfo() failed to parse providerID, trying find by name", "node", klog.KObj(node), "providerID", providerID) - } - - vmID, region, err = i.c.pxpool.FindVMByUUID(ctx, node.Status.NodeInfo.SystemUUID) - if err != nil { if errors.Is(err, proxmoxpool.ErrInstanceNotFound) { return nil, cloudprovider.InstanceNotFound } - return nil, fmt.Errorf("instances.getInstanceInfo() error: %v", err) + return nil, err } } + + providerID = provider.GetProviderIDFromID(region, vmID) + + klog.V(4).InfoS("instances.getInstanceInfo() set providerID", "node", klog.KObj(node), "providerID", providerID) } px, err := i.c.pxpool.GetProxmoxCluster(region) if err != nil { - return nil, fmt.Errorf("instances.getInstanceInfo() error: %v", err) + return nil, err } - mc := metrics.NewMetricContext("getVmInfo") + mc := metrics.NewMetricContext("getVMConfig") vm, err := px.GetVMConfig(ctx, vmID) if mc.ObserveRequest(err) != nil { @@ -336,6 +344,10 @@ func (i *instances) getInstanceInfo(ctx context.Context, node *v1.Node) (*instan return nil, cloudprovider.InstanceNotFound } + if errors.Is(err, goproxmox.ErrVirtualMachineUnreachable) { + return nil, proxmoxpool.ErrNodeInaccessible + } + return nil, err } diff --git a/pkg/proxmox/instances_test.go b/pkg/proxmox/instances_test.go index b42dc52..eadff4c 100644 --- a/pkg/proxmox/instances_test.go +++ b/pkg/proxmox/instances_test.go @@ -141,7 +141,7 @@ func (ts *configuredTestSuite) TestInstanceExists() { }, }, expected: false, - expectedError: "instances.getInstanceInfo() error: region not found", + expectedError: "region not found", }, { msg: "NodeNotExists", @@ -162,7 +162,10 @@ func (ts *configuredTestSuite) TestInstanceExists() { Name: "cluster-1-node-1", }, Spec: v1.NodeSpec{ - ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, "proxmox://11833f4c-341f-4bd3-aad7-f7abed000000", "proxmox://cluster-1/100"), + ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, + "proxmox://11833f4c-341f-4bd3-aad7-f7abed000000", + "proxmox://cluster-1/100", + ), }, Status: v1.NodeStatus{ NodeInfo: v1.NodeSystemInfo{ @@ -179,7 +182,10 @@ func (ts *configuredTestSuite) TestInstanceExists() { Name: "cluster-1-node-3", }, Spec: v1.NodeSpec{ - ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, "proxmox://11833f4c-341f-4bd3-aad7-f7abed000000", "proxmox://cluster-1/100"), + ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, + "proxmox://11833f4c-341f-4bd3-aad7-f7abed000000", + "proxmox://cluster-1/100", + ), }, Status: v1.NodeStatus{ NodeInfo: v1.NodeSystemInfo{ @@ -196,7 +202,10 @@ func (ts *configuredTestSuite) TestInstanceExists() { Name: "cluster-1-node-1", }, Spec: v1.NodeSpec{ - ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, "proxmox://8af7110d-0000-0000-0000-9527d10a6583", "proxmox://cluster-1/100"), + ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, + "proxmox://8af7110d-0000-0000-0000-9527d10a6583", + "proxmox://cluster-1/100", + ), }, Status: v1.NodeStatus{ NodeInfo: v1.NodeSystemInfo{ @@ -204,7 +213,7 @@ func (ts *configuredTestSuite) TestInstanceExists() { }, }, }, - expected: false, + expected: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, true, false), }, { msg: "NodeExistsWithDifferentNameAndUUID", @@ -223,11 +232,66 @@ func (ts *configuredTestSuite) TestInstanceExists() { }, expected: false, }, + { + msg: "NodeExistsOfflinePVENode", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-1-node-4", + Annotations: map[string]string{ + cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", + AnnotationProxmoxInstanceID: "104", + }, + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + SystemUUID: "11833f4c-341f-4bd3-aad7-f7abea000002", + }, + }, + Spec: v1.NodeSpec{ + ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, + "proxmox://11833f4c-341f-4bd3-aad7-f7abea000002", + "proxmox://cluster-1/104"), + }, + }, + expected: true, + }, + { + msg: "NodeExistsOfflinePVENodeUninitialized", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-1-node-4", + Annotations: map[string]string{ + cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", + }, + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + SystemUUID: "11833f4c-341f-4bd3-aad7-f7abea000002", + }, + }, + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + { + Key: cloudproviderapi.TaintExternalCloudProvider, + Value: "true", + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + }, + expected: true, + }, { msg: "NodeUUIDNotFoundCAPMox", node: &v1.Node{ ObjectMeta: metav1.ObjectMeta{ - Name: "talos-rqa-u7y", + Name: "node-rqa-u7y", + Annotations: map[string]string{ + AnnotationProxmoxInstanceID: "105", + }, + Labels: map[string]string{ + LabelTopologyRegion: "cluster-1", + }, }, Spec: v1.NodeSpec{ ProviderID: "proxmox://d290d7f2-b179-404c-b627-6e4dccb59066", @@ -240,23 +304,6 @@ func (ts *configuredTestSuite) TestInstanceExists() { }, expected: false, }, - { - msg: "NodeUUIDNotFoundCAPMoxDifferentFormat", - node: &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "talos-missing-node", - }, - Spec: v1.NodeSpec{ - ProviderID: "proxmox://00000000-0000-0000-0000-000000000000", - }, - Status: v1.NodeStatus{ - NodeInfo: v1.NodeSystemInfo{ - SystemUUID: "00000000-0000-0000-0000-000000000000", - }, - }, - }, - expected: false, - }, { msg: "NodeUUIDFoundCAPMox", node: &v1.Node{ @@ -277,7 +324,7 @@ func (ts *configuredTestSuite) TestInstanceExists() { } for _, testCase := range tests { - ts.Run(fmt.Sprint(testCase.msg), func() { + ts.Run(fmt.Sprintf("%s/%s", ts.configCase.name, testCase.msg), func() { exists, err := ts.i.InstanceExists(ts.T().Context(), testCase.node) if testCase.expectedError != "" { @@ -420,10 +467,58 @@ func (ts *configuredTestSuite) TestInstanceShutdown() { }, expected: false, }, + { + msg: "NodeExistsOfflinePVENode", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-1-node-4", + Annotations: map[string]string{ + cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", + }, + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + SystemUUID: "11833f4c-341f-4bd3-aad7-f7abea000002", + }, + }, + Spec: v1.NodeSpec{ + ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, + "proxmox://11833f4c-341f-4bd3-aad7-f7abea000002", + "proxmox://cluster-1/104"), + }, + }, + expected: false, + }, + { + msg: "NodeExistsOfflinePVENodeUninitialized", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-1-node-4", + Annotations: map[string]string{ + cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", + }, + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + SystemUUID: "11833f4c-341f-4bd3-aad7-f7abea000002", + }, + }, + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + { + Key: cloudproviderapi.TaintExternalCloudProvider, + Value: "true", + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + }, + expected: false, + }, } for _, testCase := range tests { - ts.Run(fmt.Sprint(testCase.msg), func() { + ts.Run(fmt.Sprintf("%s/%s", ts.configCase.name, testCase.msg), func() { exists, err := ts.i.InstanceShutdown(ts.T().Context(), testCase.node) if testCase.expectedError != "" { @@ -472,6 +567,30 @@ func (ts *configuredTestSuite) TestInstanceMetadata() { }, expected: &cloudprovider.InstanceMetadata{}, }, + { + msg: "NodeForeignProviderIDWithAnnotationAndLabel", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-1-node-1", + Annotations: map[string]string{ + cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", + AnnotationProxmoxInstanceID: "100", + }, + Labels: map[string]string{ + LabelTopologyRegion: "cluster-1", + }, + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + SystemUUID: "11833f4c-341f-4bd3-aad7-f7abed000000", + }, + }, + Spec: v1.NodeSpec{ + ProviderID: "foreign://provider-id", + }, + }, + expected: &cloudprovider.InstanceMetadata{}, + }, { msg: "NodeWrongCluster", node: &v1.Node{ @@ -486,7 +605,7 @@ func (ts *configuredTestSuite) TestInstanceMetadata() { }, }, expected: &cloudprovider.InstanceMetadata{}, - expectedError: "instances.getInstanceInfo() error: region not found", + expectedError: "region not found", }, { msg: "NodeNotExists", @@ -501,8 +620,7 @@ func (ts *configuredTestSuite) TestInstanceMetadata() { ProviderID: "proxmox://cluster-1/500", }, }, - expected: &cloudprovider.InstanceMetadata{}, - expectedError: cloudprovider.InstanceNotFound.Error(), + expected: &cloudprovider.InstanceMetadata{}, }, { msg: "NodeExists", @@ -529,7 +647,10 @@ func (ts *configuredTestSuite) TestInstanceMetadata() { }, }, expected: &cloudprovider.InstanceMetadata{ - ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, "proxmox://11833f4c-341f-4bd3-aad7-f7abed000000", "proxmox://cluster-1/100"), + ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, + "proxmox://11833f4c-341f-4bd3-aad7-f7abed000000", + "proxmox://cluster-1/100", + ), NodeAddresses: []v1.NodeAddress{ { Type: v1.NodeHostName, @@ -544,8 +665,8 @@ func (ts *configuredTestSuite) TestInstanceMetadata() { Region: "cluster-1", Zone: "pve-1", AdditionalLabels: map[string]string{ - "topology.proxmox.sinextra.dev/node": "pve-1", "topology.proxmox.sinextra.dev/region": "cluster-1", + "topology.proxmox.sinextra.dev/zone": "pve-1", }, }, }, @@ -574,7 +695,10 @@ func (ts *configuredTestSuite) TestInstanceMetadata() { }, }, expected: &cloudprovider.InstanceMetadata{ - ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, "proxmox://11833f4c-341f-4bd3-aad7-f7abed000000", "proxmox://cluster-1/100"), + ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, + "proxmox://11833f4c-341f-4bd3-aad7-f7abed000000", + "proxmox://cluster-1/100", + ), NodeAddresses: []v1.NodeAddress{ { Type: v1.NodeHostName, @@ -593,11 +717,60 @@ func (ts *configuredTestSuite) TestInstanceMetadata() { Region: "cluster-1", Zone: "pve-1", AdditionalLabels: map[string]string{ - "topology.proxmox.sinextra.dev/node": "pve-1", "topology.proxmox.sinextra.dev/region": "cluster-1", + "topology.proxmox.sinextra.dev/zone": "pve-1", }, }, }, + { + msg: "NodeExistsOfflinePVENode", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-1-node-4", + Annotations: map[string]string{ + cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", + AnnotationProxmoxInstanceID: "104", + }, + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + SystemUUID: "11833f4c-341f-4bd3-aad7-f7abea000002", + }, + }, + Spec: v1.NodeSpec{ + ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, + "proxmox://11833f4c-341f-4bd3-aad7-f7abea000002", + "proxmox://cluster-1/104"), + }, + }, + expected: &cloudprovider.InstanceMetadata{}, + }, + { + msg: "NodeExistsOfflinePVENodeUninitialized", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-1-node-4", + Annotations: map[string]string{ + cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", + }, + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + SystemUUID: "11833f4c-341f-4bd3-aad7-f7abea000002", + }, + }, + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + { + Key: cloudproviderapi.TaintExternalCloudProvider, + Value: "true", + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + }, + expected: &cloudprovider.InstanceMetadata{}, + }, { msg: "NodeExistsCluster2", node: &v1.Node{ @@ -623,7 +796,10 @@ func (ts *configuredTestSuite) TestInstanceMetadata() { }, }, expected: &cloudprovider.InstanceMetadata{ - ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, "proxmox://11833f4c-341f-4bd3-aad7-f7abea000000", "proxmox://cluster-2/103"), + ProviderID: lo.Ternary(ts.i.provider == providerconfig.ProviderCapmox, + "proxmox://11833f4c-341f-4bd3-aad7-f7abea000000", + "proxmox://cluster-2/103", + ), NodeAddresses: []v1.NodeAddress{ { Type: v1.NodeHostName, @@ -638,15 +814,15 @@ func (ts *configuredTestSuite) TestInstanceMetadata() { Region: "cluster-2", Zone: "pve-3", AdditionalLabels: map[string]string{ - "topology.proxmox.sinextra.dev/node": "pve-3", "topology.proxmox.sinextra.dev/region": "cluster-2", + "topology.proxmox.sinextra.dev/zone": "pve-3", }, }, }, } for _, testCase := range tests { - ts.Run(fmt.Sprint(testCase.msg), func() { + ts.Run(fmt.Sprintf("%s/%s", ts.configCase.name, testCase.msg), func() { meta, err := ts.i.InstanceMetadata(ts.T().Context(), testCase.node) if testCase.expectedError != "" { diff --git a/pkg/proxmox/labels.go b/pkg/proxmox/labels.go index 4ff38b9..7794a3a 100644 --- a/pkg/proxmox/labels.go +++ b/pkg/proxmox/labels.go @@ -23,9 +23,6 @@ const ( // LabelTopologyZone is the label used to store the Proxmox zone name. LabelTopologyZone = "topology." + Group + "/zone" - // LabelTopologyNode is the label used to store the Proxmox node name. - LabelTopologyNode = "topology." + Group + "/node" - // LabelTopologyHAGroup is the label used to store the Proxmox HA group name. LabelTopologyHAGroup = "topology." + Group + "/ha-group" ) diff --git a/pkg/proxmoxpool/errors.go b/pkg/proxmoxpool/errors.go index 0ac8de5..8058650 100644 --- a/pkg/proxmoxpool/errors.go +++ b/pkg/proxmoxpool/errors.go @@ -29,4 +29,7 @@ var ( ErrZoneNotFound = errors.New("zone not found") // ErrInstanceNotFound is returned when an instance is not found in the Proxmox ErrInstanceNotFound = errors.New("instance not found") + + // ErrNodeInaccessible is returned when a Proxmox node cannot be reached or accessed + ErrNodeInaccessible = errors.New("node is inaccessible") ) diff --git a/pkg/proxmoxpool/pool.go b/pkg/proxmoxpool/pool.go index e11beb5..315e544 100644 --- a/pkg/proxmoxpool/pool.go +++ b/pkg/proxmoxpool/pool.go @@ -28,6 +28,7 @@ import ( "strings" proxmox "github.com/luthermonson/go-proxmox" + "go.uber.org/multierr" goproxmox "github.com/sergelogvinov/go-proxmox" @@ -220,6 +221,8 @@ func (c *ProxmoxPool) GetNodeGroup(ctx context.Context, region string, node stri // FindVMByNode find a VM by kubernetes node resource in all Proxmox clusters. func (c *ProxmoxPool) FindVMByNode(ctx context.Context, node *v1.Node) (vmID int, region string, err error) { + var errs error + for region, px := range c.clients { vmid, err := px.FindVMByFilter(ctx, func(rs *proxmox.ClusterResource) (bool, error) { if rs.Type != "qemu" { @@ -230,9 +233,17 @@ func (c *ProxmoxPool) FindVMByNode(ctx context.Context, node *v1.Node) (vmID int return false, nil } + if rs.Status == "unknown" { + errs = multierr.Append(errs, fmt.Errorf("region %s node %s: %w", region, rs.Node, ErrNodeInaccessible)) + + return false, nil //nolint: nilerr + } + pxnode, err := px.Client.Node(ctx, rs.Node) if err != nil { - return false, err + errs = multierr.Append(errs, fmt.Errorf("region %s node %s: %v: %w", region, rs.Node, err, ErrNodeInaccessible)) + + return false, nil //nolint: nilerr } vm, err := pxnode.VirtualMachine(ctx, int(rs.VMID)) @@ -264,20 +275,34 @@ func (c *ProxmoxPool) FindVMByNode(ctx context.Context, node *v1.Node) (vmID int return vmid, region, nil } + if errs != nil { + return 0, "", errs + } + return 0, "", ErrInstanceNotFound } // FindVMByUUID find a VM by uuid in all Proxmox clusters. func (c *ProxmoxPool) FindVMByUUID(ctx context.Context, uuid string) (vmID int, region string, err error) { + var errs error + for region, px := range c.clients { vmid, err := px.FindVMByFilter(ctx, func(rs *proxmox.ClusterResource) (bool, error) { if rs.Type != "qemu" { return false, nil } + if rs.Status == "unknown" { + errs = multierr.Append(errs, fmt.Errorf("region %s node %s: %w", region, rs.Node, ErrNodeInaccessible)) + + return false, nil //nolint: nilerr + } + pxnode, err := px.Client.Node(ctx, rs.Node) if err != nil { - return false, err + errs = multierr.Append(errs, fmt.Errorf("region %s node %s: %v: %w", region, rs.Node, err, ErrNodeInaccessible)) + + return false, nil //nolint: nilerr } vm, err := pxnode.VirtualMachine(ctx, int(rs.VMID)) @@ -302,6 +327,10 @@ func (c *ProxmoxPool) FindVMByUUID(ctx context.Context, uuid string) (vmID int, return vmid, region, nil } + if errs != nil { + return 0, "", errs + } + return 0, "", ErrInstanceNotFound } diff --git a/test/cluster/cluster.go b/test/cluster/cluster.go index 0420407..8044af8 100644 --- a/test/cluster/cluster.go +++ b/test/cluster/cluster.go @@ -35,7 +35,7 @@ func SetupMockResponders() { httpmock.RegisterResponder(http.MethodGet, `=~/cluster/status`, func(_ *http.Request) (*http.Response, error) { return httpmock.NewJsonResponse(200, map[string]any{ - "data": proxmox.NodeStatuses{{Name: "pve-1"}, {Name: "pve-2"}, {Name: "pve-3"}}, + "data": proxmox.NodeStatuses{{Name: "pve-1"}, {Name: "pve-2"}, {Name: "pve-3"}, {Name: "pve-4"}}, }) }) @@ -78,6 +78,15 @@ func SetupMockResponders() { MaxMem: 5 * 1024 * 1024 * 1024, Status: "running", }, + &proxmox.ClusterResource{ + Node: "pve-4", + Type: "qemu", + VMID: 104, + Name: "cluster-1-node-4", + MaxCPU: 2, + MaxMem: 4 * 1024 * 1024 * 1024, + Status: "unknown", + }, &proxmox.ClusterResource{ ID: "storage/smb", @@ -117,6 +126,15 @@ func SetupMockResponders() { Content: "images", Status: "available", }, + &proxmox.ClusterResource{ + ID: "storage/zfs", + Type: "storage", + PluginType: "zfspool", + Node: "pve-4", + Storage: "zfs", + Content: "images", + Status: "unknown", + }, &proxmox.ClusterResource{ ID: "storage/lvm", Type: "storage", @@ -135,6 +153,15 @@ func SetupMockResponders() { Content: "images", Status: "available", }, + &proxmox.ClusterResource{ + ID: "storage/lvm", + Type: "storage", + PluginType: "lvm", + Node: "pve-4", + Storage: "local-lvm", + Content: "images", + Status: "unknown", + }, }, }) }, @@ -158,6 +185,10 @@ func SetupMockResponders() { "data": proxmox.Node{}, }) }) + httpmock.RegisterResponder(http.MethodGet, `=~/nodes/pve-4/status`, + func(_ *http.Request) (*http.Response, error) { + return httpmock.NewBytesResponse(595, []byte{}), nil + }) httpmock.RegisterResponder(http.MethodGet, "=~/nodes$", func(_ *http.Request) (*http.Response, error) { @@ -175,6 +206,10 @@ func SetupMockResponders() { Node: "pve-3", Status: "online", }, + { + Node: "pve-4", + Status: "offline", + }, }, }) }) @@ -305,6 +340,10 @@ func SetupMockResponders() { }, }) }) + httpmock.RegisterResponder(http.MethodGet, `=~/nodes/pve-4/qemu$`, + func(_ *http.Request) (*http.Response, error) { + return httpmock.NewBytesResponse(595, []byte{}), nil + }) httpmock.RegisterResponder(http.MethodGet, `=~/nodes/pve-1/qemu/100/status/current`, func(_ *http.Request) (*http.Response, error) { @@ -377,7 +416,7 @@ func SetupMockResponders() { }) }, ) - httpmock.RegisterResponder("GET", `=~/nodes/pve-3/qemu/103/config`, + httpmock.RegisterResponder(http.MethodGet, `=~/nodes/pve-3/qemu/103/config`, func(_ *http.Request) (*http.Response, error) { return httpmock.NewJsonResponse(200, map[string]any{ "data": map[string]any{ @@ -387,8 +426,13 @@ func SetupMockResponders() { }) }, ) + httpmock.RegisterResponder(http.MethodGet, `=~/nodes/pve-4/qemu/`, + func(_ *http.Request) (*http.Response, error) { + return httpmock.NewBytesResponse(595, []byte{}), nil + }, + ) - httpmock.RegisterResponder("PUT", "https://127.0.0.1:8006/api2/json/nodes/pve-1/qemu/100/resize", + httpmock.RegisterResponder(http.MethodPut, "https://127.0.0.1:8006/api2/json/nodes/pve-1/qemu/100/resize", func(_ *http.Request) (*http.Response, error) { return httpmock.NewJsonResponse(200, map[string]any{ "data": "",