feat: add new network addressing features

Changes:
- Increase test coverage of config
- Add networking feature config
- Add ability to find node ip addresses via qemu and specify ips that
  should be treated as ExternalIPAddresses

Signed-off-by: Daniel J. Holmes (jaitaiwan) <dan@jaitaiwan.dev>
Signed-off-by: Serge Logvinov <serge.logvinov@sinextra.dev>

.golangci.yml

@@ -60,6 +60,20 @@ linters:
     - intrange
     - noinlineerr
   settings:
+    importas:
+      alias:
+        - pkg: github.com/Telmate/proxmox-api-go/proxmox
+          alias: proxmoxapi
+        - pkg: github.com/sergelogvinov/proxmox-cloud-controller/manager/metrics
+          alias: metrics
+        - pkg: github.com/sergelogvinov/proxmox-cloud-controller/proxmoxpool
+          alias: proxmoxpool
+        - pkg: github.com/sergelogvinov/proxmox-cloud-controller/proxmox
+          alias: proxmox
+        - pkg: github.com/sergelogvinov/proxmox-cloud-controller/provider
+          alias: provider
+        - pkg: github.com/sergelogvinov/proxmox-cloud-controller/config
+          alias: providerconfig
     wsl_v5:
       allow-first-in-block: true
       allow-whole-block: false
@@ -69,7 +83,7 @@ linters:
     cyclop:
       max-complexity: 30
     dupl:
-      threshold: 100
+      threshold: 150
     errcheck:
       check-type-assertions: false
       check-blank: true

docs/config.md

@@ -0,0 +1,58 @@
+# Cloud controller manager configuration file
+
+This file is used to configure the Proxmox CCM.
+
+```yaml
+features:
+  # Provider type
+  provider: default|capmox
+  # Network mode
+  network: default|qemu|auto
+  # Enable or disable the IPv6 support
+  ipv6_support_disabled: true|false
+  # External IP address CIDRs list, comma-separated
+  # Use `!` to exclude a CIDR
+  external_ip_cidrs: '192.168.0.0/16,2001:db8:85a3::8a2e:370:7334/112,!fd00:1234:5678::/64'
+  # IP addresses sort order, comma-separated
+  # The IPs that do not match the CIDRs will be kept in the order they
+  # were detected.
+  ip_sort_order: '192.168.0.0/16,2001:db8:85a3::8a2e:370:7334/112'
+
+clusters:
+  # List of Proxmox clusters
+  - url: https://cluster-api-1.exmple.com:8006/api2/json
+    # Skip the certificate verification, if needed
+    insecure: false
+    # Proxmox api token
+    token_id: "kubernetes-csi@pve!csi"
+    token_secret: "secret"
+    # Region name, which is cluster name
+    region: Region-1
+
+  # Add more clusters if needed
+  - url: https://cluster-api-2.exmple.com:8006/api2/json
+    insecure: false
+    token_id: "kubernetes-csi@pve!csi"
+    token_secret: "secret"
+    region: Region-2
+```
+
+## Cluster list
+
+You can define multiple clusters in the `clusters` section.
+
+* `url` - The URL of the Proxmox cluster API.
+* `insecure` - Set to `true` to skip TLS certificate verification.
+* `token_id` - The Proxmox API token ID.
+* `token_secret` - The name of the Kubernetes Secret that contains the Proxmox API token.
+* `region` - The name of the region, which is also used as `topology.kubernetes.io/region` label.
+
+## Feature flags
+
+* `provider` - Set the provider type. The default is `default`, which uses provider-id to define the Proxmox VM ID. The `capmox` value is used for working with the Cluster API for Proxmox (CAPMox).
+* `network` - Defines how the network addresses are handled by the CCM. The default value is `default`, which uses the kubelet argument `--node-ips` to assign IPs to the node resource. The `qemu` mode uses the QEMU agent API to retrieve network addresses from the virtual machine, while auto attempts to detect the best mode automatically.
+* `ipv6_support_disabled` - Set to `true` to ignore any IPv6 addresses. The default is `false`.
+* `external_ip_cidrs` - A comma-separated list of external IP address CIDRs. You can use `!` to exclude a CIDR from the list. This is useful for defining which IPs should be considered external and not included in the node addresses.
+
+
+For more information about the network modes, see the [Networking documentation](networking.md).

docs/install.md

@@ -71,6 +71,8 @@ clusters:
     region: cluster-1
 ```
 
+See [configuration documentation](config.md) for more details.
+
 ### Method 1: kubectl
 
 Upload it to the kubernetes:

docs/networking.md

@@ -0,0 +1,69 @@
+# Networking
+
+## Node Addressing modes
+
+There are three node addressing modes that Proxmox CCM supports:
+ - Default mode (only mode available till v0.9.0)
+ - Auto mode (available from vX.X.X)
+ - QEMU-only Mode
+
+In Default mode Proxmox CCM expects nodes to be provided with their private IP Address via the `--node-ip` kubelet flag. Default mode
+*does not* set the External IP of the node.
+
+In Auto mode, Proxmox CCM makes use of both the host-networking access (if available) and the QEMU guest agent API (if available) to determine the available IP Addresses. At a minimum Auto mode will set only the Internal IP addresses of the node but can be configured to know which IP Addresses should be treated as external based on provided CIDRs and what order ALL IP addresses should be sorted in according to a sort order CIDR.
+
+> [!NOTE]
+> All modes, including Default Mode, will use any IPs provided via the `alpha.kubernetes.io/provided-node-ip` annotation, unless they are part of the ignored cidrs list (non-default modes only).
+
+### Default Mode
+
+In Default Mode, Proxmox CCM assumes that the private IP of the node will be set using the kubelet arg `--node-ip`. Setting this flag adds an annotation to the node `alpha.kubernetes.io/provided-node-ip` which is used to then set the Node's `status.Addresses` field.
+
+In this mode there is no validation of the IP address.
+
+### Auto Mode
+
+In Auto mode, Proxmox CCM uses access to the QEMU guest agent API (if available) to get a list of interfaces and IP Addresses as well as any IP addresses provided via `--node-ip`. From there depending on configuration it will setup all detected addresses as private and set any addresses matching a configured set of external CIDRs as external.
+
+Enabling auto mode is done by setting the network feature mode to `auto`:
+
+```yaml
+features:
+  network:
+    mode: auto
+```
+
+### QEMU-only Mode
+
+In QEMU Mode, Proxmox CCM uses the QEMU guest agent API to retrieve a list of IP addresses and set them as Node Addresses. Any node addresses provided via the `alpha.kubernetes.io/provided-node-ip` node annotation will also be available.
+
+Enabling qemu-only mode is done by setting the network feature mode to `qemu`:
+
+```yaml
+features:
+  network:
+    mode: qemu
+```
+
+## Example configuration
+
+The following is example configuration which sets IP addresses from 192.168.0.1 - 192.168.255.254 and 2001:0db8:85a3:0000:0000:8a2e:0370:0000 - 2001:0db8:85a3:0000:0000:8a2e:0370:ffff as "external" addresses. All other IPs from subnet 10.0.0.0/8 will be ignored.
+
+To use any mode other than default specify the following configuration:
+
+```yaml
+features:
+  network:
+    mode: auto
+    external_ip_cidrs: '192.168.0.0/16,2001:db8:85a3::8a2e:370:7334/112,!10.0.0.0/8'
+```
+
+Further configuration options are available as well. We can disable ipv6 support entirely and provide an order to sort IP addresses in (with any that don't match just being kept in whatever order the make it into the list):
+
+```yaml
+features:
+  network:
+    mode: auto
+    ipv6_support_disabled: true
+    ip_sort_order: '192.168.0.0/16,2001:db8:85a3::8a2e:370:7334/112'
+```

pkg/config/config.go

@@ -22,11 +22,12 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	yaml "gopkg.in/yaml.v3"
 
-	pxpool "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/proxmoxpool"
+	"github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/proxmoxpool"
 )
 
 // Provider specifies the provider. Can be 'default' or 'capmox'
@@ -38,12 +39,36 @@ const ProviderDefault Provider = "default"
 // ProviderCapmox is the Provider for capmox
 const ProviderCapmox Provider = "capmox"
 
+// NetworkMode specifies the network mode.
+type NetworkMode string
+
+const (
+	// NetworkModeDefault 'default' mode uses ips provided to the kubelet via --node-ip flags
+	NetworkModeDefault NetworkMode = "default"
+	// NetworkModeOnlyQemu 'qemu' mode tries to determine the ip addresses via the QEMU agent.
+	NetworkModeOnlyQemu NetworkMode = "qemu"
+	// NetworkModeAuto 'auto' attempts to use a combination of the above modes
+	NetworkModeAuto NetworkMode = "auto"
+)
+
+// ValidNetworkModes is a list of valid network modes.
+var ValidNetworkModes = []NetworkMode{NetworkModeDefault, NetworkModeOnlyQemu, NetworkModeAuto}
+
+// NetworkOpts specifies the network options for the cloud provider.
+type NetworkOpts struct {
+	ExternalIPCIDRS     string      `yaml:"external_ip_cidrs,omitempty"`
+	IPv6SupportDisabled bool        `yaml:"ipv6_support_disabled,omitempty"`
+	IPSortOrder         string      `yaml:"ip_sort_order,omitempty"`
+	Mode                NetworkMode `yaml:"mode,omitempty"`
+}
+
 // ClustersConfig is proxmox multi-cluster cloud config.
 type ClustersConfig struct {
 	Features struct {
 		Provider Provider    `yaml:"provider,omitempty"`
+		Network  NetworkOpts `yaml:"network,omitempty"`
 	} `yaml:"features,omitempty"`
-	Clusters []*pxpool.ProxmoxCluster `yaml:"clusters,omitempty"`
+	Clusters []*proxmoxpool.ProxmoxCluster `yaml:"clusters,omitempty"`
 }
 
 // ReadCloudConfig reads cloud config from a reader.
@@ -78,6 +103,15 @@ func ReadCloudConfig(config io.Reader) (ClustersConfig, error) {
 		cfg.Features.Provider = ProviderDefault
 	}
 
+	if cfg.Features.Network.Mode == "" {
+		cfg.Features.Network.Mode = NetworkModeDefault
+	}
+
+	// Validate network mode is valid
+	if !slices.Contains(ValidNetworkModes, cfg.Features.Network.Mode) {
+		return ClustersConfig{}, fmt.Errorf("invalid network mode: %s, valid modes are %v", cfg.Features.Network.Mode, ValidNetworkModes)
+	}
+
 	return cfg, nil
 }
 

pkg/config/config_test.go

@@ -22,23 +22,23 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
-	ccmConfig "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/config"
+	providerconfig "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/config"
 )
 
 func TestReadCloudConfig(t *testing.T) {
-	cfg, err := ccmConfig.ReadCloudConfig(nil)
+	cfg, err := providerconfig.ReadCloudConfig(nil)
 	assert.Nil(t, err)
 	assert.NotNil(t, cfg)
 
 	// Empty config
-	cfg, err = ccmConfig.ReadCloudConfig(strings.NewReader(`
+	cfg, err = providerconfig.ReadCloudConfig(strings.NewReader(`
 clusters:
 `))
 	assert.Nil(t, err)
 	assert.NotNil(t, cfg)
 
 	// Wrong config
-	cfg, err = ccmConfig.ReadCloudConfig(strings.NewReader(`
+	cfg, err = providerconfig.ReadCloudConfig(strings.NewReader(`
 clusters:
   test: false
 `))
@@ -47,7 +47,7 @@ clusters:
 	assert.NotNil(t, cfg)
 
 	// Non full config
-	cfg, err = ccmConfig.ReadCloudConfig(strings.NewReader(`
+	cfg, err = providerconfig.ReadCloudConfig(strings.NewReader(`
 clusters:
 - url: abcd
   region: cluster-1
@@ -57,7 +57,7 @@ clusters:
 	assert.NotNil(t, cfg)
 
 	// Valid config with one cluster
-	cfg, err = ccmConfig.ReadCloudConfig(strings.NewReader(`
+	cfg, err = providerconfig.ReadCloudConfig(strings.NewReader(`
 clusters:
   - url: https://example.com
     insecure: false
@@ -68,9 +68,10 @@ clusters:
 	assert.Nil(t, err)
 	assert.NotNil(t, cfg)
 	assert.Equal(t, 1, len(cfg.Clusters))
+	assert.Equal(t, "user!token-id", cfg.Clusters[0].TokenID)
 
 	// Valid config with one cluster (username/password), implicit default provider
-	cfg, err = ccmConfig.ReadCloudConfig(strings.NewReader(`
+	cfg, err = providerconfig.ReadCloudConfig(strings.NewReader(`
 clusters:
   - url: https://example.com
     insecure: false
@@ -81,10 +82,10 @@ clusters:
 	assert.Nil(t, err)
 	assert.NotNil(t, cfg)
 	assert.Equal(t, 1, len(cfg.Clusters))
-	assert.Equal(t, ccmConfig.ProviderDefault, cfg.Features.Provider)
+	assert.Equal(t, providerconfig.ProviderDefault, cfg.Features.Provider)
 
 	// Valid config with one cluster (username/password), explicit provider default
-	cfg, err = ccmConfig.ReadCloudConfig(strings.NewReader(`
+	cfg, err = providerconfig.ReadCloudConfig(strings.NewReader(`
 features:
   provider: 'default'
 clusters:
@@ -97,10 +98,10 @@ clusters:
 	assert.Nil(t, err)
 	assert.NotNil(t, cfg)
 	assert.Equal(t, 1, len(cfg.Clusters))
-	assert.Equal(t, ccmConfig.ProviderDefault, cfg.Features.Provider)
+	assert.Equal(t, providerconfig.ProviderDefault, cfg.Features.Provider)
 
 	// Valid config with one cluster (username/password), explicit provider capmox
-	cfg, err = ccmConfig.ReadCloudConfig(strings.NewReader(`
+	cfg, err = providerconfig.ReadCloudConfig(strings.NewReader(`
 features:
   provider: 'capmox'
 clusters:
@@ -113,16 +114,85 @@ clusters:
 	assert.Nil(t, err)
 	assert.NotNil(t, cfg)
 	assert.Equal(t, 1, len(cfg.Clusters))
-	assert.Equal(t, ccmConfig.ProviderCapmox, cfg.Features.Provider)
+	assert.Equal(t, providerconfig.ProviderCapmox, cfg.Features.Provider)
+
+	// Errors when username/password are set with token_id/token_secret
+	_, err = providerconfig.ReadCloudConfig(strings.NewReader(`
+features:
+  provider: 'capmox'
+clusters:
+  - url: https://example.com
+    insecure: false
+    username: "user@pam"
+    password: "secret"
+    token_id: "ha"
+    token_secret: "secret"
+    region: cluster-1
+`))
+	assert.NotNil(t, err)
+
+	// Errors when no region
+	_, err = providerconfig.ReadCloudConfig(strings.NewReader(`
+features:
+  provider: 'capmox'
+clusters:
+  - url: https://example.com
+    insecure: false
+    username: "user@pam"
+    password: "secret"
+`))
+	assert.NotNil(t, err)
+
+	// Errors when empty url
+	_, err = providerconfig.ReadCloudConfig(strings.NewReader(`
+features:
+  provider: 'capmox'
+clusters:
+  - url: ""
+    region: test
+    insecure: false
+    username: "user@pam"
+    password: "secret"
+`))
+	assert.NotNil(t, err)
+
+	// Errors when invalid url protocol
+	_, err = providerconfig.ReadCloudConfig(strings.NewReader(`
+features:
+  provider: 'capmox'
+clusters:
+  - url: quic://example.com
+    insecure: false
+    region: test
+    username: "user@pam"
+    password: "secret"
+`))
+	assert.NotNil(t, err)
+}
+
+func TestNetworkConfig(t *testing.T) {
+	// Empty config results in default network mode
+	cfg, err := providerconfig.ReadCloudConfig(strings.NewReader(`---`))
+	assert.Nil(t, err)
+	assert.NotNil(t, cfg)
+	assert.Equal(t, providerconfig.NetworkModeDefault, cfg.Features.Network.Mode)
+
+	// Invalid network mode value results in error
+	_, err = providerconfig.ReadCloudConfig(strings.NewReader(`
+features:
+  network:
+    mode: 'invalid-mode'
+`))
+	assert.NotNil(t, err)
 }
 
 func TestReadCloudConfigFromFile(t *testing.T) {
-	cfg, err := ccmConfig.ReadCloudConfigFromFile("testdata/cloud-config.yaml")
+	cfg, err := providerconfig.ReadCloudConfigFromFile("testdata/cloud-config.yaml")
 	assert.NotNil(t, err)
 	assert.EqualError(t, err, "error reading testdata/cloud-config.yaml: open testdata/cloud-config.yaml: no such file or directory")
 	assert.NotNil(t, cfg)
 
-	cfg, err = ccmConfig.ReadCloudConfigFromFile("../../hack/proxmox-config.yaml")
+	cfg, err = providerconfig.ReadCloudConfigFromFile("../../hack/proxmox-config.yaml")
 	assert.Nil(t, err)
 	assert.NotNil(t, cfg)
 	assert.Equal(t, 2, len(cfg.Clusters))

pkg/provider/provider.go

@@ -23,7 +23,7 @@ import (
 	"strconv"
 	"strings"
 
-	pxapi "github.com/Telmate/proxmox-api-go/proxmox"
+	"github.com/Telmate/proxmox-api-go/proxmox"
 )
 
 const (
@@ -34,7 +34,7 @@ const (
 var providerIDRegexp = regexp.MustCompile(`^` + ProviderName + `://([^/]*)/([^/]+)$`)
 
 // GetProviderID returns the magic providerID for kubernetes node.
-func GetProviderID(region string, vmr *pxapi.VmRef) string {
+func GetProviderID(region string, vmr *proxmox.VmRef) string {
 	return fmt.Sprintf("%s://%s/%d", ProviderName, region, vmr.VmId())
 }
 
@@ -63,7 +63,7 @@ func GetVMID(providerID string) (int, error) {
 }
 
 // ParseProviderID returns the VmRef and region from the providerID.
-func ParseProviderID(providerID string) (*pxapi.VmRef, string, error) {
+func ParseProviderID(providerID string) (*proxmox.VmRef, string, error) {
 	if !strings.HasPrefix(providerID, ProviderName) {
 		return nil, "", fmt.Errorf("foreign providerID or empty \"%s\"", providerID)
 	}
@@ -78,5 +78,5 @@ func ParseProviderID(providerID string) (*pxapi.VmRef, string, error) {
 		return nil, "", fmt.Errorf("InstanceID have to be a number, but got \"%s\"", matches[2])
 	}
 
-	return pxapi.NewVmRef(vmID), matches[1], nil
+	return proxmox.NewVmRef(vmID), matches[1], nil
 }

pkg/provider/provider_test.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 	"testing"
 
-	pxapi "github.com/Telmate/proxmox-api-go/proxmox"
+	"github.com/Telmate/proxmox-api-go/proxmox"
 	"github.com/stretchr/testify/assert"
 
 	provider "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/provider"
@@ -55,7 +55,7 @@ func TestGetProviderID(t *testing.T) {
 		t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) {
 			t.Parallel()
 
-			providerID := provider.GetProviderID(testCase.region, pxapi.NewVmRef(testCase.vmID))
+			providerID := provider.GetProviderID(testCase.region, proxmox.NewVmRef(testCase.vmID))
 
 			assert.Equal(t, testCase.expectedProviderID, providerID)
 		})

pkg/proxmox/cloud.go

@@ -66,7 +66,7 @@ func newCloud(config *ccmConfig.ClustersConfig) (cloudprovider.Interface, error)
 		return nil, err
 	}
 
-	instancesInterface := newInstances(client, config.Features.Provider)
+	instancesInterface := newInstances(client, config.Features.Provider, config.Features.Network)
 
 	return &cloud{
 		client:      client,

pkg/proxmox/instance_addresses.go

@@ -0,0 +1,282 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package proxmox
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"sort"
+	"strings"
+
+	"github.com/Telmate/proxmox-api-go/proxmox"
+
+	providerconfig "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/config"
+	metrics "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/metrics"
+
+	v1 "k8s.io/api/core/v1"
+	cloudproviderapi "k8s.io/cloud-provider/api"
+	"k8s.io/klog/v2"
+)
+
+const (
+	noSortPriority = 0
+)
+
+func (i *instances) addresses(ctx context.Context, node *v1.Node, vmRef *proxmox.VmRef, region string) []v1.NodeAddress {
+	klog.V(4).InfoS("instances.addresses() called", "node", klog.KObj(node))
+
+	var (
+		providedIP string
+		ok         bool
+	)
+
+	if providedIP, ok = node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr]; !ok {
+		klog.InfoS(fmt.Sprintf(
+			"instances.InstanceMetadata() called: annotation %s missing from node. Was kubelet started without --cloud-provider=external or --node-ip?",
+			cloudproviderapi.AnnotationAlphaProvidedIPAddr),
+			node, klog.KRef("", node.Name))
+	}
+
+	// providedIP is supposed to be a single IP but some kubelets might set a comma separated list of IPs.
+	providedAddresses := []string{}
+	if providedIP != "" {
+		providedAddresses = strings.Split(providedIP, ",")
+	}
+
+	addresses := []v1.NodeAddress{
+		{Type: v1.NodeHostName, Address: node.Name},
+	}
+
+	for _, address := range providedAddresses {
+		if address = strings.TrimSpace(address); address != "" {
+			parsedAddress := net.ParseIP(address)
+			if parsedAddress != nil {
+				addresses = append(addresses, v1.NodeAddress{
+					Type:    v1.NodeInternalIP,
+					Address: parsedAddress.String(),
+				})
+			} else {
+				klog.Warningf("Ignoring invalid provided address '%s' for node %s", address, node.Name)
+			}
+		}
+	}
+
+	if i.networkOpts.Mode == providerconfig.NetworkModeDefault {
+		// If the network mode is 'default', we only return the provided IPs.
+		klog.V(4).InfoS("instances.addresses() returning provided IPs", "node", klog.KObj(node))
+
+		return addresses
+	}
+
+	if i.networkOpts.Mode == providerconfig.NetworkModeOnlyQemu || i.networkOpts.Mode == providerconfig.NetworkModeAuto {
+		newAddresses, err := i.retrieveQemuAddresses(ctx, vmRef, region)
+		if err != nil {
+			klog.ErrorS(err, "Failed to retrieve host addresses")
+		} else {
+			addToNodeAddresses(&addresses, newAddresses...)
+		}
+	}
+
+	// Remove addresses that match the ignored CIDRs
+	if len(i.networkOpts.IgnoredCIDRs) > 0 {
+		var removableAddresses []v1.NodeAddress
+
+		for _, addr := range addresses {
+			ip := net.ParseIP(addr.Address)
+			if ip != nil && isAddressInCIDRList(i.networkOpts.IgnoredCIDRs, ip) {
+				removableAddresses = append(removableAddresses, addr)
+			}
+		}
+
+		removeFromNodeAddresses(&addresses, removableAddresses...)
+	}
+
+	sortNodeAddresses(addresses, i.networkOpts.SortOrder)
+
+	klog.InfoS("instances.addresses() returning addresses", "addresses", addresses, "node", klog.KObj(node))
+
+	return addresses
+}
+
+// retrieveQemuAddresses retrieves the addresses from the QEMU agent
+func (i *instances) retrieveQemuAddresses(ctx context.Context, vmRef *proxmox.VmRef, region string) ([]v1.NodeAddress, error) {
+	var addresses []v1.NodeAddress
+
+	klog.V(4).InfoS("retrieveQemuAddresses() retrieving addresses from QEMU agent")
+
+	r, err := i.getInstanceNics(ctx, vmRef, region)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, nic := range r {
+		for _, ip := range nic.IpAddresses {
+			i.processIP(ctx, &addresses, ip)
+		}
+	}
+
+	klog.V(4).InfoS("retrieveQemuAddresses() retrieved instance nics", "nics", r)
+
+	return addresses, nil
+}
+
+func (i *instances) processIP(_ context.Context, addresses *[]v1.NodeAddress, ip net.IP) {
+	if ip == nil || ip.IsLoopback() {
+		return
+	}
+
+	var isIPv6 bool
+
+	addressType := v1.NodeInternalIP
+
+	if isIPv6 = ip.To4() == nil; isIPv6 && i.networkOpts.IPv6SupportDisabled {
+		klog.V(4).InfoS("Skipping IPv6 address due to IPv6 support being disabled", "address", ip.String())
+
+		return // skip IPv6 addresses if IPv6 support is disabled
+	}
+
+	ipStr := ip.String()
+
+	// Check if the address is an external CIDR
+	if len(i.networkOpts.ExternalCIDRs) != 0 && isAddressInCIDRList(i.networkOpts.ExternalCIDRs, ip) {
+		addressType = v1.NodeExternalIP
+	}
+
+	*addresses = append(*addresses, v1.NodeAddress{
+		Type:    addressType,
+		Address: ipStr,
+	})
+}
+
+func (i *instances) getInstanceNics(ctx context.Context, vmRef *proxmox.VmRef, region string) ([]proxmox.AgentNetworkInterface, error) {
+	px, err := i.c.GetProxmoxCluster(region)
+	result := make([]proxmox.AgentNetworkInterface, 0)
+
+	if err != nil {
+		return result, err
+	}
+
+	mc := metrics.NewMetricContext("getVmInfo")
+	nicset, err := px.GetVmAgentNetworkInterfaces(ctx, vmRef)
+
+	if mc.ObserveRequest(err) != nil {
+		return result, err
+	}
+
+	klog.V(4).InfoS("getInstanceNics() retrieved IP set", "nicset", nicset)
+
+	return nicset, nil
+}
+
+// getSortPriority returns the priority as int of an address.
+//
+// The priority depends on the index of the CIDR in the list the address is matching,
+// where the first item of the list has higher priority than the last.
+//
+// If the address does not match any CIDR or is not an IP address the function returns noSortPriority.
+func getSortPriority(list []*net.IPNet, address string) int {
+	parsedAddress := net.ParseIP(address)
+	if parsedAddress == nil {
+		return noSortPriority
+	}
+
+	for i, cidr := range list {
+		if cidr.Contains(parsedAddress) {
+			return len(list) - i
+		}
+	}
+
+	return noSortPriority
+}
+
+// sortNodeAddresses sorts node addresses based on comma separated list of CIDRs represented by addressSortOrder.
+//
+// The function only sorts addresses which match the CIDR and leaves the other addresses in the same order they are in.
+// Essentially, it will also group the addresses matching a CIDR together and sort them ascending in this group,
+// whereas the inter-group sorting depends on the priority.
+//
+// The priority depends on the order of the item in addressSortOrder, where the first item has higher priority than the last.
+func sortNodeAddresses(addresses []v1.NodeAddress, addressSortOrder []*net.IPNet) {
+	sort.SliceStable(addresses, func(i int, j int) bool {
+		addressLeft := addresses[i]
+		addressRight := addresses[j]
+
+		priorityLeft := getSortPriority(addressSortOrder, addressLeft.Address)
+		priorityRight := getSortPriority(addressSortOrder, addressRight.Address)
+
+		// ignore priorities of value 0 since this means the address has noSortPriority and we need to sort by priority
+		if priorityLeft > noSortPriority && priorityLeft == priorityRight {
+			return bytes.Compare(net.ParseIP(addressLeft.Address), net.ParseIP(addressRight.Address)) < 0
+		}
+
+		return priorityLeft > priorityRight
+	})
+}
+
+// addToNodeAddresses appends the NodeAddresses to the passed-by-pointer slice,
+// only if they do not already exist
+func addToNodeAddresses(addresses *[]v1.NodeAddress, addAddresses ...v1.NodeAddress) {
+	for _, add := range addAddresses {
+		exists := false
+
+		for _, existing := range *addresses {
+			if existing.Address == add.Address && existing.Type == add.Type {
+				exists = true
+
+				break
+			}
+		}
+
+		if !exists {
+			*addresses = append(*addresses, add)
+		}
+	}
+}
+
+// removeFromNodeAddresses removes the NodeAddresses from the passed-by-pointer
+// slice if they already exist.
+func removeFromNodeAddresses(addresses *[]v1.NodeAddress, removeAddresses ...v1.NodeAddress) {
+	var indexesToRemove []int
+
+	for _, remove := range removeAddresses {
+		for i := len(*addresses) - 1; i >= 0; i-- {
+			existing := (*addresses)[i]
+			if existing.Address == remove.Address && (existing.Type == remove.Type || remove.Type == "") {
+				indexesToRemove = append(indexesToRemove, i)
+			}
+		}
+	}
+
+	for _, i := range indexesToRemove {
+		if i < len(*addresses) {
+			*addresses = append((*addresses)[:i], (*addresses)[i+1:]...)
+		}
+	}
+}
+
+// isAddressInCIDRList checks if the given address is contained in any of the CIDRs in the list.
+func isAddressInCIDRList(cidrs []*net.IPNet, address net.IP) bool {
+	for _, cidr := range cidrs {
+		if cidr.Contains(address) {
+			return true
+		}
+	}
+
+	return false
+}

pkg/proxmox/instances.go

@@ -19,16 +19,17 @@ package proxmox
 import (
 	"context"
 	"fmt"
+	"net"
 	"regexp"
 	"strconv"
 	"strings"
 
-	pxapi "github.com/Telmate/proxmox-api-go/proxmox"
+	"github.com/Telmate/proxmox-api-go/proxmox"
 
-	ccmConfig "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/config"
+	providerconfig "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/config"
 	metrics "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/metrics"
 	provider "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/provider"
-	pxpool "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/proxmoxpool"
+	"github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/proxmoxpool"
 
 	v1 "k8s.io/api/core/v1"
 	cloudprovider "k8s.io/cloud-provider"
@@ -36,17 +37,49 @@ import (
 	"k8s.io/klog/v2"
 )
 
+type instanceNetops struct {
+	ExternalCIDRs       []*net.IPNet
+	SortOrder           []*net.IPNet
+	IgnoredCIDRs        []*net.IPNet
+	Mode                providerconfig.NetworkMode
+	IPv6SupportDisabled bool
+}
+
 type instances struct {
-	c        *pxpool.ProxmoxPool
+	c           *proxmoxpool.ProxmoxPool
-	provider ccmConfig.Provider
+	provider    providerconfig.Provider
+	networkOpts instanceNetops
 }
 
 var instanceTypeNameRegexp = regexp.MustCompile(`(^[a-zA-Z0-9_.-]+)$`)
 
-func newInstances(client *pxpool.ProxmoxPool, provider ccmConfig.Provider) *instances {
+func newInstances(client *proxmoxpool.ProxmoxPool, provider providerconfig.Provider, networkOpts providerconfig.NetworkOpts) *instances {
+	externalIPCIDRs := ParseCIDRList(networkOpts.ExternalIPCIDRS)
+	if len(networkOpts.ExternalIPCIDRS) > 0 && len(externalIPCIDRs) == 0 {
+		klog.Warningf("Failed to parse external CIDRs: %v", networkOpts.ExternalIPCIDRS)
+	}
+
+	sortOrderCIDRs, ignoredCIDRs, err := ParseCIDRRuleset(networkOpts.IPSortOrder)
+	if err != nil {
+		klog.Errorf("Failed to parse sort order CIDRs: %v", err)
+	}
+
+	if len(networkOpts.IPSortOrder) > 0 && (len(sortOrderCIDRs)+len(ignoredCIDRs)) == 0 {
+		klog.Warningf("Failed to parse sort order CIDRs: %v", networkOpts.IPSortOrder)
+	}
+
+	netOps := instanceNetops{
+		ExternalCIDRs:       externalIPCIDRs,
+		SortOrder:           sortOrderCIDRs,
+		IgnoredCIDRs:        ignoredCIDRs,
+		Mode:                networkOpts.Mode,
+		IPv6SupportDisabled: networkOpts.IPv6SupportDisabled,
+	}
+
 	return &instances{
 		c:           client,
 		provider:    provider,
+		networkOpts: netOps,
 	}
 }
 
@@ -132,15 +165,20 @@ func (i *instances) InstanceShutdown(ctx context.Context, node *v1.Node) (bool,
 func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloudprovider.InstanceMetadata, error) {
 	klog.V(4).InfoS("instances.InstanceMetadata() called", "node", klog.KRef("", node.Name))
 
-	if providedIP, ok := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr]; ok {
 	var (
-			vmRef  *pxapi.VmRef
+		vmRef  *proxmox.VmRef
 		region string
 		err    error
 	)
 
 	providerID := node.Spec.ProviderID
-		if providerID == "" {
+	if providerID != "" && !strings.HasPrefix(providerID, provider.ProviderName) {
+		klog.V(4).InfoS("instances.InstanceMetadata() omitting unmanaged node", "node", klog.KObj(node), "providerID", providerID)
+
+		return &cloudprovider.InstanceMetadata{}, nil
+	}
+
+	if providerID == "" && HasTaintWithEffect(node, cloudproviderapi.TaintExternalCloudProvider, "") {
 		uuid := node.Status.NodeInfo.SystemUUID
 
 		klog.V(4).InfoS("instances.InstanceMetadata() empty providerID, trying find node", "node", klog.KObj(node), "uuid", uuid)
@@ -157,13 +195,15 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud
 			}
 		}
 
-			if i.provider == ccmConfig.ProviderCapmox {
+		if i.provider == providerconfig.ProviderCapmox {
 			providerID = provider.GetProviderIDFromUUID(uuid)
 		} else {
 			providerID = provider.GetProviderID(region, vmRef)
 		}
-		} else if !strings.HasPrefix(node.Spec.ProviderID, provider.ProviderName) {
+	}
-			klog.V(4).InfoS("instances.InstanceMetadata() omitting unmanaged node", "node", klog.KObj(node), "providerID", node.Spec.ProviderID)
+
+	if providerID == "" {
+		klog.V(4).InfoS("instances.InstanceMetadata() empty providerID, omitting unmanaged node", "node", klog.KObj(node))
 
 		return &cloudprovider.InstanceMetadata{}, nil
 	}
@@ -177,13 +217,7 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud
 		}
 	}
 
-		addresses := []v1.NodeAddress{}
+	addresses := i.addresses(ctx, node, vmRef, region)
-
-		for _, ip := range strings.Split(providedIP, ",") {
-			addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: ip})
-		}
-
-		addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: node.Name})
 
 	instanceType, err := i.getInstanceType(ctx, vmRef, region)
 	if err != nil {
@@ -199,18 +233,10 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud
 	}, nil
 }
 
-	klog.InfoS(fmt.Sprintf(
+func (i *instances) getInstance(ctx context.Context, node *v1.Node) (*proxmox.VmRef, string, error) {
-		"instances.InstanceMetadata() called: label %s missing from node. Was kubelet started without --cloud-provider=external?",
-		cloudproviderapi.AnnotationAlphaProvidedIPAddr),
-		node, klog.KRef("", node.Name))
-
-	return &cloudprovider.InstanceMetadata{}, nil
-}
-
-func (i *instances) getInstance(ctx context.Context, node *v1.Node) (*pxapi.VmRef, string, error) {
 	klog.V(4).InfoS("instances.getInstance() called", "node", klog.KRef("", node.Name), "provider", i.provider)
 
-	if i.provider == ccmConfig.ProviderCapmox {
+	if i.provider == providerconfig.ProviderCapmox {
 		uuid := node.Status.NodeInfo.SystemUUID
 
 		vmRef, region, err := i.c.FindVMByUUID(ctx, uuid)
@@ -253,7 +279,7 @@ func (i *instances) getInstance(ctx context.Context, node *v1.Node) (*pxapi.VmRe
 	return vmRef, region, nil
 }
 
-func (i *instances) getInstanceType(ctx context.Context, vmRef *pxapi.VmRef, region string) (string, error) {
+func (i *instances) getInstanceType(ctx context.Context, vmRef *proxmox.VmRef, region string) (string, error) {
 	px, err := i.c.GetProxmoxCluster(region)
 	if err != nil {
 		return "", err

pkg/proxmox/instances_test.go

@@ -23,14 +23,14 @@ import (
 	"strings"
 	"testing"
 
-	pxapi "github.com/Telmate/proxmox-api-go/proxmox"
+	"github.com/Telmate/proxmox-api-go/proxmox"
 	"github.com/jarcoal/httpmock"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 
-	ccmConfig "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/config"
+	providerconfig "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/config"
 	"github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/provider"
-	pxpool "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/proxmoxpool"
+	"github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/proxmoxpool"
 
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -45,7 +45,7 @@ type ccmTestSuite struct {
 }
 
 func (ts *ccmTestSuite) SetupTest() {
-	cfg, err := ccmConfig.ReadCloudConfig(strings.NewReader(`
+	cfg, err := providerconfig.ReadCloudConfig(strings.NewReader(`
 clusters:
 - url: https://127.0.0.1:8006/api2/json
   insecure: false
@@ -172,12 +172,12 @@ clusters:
 		},
 	)
 
-	cluster, err := pxpool.NewProxmoxPool(cfg.Clusters, &http.Client{})
+	cluster, err := proxmoxpool.NewProxmoxPool(cfg.Clusters, &http.Client{})
 	if err != nil {
 		ts.T().Fatalf("failed to create cluster client: %v", err)
 	}
 
-	ts.i = newInstances(cluster, ccmConfig.ProviderDefault)
+	ts.i = newInstances(cluster, providerconfig.ProviderDefault, providerconfig.NetworkOpts{})
 }
 
 func (ts *ccmTestSuite) TearDownTest() {
@@ -551,18 +551,27 @@ func (ts *ccmTestSuite) TestInstanceMetadata() {
 						SystemUUID: "8af7110d-bfad-407a-a663-9527d10a6583",
 					},
 				},
+				Spec: v1.NodeSpec{
+					Taints: []v1.Taint{
+						{
+							Key:    cloudproviderapi.TaintExternalCloudProvider,
+							Value:  "true",
+							Effect: v1.TaintEffectNoSchedule,
+						},
+					},
+				},
 			},
 			expected: &cloudprovider.InstanceMetadata{
 				ProviderID: "proxmox://cluster-1/100",
 				NodeAddresses: []v1.NodeAddress{
-					{
-						Type:    v1.NodeInternalIP,
-						Address: "1.2.3.4",
-					},
 					{
 						Type:    v1.NodeHostName,
 						Address: "cluster-1-node-1",
 					},
+					{
+						Type:    v1.NodeInternalIP,
+						Address: "1.2.3.4",
+					},
 				},
 				InstanceType: "4VCPU-10GB",
 				Region:       "cluster-1",
@@ -583,10 +592,23 @@ func (ts *ccmTestSuite) TestInstanceMetadata() {
 						SystemUUID: "8af7110d-bfad-407a-a663-9527d10a6583",
 					},
 				},
+				Spec: v1.NodeSpec{
+					Taints: []v1.Taint{
+						{
+							Key:    cloudproviderapi.TaintExternalCloudProvider,
+							Value:  "true",
+							Effect: v1.TaintEffectNoSchedule,
+						},
+					},
+				},
 			},
 			expected: &cloudprovider.InstanceMetadata{
 				ProviderID: "proxmox://cluster-1/100",
 				NodeAddresses: []v1.NodeAddress{
+					{
+						Type:    v1.NodeHostName,
+						Address: "cluster-1-node-1",
+					},
 					{
 						Type:    v1.NodeInternalIP,
 						Address: "1.2.3.4",
@@ -595,10 +617,6 @@ func (ts *ccmTestSuite) TestInstanceMetadata() {
 						Type:    v1.NodeInternalIP,
 						Address: "2001::1",
 					},
-					{
-						Type:    v1.NodeHostName,
-						Address: "cluster-1-node-1",
-					},
 				},
 				InstanceType: "4VCPU-10GB",
 				Region:       "cluster-1",
@@ -619,18 +637,27 @@ func (ts *ccmTestSuite) TestInstanceMetadata() {
 						SystemUUID: "3d3db687-89dd-473e-8463-6599f25b36a8",
 					},
 				},
+				Spec: v1.NodeSpec{
+					Taints: []v1.Taint{
+						{
+							Key:    cloudproviderapi.TaintExternalCloudProvider,
+							Value:  "true",
+							Effect: v1.TaintEffectNoSchedule,
+						},
+					},
+				},
 			},
 			expected: &cloudprovider.InstanceMetadata{
 				ProviderID: "proxmox://cluster-2/100",
 				NodeAddresses: []v1.NodeAddress{
-					{
-						Type:    v1.NodeInternalIP,
-						Address: "1.2.3.4",
-					},
 					{
 						Type:    v1.NodeHostName,
 						Address: "cluster-2-node-1",
 					},
+					{
+						Type:    v1.NodeInternalIP,
+						Address: "1.2.3.4",
+					},
 				},
 				InstanceType: "c1.medium",
 				Region:       "cluster-2",
@@ -662,19 +689,19 @@ func TestGetProviderID(t *testing.T) {
 	tests := []struct {
 		msg      string
 		region   string
-		vmr      *pxapi.VmRef
+		vmr      *proxmox.VmRef
 		expected string
 	}{
 		{
 			msg:      "empty region",
 			region:   "",
-			vmr:      pxapi.NewVmRef(100),
+			vmr:      proxmox.NewVmRef(100),
 			expected: "proxmox:///100",
 		},
 		{
 			msg:      "region",
 			region:   "cluster1",
-			vmr:      pxapi.NewVmRef(100),
+			vmr:      proxmox.NewVmRef(100),
 			expected: "proxmox://cluster1/100",
 		},
 	}
@@ -698,7 +725,7 @@ func TestParseProviderID(t *testing.T) {
 		msg             string
 		magic           string
 		expectedCluster string
-		expectedVmr     *pxapi.VmRef
+		expectedVmr     *proxmox.VmRef
 		expectedError   error
 	}{
 		{
@@ -715,7 +742,7 @@ func TestParseProviderID(t *testing.T) {
 			msg:             "Empty region",
 			magic:           "proxmox:///100",
 			expectedCluster: "",
-			expectedVmr:     pxapi.NewVmRef(100),
+			expectedVmr:     proxmox.NewVmRef(100),
 		},
 		{
 			msg:           "Empty region",
@@ -726,7 +753,7 @@ func TestParseProviderID(t *testing.T) {
 			msg:             "Cluster and InstanceID",
 			magic:           "proxmox://cluster/100",
 			expectedCluster: "cluster",
-			expectedVmr:     pxapi.NewVmRef(100),
+			expectedVmr:     proxmox.NewVmRef(100),
 		},
 		{
 			msg:           "Cluster and wrong InstanceID",

pkg/proxmox/utils.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package proxmox
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"unicode"
+
+	v1 "k8s.io/api/core/v1"
+)
+
+// ErrorCIDRConflict is the error message formatting string for CIDR conflicts
+const ErrorCIDRConflict = "CIDR %s intersects with ignored CIDR %s"
+
+// SplitTrim splits a string of values separated by sep rune into a slice of
+// strings with trimmed spaces.
+func SplitTrim(s string, sep rune) []string {
+	f := func(c rune) bool {
+		return unicode.IsSpace(c) || c == sep
+	}
+
+	return strings.FieldsFunc(s, f)
+}
+
+// ParseCIDRRuleset parses a comma separated list of CIDRs and returns two slices of *net.IPNet, the first being the allow list, the second be the disallow list
+func ParseCIDRRuleset(cidrList string) (allowList, ignoreList []*net.IPNet, err error) {
+	cidrlist := SplitTrim(cidrList, ',')
+	if len(cidrlist) == 0 {
+		return []*net.IPNet{}, []*net.IPNet{}, nil
+	}
+
+	for _, item := range cidrlist {
+		isIgnore := false
+
+		if strings.HasPrefix(item, "!") {
+			item = strings.TrimPrefix(item, "!")
+			isIgnore = true
+		}
+
+		_, cidr, err := net.ParseCIDR(item)
+		if err != nil {
+			continue
+		}
+
+		if isIgnore {
+			ignoreList = append(ignoreList, cidr)
+
+			continue
+		}
+
+		allowList = append(allowList, cidr)
+	}
+
+	// Check for no interactions
+	for _, n1 := range allowList {
+		for _, n2 := range ignoreList {
+			if checkIPIntersects(n1, n2) {
+				return nil, nil, fmt.Errorf(ErrorCIDRConflict, n1.String(), n2.String())
+			}
+		}
+	}
+
+	return ignoreList, allowList, nil
+}
+
+// ParseCIDRList parses a comma separated list of CIDRs and returns a slice of *net.IPNet ignoring errors
+func ParseCIDRList(cidrList string) []*net.IPNet {
+	cidrlist := SplitTrim(cidrList, ',')
+	if len(cidrlist) == 0 {
+		return []*net.IPNet{}
+	}
+
+	cidrs := make([]*net.IPNet, 0, len(cidrlist))
+
+	for _, item := range cidrlist {
+		_, cidr, err := net.ParseCIDR(item)
+		if err != nil {
+			continue
+		}
+
+		cidrs = append(cidrs, cidr)
+	}
+
+	return cidrs
+}
+
+// HasTaintWithEffect checks if a node has a specific taint with the given key and effect.
+// An empty effect string will match any effect for the specified key
+func HasTaintWithEffect(node *v1.Node, key, effect string) bool {
+	for _, taint := range node.Spec.Taints {
+		if taint.Key == key {
+			if effect != "" {
+				return string(taint.Effect) == effect
+			}
+
+			return true
+		}
+	}
+
+	return false
+}
+
+func checkIPIntersects(n1, n2 *net.IPNet) bool {
+	return n2.Contains(n1.IP) || n1.Contains(n2.IP)
+}

pkg/proxmox/utils_test.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package proxmox_test
+
+import (
+	"fmt"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	proxmox "github.com/sergelogvinov/proxmox-cloud-controller-manager/pkg/proxmox"
+)
+
+func TestParseCIDRRuleset(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		msg                string
+		cidrs              string
+		expectedAllowList  []*net.IPNet
+		expectedIgnoreList []*net.IPNet
+		expectedError      []interface{}
+	}{
+		{
+			msg:                "Empty CIDR ruleset",
+			cidrs:              "",
+			expectedAllowList:  []*net.IPNet{},
+			expectedIgnoreList: []*net.IPNet{},
+			expectedError:      []interface{}{},
+		},
+		{
+			msg:                "Conflicting CIDRs",
+			cidrs:              "192.168.0.1/16,!192.168.0.1/24",
+			expectedAllowList:  []*net.IPNet{},
+			expectedIgnoreList: []*net.IPNet{},
+			expectedError:      []interface{}{"192.168.0.0/16", "192.168.0.0/24"},
+		},
+		{
+			msg:                "Ignores invalid CIDRs",
+			cidrs:              "722.887.0.1/16,!588.0.1/24",
+			expectedAllowList:  []*net.IPNet{},
+			expectedIgnoreList: []*net.IPNet{},
+			expectedError:      []interface{}{},
+		},
+		{
+			msg:                "Valid CIDRs with ignore",
+			cidrs:              "192.168.0.1/16,!10.0.0.5/8,144.0.0.7/16,!13.0.0.9/8",
+			expectedAllowList:  []*net.IPNet{mustParseCIDR("192.168.0.0/16"), mustParseCIDR("144.0.0.0/16")},
+			expectedIgnoreList: []*net.IPNet{mustParseCIDR("10.0.0.0/8"), mustParseCIDR("13.0.0.0/8")},
+			expectedError:      []interface{}{},
+		},
+	}
+
+	for _, testCase := range tests {
+		testCase := testCase
+
+		t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) {
+			t.Parallel()
+
+			allowList, ignoreList, err := proxmox.ParseCIDRRuleset(testCase.cidrs)
+
+			assert.Equal(t, len(testCase.expectedAllowList), len(allowList), "Allow list length mismatch")
+			assert.Equal(t, len(testCase.expectedIgnoreList), len(ignoreList), "Allow list length mismatch")
+
+			if len(testCase.expectedError) != 0 {
+				assert.EqualError(t, err, fmt.Sprintf(proxmox.ErrorCIDRConflict, testCase.expectedError...), "Error mismatch")
+			} else {
+				assert.NoError(t, err, "Unexpected error")
+			}
+		})
+	}
+}
+
+func mustParseCIDR(cidr string) *net.IPNet {
+	_, parsedCIDR, err := net.ParseCIDR(cidr)
+	if err != nil {
+		panic(fmt.Sprintf("Failed to parse CIDR %s: %v", cidr, err))
+	}
+
+	return parsedCIDR
+}

pkg/proxmoxpool/pool.go

@@ -27,7 +27,7 @@ import (
 	"os"
 	"strings"
 
-	pxapi "github.com/Telmate/proxmox-api-go/proxmox"
+	"github.com/Telmate/proxmox-api-go/proxmox"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/klog/v2"
@@ -46,14 +46,14 @@ type ProxmoxCluster struct {
 
 // ProxmoxPool is a Proxmox client.
 type ProxmoxPool struct {
-	clients map[string]*pxapi.Client
+	clients map[string]*proxmox.Client
 }
 
 // NewProxmoxPool creates a new Proxmox cluster client.
 func NewProxmoxPool(config []*ProxmoxCluster, hClient *http.Client) (*ProxmoxPool, error) {
 	clusters := len(config)
 	if clusters > 0 {
-		proxmox := make(map[string]*pxapi.Client, clusters)
+		clients := make(map[string]*proxmox.Client, clusters)
 
 		for _, cfg := range config {
 			tlsconf := &tls.Config{InsecureSkipVerify: true}
@@ -61,7 +61,7 @@ func NewProxmoxPool(config []*ProxmoxCluster, hClient *http.Client) (*ProxmoxPoo
 				tlsconf = nil
 			}
 
-			pClient, err := pxapi.NewClient(cfg.URL, hClient, os.Getenv("PM_HTTP_HEADERS"), tlsconf, "", 600)
+			pClient, err := proxmox.NewClient(cfg.URL, hClient, os.Getenv("PM_HTTP_HEADERS"), tlsconf, "", 600)
 			if err != nil {
 				return nil, err
 			}
@@ -74,11 +74,11 @@ func NewProxmoxPool(config []*ProxmoxCluster, hClient *http.Client) (*ProxmoxPoo
 				pClient.SetAPIToken(cfg.TokenID, cfg.TokenSecret)
 			}
 
-			proxmox[cfg.Region] = pClient
+			clients[cfg.Region] = pClient
 		}
 
 		return &ProxmoxPool{
-			clients: proxmox,
+			clients: clients,
 		}, nil
 	}
 
@@ -113,7 +113,7 @@ func (c *ProxmoxPool) CheckClusters(ctx context.Context) error {
 }
 
 // GetProxmoxCluster returns a Proxmox cluster client in a given region.
-func (c *ProxmoxPool) GetProxmoxCluster(region string) (*pxapi.Client, error) {
+func (c *ProxmoxPool) GetProxmoxCluster(region string) (*proxmox.Client, error) {
 	if c.clients[region] != nil {
 		return c.clients[region], nil
 	}
@@ -122,7 +122,7 @@ func (c *ProxmoxPool) GetProxmoxCluster(region string) (*pxapi.Client, error) {
 }
 
 // FindVMByNode find a VM by kubernetes node resource in all Proxmox clusters.
-func (c *ProxmoxPool) FindVMByNode(ctx context.Context, node *v1.Node) (*pxapi.VmRef, string, error) {
+func (c *ProxmoxPool) FindVMByNode(ctx context.Context, node *v1.Node) (*proxmox.VmRef, string, error) {
 	for region, px := range c.clients {
 		vmrs, err := px.GetVmRefsByName(ctx, node.Name)
 		if err != nil {
@@ -149,7 +149,7 @@ func (c *ProxmoxPool) FindVMByNode(ctx context.Context, node *v1.Node) (*pxapi.V
 }
 
 // FindVMByName find a VM by name in all Proxmox clusters.
-func (c *ProxmoxPool) FindVMByName(ctx context.Context, name string) (*pxapi.VmRef, string, error) {
+func (c *ProxmoxPool) FindVMByName(ctx context.Context, name string) (*proxmox.VmRef, string, error) {
 	for region, px := range c.clients {
 		vmr, err := px.GetVmRefByName(ctx, name)
 		if err != nil {
@@ -167,7 +167,7 @@ func (c *ProxmoxPool) FindVMByName(ctx context.Context, name string) (*pxapi.VmR
 }
 
 // FindVMByUUID find a VM by uuid in all Proxmox clusters.
-func (c *ProxmoxPool) FindVMByUUID(ctx context.Context, uuid string) (*pxapi.VmRef, string, error) {
+func (c *ProxmoxPool) FindVMByUUID(ctx context.Context, uuid string) (*proxmox.VmRef, string, error) {
 	for region, px := range c.clients {
 		vms, err := px.GetResourceList(ctx, "vm")
 		if err != nil {
@@ -184,7 +184,7 @@ func (c *ProxmoxPool) FindVMByUUID(ctx context.Context, uuid string) (*pxapi.VmR
 				continue
 			}
 
-			vmr := pxapi.NewVmRef(int(vm["vmid"].(float64))) //nolint:errcheck
+			vmr := proxmox.NewVmRef(int(vm["vmid"].(float64))) //nolint:errcheck
 			vmr.SetNode(vm["node"].(string))                   //nolint:errcheck
 			vmr.SetVmType("qemu")