feat: add extra labels

Add labels:
* topology.proxmox.sinextra.dev/node
* topology.proxmox.sinextra.dev/region

These labels represent the default topology labels.
They make it possible to use different topologies on the Proxmox side.

Signed-off-by: Serge Logvinov <serge.logvinov@sinextra.dev>

docs/config.md

@@ -52,7 +52,7 @@ You can define multiple clusters in the `clusters` section.
 
 ## Feature flags
 
-* `provider` - Set the provider type. The default is `default`, which uses provider-id to define the Proxmox VM ID. The `capmox` value is used for working with the Cluster API for Proxmox (CAPMox).
+* `provider` - Set the provider type. The default is `default`, which uses provider-id format `proxmox://<region>/<vm-id>`. The `capmox` value is used for working with the Cluster API for Proxmox (CAPMox), which uses provider-id format `proxmox://<SystemUUID>`.
 * `network` - Defines how the network addresses are handled by the CCM. The default value is `default`, which uses the kubelet argument `--node-ips` to assign IPs to the node resource. The `qemu` mode uses the QEMU agent API to retrieve network addresses from the virtual machine, while auto attempts to detect the best mode automatically.
 * `ipv6_support_disabled` - Set to `true` to ignore any IPv6 addresses. The default is `false`.
 * `external_ip_cidrs` - A comma-separated list of external IP address CIDRs. You can use `!` to exclude a CIDR from the list. This is useful for defining which IPs should be considered external and not included in the node addresses.

pkg/proxmox/instances.go

@@ -190,9 +190,6 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud
 		return &cloudprovider.InstanceMetadata{}, nil
 	}
 
-	// if providerID == "" && HasTaintWithEffect(node, cloudproviderapi.TaintExternalCloudProvider, "") {
-	// }
-
 	mc := metrics.NewMetricContext("getInstanceInfo")
 
 	info, err = i.getInstanceInfo(ctx, node)
@@ -206,6 +203,11 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud
 		return nil, err
 	}
 
+	additionalLabels := map[string]string{
+		LabelTopologyRegion: info.Region,
+		LabelTopologyNode:   info.Node,
+	}
+
 	if providerID == "" {
 		if i.provider == providerconfig.ProviderCapmox {
 			providerID = provider.GetProviderIDFromUUID(info.UUID)
@@ -222,12 +224,19 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud
 		}
 	}
 
+	if len(additionalLabels) > 0 && !hasUninitializedTaint(node) {
+		if err := syncNodeLabels(i.c, node, additionalLabels); err != nil {
+			klog.ErrorS(err, "error updating labels for the node", "node", klog.KRef("", node.Name))
+		}
+	}
+
 	metadata := &cloudprovider.InstanceMetadata{
-		ProviderID:    providerID,
-		NodeAddresses: i.addresses(ctx, node, info),
-		InstanceType:  info.Type,
-		Zone:          info.Zone,
-		Region:        info.Region,
+		ProviderID:       providerID,
+		NodeAddresses:    i.addresses(ctx, node, info),
+		InstanceType:     info.Type,
+		Zone:             info.Zone,
+		Region:           info.Region,
+		AdditionalLabels: additionalLabels,
 	}
 
 	klog.V(5).InfoS("instances.InstanceMetadata()", "info", info, "metadata", metadata)
@@ -246,7 +255,10 @@ func (i *instances) getInstanceInfo(ctx context.Context, node *v1.Node) (*instan
 
 	providerID := node.Spec.ProviderID
 	if providerID == "" && node.Annotations[AnnotationProxmoxInstanceID] != "" {
-		region = node.Annotations[v1.LabelTopologyRegion]
+		region = node.Labels[LabelTopologyRegion]
+		if region == "" {
+			region = node.Labels[v1.LabelTopologyRegion]
+		}
 
 		vmID, err := strconv.Atoi(node.Annotations[AnnotationProxmoxInstanceID])
 		if err != nil {

pkg/proxmox/instances_test.go

@@ -574,6 +574,10 @@ func (ts *ccmTestSuite) TestInstanceMetadata() {
 				InstanceType: "4VCPU-10GB",
 				Region:       "cluster-1",
 				Zone:         "pve-1",
+				AdditionalLabels: map[string]string{
+					"topology.proxmox.sinextra.dev/node":   "pve-1",
+					"topology.proxmox.sinextra.dev/region": "cluster-1",
+				},
 			},
 		},
 		{
@@ -619,6 +623,10 @@ func (ts *ccmTestSuite) TestInstanceMetadata() {
 				InstanceType: "4VCPU-10GB",
 				Region:       "cluster-1",
 				Zone:         "pve-1",
+				AdditionalLabels: map[string]string{
+					"topology.proxmox.sinextra.dev/node":   "pve-1",
+					"topology.proxmox.sinextra.dev/region": "cluster-1",
+				},
 			},
 		},
 		{
@@ -660,6 +668,10 @@ func (ts *ccmTestSuite) TestInstanceMetadata() {
 				InstanceType: "c1.medium",
 				Region:       "cluster-2",
 				Zone:         "pve-3",
+				AdditionalLabels: map[string]string{
+					"topology.proxmox.sinextra.dev/node":   "pve-3",
+					"topology.proxmox.sinextra.dev/region": "cluster-2",
+				},
 			},
 		},
 	}

pkg/proxmox/labels.go

@@ -0,0 +1,31 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package proxmox
+
+const (
+	// LabelTopologyRegion is the label used to store the Proxmox region name.
+	LabelTopologyRegion = "topology." + Group + "/region"
+
+	// LabelTopologyZone is the label used to store the Proxmox zone name.
+	LabelTopologyZone = "topology." + Group + "/zone"
+
+	// LabelTopologyNode is the label used to store the Proxmox node name.
+	LabelTopologyNode = "topology." + Group + "/node"
+
+	// LabelTopologyHAGroup is the label used to store the Proxmox HA group name.
+	LabelTopologyHAGroup = "topology." + Group + "/ha-group"
+)

pkg/proxmox/utils.go

@@ -29,11 +29,18 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	clientkubernetes "k8s.io/client-go/kubernetes"
+	cloudproviderapi "k8s.io/cloud-provider/api"
+	cloudnodeutil "k8s.io/cloud-provider/node/helpers"
 )
 
 // ErrorCIDRConflict is the error message formatting string for CIDR conflicts
 const ErrorCIDRConflict = "CIDR %s intersects with ignored CIDR %s"
 
+var uninitializedTaint = &corev1.Taint{
+	Key:    cloudproviderapi.TaintExternalCloudProvider,
+	Effect: corev1.TaintEffectNoSchedule,
+}
+
 // SplitTrim splits a string of values separated by sep rune into a slice of
 // strings with trimmed spaces.
 func SplitTrim(s string, sep rune) []string {
@@ -106,15 +113,13 @@ func ParseCIDRList(cidrList string) []*net.IPNet {
 	return cidrs
 }
 
-// HasTaintWithEffect checks if a node has a specific taint with the given key and effect.
-// An empty effect string will match any effect for the specified key
-func HasTaintWithEffect(node *corev1.Node, key, effect string) bool {
-	for _, taint := range node.Spec.Taints {
-		if taint.Key == key {
-			if effect != "" {
-				return string(taint.Effect) == effect
-			}
+func checkIPIntersects(n1, n2 *net.IPNet) bool {
+	return n2.Contains(n1.IP) || n1.Contains(n2.IP)
+}
 
+func hasUninitializedTaint(node *corev1.Node) bool {
+	for _, taint := range node.Spec.Taints {
+		if taint.MatchTaint(uninitializedTaint) {
 			return true
 		}
 	}
@@ -122,10 +127,6 @@ func HasTaintWithEffect(node *corev1.Node, key, effect string) bool {
 	return false
 }
 
-func checkIPIntersects(n1, n2 *net.IPNet) bool {
-	return n2.Contains(n1.IP) || n1.Contains(n2.IP)
-}
-
 func syncNodeAnnotations(ctx context.Context, kclient clientkubernetes.Interface, node *corev1.Node, nodeAnnotations map[string]string) error {
 	nodeAnnotationsOrig := node.ObjectMeta.Annotations
 	annotationsToUpdate := map[string]string{}
@@ -168,3 +169,22 @@ func syncNodeAnnotations(ctx context.Context, kclient clientkubernetes.Interface
 
 	return nil
 }
+
+func syncNodeLabels(c *client, node *corev1.Node, nodeLabels map[string]string) error {
+	nodeLabelsOrig := node.ObjectMeta.Labels
+	labelsToUpdate := map[string]string{}
+
+	for k, v := range nodeLabels {
+		if r, ok := nodeLabelsOrig[k]; !ok || r != v {
+			labelsToUpdate[k] = v
+		}
+	}
+
+	if len(labelsToUpdate) > 0 {
+		if !cloudnodeutil.AddOrUpdateLabelsOnNode(c.kclient, labelsToUpdate, node) {
+			return fmt.Errorf("failed update labels for node %s", node.Name)
+		}
+	}
+
+	return nil
+}