From 229be1432a8922c310678c89d12c1d1a087c90aa Mon Sep 17 00:00:00 2001 From: Serge Logvinov Date: Mon, 8 Sep 2025 11:26:04 +0700 Subject: [PATCH] feat: add extra labels Add labels: * topology.proxmox.sinextra.dev/node * topology.proxmox.sinextra.dev/region These labels represent the default topology labels. They make it possible to use different topologies on the Proxmox side. Signed-off-by: Serge Logvinov --- docs/config.md | 2 +- pkg/proxmox/instances.go | 30 +++++++++++++++++------- pkg/proxmox/instances_test.go | 12 ++++++++++ pkg/proxmox/labels.go | 31 ++++++++++++++++++++++++ pkg/proxmox/utils.go | 44 +++++++++++++++++++++++++---------- 5 files changed, 97 insertions(+), 22 deletions(-) create mode 100644 pkg/proxmox/labels.go diff --git a/docs/config.md b/docs/config.md index e145adf..3217f0d 100644 --- a/docs/config.md +++ b/docs/config.md @@ -52,7 +52,7 @@ You can define multiple clusters in the `clusters` section. ## Feature flags -* `provider` - Set the provider type. The default is `default`, which uses provider-id to define the Proxmox VM ID. The `capmox` value is used for working with the Cluster API for Proxmox (CAPMox). +* `provider` - Set the provider type. The default is `default`, which uses provider-id format `proxmox:///`. The `capmox` value is used for working with the Cluster API for Proxmox (CAPMox), which uses provider-id format `proxmox://`. * `network` - Defines how the network addresses are handled by the CCM. The default value is `default`, which uses the kubelet argument `--node-ips` to assign IPs to the node resource. The `qemu` mode uses the QEMU agent API to retrieve network addresses from the virtual machine, while auto attempts to detect the best mode automatically. * `ipv6_support_disabled` - Set to `true` to ignore any IPv6 addresses. The default is `false`. * `external_ip_cidrs` - A comma-separated list of external IP address CIDRs. You can use `!` to exclude a CIDR from the list. This is useful for defining which IPs should be considered external and not included in the node addresses. diff --git a/pkg/proxmox/instances.go b/pkg/proxmox/instances.go index 6e3c15a..ae1a3de 100644 --- a/pkg/proxmox/instances.go +++ b/pkg/proxmox/instances.go @@ -190,9 +190,6 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud return &cloudprovider.InstanceMetadata{}, nil } - // if providerID == "" && HasTaintWithEffect(node, cloudproviderapi.TaintExternalCloudProvider, "") { - // } - mc := metrics.NewMetricContext("getInstanceInfo") info, err = i.getInstanceInfo(ctx, node) @@ -206,6 +203,11 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud return nil, err } + additionalLabels := map[string]string{ + LabelTopologyRegion: info.Region, + LabelTopologyNode: info.Node, + } + if providerID == "" { if i.provider == providerconfig.ProviderCapmox { providerID = provider.GetProviderIDFromUUID(info.UUID) @@ -222,12 +224,19 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud } } + if len(additionalLabels) > 0 && !hasUninitializedTaint(node) { + if err := syncNodeLabels(i.c, node, additionalLabels); err != nil { + klog.ErrorS(err, "error updating labels for the node", "node", klog.KRef("", node.Name)) + } + } + metadata := &cloudprovider.InstanceMetadata{ - ProviderID: providerID, - NodeAddresses: i.addresses(ctx, node, info), - InstanceType: info.Type, - Zone: info.Zone, - Region: info.Region, + ProviderID: providerID, + NodeAddresses: i.addresses(ctx, node, info), + InstanceType: info.Type, + Zone: info.Zone, + Region: info.Region, + AdditionalLabels: additionalLabels, } klog.V(5).InfoS("instances.InstanceMetadata()", "info", info, "metadata", metadata) @@ -246,7 +255,10 @@ func (i *instances) getInstanceInfo(ctx context.Context, node *v1.Node) (*instan providerID := node.Spec.ProviderID if providerID == "" && node.Annotations[AnnotationProxmoxInstanceID] != "" { - region = node.Annotations[v1.LabelTopologyRegion] + region = node.Labels[LabelTopologyRegion] + if region == "" { + region = node.Labels[v1.LabelTopologyRegion] + } vmID, err := strconv.Atoi(node.Annotations[AnnotationProxmoxInstanceID]) if err != nil { diff --git a/pkg/proxmox/instances_test.go b/pkg/proxmox/instances_test.go index 46ca7e7..a08eafe 100644 --- a/pkg/proxmox/instances_test.go +++ b/pkg/proxmox/instances_test.go @@ -574,6 +574,10 @@ func (ts *ccmTestSuite) TestInstanceMetadata() { InstanceType: "4VCPU-10GB", Region: "cluster-1", Zone: "pve-1", + AdditionalLabels: map[string]string{ + "topology.proxmox.sinextra.dev/node": "pve-1", + "topology.proxmox.sinextra.dev/region": "cluster-1", + }, }, }, { @@ -619,6 +623,10 @@ func (ts *ccmTestSuite) TestInstanceMetadata() { InstanceType: "4VCPU-10GB", Region: "cluster-1", Zone: "pve-1", + AdditionalLabels: map[string]string{ + "topology.proxmox.sinextra.dev/node": "pve-1", + "topology.proxmox.sinextra.dev/region": "cluster-1", + }, }, }, { @@ -660,6 +668,10 @@ func (ts *ccmTestSuite) TestInstanceMetadata() { InstanceType: "c1.medium", Region: "cluster-2", Zone: "pve-3", + AdditionalLabels: map[string]string{ + "topology.proxmox.sinextra.dev/node": "pve-3", + "topology.proxmox.sinextra.dev/region": "cluster-2", + }, }, }, } diff --git a/pkg/proxmox/labels.go b/pkg/proxmox/labels.go new file mode 100644 index 0000000..4ff38b9 --- /dev/null +++ b/pkg/proxmox/labels.go @@ -0,0 +1,31 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxmox + +const ( + // LabelTopologyRegion is the label used to store the Proxmox region name. + LabelTopologyRegion = "topology." + Group + "/region" + + // LabelTopologyZone is the label used to store the Proxmox zone name. + LabelTopologyZone = "topology." + Group + "/zone" + + // LabelTopologyNode is the label used to store the Proxmox node name. + LabelTopologyNode = "topology." + Group + "/node" + + // LabelTopologyHAGroup is the label used to store the Proxmox HA group name. + LabelTopologyHAGroup = "topology." + Group + "/ha-group" +) diff --git a/pkg/proxmox/utils.go b/pkg/proxmox/utils.go index def5fcb..66c1992 100644 --- a/pkg/proxmox/utils.go +++ b/pkg/proxmox/utils.go @@ -29,11 +29,18 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/strategicpatch" clientkubernetes "k8s.io/client-go/kubernetes" + cloudproviderapi "k8s.io/cloud-provider/api" + cloudnodeutil "k8s.io/cloud-provider/node/helpers" ) // ErrorCIDRConflict is the error message formatting string for CIDR conflicts const ErrorCIDRConflict = "CIDR %s intersects with ignored CIDR %s" +var uninitializedTaint = &corev1.Taint{ + Key: cloudproviderapi.TaintExternalCloudProvider, + Effect: corev1.TaintEffectNoSchedule, +} + // SplitTrim splits a string of values separated by sep rune into a slice of // strings with trimmed spaces. func SplitTrim(s string, sep rune) []string { @@ -106,15 +113,13 @@ func ParseCIDRList(cidrList string) []*net.IPNet { return cidrs } -// HasTaintWithEffect checks if a node has a specific taint with the given key and effect. -// An empty effect string will match any effect for the specified key -func HasTaintWithEffect(node *corev1.Node, key, effect string) bool { - for _, taint := range node.Spec.Taints { - if taint.Key == key { - if effect != "" { - return string(taint.Effect) == effect - } +func checkIPIntersects(n1, n2 *net.IPNet) bool { + return n2.Contains(n1.IP) || n1.Contains(n2.IP) +} +func hasUninitializedTaint(node *corev1.Node) bool { + for _, taint := range node.Spec.Taints { + if taint.MatchTaint(uninitializedTaint) { return true } } @@ -122,10 +127,6 @@ func HasTaintWithEffect(node *corev1.Node, key, effect string) bool { return false } -func checkIPIntersects(n1, n2 *net.IPNet) bool { - return n2.Contains(n1.IP) || n1.Contains(n2.IP) -} - func syncNodeAnnotations(ctx context.Context, kclient clientkubernetes.Interface, node *corev1.Node, nodeAnnotations map[string]string) error { nodeAnnotationsOrig := node.ObjectMeta.Annotations annotationsToUpdate := map[string]string{} @@ -168,3 +169,22 @@ func syncNodeAnnotations(ctx context.Context, kclient clientkubernetes.Interface return nil } + +func syncNodeLabels(c *client, node *corev1.Node, nodeLabels map[string]string) error { + nodeLabelsOrig := node.ObjectMeta.Labels + labelsToUpdate := map[string]string{} + + for k, v := range nodeLabels { + if r, ok := nodeLabelsOrig[k]; !ok || r != v { + labelsToUpdate[k] = v + } + } + + if len(labelsToUpdate) > 0 { + if !cloudnodeutil.AddOrUpdateLabelsOnNode(c.kclient, labelsToUpdate, node) { + return fmt.Errorf("failed update labels for node %s", node.Name) + } + } + + return nil +}