From b3d55f0810bd48659ee3f151669e07fc88406911 Mon Sep 17 00:00:00 2001 From: Serge Logvinov Date: Sat, 27 May 2023 15:15:00 +0300 Subject: [PATCH] test: add basic tests Tests: * cloud-config * helper funcs Signed-off-by: Serge Logvinov --- .github/workflows/build-test.yaml | 8 + Makefile | 2 +- hack/{talos-config.yaml => ccm-config.yaml} | 0 hack/talosconfig | 5 + pkg/talos/client.go | 20 +- pkg/talos/cloud_config.go | 25 +- pkg/talos/cloud_config_test.go | 22 +- pkg/talos/cloud_test.go | 77 +++- pkg/talos/helper.go | 126 ++++++ pkg/talos/helper_test.go | 417 ++++++++++++++++++++ pkg/talos/instances.go | 117 ------ pkg/talos/instances_test.go | 133 ++----- 12 files changed, 694 insertions(+), 258 deletions(-) rename hack/{talos-config.yaml => ccm-config.yaml} (100%) create mode 100644 hack/talosconfig create mode 100644 pkg/talos/helper.go create mode 100644 pkg/talos/helper_test.go diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 8d0d35f..ddf4fac 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -11,9 +11,14 @@ on: - 'pkg/**' - 'Dockerfile' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: name: Build + timeout-minutes: 10 runs-on: ubuntu-22.04 permissions: contents: read @@ -24,6 +29,7 @@ jobs: run: git fetch --prune --unshallow - name: Set up go + timeout-minutes: 5 uses: actions/setup-go@v3 with: go-version-file: 'go.mod' @@ -35,3 +41,5 @@ jobs: args: --config=.golangci.yml - name: Build run: make build + - name: Test + run: make unit diff --git a/Makefile b/Makefile index 932dec9..c829930 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ build: ## Build .PHONY: run run: build - ./talos-cloud-controller-manager-$(ARCH) --v=5 --kubeconfig=kubeconfig --cloud-config=hack/talos-config.yaml --controllers=cloud-node \ + ./talos-cloud-controller-manager-$(ARCH) --v=5 --kubeconfig=kubeconfig --cloud-config=hack/ccm-config.yaml --controllers=cloud-node \ --use-service-account-credentials --leader-elect=false --bind-address=127.0.0.1 .PHONY: lint diff --git a/hack/talos-config.yaml b/hack/ccm-config.yaml similarity index 100% rename from hack/talos-config.yaml rename to hack/ccm-config.yaml diff --git a/hack/talosconfig b/hack/talosconfig new file mode 100644 index 0000000..aa77732 --- /dev/null +++ b/hack/talosconfig @@ -0,0 +1,5 @@ +context: talos +contexts: + talos: + endpoints: + - 1.2.3.4 diff --git a/pkg/talos/client.go b/pkg/talos/client.go index 2349b7f..0aba0b4 100644 --- a/pkg/talos/client.go +++ b/pkg/talos/client.go @@ -11,7 +11,6 @@ import ( "github.com/siderolabs/talos/pkg/machinery/resources/runtime" clientkubernetes "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" ) type client struct { @@ -23,10 +22,18 @@ type client struct { func newClient(ctx context.Context, config *cloudConfig) (*client, error) { clientOpts := []clienttalos.OptionFunc{} + if config == nil { + return nil, fmt.Errorf("talos cloudConfig is nil") + } + + clientOpts = append(clientOpts, clienttalos.WithDefaultConfig()) + if len(config.Global.Endpoints) > 0 { clientOpts = append(clientOpts, clienttalos.WithEndpoints(config.Global.Endpoints...)) - } else { - clientOpts = append(clientOpts, clienttalos.WithDefaultConfig()) + } + + if config.Global.ClusterName != "" { + clientOpts = append(clientOpts, clienttalos.WithCluster(config.Global.ClusterName)) } talos, err := clienttalos.New(ctx, clientOpts...) @@ -34,13 +41,6 @@ func newClient(ctx context.Context, config *cloudConfig) (*client, error) { return nil, err } - //nolint:revive - if ver, err := talos.Version(ctx); err != nil { - return nil, fmt.Errorf("failed to initialized talos client: %w", err) - } else { - klog.V(4).Infof("talos api version: %s", ver.String()) - } - return &client{ config: config, talos: talos, diff --git a/pkg/talos/cloud_config.go b/pkg/talos/cloud_config.go index c9b8c98..29d4aac 100644 --- a/pkg/talos/cloud_config.go +++ b/pkg/talos/cloud_config.go @@ -11,16 +11,21 @@ import ( ) type cloudConfig struct { - Global struct { - // Approve Node Certificate Signing Request. - ApproveNodeCSR bool `yaml:"approveNodeCSR,omitempty"` - // Talos API endpoints. - Endpoints []string `yaml:"endpoints,omitempty"` - // Do not update foreign initialized node. - SkipForeignNode bool `yaml:"skipForeignNode,omitempty"` - // Prefer IPv6. - PreferIPv6 bool `yaml:"preferIPv6,omitempty"` - } `yaml:"global,omitempty"` + // Global configuration. + Global cloudConfigGlobal `yaml:"global,omitempty"` +} + +type cloudConfigGlobal struct { + // Approve Node Certificate Signing Request. + ApproveNodeCSR bool `yaml:"approveNodeCSR,omitempty"` + // Talos cluster name. + ClusterName string `yaml:"clusterName,omitempty"` + // Talos API endpoints. + Endpoints []string `yaml:"endpoints,omitempty"` + // Do not update foreign initialized node. + SkipForeignNode bool `yaml:"skipForeignNode,omitempty"` + // Prefer IPv6. + PreferIPv6 bool `yaml:"preferIPv6,omitempty"` } func readCloudConfig(config io.Reader) (cloudConfig, error) { diff --git a/pkg/talos/cloud_config_test.go b/pkg/talos/cloud_config_test.go index c3f495d..03754c0 100644 --- a/pkg/talos/cloud_config_test.go +++ b/pkg/talos/cloud_config_test.go @@ -5,14 +5,28 @@ import ( "testing" ) -func TestReadCloudConfig(t *testing.T) { - t.Setenv("TALOS_ENDPOINTS", "127.0.0.1,127.0.0.2") - - _, err := readCloudConfig(nil) +func TestReadCloudConfigEmpty(t *testing.T) { + cfg, err := readCloudConfig(nil) if err != nil { t.Errorf("Should not fail when no config is provided: %s", err) } + if len(cfg.Global.Endpoints) != 0 { + t.Errorf("incorrect endpoints: %s", cfg.Global.Endpoints) + } + + if cfg.Global.PreferIPv6 { + t.Errorf("%v is not default value of preferIPv6", cfg.Global.PreferIPv6) + } + + if cfg.Global.ApproveNodeCSR { + t.Errorf("%v is not default value of ApproveNodeCSR", cfg.Global.ApproveNodeCSR) + } +} + +func TestReadCloudConfig(t *testing.T) { + t.Setenv("TALOS_ENDPOINTS", "127.0.0.1,127.0.0.2") + cfg, err := readCloudConfig(strings.NewReader(` global: approveNodeCSR: true diff --git a/pkg/talos/cloud_test.go b/pkg/talos/cloud_test.go index 6350307..4f9c173 100644 --- a/pkg/talos/cloud_test.go +++ b/pkg/talos/cloud_test.go @@ -1,24 +1,65 @@ package talos -import "testing" +import ( + "testing" -// func config() cloudConfig { -// cfg := cloudConfig{} -// cfg.Global.Endpoints = []string{"127.0.0.1"} + "github.com/stretchr/testify/assert" +) -// return cfg -// } +func config() cloudConfig { + cfg := cloudConfig{} + cfg.Global.Endpoints = []string{"127.0.0.1"} -func TestNewCloud(*testing.T) { - // cfg := Config() - - // ccm, err := newCloud(&cfg) - // if err != nil { - // t.Fatalf("Failed to create Talos CCM: %s", err) - // } - - // _, ok := ccm.InstancesV2() - // if !ok { - // t.Fatalf("InstancesV2() returned false") - // } + return cfg +} + +func TestNewCloudError(t *testing.T) { + ccm, err := newCloud(nil) + assert.NotNil(t, err) + assert.Nil(t, ccm) + assert.EqualError(t, err, "talos cloudConfig is nil") +} + +func TestNewCloud(t *testing.T) { + t.Setenv("TALOSCONFIG", "../../hack/talosconfig") + + cfg := config() + + ccm, err := newCloud(&cfg) + if err != nil { + t.Fatalf("Failed to create Talos CCM: %s", err) + } + + assert.Nil(t, err) + assert.NotNil(t, ccm) + + lb, res := ccm.LoadBalancer() + assert.Nil(t, lb) + assert.Equal(t, res, false) + + ins, res := ccm.Instances() + assert.Nil(t, ins) + assert.Equal(t, res, false) + + ins2, res := ccm.InstancesV2() + assert.NotNil(t, ins2) + assert.Equal(t, res, true) + + zone, res := ccm.Zones() + assert.Nil(t, zone) + assert.Equal(t, res, false) + + cl, res := ccm.Clusters() + assert.Nil(t, cl) + assert.Equal(t, res, false) + + route, res := ccm.Routes() + assert.Nil(t, route) + assert.Equal(t, res, false) + + pName := ccm.ProviderName() + assert.Equal(t, pName, ProviderName) + + clID := ccm.HasClusterID() + assert.Equal(t, clID, true) } diff --git a/pkg/talos/helper.go b/pkg/talos/helper.go new file mode 100644 index 0000000..6bdddc1 --- /dev/null +++ b/pkg/talos/helper.go @@ -0,0 +1,126 @@ +package talos + +import ( + "context" + "crypto/x509" + "fmt" + + utilsnet "github.com/siderolabs/talos-cloud-controller-manager/pkg/utils/net" + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientkubernetes "k8s.io/client-go/kubernetes" + cloudproviderapi "k8s.io/cloud-provider/api" + cloudnodeutil "k8s.io/cloud-provider/node/helpers" + "k8s.io/utils/strings/slices" +) + +func getNodeAddresses(config *cloudConfig, platform, nodeIP string, ifaces []network.AddressStatusSpec) []v1.NodeAddress { + var publicIPv4s, publicIPv6s, publicIPs []string + + switch platform { + case "nocloud", "metal": + for _, iface := range ifaces { + if iface.LinkName == "kubespan" { + continue + } + + ip := iface.Address.Addr() + if ip.IsGlobalUnicast() && !ip.IsPrivate() { + if ip.Is6() { + publicIPv6s = append(publicIPv6s, ip.String()) + } else { + publicIPv4s = append(publicIPv4s, ip.String()) + } + } + } + default: + for _, iface := range ifaces { + if iface.LinkName == "external" { + ip := iface.Address.Addr() + + if ip.Is6() { + publicIPv6s = append(publicIPv6s, ip.String()) + } else { + publicIPv4s = append(publicIPv4s, ip.String()) + } + } + } + } + + addresses := []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: nodeIP}} + + if config.Global.PreferIPv6 { + publicIPs = utilsnet.SortedNodeIPs(nodeIP, publicIPv6s, publicIPv4s) + } else { + publicIPs = utilsnet.SortedNodeIPs(nodeIP, publicIPv4s, publicIPv6s) + } + + for _, ip := range publicIPs { + addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: ip}) + } + + return addresses +} + +func syncNodeLabels(c *client, node *v1.Node, meta *runtime.PlatformMetadataSpec) error { + nodeLabels := node.ObjectMeta.Labels + labelsToUpdate := map[string]string{} + + if nodeLabels == nil { + nodeLabels = map[string]string{} + } + + if meta.Platform != "" && nodeLabels[ClusterNodePlatformLabel] != meta.Platform { + labelsToUpdate[ClusterNodePlatformLabel] = meta.Platform + } + + if meta.Spot && nodeLabels[ClusterNodeLifeCycleLabel] != "spot" { + labelsToUpdate[ClusterNodeLifeCycleLabel] = "spot" + } + + if clusterName := c.talos.GetClusterName(); clusterName != "" && nodeLabels[ClusterNameNodeLabel] != clusterName { + labelsToUpdate[ClusterNameNodeLabel] = clusterName + } + + if len(labelsToUpdate) > 0 { + if !cloudnodeutil.AddOrUpdateLabelsOnNode(c.kclient, labelsToUpdate, node) { + return fmt.Errorf("failed update labels for node %s", node.Name) + } + } + + return nil +} + +// TODO: add more checks, like domain name, worker nodes don't have controlplane IPs, etc... +func csrNodeChecks(ctx context.Context, kclient clientkubernetes.Interface, x509cr *x509.CertificateRequest) (bool, error) { + node, err := kclient.CoreV1().Nodes().Get(ctx, x509cr.DNSNames[0], metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("failed to get node %s: %w", x509cr.DNSNames[0], err) + } + + var nodeAddrs []string + + if node != nil { + if providedIP, ok := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr]; ok { + nodeAddrs = append(nodeAddrs, providedIP) + } + + for _, ip := range node.Status.Addresses { + nodeAddrs = append(nodeAddrs, ip.Address) + } + + for _, ip := range x509cr.IPAddresses { + if !slices.Contains(nodeAddrs, ip.String()) { + return false, fmt.Errorf("csrNodeChecks: CSR %s Node IP addresses don't match corresponding "+ + "Node IP addresses %q, got %q", x509cr.DNSNames[0], nodeAddrs, ip) + } + } + + return true, nil + } + + return false, fmt.Errorf("failed to get node %s", x509cr.DNSNames[0]) +} diff --git a/pkg/talos/helper_test.go b/pkg/talos/helper_test.go new file mode 100644 index 0000000..22f3e68 --- /dev/null +++ b/pkg/talos/helper_test.go @@ -0,0 +1,417 @@ +package talos + +import ( + "context" + "crypto/x509" + "fmt" + "net" + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/siderolabs/talos/pkg/machinery/resources/runtime" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + cloudproviderapi "k8s.io/cloud-provider/api" +) + +func TestGetNodeAddresses(t *testing.T) { + cfg := cloudConfig{} + + for _, tt := range []struct { + name string + cfg cloudConfig + platform string + providedIP string + ifaces []network.AddressStatusSpec + expected []v1.NodeAddress + }{ + { + name: "nocloud has no PublicIPs", + cfg: cfg, + platform: "nocloud", + providedIP: "192.168.0.1", + ifaces: []network.AddressStatusSpec{ + {Address: netip.MustParsePrefix("192.168.0.1/24")}, + {Address: netip.MustParsePrefix("fe80::e0b5:71ff:fe24:7e60/64")}, + {Address: netip.MustParsePrefix("fd15:1:2::192:168:0:1/64")}, + {Address: netip.MustParsePrefix("fd43:fe8a:be2:ab02:dc3c:38ff:fe51:5022/64"), LinkName: "kubespan"}, + }, + expected: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "192.168.0.1"}, + }, + }, + { + name: "nocloud has many PublicIPs", + cfg: cfg, + platform: "nocloud", + providedIP: "192.168.0.1", + ifaces: []network.AddressStatusSpec{ + {Address: netip.MustParsePrefix("192.168.0.1/24")}, + {Address: netip.MustParsePrefix("fe80::e0b5:71ff:fe24:7e60/64")}, + {Address: netip.MustParsePrefix("fd15:1:2::192:168:0:1/64")}, + {Address: netip.MustParsePrefix("1.2.3.4/24")}, + {Address: netip.MustParsePrefix("4.3.2.1/24")}, + {Address: netip.MustParsePrefix("2001:1234::1/64")}, + {Address: netip.MustParsePrefix("2001:1234:4321::32/64")}, + }, + expected: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "192.168.0.1"}, + {Type: v1.NodeExternalIP, Address: "1.2.3.4"}, + {Type: v1.NodeExternalIP, Address: "2001:1234:4321::32"}, + }, + }, + { + name: "nocloud has many PublicIPs (IPv6 preferred)", + cfg: cloudConfig{Global: cloudConfigGlobal{PreferIPv6: true}}, + platform: "nocloud", + providedIP: "192.168.0.1", + ifaces: []network.AddressStatusSpec{ + {Address: netip.MustParsePrefix("192.168.0.1/24")}, + {Address: netip.MustParsePrefix("fe80::e0b5:71ff:fe24:7e60/64")}, + {Address: netip.MustParsePrefix("fd15:1:2::192:168:0:1/64")}, + {Address: netip.MustParsePrefix("1.2.3.4/24")}, + {Address: netip.MustParsePrefix("4.3.2.1/24")}, + {Address: netip.MustParsePrefix("2001:1234::1/64")}, + {Address: netip.MustParsePrefix("2001:1234:4321::32/64")}, + }, + expected: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "192.168.0.1"}, + {Type: v1.NodeExternalIP, Address: "2001:1234:4321::32"}, + {Type: v1.NodeExternalIP, Address: "1.2.3.4"}, + }, + }, + { + name: "metal has PublicIPs", + cfg: cfg, + platform: "metal", + providedIP: "192.168.0.1", + ifaces: []network.AddressStatusSpec{ + {Address: netip.MustParsePrefix("192.168.0.1/24")}, + {Address: netip.MustParsePrefix("fe80::e0b5:71ff:fe24:7e60/64")}, + {Address: netip.MustParsePrefix("fd15:1:2::192:168:0:1/64")}, + {Address: netip.MustParsePrefix("1.2.3.4/24")}, + {Address: netip.MustParsePrefix("2001:1234::1/128")}, + }, + expected: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "192.168.0.1"}, + {Type: v1.NodeExternalIP, Address: "1.2.3.4"}, + {Type: v1.NodeExternalIP, Address: "2001:1234::1"}, + }, + }, + { + name: "gcp has provided PublicIPs", + cfg: cfg, + platform: "gcp", + providedIP: "192.168.0.1", + ifaces: []network.AddressStatusSpec{ + {Address: netip.MustParsePrefix("192.168.0.1/24")}, + {Address: netip.MustParsePrefix("fe80::e0b5:71ff:fe24:7e60/64")}, + {Address: netip.MustParsePrefix("1.2.3.4/24"), LinkName: "external"}, + {Address: netip.MustParsePrefix("4.3.2.1/24")}, + {Address: netip.MustParsePrefix("2001:1234::1/128"), LinkName: "external"}, + {Address: netip.MustParsePrefix("2001:1234::123/64")}, + }, + expected: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: "192.168.0.1"}, + {Type: v1.NodeExternalIP, Address: "1.2.3.4"}, + {Type: v1.NodeExternalIP, Address: "2001:1234::1"}, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + addresses := getNodeAddresses(&tt.cfg, tt.platform, tt.providedIP, tt.ifaces) + + assert.Equal(t, tt.expected, addresses) + }) + } +} + +func TestSyncNodeLabels(t *testing.T) { + t.Setenv("TALOSCONFIG", "../../hack/talosconfig") + + cfg := cloudConfig{Global: cloudConfigGlobal{ + ClusterName: "test-cluster", + Endpoints: []string{"127.0.0.1"}, + }} + ctx := context.Background() + nodes := &v1.NodeList{ + Items: []v1.Node{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + }, + }, + } + + client, err := newClient(ctx, &cfg) + assert.NoError(t, err) + + client.kclient = fake.NewSimpleClientset(nodes) + + for _, tt := range []struct { + name string + node *v1.Node + meta *runtime.PlatformMetadataSpec + expectedError error + expectedNode *v1.Node + }{ + { + name: "node has no metadata", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + }, + meta: &runtime.PlatformMetadataSpec{}, + expectedError: nil, + expectedNode: &v1.Node{ + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{ + ClusterNameNodeLabel: "test-cluster", + }, + }, + }, + }, + { + name: "node with platform name", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + }, + meta: &runtime.PlatformMetadataSpec{ + Platform: "metal", + Hostname: "node1", + }, + expectedError: nil, + expectedNode: &v1.Node{ + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{ + ClusterNameNodeLabel: "test-cluster", + ClusterNodePlatformLabel: "metal", + }, + }, + }, + }, + { + name: "spot node", + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + }, + meta: &runtime.PlatformMetadataSpec{ + Platform: "metal", + Hostname: "node1", + Spot: true, + }, + expectedError: nil, + expectedNode: &v1.Node{ + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{ + ClusterNameNodeLabel: "test-cluster", + ClusterNodePlatformLabel: "metal", + ClusterNodeLifeCycleLabel: "spot", + }, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + err := syncNodeLabels(client, tt.node, tt.meta) + + assert.Equal(t, tt.expectedError, err) + + node, err := client.kclient.CoreV1().Nodes().Get(ctx, tt.node.Name, metav1.GetOptions{}) + assert.NoError(t, err) + assert.Equal(t, tt.expectedNode, node) + }) + } +} + +func TestCsrNodeChecks(t *testing.T) { + ctx := context.Background() + nodes := &v1.NodeList{ + Items: []v1.Node{ + { + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node2", + Annotations: map[string]string{ + cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", + }, + }, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node-int", + Annotations: map[string]string{ + cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", + }, + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + { + Type: v1.NodeInternalIP, + Address: "1.2.3.4", + }, + }, + }, + }, + { + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "node-int-ext", + Annotations: map[string]string{ + cloudproviderapi.AnnotationAlphaProvidedIPAddr: "1.2.3.4", + }, + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + { + Type: v1.NodeInternalIP, + Address: "1.2.3.4", + }, + { + Type: v1.NodeExternalIP, + Address: "2000::1", + }, + }, + }, + }, + }, + } + + for _, tt := range []struct { + name string + cert *x509.CertificateRequest + expectedError error + expected bool + }{ + { + name: "fake node", + cert: &x509.CertificateRequest{ + DNSNames: []string{"node-non-existing"}, + }, + expectedError: fmt.Errorf("failed to get node node-non-existing: nodes \"node-non-existing\" not found"), + expected: false, + }, + { + name: "empty node", + cert: &x509.CertificateRequest{ + DNSNames: []string{"node1"}, + }, + expectedError: nil, + expected: true, + }, + { + name: "empty node", + cert: &x509.CertificateRequest{ + DNSNames: []string{"node2"}, + }, + expectedError: nil, + expected: true, + }, + { + name: "node with IP", + cert: &x509.CertificateRequest{ + DNSNames: []string{"node2"}, + IPAddresses: []net.IP{ + net.ParseIP("1.2.3.4"), + }, + }, + expectedError: nil, + expected: true, + }, + { + name: "node with fake IPs", + cert: &x509.CertificateRequest{ + DNSNames: []string{"node2"}, + IPAddresses: []net.IP{ + net.ParseIP("1.2.3.4"), + net.ParseIP("2000::1"), + }, + }, + expectedError: nil, + expected: false, + }, + { + name: "node with node-IP", + cert: &x509.CertificateRequest{ + DNSNames: []string{"node-int"}, + IPAddresses: []net.IP{ + net.ParseIP("1.2.3.4"), + }, + }, + expectedError: nil, + expected: true, + }, + { + name: "node with node-IPs", + cert: &x509.CertificateRequest{ + DNSNames: []string{"node-int-ext"}, + IPAddresses: []net.IP{ + net.ParseIP("1.2.3.4"), + net.ParseIP("2000::1"), + }, + }, + expectedError: nil, + expected: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + kclient := fake.NewSimpleClientset(nodes) + approve, err := csrNodeChecks(ctx, kclient, tt.cert) + + if tt.expectedError != nil { + assert.Equal(t, tt.expectedError.Error(), err.Error()) + } else { + assert.Equal(t, tt.expected, approve) + } + }) + } +} diff --git a/pkg/talos/instances.go b/pkg/talos/instances.go index 40c6fc6..56dde22 100644 --- a/pkg/talos/instances.go +++ b/pkg/talos/instances.go @@ -2,22 +2,13 @@ package talos import ( "context" - "crypto/x509" "fmt" "strings" - utilsnet "github.com/siderolabs/talos-cloud-controller-manager/pkg/utils/net" - "github.com/siderolabs/talos/pkg/machinery/resources/network" - "github.com/siderolabs/talos/pkg/machinery/resources/runtime" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - clientkubernetes "k8s.io/client-go/kubernetes" cloudprovider "k8s.io/cloud-provider" cloudproviderapi "k8s.io/cloud-provider/api" - cloudnodeutil "k8s.io/cloud-provider/node/helpers" "k8s.io/klog/v2" - "k8s.io/utils/strings/slices" ) type instances struct { @@ -104,111 +95,3 @@ func (i *instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud return &cloudprovider.InstanceMetadata{}, nil } - -func getNodeAddresses(config *cloudConfig, platform, nodeIP string, ifaces []network.AddressStatusSpec) []v1.NodeAddress { - var publicIPv4s, publicIPv6s, publicIPs []string - - switch platform { - case "nocloud", "metal": - for _, iface := range ifaces { - if iface.LinkName == "kubespan" { - continue - } - - ip := iface.Address.Addr() - if ip.IsGlobalUnicast() && !ip.IsPrivate() { - if ip.Is6() { - publicIPv6s = append(publicIPv6s, ip.String()) - } else { - publicIPv4s = append(publicIPv4s, ip.String()) - } - } - } - default: - for _, iface := range ifaces { - if iface.LinkName == "external" { - ip := iface.Address.Addr() - - if ip.Is6() { - publicIPv6s = append(publicIPv6s, ip.String()) - } else { - publicIPv4s = append(publicIPv4s, ip.String()) - } - } - } - } - - addresses := []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: nodeIP}} - - if config.Global.PreferIPv6 { - publicIPs = utilsnet.SortedNodeIPs(nodeIP, publicIPv6s, publicIPv4s) - } else { - publicIPs = utilsnet.SortedNodeIPs(nodeIP, publicIPv4s, publicIPv6s) - } - - for _, ip := range publicIPs { - addresses = append(addresses, v1.NodeAddress{Type: v1.NodeExternalIP, Address: ip}) - } - - return addresses -} - -func syncNodeLabels(c *client, node *v1.Node, meta *runtime.PlatformMetadataSpec) error { - nodeLabels := node.ObjectMeta.Labels - labelsToUpdate := map[string]string{} - - if nodeLabels == nil { - nodeLabels = map[string]string{} - } - - if meta.Platform != "" && nodeLabels[ClusterNodePlatformLabel] != meta.Platform { - labelsToUpdate[ClusterNodePlatformLabel] = meta.Platform - } - - if meta.Spot && nodeLabels[ClusterNodeLifeCycleLabel] != "spot" { - labelsToUpdate[ClusterNodeLifeCycleLabel] = "spot" - } - - if clusterName := c.talos.GetClusterName(); clusterName != "" && nodeLabels[ClusterNameNodeLabel] != clusterName { - labelsToUpdate[ClusterNameNodeLabel] = clusterName - } - - if len(labelsToUpdate) > 0 { - if !cloudnodeutil.AddOrUpdateLabelsOnNode(c.kclient, labelsToUpdate, node) { - return fmt.Errorf("failed update labels for node %s", node.Name) - } - } - - return nil -} - -// TODO: add more checks, like domain name, worker nodes don't have controlplane IPs, etc... -func csrNodeChecks(ctx context.Context, kclient clientkubernetes.Interface, x509cr *x509.CertificateRequest) (bool, error) { - node, err := kclient.CoreV1().Nodes().Get(ctx, x509cr.DNSNames[0], metav1.GetOptions{}) - if err != nil { - return false, fmt.Errorf("failed to get node %s: %w", x509cr.DNSNames[0], err) - } - - var nodeAddrs []string - - if node != nil { - if providedIP, ok := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr]; ok { - nodeAddrs = append(nodeAddrs, providedIP) - } - - for _, ip := range node.Status.Addresses { - nodeAddrs = append(nodeAddrs, ip.Address) - } - - for _, ip := range x509cr.IPAddresses { - if !slices.Contains(nodeAddrs, ip.String()) { - return false, fmt.Errorf("csrNodeChecks: CSR %s Node IP addresses don't match corresponding "+ - "Node IP addresses %q, got %q", x509cr.DNSNames[0], nodeAddrs, ip) - } - } - - return true, nil - } - - return false, fmt.Errorf("failed to get node %s", x509cr.DNSNames[0]) -} diff --git a/pkg/talos/instances_test.go b/pkg/talos/instances_test.go index b04b9bb..316167b 100644 --- a/pkg/talos/instances_test.go +++ b/pkg/talos/instances_test.go @@ -2,107 +2,44 @@ package talos import ( "context" - "net/netip" "testing" "github.com/stretchr/testify/assert" - - "github.com/siderolabs/talos/pkg/machinery/resources/network" + "github.com/stretchr/testify/suite" v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" cloudprovider "k8s.io/cloud-provider" - cloudproviderapi "k8s.io/cloud-provider/api" ) -func TestGetNodeAddresses(t *testing.T) { - cfg := cloudConfig{} +type ccmTestSuite struct { + suite.Suite - for _, tt := range []struct { - name string - platform string - providedIP string - ifaces []network.AddressStatusSpec - expected []v1.NodeAddress - }{ - { - name: "nocloud has no PublicIPs", - platform: "nocloud", - providedIP: "192.168.0.1", - ifaces: []network.AddressStatusSpec{ - {Address: netip.MustParsePrefix("192.168.0.1/24")}, - {Address: netip.MustParsePrefix("fe80::e0b5:71ff:fe24:7e60/64")}, - {Address: netip.MustParsePrefix("fd15:1:2::192:168:0:1/64")}, - {Address: netip.MustParsePrefix("fd43:fe8a:be2:ab02:dc3c:38ff:fe51:5022/64"), LinkName: "kubespan"}, - }, - expected: []v1.NodeAddress{ - {Type: v1.NodeInternalIP, Address: "192.168.0.1"}, - }, - }, - { - name: "nocloud has many PublicIPs", - platform: "nocloud", - providedIP: "192.168.0.1", - ifaces: []network.AddressStatusSpec{ - {Address: netip.MustParsePrefix("192.168.0.1/24")}, - {Address: netip.MustParsePrefix("fe80::e0b5:71ff:fe24:7e60/64")}, - {Address: netip.MustParsePrefix("fd15:1:2::192:168:0:1/64")}, - {Address: netip.MustParsePrefix("1.2.3.4/24")}, - {Address: netip.MustParsePrefix("4.3.2.1/24")}, - {Address: netip.MustParsePrefix("2001:1234::1/64")}, - {Address: netip.MustParsePrefix("2001:1234:4321::32/64")}, - }, - expected: []v1.NodeAddress{ - {Type: v1.NodeInternalIP, Address: "192.168.0.1"}, - {Type: v1.NodeExternalIP, Address: "1.2.3.4"}, - {Type: v1.NodeExternalIP, Address: "2001:1234:4321::32"}, - }, - }, - { - name: "metal has PublicIPs", - platform: "metal", - providedIP: "192.168.0.1", - ifaces: []network.AddressStatusSpec{ - {Address: netip.MustParsePrefix("192.168.0.1/24")}, - {Address: netip.MustParsePrefix("fe80::e0b5:71ff:fe24:7e60/64")}, - {Address: netip.MustParsePrefix("fd15:1:2::192:168:0:1/64")}, - {Address: netip.MustParsePrefix("1.2.3.4/24")}, - {Address: netip.MustParsePrefix("2001:1234::1/128")}, - }, - expected: []v1.NodeAddress{ - {Type: v1.NodeInternalIP, Address: "192.168.0.1"}, - {Type: v1.NodeExternalIP, Address: "1.2.3.4"}, - {Type: v1.NodeExternalIP, Address: "2001:1234::1"}, - }, - }, - { - name: "gcp has provided PublicIPs", - platform: "gcp", - providedIP: "192.168.0.1", - ifaces: []network.AddressStatusSpec{ - {Address: netip.MustParsePrefix("192.168.0.1/24")}, - {Address: netip.MustParsePrefix("fe80::e0b5:71ff:fe24:7e60/64")}, - {Address: netip.MustParsePrefix("1.2.3.4/24"), LinkName: "external"}, - {Address: netip.MustParsePrefix("4.3.2.1/24")}, - {Address: netip.MustParsePrefix("2001:1234::1/128"), LinkName: "external"}, - {Address: netip.MustParsePrefix("2001:1234::123/64")}, - }, - expected: []v1.NodeAddress{ - {Type: v1.NodeInternalIP, Address: "192.168.0.1"}, - {Type: v1.NodeExternalIP, Address: "1.2.3.4"}, - {Type: v1.NodeExternalIP, Address: "2001:1234::1"}, - }, - }, - } { - t.Run(tt.name, func(t *testing.T) { - addresses := getNodeAddresses(&cfg, tt.platform, tt.providedIP, tt.ifaces) + i *instances +} - assert.Equal(t, tt.expected, addresses) - }) - } +func (ts *ccmTestSuite) SetupTest() { + ts.i = newInstances(nil) +} + +func TestSuiteCCM(t *testing.T) { + suite.Run(t, new(ccmTestSuite)) +} + +func (ts *ccmTestSuite) TestInstanceExists() { + exists, err := ts.i.InstanceExists(context.Background(), &v1.Node{}) + ts.Require().NoError(err) + ts.Require().True(exists) +} + +func (ts *ccmTestSuite) TestInstanceShutdown() { + exists, err := ts.i.InstanceShutdown(context.Background(), &v1.Node{}) + ts.Require().NoError(err) + ts.Require().False(exists) } func TestInstanceMetadata(t *testing.T) { + t.Setenv("TALOSCONFIG", "../../hack/talosconfig") + cfg := cloudConfig{} cfg.Global.SkipForeignNode = true @@ -117,16 +54,6 @@ func TestInstanceMetadata(t *testing.T) { node *v1.Node expected *cloudprovider.InstanceMetadata }{ - { - name: "node has providerID", - node: &v1.Node{ - Spec: v1.NodeSpec{ProviderID: "provider:///id"}, - ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ - cloudproviderapi.AnnotationAlphaProvidedIPAddr: "192.168.1.1", - }}, - }, - expected: &cloudprovider.InstanceMetadata{}, - }, { name: "node does not have --cloud-provider=external", node: &v1.Node{ @@ -134,6 +61,16 @@ func TestInstanceMetadata(t *testing.T) { }, expected: &cloudprovider.InstanceMetadata{}, }, + // { + // name: "node has providerID", + // node: &v1.Node{ + // Spec: v1.NodeSpec{ProviderID: "provider:///id"}, + // ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + // cloudproviderapi.AnnotationAlphaProvidedIPAddr: "127.0.0.1", + // }}, + // }, + // expected: &cloudprovider.InstanceMetadata{}, + // }, } { t.Run(tt.name, func(t *testing.T) { metadata, err := i.InstanceMetadata(ctx, tt.node)