github-personal/talos-cloud-controller-manager
1
0
Fork 0
mirror of https://github.com/outbackdingo/talos-cloud-controller-manager.git synced 2026-01-27 10:20:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add node certificate approval

Browse Source 
TalosCCM now can verify node CSR, and approve it if it OK.

Signed-off-by: Serge Logvinov <serge.logvinov@sinextra.dev>
This commit is contained in:
Serge Logvinov
2023-04-16 18:32:36 +03:00
parent 11e77e8f83
commit 2b53c2b9e7
17 changed files with 915 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -3,6 +3,8 @@
#
/talos-cloud-controller-manager*
/kubeconfig
/kubeconfig.*
/talosconfig
/talosconfig.*
#
/docs/*.go

3
charts/talos-cloud-controller-manager/templates/configmap.yaml
View File

@@ -8,3 +8,6 @@ metadata:
data:
ccm-config.yaml: |
global:
{{- if .Values.features.approveNodeCSR }}
approveNodeCSR: true
{{- end }}

42
charts/talos-cloud-controller-manager/templates/role.yaml
View File

@@ -50,24 +50,24 @@ rules:
- serviceaccounts/token
verbs:
- create
# - apiGroups:
# - certificates.k8s.io
# resources:
# - certificatesigningrequests
# verbs:
# - list
# - watch
# - apiGroups:
# - certificates.k8s.io
# resources:
# - certificatesigningrequests/approval
# verbs:
# - update
# - apiGroups:
# - certificates.k8s.io
# resources:
# - signers
# resourceNames:
# - kubernetes.io/kubelet-serving
# verbs:
# - approve
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests
verbs:
- list
- watch
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests/approval
verbs:
- update
- apiGroups:
- certificates.k8s.io
resources:
- signers
resourceNames:
- kubernetes.io/kubelet-serving
verbs:
- approve

5
charts/talos-cloud-controller-manager/values.yaml
View File

@@ -35,6 +35,11 @@ enabledControllers:
# - route
# - service
# -- List of CCM features.
# `approveNodeCSR` - check and approve node CSR.
features:
approveNodeCSR: true
# -- Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md
# for description of individual verbosity levels.
logVerbosityLevel: 2

43
docs/deploy/cloud-controller-manager-daemonset.yml
View File

@@ -43,6 +43,7 @@ metadata:
data:
ccm-config.yaml: |
global:
approveNodeCSR: true
---
# Source: talos-cloud-controller-manager/templates/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
@@ -101,27 +102,27 @@ rules:
- serviceaccounts/token
verbs:
- create
# - apiGroups:
# - certificates.k8s.io
# resources:
# - certificatesigningrequests
# verbs:
# - list
# - watch
# - apiGroups:
# - certificates.k8s.io
# resources:
# - certificatesigningrequests/approval
# verbs:
# - update
# - apiGroups:
# - certificates.k8s.io
# resources:
# - signers
# resourceNames:
# - kubernetes.io/kubelet-serving
# verbs:
# - approve
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests
verbs:
- list
- watch
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests/approval
verbs:
- update
- apiGroups:
- certificates.k8s.io
resources:
- signers
resourceNames:
- kubernetes.io/kubelet-serving
verbs:
- approve
---
# Source: talos-cloud-controller-manager/templates/rolebinding.yaml
kind: ClusterRoleBinding

43
docs/deploy/cloud-controller-manager.yml
View File

@@ -43,6 +43,7 @@ metadata:
data:
ccm-config.yaml: |
global:
approveNodeCSR: true
---
# Source: talos-cloud-controller-manager/templates/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
@@ -101,27 +102,27 @@ rules:
- serviceaccounts/token
verbs:
- create
# - apiGroups:
# - certificates.k8s.io
# resources:
# - certificatesigningrequests
# verbs:
# - list
# - watch
# - apiGroups:
# - certificates.k8s.io
# resources:
# - certificatesigningrequests/approval
# verbs:
# - update
# - apiGroups:
# - certificates.k8s.io
# resources:
# - signers
# resourceNames:
# - kubernetes.io/kubelet-serving
# verbs:
# - approve
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests
verbs:
- list
- watch
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests/approval
verbs:
- update
- apiGroups:
- certificates.k8s.io
resources:
- signers
resourceNames:
- kubernetes.io/kubelet-serving
verbs:
- approve
---
# Source: talos-cloud-controller-manager/templates/rolebinding.yaml
kind: ClusterRoleBinding

4
go.mod
View File

@@ -5,7 +5,7 @@ go 1.20
require (
github.com/cosi-project/runtime v0.2.1
github.com/siderolabs/net v0.4.0
github.com/siderolabs/talos/pkg/machinery v1.3.6
github.com/siderolabs/talos/pkg/machinery v1.3.7
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.2
gopkg.in/yaml.v3 v3.0.1
@@ -16,6 +16,7 @@ require (
k8s.io/cloud-provider v0.26.3
k8s.io/component-base v0.26.3
k8s.io/klog/v2 v2.90.0
k8s.io/utils v0.0.0-20221107191617-1a15be271d1d
)
require (
@@ -131,7 +132,6 @@ require (
k8s.io/controller-manager v0.26.3 // indirect
k8s.io/kms v0.26.3 // indirect
k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
k8s.io/utils v0.0.0-20221107191617-1a15be271d1d // indirect
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.36 // indirect
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect

4
go.sum
View File

@@ -387,8 +387,8 @@ github.com/siderolabs/net v0.4.0 h1:1bOgVay/ijPkJz4qct98nHsiB/ysLQU0KLoBC4qLm7I=
github.com/siderolabs/net v0.4.0/go.mod h1:/ibG+Hm9HU27agp5r9Q3eZicEfjquzNzQNux5uEk0kM=
github.com/siderolabs/protoenc v0.2.0 h1:QFxWIAo//12+/bm27GNYoK/TpQGTYsRrrZCu9jSghvU=
github.com/siderolabs/protoenc v0.2.0/go.mod h1:mu4gc6pJxhdJYpuloacKE4jsJojj87qDXwn8LUvs2bY=
github.com/siderolabs/talos/pkg/machinery v1.3.6 h1:JyMLQNHV+WzHRSav4IUdjPwTz6q4miilVG/iCr3Nq/I=
github.com/siderolabs/talos/pkg/machinery v1.3.6/go.mod h1:z8sMFHG7ert6wMIVFlwF65q+MxMpleGuvjk37nmdD7Y=
github.com/siderolabs/talos/pkg/machinery v1.3.7 h1:MkO6APav3q7+ZRrgwMZH+J1uLMmASwUp8eYZgoWX0gc=
github.com/siderolabs/talos/pkg/machinery v1.3.7/go.mod h1:z8sMFHG7ert6wMIVFlwF65q+MxMpleGuvjk37nmdD7Y=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=

1
hack/talos-config.yaml
View File

@@ -1,4 +1,5 @@
global:
approveNodeCSR: true
# endpoints:
# - 1.2.3.4
# - 4.3.2.1

168
pkg/certificatesigningrequest/controller.go Normal file
View File

@@ -0,0 +1,168 @@
// Package certificatesigningrequest implements the controller for Node Certificate Signing Request.
package certificatesigningrequest
import (
"context"
"crypto/x509"
"fmt"
"strings"
"time"
certificatesv1 "k8s.io/api/certificates/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8swatch "k8s.io/apimachinery/pkg/watch"
clientkubernetes "k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
)
// ProviderChecks is a function that checks if the CertificateSigningRequest is valid in the provider.
type ProviderChecks func(context.Context, clientkubernetes.Interface, *x509.CertificateRequest) (bool, error)
// Reconciler is the controller for CertificateSigningRequest.
type Reconciler struct {
kclient clientkubernetes.Interface
providerChecks ProviderChecks
}
// NewCsrController returns a new CertificateSigningRequest controller.
func NewCsrController(kclient clientkubernetes.Interface, fn ProviderChecks) *Reconciler {
return &Reconciler{
kclient: kclient,
providerChecks: fn,
}
}
// Run the CertificateSigningRequest controller.
//
//nolint:gocyclo
func (r *Reconciler) Run(ctx context.Context) {
watchTimeoutSeconds := int64(time.Minute * 5)
for {
watcher, err := r.kclient.
CertificatesV1().
CertificateSigningRequests().
Watch(ctx, metav1.ListOptions{
Watch: true,
TimeoutSeconds: &watchTimeoutSeconds, // Default timeout: 20 minutes.
})
if err != nil {
klog.Errorf("CertificateSigningRequestReconciler: failed to list CSR resources: %v", err)
time.Sleep(10 * time.Second) // Pause for a while before retrying, otherwise we'll spam error logs.
continue
}
csrWatcher := k8swatch.Filter(watcher, func(in k8swatch.Event) (out k8swatch.Event, keep bool) {
if in.Type != k8swatch.Added {
return in, false
}
return in, true
})
watch:
for {
select {
case <-ctx.Done():
klog.V(4).Infof("CertificateSigningRequestReconciler: context canceled, terminating")
return
case event, ok := <-csrWatcher.ResultChan():
if !ok {
// Server timeout closed the watcher channel, loop again to re-create a new one.
klog.V(5).Infof("CertificateSigningRequestReconciler: API server closed watcher channel")
break watch
}
csr, ok := event.Object.DeepCopyObject().(*certificatesv1.CertificateSigningRequest)
if !ok {
klog.Errorf("CertificateSigningRequestReconciler: expected event of type *CertificateSigningRequest, got %v",
event.Object.GetObjectKind())
continue
}
valid, err := r.Reconcile(ctx, csr)
if err != nil {
klog.Errorf("CertificateSigningRequestReconciler: failed to reconcile CSR %s: %v", csr.Name, err)
continue
}
_, err = r.kclient.CertificatesV1().CertificateSigningRequests().UpdateApproval(ctx, csr.Name, csr, metav1.UpdateOptions{})
if err != nil {
klog.Errorf("CertificateSigningRequestReconciler: failed to approve/deny CSR %s: %v", csr.Name, err)
}
if !valid {
klog.V(3).Infof("CertificateSigningRequestReconciler: has been denied: %s, %+v", csr.Name, err.Error())
} else {
klog.V(3).Infof("CertificateSigningRequestReconciler: has been approved: %s", csr.Name)
}
}
}
}
}
// Reconcile the CertificateSigningRequest.
func (r *Reconciler) Reconcile(ctx context.Context, csr *certificatesv1.CertificateSigningRequest) (bool, error) {
switch {
case len(csr.Status.Conditions) > 0:
return false, fmt.Errorf("already been approved or denied, signer %s", csr.Spec.SignerName)
case csr.Spec.SignerName != certificatesv1.KubeletServingSignerName:
return false, fmt.Errorf("is not Kubelet serving certificate, signer %s", csr.Spec.SignerName)
case !strings.HasPrefix(csr.Spec.Username, "system:node:"):
return false, fmt.Errorf("ignoring, %s, signer %s", errCommonNameNotSystemNode, csr.Spec.SignerName)
case csr.Status.Certificate != nil:
return false, fmt.Errorf("ignoring, already signed, username %s", csr.Spec.Username)
default:
x509cr, err := parseCSR(csr.Spec.Request)
if err != nil {
return false, err
}
err = validateKubeletServingCSR(x509cr, csr.Spec.Usages)
if err != nil {
r.updateApproval(csr, false, err.Error())
return false, nil
}
valid, err := r.providerChecks(ctx, r.kclient, x509cr)
if err != nil {
return valid, fmt.Errorf("providerChecks has an error: %v", err)
}
if valid {
r.updateApproval(csr, valid, "all checks passed")
} else {
r.updateApproval(csr, valid, "providerChecks failed")
}
return valid, nil
}
}
func (r *Reconciler) updateApproval(csr *certificatesv1.CertificateSigningRequest, approved bool, reason string) {
if approved {
csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{
Type: certificatesv1.CertificateApproved,
Status: corev1.ConditionTrue,
Reason: "Approved by TalosCloudControllerManager",
Message: "This CSR was approved by Talos Cloud Controller Manager",
LastUpdateTime: metav1.Time{Time: time.Now().UTC()},
})
} else {
csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{
Type: certificatesv1.CertificateDenied,
Status: corev1.ConditionTrue,
Reason: "Denied by TalosCloudControllerManager",
Message: "This CSR was denied by Talos Cloud Controller Manager, Reason: " + reason,
LastUpdateTime: metav1.Time{Time: time.Now().UTC()},
})
}
}

265
pkg/certificatesigningrequest/controller_test.go Normal file
View File

@@ -0,0 +1,265 @@
package certificatesigningrequest_test
import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"
"net"
"reflect"
"testing"
"github.com/stretchr/testify/assert"
"github.com/siderolabs/talos-cloud-controller-manager/pkg/certificatesigningrequest"
certificatesv1 "k8s.io/api/certificates/v1"
clientkubernetes "k8s.io/client-go/kubernetes"
)
const (
hostname = "talos-1"
organization = "system:nodes"
username = "system:node:" + hostname
)
var rsaKey *rsa.PrivateKey
func init() {
res, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
panic(err)
}
rsaKey = res
}
func generateCSR(t *testing.T, csrTemplate *x509.CertificateRequest) []byte {
t.Helper()
csrCertificate, err := x509.CreateCertificateRequest(rand.Reader, csrTemplate, rsaKey)
if err != nil {
t.Fatalf("Can not create Certificate Request %v", err)
}
csr := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE REQUEST",
Bytes: csrCertificate,
})
return csr
}
func TestNewCsrController(t *testing.T) {
t.Parallel()
kclient := &clientkubernetes.Clientset{}
controller := certificatesigningrequest.NewCsrController(kclient,
func(context.Context, clientkubernetes.Interface, *x509.CertificateRequest) (bool, error) {
return true, nil
})
assert.NotNil(t, controller)
}
func TestControllerReconcileCSR(t *testing.T) {
t.Parallel()
controller := certificatesigningrequest.NewCsrController(&clientkubernetes.Clientset{},
func(_ context.Context, _ clientkubernetes.Interface, x509cr *x509.CertificateRequest) (bool, error) {
if reflect.DeepEqual(x509cr.DNSNames, []string{"error"}) {
return false, fmt.Errorf("someting went wrong")
}
if !reflect.DeepEqual(x509cr.DNSNames, []string{hostname}) {
return false, nil
}
return true, nil
})
assert.NotNil(t, controller)
tests := []struct {
msg string
csr certificatesv1.CertificateSigningRequest
x509cr x509.CertificateRequest
expectedValid bool
expectedError error
expectedMessage string
}{
{
msg: "Not Kubelet CSR",
csr: certificatesv1.CertificateSigningRequest{
Spec: certificatesv1.CertificateSigningRequestSpec{
SignerName: certificatesv1.KubeletServingSignerName,
},
Status: certificatesv1.CertificateSigningRequestStatus{
Conditions: []certificatesv1.CertificateSigningRequestCondition{
{},
},
},
},
expectedError: fmt.Errorf("already been approved or denied, signer kubernetes.io/kubelet-serving"),
},
{
msg: "Not Kubelet CSR",
csr: certificatesv1.CertificateSigningRequest{
Spec: certificatesv1.CertificateSigningRequestSpec{
SignerName: "someothername",
},
},
expectedError: fmt.Errorf("is not Kubelet serving certificate, signer someothername"),
},
{
msg: "Not Kubelet CSR, wrong username",
csr: certificatesv1.CertificateSigningRequest{
Spec: certificatesv1.CertificateSigningRequestSpec{
SignerName: certificatesv1.KubeletServingSignerName,
Username: "invalid",
},
Status: certificatesv1.CertificateSigningRequestStatus{
Certificate: []byte("somecert"),
},
},
expectedError: fmt.Errorf("ignoring, subject common name does not begin with system:node: , signer kubernetes.io/kubelet-serving"),
},
{
msg: "Already signed CSR",
csr: certificatesv1.CertificateSigningRequest{
Spec: certificatesv1.CertificateSigningRequestSpec{
SignerName: certificatesv1.KubeletServingSignerName,
Username: username,
},
Status: certificatesv1.CertificateSigningRequestStatus{
Certificate: []byte("somecert"),
},
},
expectedError: fmt.Errorf("ignoring, already signed, username %s", username),
},
{
msg: "Wrong CSR body",
csr: certificatesv1.CertificateSigningRequest{
Spec: certificatesv1.CertificateSigningRequestSpec{
SignerName: certificatesv1.KubeletServingSignerName,
Username: username,
Request: []byte("somecert"),
},
},
expectedError: fmt.Errorf("PEM block type must be CERTIFICATE REQUEST"),
},
{
msg: "Wrong CSR DNS-IP",
csr: certificatesv1.CertificateSigningRequest{
Spec: certificatesv1.CertificateSigningRequestSpec{
SignerName: certificatesv1.KubeletServingSignerName,
Username: username,
Request: generateCSR(t, &x509.CertificateRequest{
Subject: pkix.Name{
Organization: []string{organization},
CommonName: username,
},
SignatureAlgorithm: x509.SHA256WithRSA,
}),
},
},
expectedValid: false,
expectedMessage: "This CSR was denied by Talos Cloud Controller Manager, Reason: DNS or IP subjectAltName is required",
},
{
msg: "Approved CSR",
csr: certificatesv1.CertificateSigningRequest{
Spec: certificatesv1.CertificateSigningRequestSpec{
SignerName: certificatesv1.KubeletServingSignerName,
Username: username,
Request: generateCSR(t, &x509.CertificateRequest{
Subject: pkix.Name{
Organization: []string{organization},
CommonName: username,
},
DNSNames: []string{hostname},
IPAddresses: []net.IP{net.ParseIP("1.2.3.4")},
SignatureAlgorithm: x509.SHA256WithRSA,
}),
Usages: []certificatesv1.KeyUsage{
certificatesv1.UsageDigitalSignature,
certificatesv1.UsageServerAuth,
},
},
},
expectedValid: true,
expectedMessage: "This CSR was approved by Talos Cloud Controller Manager",
},
{
msg: "Denied CSR with invalid DNS",
csr: certificatesv1.CertificateSigningRequest{
Spec: certificatesv1.CertificateSigningRequestSpec{
SignerName: certificatesv1.KubeletServingSignerName,
Username: username,
Request: generateCSR(t, &x509.CertificateRequest{
Subject: pkix.Name{
Organization: []string{organization},
CommonName: username,
},
DNSNames: []string{"invalid"},
IPAddresses: []net.IP{net.ParseIP("1.2.3.4")},
SignatureAlgorithm: x509.SHA256WithRSA,
}),
Usages: []certificatesv1.KeyUsage{
certificatesv1.UsageDigitalSignature,
certificatesv1.UsageServerAuth,
},
},
},
expectedValid: false,
expectedMessage: "This CSR was denied by Talos Cloud Controller Manager, Reason: providerChecks failed",
},
{
msg: "ProviderChecks has an error",
csr: certificatesv1.CertificateSigningRequest{
Spec: certificatesv1.CertificateSigningRequestSpec{
SignerName: certificatesv1.KubeletServingSignerName,
Username: username,
Request: generateCSR(t, &x509.CertificateRequest{
Subject: pkix.Name{
Organization: []string{organization},
CommonName: username,
},
DNSNames: []string{"error"},
IPAddresses: []net.IP{net.ParseIP("1.2.3.4")},
SignatureAlgorithm: x509.SHA256WithRSA,
}),
Usages: []certificatesv1.KeyUsage{
certificatesv1.UsageDigitalSignature,
certificatesv1.UsageServerAuth,
},
},
},
expectedError: fmt.Errorf("providerChecks has an error: someting went wrong"),
},
}
for _, testCase := range tests {
testCase := testCase
t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) {
t.Parallel()
valid, err := controller.Reconcile(context.Background(), &testCase.csr)
if testCase.expectedError != nil {
assert.NotNil(t, err)
assert.Contains(t, err.Error(), testCase.expectedError.Error())
} else {
assert.Nil(t, err)
assert.Equal(t, testCase.expectedValid, valid)
assert.Len(t, testCase.csr.Status.Conditions, 1)
assert.Contains(t, testCase.expectedMessage, testCase.csr.Status.Conditions[0].Message)
}
})
}
}

93
pkg/certificatesigningrequest/utils.go Normal file
View File

@@ -0,0 +1,93 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package certificatesigningrequest
import (
"crypto/x509"
"encoding/pem"
"fmt"
"reflect"
"strings"
certificatesv1 "k8s.io/api/certificates/v1"
)
// Source(08/2022): https://github.com/kubernetes/kubernetes/blob/master/pkg/apis/certificates/helpers.go 160f015
func parseCSR(pemBytes []byte) (*x509.CertificateRequest, error) {
block, _ := pem.Decode(pemBytes)
if block == nil || block.Type != "CERTIFICATE REQUEST" {
return nil, errNotCertificateRequest
}
csr, err := x509.ParseCertificateRequest(block.Bytes)
if err != nil {
return nil, err
}
return csr, nil
}
var (
errNotCertificateRequest = fmt.Errorf("PEM block type must be CERTIFICATE REQUEST")
errOrganizationNotSystemNodes = fmt.Errorf("subject organization is not system:nodes")
errCommonNameNotSystemNode = fmt.Errorf("subject common name does not begin with system:node: ")
errDNSOrIPSANRequired = fmt.Errorf("DNS or IP subjectAltName is required")
errEmailSANNotAllowed = fmt.Errorf("email subjectAltNames are not allowed")
errURISANNotAllowed = fmt.Errorf("URI subjectAltNames are not allowed")
errKeyUsageMismatch = fmt.Errorf("key usage does not match")
)
var (
kubeletServingRequiredUsages = []certificatesv1.KeyUsage{
certificatesv1.UsageKeyEncipherment,
certificatesv1.UsageDigitalSignature,
certificatesv1.UsageServerAuth,
}
kubeletServingRequiredUsagesNoRSA = []certificatesv1.KeyUsage{
certificatesv1.UsageDigitalSignature,
certificatesv1.UsageServerAuth,
}
)
func validateKubeletServingCSR(req *x509.CertificateRequest, keyUsages []certificatesv1.KeyUsage) error {
if len(req.DNSNames) == 0 && len(req.IPAddresses) == 0 {
return errDNSOrIPSANRequired
}
if len(req.EmailAddresses) > 0 {
return errEmailSANNotAllowed
}
if len(req.URIs) > 0 {
return errURISANNotAllowed
}
if !reflect.DeepEqual([]string{"system:nodes"}, req.Subject.Organization) {
return errOrganizationNotSystemNodes
}
if !strings.HasPrefix(req.Subject.CommonName, "system:node:") {
return errCommonNameNotSystemNode
}
if !reflect.DeepEqual(kubeletServingRequiredUsages, keyUsages) && !reflect.DeepEqual(kubeletServingRequiredUsagesNoRSA, keyUsages) {
return errKeyUsageMismatch
}
return nil
}

255
pkg/certificatesigningrequest/utils_test.go Normal file
View File

@@ -0,0 +1,255 @@
//nolint:testpackage // Need to reach functions.
package certificatesigningrequest
import (
"crypto/x509"
"crypto/x509/pkix"
"encoding/asn1"
"fmt"
"net"
"net/url"
"testing"
"github.com/stretchr/testify/assert"
certificatesv1 "k8s.io/api/certificates/v1"
)
func TestParseCSRValid(t *testing.T) {
t.Parallel()
tests := []struct {
msg string
pemCSR []byte
csr certificatesv1.CertificateSigningRequest
expectedError error
}{
{
msg: "empty PEM CSR",
pemCSR: []byte(""),
expectedError: errNotCertificateRequest,
},
{
msg: "empty PEM data",
pemCSR: []byte("-----BEGIN CERTIFICATE REQUEST-----\n-----END CERTIFICATE REQUEST-----\n"),
expectedError: asn1.SyntaxError{Msg: "sequence truncated"},
},
{
msg: "wrong PEM data",
pemCSR: []byte("-----BEGIN CERTIFICATE REQUEST-----\n1234567890\n-----END CERTIFICATE REQUEST-----\n"),
expectedError: errNotCertificateRequest,
},
}
for _, testCase := range tests {
testCase := testCase
t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) {
t.Parallel()
csr, err := parseCSR(testCase.pemCSR)
assert.NotNil(t, err)
assert.Nil(t, csr)
assert.Contains(t, err.Error(), testCase.expectedError.Error())
})
}
}
func TestValidateKubeletServingCSRValid(t *testing.T) {
t.Parallel()
org := "system:nodes"
cname := "system:node:valid"
usages := []certificatesv1.KeyUsage{
certificatesv1.UsageKeyEncipherment,
certificatesv1.UsageDigitalSignature,
certificatesv1.UsageServerAuth,
}
tests := []struct {
msg string
x509cr x509.CertificateRequest
keyUsages []certificatesv1.KeyUsage
}{
{
msg: "Only DNSNames",
x509cr: x509.CertificateRequest{
Subject: pkix.Name{
CommonName: cname,
Organization: []string{org},
},
DNSNames: []string{"valid"},
},
keyUsages: usages,
},
{
msg: "Only IPAddresses",
x509cr: x509.CertificateRequest{
Subject: pkix.Name{
CommonName: cname,
Organization: []string{org},
},
IPAddresses: []net.IP{net.ParseIP("1.2.3.4")},
},
keyUsages: usages,
},
{
msg: "Key usages RSA",
x509cr: x509.CertificateRequest{
Subject: pkix.Name{
CommonName: cname,
Organization: []string{org},
},
DNSNames: []string{"valid"},
IPAddresses: []net.IP{net.ParseIP("1.2.3.4")},
},
keyUsages: []certificatesv1.KeyUsage{
certificatesv1.UsageKeyEncipherment,
certificatesv1.UsageDigitalSignature,
certificatesv1.UsageServerAuth,
},
},
{
msg: "Key usages ECDSA",
x509cr: x509.CertificateRequest{
Subject: pkix.Name{
CommonName: cname,
Organization: []string{org},
},
DNSNames: []string{"valid"},
IPAddresses: []net.IP{net.ParseIP("1.2.3.4")},
},
keyUsages: []certificatesv1.KeyUsage{
certificatesv1.UsageDigitalSignature,
certificatesv1.UsageServerAuth,
},
},
}
for _, testCase := range tests {
testCase := testCase
t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) {
t.Parallel()
err := validateKubeletServingCSR(&testCase.x509cr, usages)
assert.NoError(t, err)
})
}
}
func TestValidateKubeletServingCSRInvalid(t *testing.T) {
t.Parallel()
org := "system:nodes"
cname := "system:node:invalid"
dnsNames := []string{"valid"}
ipAddresses := []net.IP{net.ParseIP("1.2.3.4")}
usages := []certificatesv1.KeyUsage{
certificatesv1.UsageDigitalSignature,
certificatesv1.UsageServerAuth,
}
tests := []struct {
msg string
x509cr x509.CertificateRequest
keyUsages []certificatesv1.KeyUsage
expectedError error
}{
{
msg: "DNSNames or IPAddresses required",
x509cr: x509.CertificateRequest{
Subject: pkix.Name{
CommonName: cname,
Organization: []string{org},
},
},
keyUsages: usages,
expectedError: errDNSOrIPSANRequired,
},
{
msg: "Invalid Organization",
x509cr: x509.CertificateRequest{
Subject: pkix.Name{
CommonName: cname,
Organization: []string{"invalid"},
},
DNSNames: dnsNames,
IPAddresses: ipAddresses,
},
keyUsages: usages,
expectedError: errOrganizationNotSystemNodes,
},
{
msg: "Invalid CommonName",
x509cr: x509.CertificateRequest{
Subject: pkix.Name{
CommonName: "invalid",
Organization: []string{org},
},
DNSNames: dnsNames,
IPAddresses: ipAddresses,
},
keyUsages: usages,
expectedError: errCommonNameNotSystemNode,
},
{
msg: "Has email addresses",
x509cr: x509.CertificateRequest{
Subject: pkix.Name{
CommonName: cname,
Organization: []string{org},
},
EmailAddresses: []string{"invalid"},
DNSNames: dnsNames,
IPAddresses: ipAddresses,
},
keyUsages: usages,
expectedError: errEmailSANNotAllowed,
},
{
msg: "Has URI addresses",
x509cr: x509.CertificateRequest{
Subject: pkix.Name{
CommonName: cname,
Organization: []string{org},
},
URIs: []*url.URL{{Scheme: "https", Host: "invalid"}},
DNSNames: dnsNames,
IPAddresses: ipAddresses,
},
keyUsages: usages,
expectedError: errURISANNotAllowed,
},
{
msg: "Invalid key usages",
x509cr: x509.CertificateRequest{
Subject: pkix.Name{
CommonName: cname,
Organization: []string{org},
},
DNSNames: dnsNames,
IPAddresses: ipAddresses,
},
keyUsages: []certificatesv1.KeyUsage{
certificatesv1.UsageDigitalSignature,
certificatesv1.UsageServerAuth,
certificatesv1.UsageClientAuth,
},
expectedError: errKeyUsageMismatch,
},
}
for _, testCase := range tests {
testCase := testCase
t.Run(fmt.Sprint(testCase.msg), func(t *testing.T) {
t.Parallel()
err := validateKubeletServingCSR(&testCase.x509cr, testCase.keyUsages)
assert.NotNil(t, err)
assert.Contains(t, err.Error(), testCase.expectedError.Error())
})
}
}

14
pkg/talos/cloud.go
View File

@@ -5,6 +5,8 @@ import (
"context"
"io"
"github.com/siderolabs/talos-cloud-controller-manager/pkg/certificatesigningrequest"
cloudprovider "k8s.io/cloud-provider"
"k8s.io/klog/v2"
)
@@ -24,9 +26,10 @@ const (
)
type cloud struct {
cfg *cloudConfig
client *client
instancesV2 cloudprovider.InstancesV2
cfg *cloudConfig
client *client
instancesV2 cloudprovider.InstancesV2
csrController *certificatesigningrequest.Reconciler
ctx context.Context //nolint:containedctx
stop func()
@@ -86,6 +89,11 @@ func (c *cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder,
provider.stop()
}(c)
if c.cfg.Global.ApproveNodeCSR {
c.csrController = certificatesigningrequest.NewCsrController(c.client.kclient, csrNodeChecks)
go c.csrController.Run(c.ctx)
}
klog.Infof("talos initialized")
}

2
pkg/talos/cloud_config.go
View File

@@ -12,6 +12,8 @@ import (
type cloudConfig struct {
Global struct {
// Approve Node Certificate Signing Request.
ApproveNodeCSR bool `yaml:"approveNodeCSR,omitempty"`
// Talos API endpoints.
Endpoints []string `yaml:"endpoints,omitempty"`
// Do not update foreign initialized node.

5
pkg/talos/cloud_config_test.go
View File

@@ -15,6 +15,7 @@ func TestReadCloudConfig(t *testing.T) {
cfg, err := readCloudConfig(strings.NewReader(`
global:
approveNodeCSR: true
preferIPv6: true
`))
if err != nil {
@@ -28,4 +29,8 @@ global:
if !cfg.Global.PreferIPv6 {
t.Errorf("incorrect preferIPv6: %v", cfg.Global.PreferIPv6)
}
if !cfg.Global.ApproveNodeCSR {
t.Errorf("incorrect ApproveNodeCSR: %v", cfg.Global.ApproveNodeCSR)
}
}

37
pkg/talos/instances.go
View File

@@ -2,6 +2,7 @@ package talos
import (
"context"
"crypto/x509"
"fmt"
"strings"
@@ -10,10 +11,13 @@ import (
"github.com/siderolabs/talos/pkg/machinery/resources/runtime"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientkubernetes "k8s.io/client-go/kubernetes"
cloudprovider "k8s.io/cloud-provider"
cloudproviderapi "k8s.io/cloud-provider/api"
cloudnodeutil "k8s.io/cloud-provider/node/helpers"
"k8s.io/klog/v2"
"k8s.io/utils/strings/slices"
)
type instances struct {
@@ -39,7 +43,7 @@ func (i *instances) InstanceExists(_ context.Context, node *v1.Node) (bool, erro
func (i *instances) InstanceShutdown(_ context.Context, node *v1.Node) (bool, error) {
klog.V(4).Info("instances.InstanceShutdown() called, node: ", node.Name)
return true, nil
return false, nil
}
// InstanceMetadata returns the instance's metadata. The values returned in InstanceMetadata are
@@ -173,3 +177,34 @@ func syncNodeLabels(c *client, node *v1.Node, meta *runtime.PlatformMetadataSpec
return nil
}
// TODO: add more checks, like domain name, worker nodes don't have controlplane IPs, etc...
func csrNodeChecks(ctx context.Context, kclient clientkubernetes.Interface, x509cr *x509.CertificateRequest) (bool, error) {
node, err := kclient.CoreV1().Nodes().Get(ctx, x509cr.DNSNames[0], metav1.GetOptions{})
if err != nil {
return false, fmt.Errorf("failed to get node %s: %w", x509cr.DNSNames[0], err)
}
var nodeAddrs []string
if node != nil {
if providedIP, ok := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr]; ok {
nodeAddrs = append(nodeAddrs, providedIP)
}
for _, ip := range node.Status.Addresses {
nodeAddrs = append(nodeAddrs, ip.Address)
}
for _, ip := range x509cr.IPAddresses {
if !slices.Contains(nodeAddrs, ip.String()) {
return false, fmt.Errorf("csrNodeChecks: CSR %s Node IP addresses don't match corresponding "+
"Node IP addresses %q, got %q", x509cr.DNSNames[0], nodeAddrs, ip)
}
}
return true, nil
}
return false, fmt.Errorf("failed to get node %s", x509cr.DNSNames[0])
}