From a4ecf168df1673288f067b9c5966037fe2d3b5ae Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 10 Apr 2021 16:18:14 -0700 Subject: [PATCH] Update static Pod manifests for Kubernetes v1.21.0 * Set `kube-apiserver` `service-account-jwks-uri` because conformance ServiceAccountIssuerDiscovery OIDC discovery will access a JWT endpoint using the kube-apiserver's advertise address by default, instead of using the intended in-cluster service (10.3.0.1) resolved by cluster DNS `kubernetes.default.svc.cluster.local`, which causes a cert SAN error * Set the authentication and authorization kubeconfig for kube-scheduler and kube-controller-manager. Here, authn/z refer to aggregated API use cases only, so its not strictly neccessary and warnings about missing `extension-apiserver-authentication` when enable_aggregation is false can be ignored * Mount `/var/lib/kubelet/volumeplugins` to to the default location expected within kube-controller-manager to remove the need for a flag * Enable `tokencleaner` controller to automatically delete expired bootstrap tokens (default node token is good 1 year, so cleanup won't really matter at that point, but enable regardless) * Remove unused `cloud-provider` flag, we never intend to use in-tree cloud providers or support custom providers --- manifests.tf | 1 - resources/static-manifests/kube-apiserver.yaml | 2 +- .../static-manifests/kube-controller-manager.yaml | 13 +++++++++++-- resources/static-manifests/kube-scheduler.yaml | 2 ++ variables.tf | 6 ------ 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/manifests.tf b/manifests.tf index adaeefb..c5e6a20 100644 --- a/manifests.tf +++ b/manifests.tf @@ -11,7 +11,6 @@ locals { kube_scheduler_image = var.container_images["kube_scheduler"] etcd_servers = join(",", formatlist("https://%s:2379", var.etcd_servers)) - cloud_provider = var.cloud_provider pod_cidr = var.pod_cidr service_cidr = var.service_cidr trusted_certs_dir = var.trusted_certs_dir diff --git a/resources/static-manifests/kube-apiserver.yaml b/resources/static-manifests/kube-apiserver.yaml index dae4768..1cbc32c 100644 --- a/resources/static-manifests/kube-apiserver.yaml +++ b/resources/static-manifests/kube-apiserver.yaml @@ -24,7 +24,6 @@ spec: - --anonymous-auth=false - --authorization-mode=Node,RBAC - --client-ca-file=/etc/kubernetes/pki/ca.crt - - --cloud-provider=${cloud_provider} - --enable-admission-plugins=NodeRestriction - --enable-bootstrap-token-auth=true - --etcd-cafile=/etc/kubernetes/pki/etcd-client-ca.crt @@ -37,6 +36,7 @@ spec: - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname${aggregation_flags} - --secure-port=6443 - --service-account-issuer=https://kubernetes.default.svc.cluster.local + - --service-account-jwks-uri=https://kubernetes.default.svc.cluster.local/openid/v1/jwks - --service-account-key-file=/etc/kubernetes/pki/service-account.pub - --service-account-signing-key-file=/etc/kubernetes/pki/service-account.key - --service-cluster-ip-range=${service_cidr} diff --git a/resources/static-manifests/kube-controller-manager.yaml b/resources/static-manifests/kube-controller-manager.yaml index d522250..546cdce 100644 --- a/resources/static-manifests/kube-controller-manager.yaml +++ b/resources/static-manifests/kube-controller-manager.yaml @@ -19,15 +19,16 @@ spec: image: ${kube_controller_manager_image} command: - kube-controller-manager + - --authentication-kubeconfig=/etc/kubernetes/pki/controller-manager.conf + - --authorization-kubeconfig=/etc/kubernetes/pki/controller-manager.conf - --allocate-node-cidrs=true - - --cloud-provider=${cloud_provider} - --client-ca-file=/etc/kubernetes/pki/ca.crt - --cluster-cidr=${pod_cidr} - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key - --cluster-signing-duration=72h + - --controllers=*,tokencleaner - --configure-cloud-routes=false - - --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins - --kubeconfig=/etc/kubernetes/pki/controller-manager.conf - --leader-elect=true - --pod-eviction-timeout=1m @@ -43,6 +44,7 @@ spec: port: 10257 initialDelaySeconds: 15 timeoutSeconds: 15 + failureThreshold: 8 resources: requests: cpu: 150m @@ -53,6 +55,8 @@ spec: - name: ssl-host mountPath: /etc/ssl/certs readOnly: true + - name: flex + mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec volumes: - name: secrets hostPath: @@ -60,3 +64,8 @@ spec: - name: ssl-host hostPath: path: ${trusted_certs_dir} + - name: flex + hostPath: + type: DirectoryOrCreate + path: /var/lib/kubelet/volumeplugins + diff --git a/resources/static-manifests/kube-scheduler.yaml b/resources/static-manifests/kube-scheduler.yaml index 56500da..f93b8c6 100644 --- a/resources/static-manifests/kube-scheduler.yaml +++ b/resources/static-manifests/kube-scheduler.yaml @@ -19,6 +19,8 @@ spec: image: ${kube_scheduler_image} command: - kube-scheduler + - --authentication-kubeconfig=/etc/kubernetes/pki/scheduler.conf + - --authorization-kubeconfig=/etc/kubernetes/pki/scheduler.conf - --kubeconfig=/etc/kubernetes/pki/scheduler.conf - --leader-elect=true livenessProbe: diff --git a/variables.tf b/variables.tf index b502420..c3a7f99 100644 --- a/variables.tf +++ b/variables.tf @@ -13,12 +13,6 @@ variable "etcd_servers" { description = "List of URLs used to reach etcd servers." } -variable "cloud_provider" { - type = string - description = "The provider for cloud services (empty string for no provider)" - default = "" -} - variable "networking" { type = string description = "Choice of networking provider (flannel or calico or cilium)"