Update static Pod manifests for Kubernetes v1.21.0

* Set `kube-apiserver` `service-account-jwks-uri` because conformance
ServiceAccountIssuerDiscovery OIDC discovery will access a JWT endpoint
using the kube-apiserver's advertise address by default, instead of
using the intended in-cluster service (10.3.0.1) resolved by cluster DNS
`kubernetes.default.svc.cluster.local`, which causes a cert SAN error
* Set the authentication and authorization kubeconfig for kube-scheduler
and kube-controller-manager. Here, authn/z refer to aggregated API
use cases only, so its not strictly neccessary and warnings about
missing `extension-apiserver-authentication` when enable_aggregation
is false can be ignored
* Mount `/var/lib/kubelet/volumeplugins` to to the default location
expected within kube-controller-manager to remove the need for a flag
* Enable `tokencleaner` controller to automatically delete expired
bootstrap tokens (default node token is good 1 year, so cleanup won't
really matter at that point, but enable regardless)
* Remove unused `cloud-provider` flag, we never intend to use in-tree
cloud providers or support custom providers

manifests.tf

@@ -11,7 +11,6 @@ locals {
         kube_scheduler_image          = var.container_images["kube_scheduler"]
 
         etcd_servers      = join(",", formatlist("https://%s:2379", var.etcd_servers))
-        cloud_provider    = var.cloud_provider
         pod_cidr          = var.pod_cidr
         service_cidr      = var.service_cidr
         trusted_certs_dir = var.trusted_certs_dir

resources/static-manifests/kube-apiserver.yaml

@@ -24,7 +24,6 @@ spec:
     - --anonymous-auth=false
     - --authorization-mode=Node,RBAC
     - --client-ca-file=/etc/kubernetes/pki/ca.crt
-    - --cloud-provider=${cloud_provider}
     - --enable-admission-plugins=NodeRestriction
     - --enable-bootstrap-token-auth=true
     - --etcd-cafile=/etc/kubernetes/pki/etcd-client-ca.crt
@@ -37,6 +36,7 @@ spec:
     - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname${aggregation_flags}
     - --secure-port=6443
     - --service-account-issuer=https://kubernetes.default.svc.cluster.local
+    - --service-account-jwks-uri=https://kubernetes.default.svc.cluster.local/openid/v1/jwks
     - --service-account-key-file=/etc/kubernetes/pki/service-account.pub
     - --service-account-signing-key-file=/etc/kubernetes/pki/service-account.key
     - --service-cluster-ip-range=${service_cidr}

resources/static-manifests/kube-controller-manager.yaml

@@ -19,15 +19,16 @@ spec:
     image: ${kube_controller_manager_image}
     command:
     - kube-controller-manager
+    - --authentication-kubeconfig=/etc/kubernetes/pki/controller-manager.conf
+    - --authorization-kubeconfig=/etc/kubernetes/pki/controller-manager.conf
     - --allocate-node-cidrs=true
-    - --cloud-provider=${cloud_provider}
     - --client-ca-file=/etc/kubernetes/pki/ca.crt
     - --cluster-cidr=${pod_cidr}
     - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
     - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
     - --cluster-signing-duration=72h
+    - --controllers=*,tokencleaner
     - --configure-cloud-routes=false
-    - --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins
     - --kubeconfig=/etc/kubernetes/pki/controller-manager.conf
     - --leader-elect=true
     - --pod-eviction-timeout=1m
@@ -43,6 +44,7 @@ spec:
         port: 10257
       initialDelaySeconds: 15
       timeoutSeconds: 15
+      failureThreshold: 8
     resources:
       requests:
         cpu: 150m
@@ -53,6 +55,8 @@ spec:
     - name: ssl-host
       mountPath: /etc/ssl/certs
       readOnly: true
+    - name: flex
+      mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
   volumes:
   - name: secrets
     hostPath:
@@ -60,3 +64,8 @@ spec:
   - name: ssl-host
     hostPath:
       path: ${trusted_certs_dir}
+  - name: flex
+    hostPath:
+      type: DirectoryOrCreate
+      path: /var/lib/kubelet/volumeplugins
+

resources/static-manifests/kube-scheduler.yaml

@@ -19,6 +19,8 @@ spec:
     image: ${kube_scheduler_image}
     command:
     - kube-scheduler
+    - --authentication-kubeconfig=/etc/kubernetes/pki/scheduler.conf
+    - --authorization-kubeconfig=/etc/kubernetes/pki/scheduler.conf
     - --kubeconfig=/etc/kubernetes/pki/scheduler.conf
     - --leader-elect=true
     livenessProbe:

variables.tf

@@ -13,12 +13,6 @@ variable "etcd_servers" {
   description = "List of URLs used to reach etcd servers."
 }
 
-variable "cloud_provider" {
-  type        = string
-  description = "The provider for cloud services (empty string for no provider)"
-  default     = ""
-}
-
 variable "networking" {
   type        = string
   description = "Choice of networking provider (flannel or calico or cilium)"