diff --git a/README.md b/README.md index 0f5c8c2..c9b9f8f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ The goal is to create all cloud services from scratch. | Platform | Checked Talos version | Addons | Setup type | Nat | |---|---|---|---|---| -| [Azure](azure) | 1.1.0 | CCM,CSI,Autoscaler | many regions, many zones | ✓ | +| [Azure](azure) | 1.3.0 | CCM,CSI,Autoscaler | many regions, many zones | ✓ | | [Exoscale](exoscale) | 1.3.0 | CCM,Autoscaler | many regions | ✗ | | [GCP](gcp-zonal) | 0.14.0 | CCM,CSI,Autoscaler | one region, many zones | ✓ | | [Hetzner](hetzner) | 1.3.0 | CCM,CSI,Autoscaler | many regions | ✗ | diff --git a/azure/Makefile b/azure/Makefile index 9ad0c05..b2e190a 100644 --- a/azure/Makefile +++ b/azure/Makefile @@ -1,6 +1,8 @@ ENDPOINT:=${shell terraform output -raw controlplane_endpoint_public 2>/dev/null} -ENDPOINT:=$(or $(ENDPOINT),api.cluster.local) +ifeq ($(ENDPOINT),) +ENDPOINT := 127.0.0.1 +endif help: @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) @@ -13,6 +15,7 @@ create-templates: @yq ea -P '. as $$item ireduce ({}; . * $$item )' _cfgs/controlplane.yaml templates/controlplane.yaml.tpl > templates/controlplane.yaml @echo 'podSubnets: "10.32.0.0/12,fd00:10:32::/102"' > _cfgs/tfstate.vars @echo 'serviceSubnets: "10.200.0.0/22,fd40:10:200::/112"' >> _cfgs/tfstate.vars + @echo 'nodeSubnets: "172.16.0.0/12"' >> _cfgs/tfstate.vars @echo 'apiDomain: api.cluster.local' >> _cfgs/tfstate.vars @yq eval '.cluster.network.dnsDomain' _cfgs/controlplane.yaml | awk '{ print "domain: "$$1}' >> _cfgs/tfstate.vars @yq eval '.cluster.clusterName' _cfgs/controlplane.yaml | awk '{ print "clusterName: "$$1}' >> _cfgs/tfstate.vars @@ -26,9 +29,9 @@ create-templates: @yq eval -o=json '{"kubernetes": .}' _cfgs/tfstate.vars > terraform.tfvars.json create-deployments: - helm template --namespace=kube-system --version=1.12.1 -f deployments/cilium.yaml cilium \ + helm template --namespace=kube-system --version=1.12.4 -f deployments/cilium.yaml cilium \ cilium/cilium > deployments/cilium-result.yaml - helm template --namespace=ingress-nginx --version=4.2.1 -f deployments/ingress.yaml ingress-nginx \ + helm template --namespace=ingress-nginx --version=4.4.0 -f deployments/ingress.yaml ingress-nginx \ ingress-nginx/ingress-nginx > deployments/ingress-result.yaml create-network: ## Create networks @@ -43,7 +46,6 @@ create-kubeconfig: ## Download kubeconfig talosctl --talosconfig _cfgs/talosconfig --nodes 172.16.136.11 kubeconfig . kubectl --kubeconfig=kubeconfig config set clusters.talos-k8s-azure.server https://${ENDPOINT}:6443 kubectl --kubeconfig=kubeconfig config set-context --current --namespace=kube-system - kubectl --kubeconfig=kubeconfig get pods -owide -A create-infrastructure: ## Bootstrap all nodes terraform apply diff --git a/azure/deployments/azure-cloud-controller-manager.yaml b/azure/deployments/azure-cloud-controller-manager.yaml index 59cbb89..dcbabc1 100644 --- a/azure/deployments/azure-cloud-controller-manager.yaml +++ b/azure/deployments/azure-cloud-controller-manager.yaml @@ -152,25 +152,16 @@ spec: serviceAccountName: azure-cloud-controller-manager nodeSelector: node-role.kubernetes.io/control-plane: "" - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: project.io/cloudprovider-type - operator: In - values: - - azure + node.cloudprovider.kubernetes.io/platform: azure tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: "node.cloudprovider.kubernetes.io/uninitialized" + value: "true" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + effect: NoSchedule containers: - name: azure-cloud-controller-manager - image: mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.24.4 + image: mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.26.0 imagePullPolicy: IfNotPresent command: ["cloud-controller-manager"] args: diff --git a/azure/deployments/azure-cloud-node-manager.yaml b/azure/deployments/azure-cloud-node-manager.yaml index 5faed08..ce9b49c 100644 --- a/azure/deployments/azure-cloud-node-manager.yaml +++ b/azure/deployments/azure-cloud-node-manager.yaml @@ -61,30 +61,17 @@ spec: serviceAccountName: azure-cloud-node-manager hostNetwork: true # required to fetch correct hostname nodeSelector: - kubernetes.io/os: linux - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: project.io/cloudprovider-type - operator: In - values: - - azure + node-role.kubernetes.io/control-plane: "" + node.cloudprovider.kubernetes.io/platform: azure tolerations: - - key: CriticalAddonsOnly - operator: Exists - - key: node-role.kubernetes.io/master - operator: Equal - value: "true" - effect: NoSchedule - - operator: "Exists" - effect: NoExecute - - operator: "Exists" - effect: NoSchedule + - key: "node.cloudprovider.kubernetes.io/uninitialized" + value: "true" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + effect: NoSchedule containers: - name: cloud-node-manager - image: mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.24.4 + image: mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.26.0 imagePullPolicy: IfNotPresent command: - cloud-node-manager diff --git a/azure/deployments/azure-csi-node.yaml b/azure/deployments/azure-csi-node.yaml index 8532bae..7d7c790 100644 --- a/azure/deployments/azure-csi-node.yaml +++ b/azure/deployments/azure-csi-node.yaml @@ -55,22 +55,11 @@ spec: serviceAccountName: csi-azuredisk-node-sa nodeSelector: kubernetes.io/os: linux - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: type - operator: NotIn - values: - - virtual-kubelet - - key: project.io/cloudprovider-type - operator: In - values: - - azure + node.cloudprovider.kubernetes.io/platform: azure priorityClassName: system-node-critical tolerations: - - operator: "Exists" + - key: "node-role.kubernetes.io/control-plane" + effect: NoSchedule containers: - name: liveness-probe volumeMounts: diff --git a/azure/deployments/azure-csi.yaml b/azure/deployments/azure-csi.yaml index 0e8fdd6..70a22e6 100644 --- a/azure/deployments/azure-csi.yaml +++ b/azure/deployments/azure-csi.yaml @@ -214,26 +214,15 @@ spec: serviceAccountName: csi-azuredisk-controller-sa nodeSelector: kubernetes.io/os: linux - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: project.io/cloudprovider-type - operator: In - values: - - azure - priorityClassName: system-cluster-critical + node-role.kubernetes.io/control-plane: "" + node.cloudprovider.kubernetes.io/platform: azure tolerations: - - key: "node-role.kubernetes.io/master" - operator: "Exists" - effect: "NoSchedule" - - key: "node-role.kubernetes.io/controlplane" - operator: "Exists" + - key: "node.cloudprovider.kubernetes.io/uninitialized" + value: "true" effect: "NoSchedule" - key: "node-role.kubernetes.io/control-plane" - operator: "Exists" - effect: "NoSchedule" + effect: NoSchedule + priorityClassName: system-cluster-critical containers: - name: csi-provisioner image: mcr.microsoft.com/oss/kubernetes-csi/csi-provisioner:v3.1.0 diff --git a/azure/deployments/cilium-result.yaml b/azure/deployments/cilium-result.yaml index ccede59..d89d1c9 100644 --- a/azure/deployments/cilium-result.yaml +++ b/azure/deployments/cilium-result.yaml @@ -148,6 +148,7 @@ data: kube-proxy-replacement: "strict" kube-proxy-replacement-healthz-bind-address: "" bpf-lb-sock: "false" + host-reachable-services-protos: enable-health-check-nodeport: "true" node-port-bind-protection: "true" enable-auto-protect-node-port-range: "true" @@ -174,7 +175,6 @@ data: bpf-root: "/sys/fs/bpf" cgroup-root: "/sys/fs/cgroup" enable-k8s-terminating-endpoint: "true" - annotate-k8s-node: "true" remove-cilium-node-taints: "true" set-cilium-is-up-condition: "true" unmanaged-pod-watcher-interval: "15" @@ -221,13 +221,6 @@ rules: - get - list - watch -- apiGroups: - - "" - resources: - - nodes/status - verbs: - # To annotate the k8s node with Cilium's metadata - - patch - apiGroups: - apiextensions.k8s.io resources: @@ -557,7 +550,7 @@ spec: spec: containers: - name: cilium-agent - image: "quay.io/cilium/cilium:v1.12.1@sha256:ea2db1ee21b88127b5c18a96ad155c25485d0815a667ef77c2b7c7f31cab601b" + image: "quay.io/cilium/cilium:v1.12.4@sha256:4b074fcfba9325c18e97569ed1988464309a5ebf64bbc79bec6f3d58cafcb8cf" imagePullPolicy: IfNotPresent command: - cilium-agent @@ -644,7 +637,7 @@ spec: - /cni-uninstall.sh resources: limits: - cpu: 1 + cpu: 2 memory: 1Gi requests: cpu: 100m @@ -664,6 +657,7 @@ spec: protocol: TCP securityContext: privileged: true + terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - name: bpf-maps mountPath: /sys/fs/bpf @@ -691,7 +685,7 @@ spec: mountPath: /run/xtables.lock initContainers: - name: clean-cilium-state - image: "quay.io/cilium/cilium:v1.12.1@sha256:ea2db1ee21b88127b5c18a96ad155c25485d0815a667ef77c2b7c7f31cab601b" + image: "quay.io/cilium/cilium:v1.12.4@sha256:4b074fcfba9325c18e97569ed1988464309a5ebf64bbc79bec6f3d58cafcb8cf" imagePullPolicy: IfNotPresent command: - /init-container.sh @@ -712,6 +706,7 @@ spec: value: "api.cluster.local" - name: KUBERNETES_SERVICE_PORT value: "6443" + terminationMessagePolicy: FallbackToLogsOnError securityContext: privileged: true volumeMounts: @@ -817,14 +812,14 @@ spec: metadata: annotations: # ensure pods roll when configmap updates - cilium.io/cilium-configmap-checksum: "10bcfd4171cc8219b04f7404f8c9add742e0de9272cd864272e80f23ec406384" + cilium.io/cilium-configmap-checksum: "c3ffdb3de5df1007b50c84e0af5ba77bc44d069f56d62d3232573a21084f2f80" labels: io.cilium/app: operator name: cilium-operator spec: containers: - name: cilium-operator - image: quay.io/cilium/operator-generic:v1.12.1@sha256:93d5aaeda37d59e6c4325ff05030d7b48fabde6576478e3fdbfb9bb4a68ec4a1 + image: "quay.io/cilium/operator-generic:v1.12.4@sha256:071089ec5bca1f556afb8e541d9972a0dfb09d1e25504ae642ced021ecbedbd1" imagePullPolicy: IfNotPresent command: - cilium-operator-generic @@ -865,6 +860,7 @@ spec: - name: cilium-config-path mountPath: /tmp/cilium/config-map readOnly: true + terminationMessagePolicy: FallbackToLogsOnError hostNetwork: true restartPolicy: Always priorityClassName: system-cluster-critical @@ -881,8 +877,10 @@ spec: topologyKey: kubernetes.io/hostname nodeSelector: kubernetes.io/os: linux + node-role.kubernetes.io/control-plane: "" tolerations: - - operator: Exists + - effect: NoSchedule + operator: Exists volumes: # To read the configuration from the config map - name: cilium-config-path diff --git a/azure/deployments/cilium.yaml b/azure/deployments/cilium.yaml index ec2a157..8ffaf05 100644 --- a/azure/deployments/cilium.yaml +++ b/azure/deployments/cilium.yaml @@ -9,6 +9,11 @@ operator: replicas: 1 prometheus: enabled: false + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - operator: Exists + effect: NoSchedule identityAllocationMode: crd kubeProxyReplacement: strict @@ -20,7 +25,6 @@ autoDirectNodeRoutes: false devices: [eth+] healthChecking: true -annotateK8sNode: true cni: install: true @@ -37,6 +41,8 @@ ipv4: enabled: true ipv6: enabled: true +hostServices: + enabled: true hostPort: enabled: true nodePort: @@ -45,6 +51,8 @@ externalIPs: enabled: true hostFirewall: enabled: true +ingressController: + enabled: false securityContext: privileged: true @@ -62,7 +70,7 @@ cgroup: resources: limits: - cpu: 1 + cpu: 2 memory: 1Gi requests: cpu: 100m diff --git a/azure/deployments/coredns-local.yaml b/azure/deployments/coredns-local.yaml index 82f3b82..e702d9b 100644 --- a/azure/deployments/coredns-local.yaml +++ b/azure/deployments/coredns-local.yaml @@ -105,9 +105,6 @@ spec: serviceAccountName: coredns enableServiceLinks: false tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Exists @@ -117,7 +114,7 @@ spec: hostNetwork: true containers: - name: coredns - image: coredns/coredns:1.9.2 + image: coredns/coredns:1.9.4 imagePullPolicy: IfNotPresent resources: limits: diff --git a/azure/deployments/ingress-result.yaml b/azure/deployments/ingress-result.yaml index 897bf6c..ac01ee4 100644 --- a/azure/deployments/ingress-result.yaml +++ b/azure/deployments/ingress-result.yaml @@ -4,10 +4,10 @@ apiVersion: v1 kind: ServiceAccount metadata: labels: - helm.sh/chart: ingress-nginx-4.2.1 + helm.sh/chart: ingress-nginx-4.4.0 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/version: "1.5.1" app.kubernetes.io/part-of: ingress-nginx app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller @@ -20,10 +20,10 @@ apiVersion: v1 kind: ConfigMap metadata: labels: - helm.sh/chart: ingress-nginx-4.2.1 + helm.sh/chart: ingress-nginx-4.4.0 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/version: "1.5.1" app.kubernetes.io/part-of: ingress-nginx app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller @@ -66,10 +66,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: labels: - helm.sh/chart: ingress-nginx-4.2.1 + helm.sh/chart: ingress-nginx-4.4.0 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/version: "1.5.1" app.kubernetes.io/part-of: ingress-nginx app.kubernetes.io/managed-by: Helm name: ingress-nginx @@ -136,16 +136,24 @@ rules: - get - list - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get --- # Source: ingress-nginx/templates/clusterrolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: labels: - helm.sh/chart: ingress-nginx-4.2.1 + helm.sh/chart: ingress-nginx-4.4.0 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/version: "1.5.1" app.kubernetes.io/part-of: ingress-nginx app.kubernetes.io/managed-by: Helm name: ingress-nginx @@ -163,10 +171,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: labels: - helm.sh/chart: ingress-nginx-4.2.1 + helm.sh/chart: ingress-nginx-4.4.0 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/version: "1.5.1" app.kubernetes.io/part-of: ingress-nginx app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller @@ -220,12 +228,17 @@ rules: - get - list - watch + # TODO(Jintao Zhang) + # Once we release a new version of the controller, + # we will be able to remove the configmap related permissions + # We have used the Lease API for selection + # ref: https://github.com/kubernetes/ingress-nginx/pull/8921 - apiGroups: - "" resources: - configmaps resourceNames: - - ingress-controller-leader + - ingress-nginx-leader verbs: - get - update @@ -240,7 +253,7 @@ rules: resources: - leases resourceNames: - - ingress-controller-leader + - ingress-nginx-leader verbs: - get - update @@ -257,16 +270,24 @@ rules: verbs: - create - patch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get --- # Source: ingress-nginx/templates/controller-rolebinding.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: labels: - helm.sh/chart: ingress-nginx-4.2.1 + helm.sh/chart: ingress-nginx-4.4.0 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/version: "1.5.1" app.kubernetes.io/part-of: ingress-nginx app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller @@ -287,10 +308,10 @@ kind: Service metadata: annotations: labels: - helm.sh/chart: ingress-nginx-4.2.1 + helm.sh/chart: ingress-nginx-4.4.0 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/version: "1.5.1" app.kubernetes.io/part-of: ingress-nginx app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller @@ -324,10 +345,10 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: - helm.sh/chart: ingress-nginx-4.2.1 + helm.sh/chart: ingress-nginx-4.4.0 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/version: "1.5.1" app.kubernetes.io/part-of: ingress-nginx app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller @@ -358,7 +379,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: controller - image: "registry.k8s.io/ingress-nginx/controller:v1.3.0@sha256:d1707ca76d3b044ab8a28277a2466a02100ee9f58a86af1535a3edf9323ea1b5" + image: "registry.k8s.io/ingress-nginx/controller:v1.5.1@sha256:4ba73c697770664c1e00e9f968de14e08f606ff961c76e5d7033a4a9c593c629" imagePullPolicy: IfNotPresent lifecycle: preStop: @@ -367,7 +388,7 @@ spec: - /wait-shutdown args: - /nginx-ingress-controller - - --election-id=ingress-controller-leader + - --election-id=ingress-nginx-leader - --controller-class=k8s.io/ingress-nginx - --ingress-class=nginx - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller @@ -450,10 +471,10 @@ apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: labels: - helm.sh/chart: ingress-nginx-4.2.1 + helm.sh/chart: ingress-nginx-4.4.0 app.kubernetes.io/name: ingress-nginx app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/version: "1.5.1" app.kubernetes.io/part-of: ingress-nginx app.kubernetes.io/managed-by: Helm app.kubernetes.io/component: controller diff --git a/azure/deployments/kubelet-serving-cert-approver.yaml b/azure/deployments/kubelet-serving-cert-approver.yaml index eaa62c7..7ef7eca 100644 --- a/azure/deployments/kubelet-serving-cert-approver.yaml +++ b/azure/deployments/kubelet-serving-cert-approver.yaml @@ -200,7 +200,7 @@ spec: fieldRef: fieldPath: metadata.namespace image: ghcr.io/alex1989hu/kubelet-serving-cert-approver:main - imagePullPolicy: Always + imagePullPolicy: IfNotPresent livenessProbe: httpGet: path: /healthz diff --git a/azure/deployments/local-path-storage.yaml b/azure/deployments/local-path-storage.yaml index 91a7c7a..9f51a09 100644 --- a/azure/deployments/local-path-storage.yaml +++ b/azure/deployments/local-path-storage.yaml @@ -59,15 +59,15 @@ spec: labels: app: local-path-provisioner spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" tolerations: - - key: "node-role.kubernetes.io/master" - effect: NoSchedule - key: "node-role.kubernetes.io/control-plane" effect: NoSchedule serviceAccountName: local-path-provisioner-service-account containers: - name: local-path-provisioner - image: rancher/local-path-provisioner:v0.0.19 + image: rancher/local-path-provisioner:v0.0.23 imagePullPolicy: IfNotPresent command: - local-path-provisioner @@ -108,49 +108,21 @@ metadata: data: config.json: |- { - "nodePathMap":[ - { - "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", - "paths":["/var/local-path-provisioner"] - } - ] + "nodePathMap":[ + { + "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", + "paths":["/var/local-path-provisioner"] + } + ] } setup: |- #!/bin/sh - while getopts "m:s:p:" opt - do - case $opt in - p) - absolutePath=$OPTARG - ;; - s) - sizeInBytes=$OPTARG - ;; - m) - volMode=$OPTARG - ;; - esac - done - - mkdir -m 0777 -p ${absolutePath} + set -eu + mkdir -m 0777 -p "$VOL_DIR" teardown: |- #!/bin/sh - while getopts "m:s:p:" opt - do - case $opt in - p) - absolutePath=$OPTARG - ;; - s) - sizeInBytes=$OPTARG - ;; - m) - volMode=$OPTARG - ;; - esac - done - - rm -rf ${absolutePath} + set -eu + rm -rf "$VOL_DIR" helperPod.yaml: |- apiVersion: v1 kind: Pod diff --git a/azure/deployments/metrics-server.yaml b/azure/deployments/metrics-server.yaml index 89d2774..f259001 100644 --- a/azure/deployments/metrics-server.yaml +++ b/azure/deployments/metrics-server.yaml @@ -131,19 +131,16 @@ spec: kubernetes.io/os: linux node-role.kubernetes.io/control-plane: "" tolerations: - - key: "CriticalAddonsOnly" - operator: "Exists" - - key: "node-role.kubernetes.io/master" - effect: NoSchedule - key: "node-role.kubernetes.io/control-plane" effect: NoSchedule containers: - args: - --cert-dir=/tmp - - --secure-port=443 + - --secure-port=6443 - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - --kubelet-use-node-status-port - --metric-resolution=15s + - --authorization-always-allow-paths=/metrics image: k8s.gcr.io/metrics-server/metrics-server:v0.5.0 imagePullPolicy: IfNotPresent livenessProbe: @@ -155,7 +152,7 @@ spec: periodSeconds: 10 name: metrics-server ports: - - containerPort: 443 + - containerPort: 6443 name: https protocol: TCP readinessProbe: diff --git a/azure/images/README.md b/azure/images/README.md index dddb790..9a8e4cd 100644 --- a/azure/images/README.md +++ b/azure/images/README.md @@ -14,10 +14,10 @@ regions = ["uksouth", "ukwest", "westeurope"] ## Init and upload images ```shell -wget https://github.com/siderolabs/talos/releases/download/v1.2.0-beta.0/azure-amd64.tar.gz +wget https://github.com/siderolabs/talos/releases/download/v1.3.0/azure-amd64.tar.gz tar -xzf azure-amd64.tar.gz && mv disk.vhd disk-x64.vhd -wget https://github.com/siderolabs/talos/releases/download/v1.2.0-beta.0/azure-arm64.tar.gz +wget https://github.com/siderolabs/talos/releases/download/v1.3.0/azure-arm64.tar.gz tar -xzf azure-arm64.tar.gz && mv disk.vhd disk-arm64.vhd terraform init && terraform apply diff --git a/azure/images/gallery.tf b/azure/images/gallery.tf index abe95ce..01efc78 100644 --- a/azure/images/gallery.tf +++ b/azure/images/gallery.tf @@ -31,12 +31,11 @@ resource "azurerm_shared_image" "talos" { hyper_v_generation = "V2" architecture = each.key accelerated_network_support_enabled = lower(each.key) == "x64" - # specialized = true identifier { publisher = var.name offer = "Talos-${lower(each.key)}" - sku = "1.2-dev" + sku = "MPL-2.0" } tags = merge(var.tags, { type = "infra" }) @@ -101,7 +100,7 @@ resource "azurerm_image" "talos" { resource "azurerm_shared_image_version" "talos" { for_each = { for name, k in azurerm_storage_blob.talos : name => k.url } - name = "1.2.0" + name = "1.3.0" location = var.regions[0] resource_group_name = data.azurerm_resource_group.kubernetes.name gallery_name = azurerm_shared_image.talos[each.key].gallery_name diff --git a/azure/images/versions.tf b/azure/images/versions.tf index 8b3d029..4be1a68 100644 --- a/azure/images/versions.tf +++ b/azure/images/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~> 3.25.0" + version = "~> 3.36.0" } } required_version = ">= 1.2" diff --git a/azure/instances-web.tf b/azure/instances-web.tf index 8160db0..bc60621 100644 --- a/azure/instances-web.tf +++ b/azure/instances-web.tf @@ -1,6 +1,6 @@ locals { - web_labels = "project.io/cloudprovider-type=azure,project.io/node-pool=web" + web_labels = "project.io/node-pool=web" } resource "azurerm_linux_virtual_machine_scale_set" "web" { diff --git a/azure/instances-werker.tf b/azure/instances-werker.tf index 1e2013c..f7bf863 100644 --- a/azure/instances-werker.tf +++ b/azure/instances-werker.tf @@ -1,6 +1,6 @@ locals { - worker_labels = "project.io/cloudprovider-type=azure,project.io/node-pool=worker" + worker_labels = "project.io/node-pool=worker" } resource "azurerm_linux_virtual_machine_scale_set" "worker" { diff --git a/azure/modules/controlplane/main.tf b/azure/modules/controlplane/main.tf index 68aebc8..207af2b 100644 --- a/azure/modules/controlplane/main.tf +++ b/azure/modules/controlplane/main.tf @@ -72,7 +72,7 @@ resource "azurerm_network_interface_backend_address_pool_association" "controlpl } locals { - controlplane_labels = "project.io/cloudprovider-type=azure,topology.kubernetes.io/region=${var.region},kubernetes.azure.com/managed=false" + controlplane_labels = "kubernetes.azure.com/managed=false" } resource "azurerm_linux_virtual_machine" "controlplane" { diff --git a/azure/prepare/versions.tf b/azure/prepare/versions.tf index 8b3d029..4be1a68 100644 --- a/azure/prepare/versions.tf +++ b/azure/prepare/versions.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "~> 3.25.0" + version = "~> 3.36.0" } } required_version = ">= 1.2" diff --git a/azure/templates/controlplane.yaml.tpl b/azure/templates/controlplane.yaml.tpl index 14ab1de..32afad1 100644 --- a/azure/templates/controlplane.yaml.tpl +++ b/azure/templates/controlplane.yaml.tpl @@ -4,20 +4,25 @@ persist: true machine: type: controlplane certSANs: ${format("%#v",certSANs)} + features: + kubernetesTalosAPIAccess: + enabled: true + allowedRoles: + - os:reader + allowedKubernetesNamespaces: + - kube-system kubelet: extraArgs: node-labels: "${labels}" rotate-server-certificates: true - nodeIP: - validSubnets: ${format("%#v",nodeSubnets)} clusterDNS: - 169.254.2.53 - ${cidrhost(split(",",serviceSubnets)[0], 10)} + nodeIP: + validSubnets: ${format("%#v",nodeSubnets)} network: hostname: "${name}" interfaces: - - interface: eth0 - dhcp: true - interface: lo addresses: ${format("%#v",ipAliases)} - interface: dummy0 @@ -32,9 +37,28 @@ machine: sysctls: net.core.somaxconn: 65535 net.core.netdev_max_backlog: 4096 + systemDiskEncryption: + state: + provider: luks2 + keys: + - nodeID: {} + slot: 0 + ephemeral: + provider: luks2 + keys: + - nodeID: {} + slot: 0 + options: + - no_read_workqueue + - no_write_workqueue cluster: + id: ${clusterID} + secret: ${clusterSecret} controlPlane: endpoint: https://${apiDomain}:6443 + clusterName: ${clusterName} + discovery: + enabled: true network: dnsDomain: ${domain} podSubnets: ${format("%#v",split(",",podSubnets))} @@ -47,6 +71,27 @@ cluster: disabled: true apiServer: certSANs: ${format("%#v",certSANs)} + admissionControl: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1alpha1 + defaults: + audit: restricted + audit-version: latest + enforce: baseline + enforce-version: latest + warn: restricted + warn-version: latest + exemptions: + namespaces: + - kube-system + - ingress-nginx + - monitoring + - local-path-storage + - local-lvm + runtimeClasses: [] + usernames: [] + kind: PodSecurityConfiguration controllerManager: extraArgs: node-cidr-mask-size-ipv4: 24 @@ -74,8 +119,8 @@ cluster: externalCloudProvider: enabled: true manifests: + - https://raw.githubusercontent.com/siderolabs/talos-cloud-controller-manager/main/docs/deploy/cloud-controller-manager.yml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/azure/deployments/azure-cloud-controller-manager.yaml - - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/azure/deployments/azure-cloud-node-manager.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/azure/deployments/azure-csi-node.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/azure/deployments/azure-csi.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/azure/deployments/azure-storage.yaml diff --git a/azure/templates/worker.yaml.tpl b/azure/templates/worker.yaml.tpl index c002dd1..b6b327f 100644 --- a/azure/templates/worker.yaml.tpl +++ b/azure/templates/worker.yaml.tpl @@ -9,8 +9,8 @@ machine: kubelet: extraArgs: cloud-provider: external - node-labels: "${labels}" rotate-server-certificates: true + node-labels: "${labels}" nodeIP: validSubnets: ${format("%#v",nodeSubnets)} clusterDNS: @@ -36,6 +36,8 @@ cluster: controlPlane: endpoint: https://${apiDomain}:6443 clusterName: ${clusterName} + discovery: + enabled: true network: dnsDomain: ${domain} serviceSubnets: ${format("%#v",split(",",serviceSubnets))}