diff --git a/_deployments/Makefile b/_deployments/Makefile index ee170b2..59ad983 100644 --- a/_deployments/Makefile +++ b/_deployments/Makefile @@ -12,6 +12,9 @@ create-deployments: ## create templates helm template --namespace=kube-system --version=1.12.7 -f vars/cilium.yaml cilium \ cilium/cilium > vars/cilium-result.yaml + # helm template --namespace=kube-system -f vars/talos-cloud-controller-manager.yaml talos-cloud-controller-manager \ + # ~/work/sergelogvinov/talos-cloud-controller-manager/charts/talos-cloud-controller-manager > vars/talos-cloud-controller-manager-result.yaml + helm template --namespace=kube-system -f vars/metrics-server.yaml metrics-server \ metrics-server/metrics-server > vars/metrics-server-result.yaml diff --git a/_deployments/vars/talos-cloud-controller-manager-result.yaml b/_deployments/vars/talos-cloud-controller-manager-result.yaml new file mode 100644 index 0000000..156e28a --- /dev/null +++ b/_deployments/vars/talos-cloud-controller-manager-result.yaml @@ -0,0 +1,273 @@ +--- +# Source: talos-cloud-controller-manager/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: talos-cloud-controller-manager + labels: + helm.sh/chart: talos-cloud-controller-manager-0.1.0 + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/managed-by: Helm + namespace: kube-system +--- +# Source: talos-cloud-controller-manager/templates/serviceaccount.yaml +apiVersion: talos.dev/v1alpha1 +kind: ServiceAccount +metadata: + name: talos-cloud-controller-manager-talos-secrets + labels: + helm.sh/chart: talos-cloud-controller-manager-0.1.0 + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/managed-by: Helm + namespace: kube-system +spec: + roles: + - os:reader +--- +# Source: talos-cloud-controller-manager/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: talos-cloud-controller-manager + labels: + helm.sh/chart: talos-cloud-controller-manager-0.1.0 + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/managed-by: Helm + namespace: kube-system +data: + ccm-config.yaml: | + global: + approveNodeCSR: true +--- +# Source: talos-cloud-controller-manager/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:talos-cloud-controller-manager + labels: + helm.sh/chart: talos-cloud-controller-manager-0.1.0 + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + verbs: + - update +- apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - kubernetes.io/kubelet-serving + verbs: + - approve +--- +# Source: talos-cloud-controller-manager/templates/rolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:talos-cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:talos-cloud-controller-manager +subjects: +- kind: ServiceAccount + name: talos-cloud-controller-manager + namespace: kube-system +--- +# Source: talos-cloud-controller-manager/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:talos-cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: talos-cloud-controller-manager + namespace: kube-system +--- +# Source: talos-cloud-controller-manager/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: talos-cloud-controller-manager + labels: + helm.sh/chart: talos-cloud-controller-manager-0.1.0 + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/managed-by: Helm + namespace: kube-system +spec: + clusterIP: None + type: ClusterIP + ports: + - name: https + port: 50258 + targetPort: 50258 + protocol: TCP + selector: + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager +--- +# Source: talos-cloud-controller-manager/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: talos-cloud-controller-manager + labels: + helm.sh/chart: talos-cloud-controller-manager-0.1.0 + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + app.kubernetes.io/version: "1.3.0" + app.kubernetes.io/managed-by: Helm + namespace: kube-system +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + template: + metadata: + labels: + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + spec: + serviceAccountName: talos-cloud-controller-manager + securityContext: + fsGroup: 10258 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 10258 + runAsNonRoot: true + runAsUser: 10258 + containers: + - name: talos-cloud-controller-manager + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + image: "ghcr.io/siderolabs/talos-cloud-controller-manager:edge" + imagePullPolicy: Always + command: ["/talos-cloud-controller-manager"] + args: + - --v=4 + - --cloud-provider=talos + - --cloud-config=/etc/talos/ccm-config.yaml + - --controllers=cloud-node + - --leader-elect-resource-name=cloud-controller-manager-talos + - --use-service-account-credentials + - --secure-port=50258 + ports: + - containerPort: 50258 + name: https + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 30 + timeoutSeconds: 5 + resources: + requests: + cpu: 10m + memory: 64Mi + volumeMounts: + - name: cloud-config + mountPath: /etc/talos + readOnly: true + - name: talos-secrets + mountPath: /var/run/secrets/talos.dev + readOnly: true + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + volumes: + - name: cloud-config + configMap: + name: talos-cloud-controller-manager + defaultMode: 416 # 0640 + - name: talos-secrets + secret: + secretName: talos-cloud-controller-manager-talos-secrets + defaultMode: 416 # 0640 diff --git a/_deployments/vars/talos-cloud-controller-manager.yaml b/_deployments/vars/talos-cloud-controller-manager.yaml new file mode 100644 index 0000000..aee65f4 --- /dev/null +++ b/_deployments/vars/talos-cloud-controller-manager.yaml @@ -0,0 +1,6 @@ + +image: + pullPolicy: Always + tag: edge + +logVerbosityLevel: 4 diff --git a/hetzner/Makefile b/hetzner/Makefile index e7e4680..f6a5408 100644 --- a/hetzner/Makefile +++ b/hetzner/Makefile @@ -60,3 +60,11 @@ create-secrets: dd if=/dev/urandom bs=1 count=16 2>/dev/null | hexdump -e '"%00x"' > hcloud-csi-secret.secret kubectl --kubeconfig=kubeconfig create secret generic hcloud-csi-secret --from-file=encryptionPassphrase=hcloud-csi-secret.secret rm -f hcloud-csi-secret.secret + +helm-repos: ## add helm repos + helm repo add hcloud https://charts.hetzner.cloud + helm repo update + +create-deployments: + helm template --namespace=kube-system -f deployments/hcloud-cloud-controller-manager.yaml \ + hcloud-cloud-controller-manager hcloud/hcloud-cloud-controller-manager > deployments/hcloud-cloud-controller-manager-result.yaml diff --git a/hetzner/deployments/hcloud-cloud-controller-manager-result.yaml b/hetzner/deployments/hcloud-cloud-controller-manager-result.yaml new file mode 100644 index 0000000..0dd8fbe --- /dev/null +++ b/hetzner/deployments/hcloud-cloud-controller-manager-result.yaml @@ -0,0 +1,88 @@ +--- +# Source: hcloud-cloud-controller-manager/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-controller-manager + namespace: kube-system +--- +# Source: hcloud-cloud-controller-manager/templates/clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +--- +# Source: hcloud-cloud-controller-manager/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app.kubernetes.io/instance: 'hcloud-cloud-controller-manager' + app.kubernetes.io/name: 'hcloud-cloud-controller-manager' + template: + metadata: + labels: + app.kubernetes.io/instance: 'hcloud-cloud-controller-manager' + app.kubernetes.io/name: 'hcloud-cloud-controller-manager' + spec: + serviceAccountName: cloud-controller-manager + dnsPolicy: Default + tolerations: + # Allow HCCM itself to schedule on nodes that have not yet been initialized by HCCM. + - key: "node.cloudprovider.kubernetes.io/uninitialized" + value: "true" + effect: "NoSchedule" + - key: "CriticalAddonsOnly" + operator: "Exists" + + # Allow HCCM to schedule on control plane nodes. + - key: "node-role.kubernetes.io/master" + effect: NoSchedule + operator: Exists + - key: "node-role.kubernetes.io/control-plane" + effect: NoSchedule + operator: Exists + + - key: "node.kubernetes.io/not-ready" + effect: "NoExecute" + containers: + - name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--allow-untagged-cloud" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--route-reconciliation-period=30s" + env: + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + key: token + name: hcloud + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: HCLOUD_METRICS_ENABLED + value: "false" + image: hetznercloud/hcloud-cloud-controller-manager:v1.15.0 + ports: + resources: + requests: + cpu: 100m + memory: 50Mi + priorityClassName: system-cluster-critical diff --git a/hetzner/deployments/hcloud-cloud-controller-manager.yaml b/hetzner/deployments/hcloud-cloud-controller-manager.yaml index 43b2067..d6d6015 100644 --- a/hetzner/deployments/hcloud-cloud-controller-manager.yaml +++ b/hetzner/deployments/hcloud-cloud-controller-manager.yaml @@ -48,8 +48,7 @@ spec: - key: "node-role.kubernetes.io/control-plane" effect: NoSchedule containers: - - image: hetznercloud/hcloud-cloud-controller-manager:v1.13.2 - # - image: ghcr.io/sergelogvinov/hetzner-cloud-controller-manager:v1.13.2-dev + - image: hetznercloud/hcloud-cloud-controller-manager:v1.15.0 name: hcloud-cloud-controller-manager args: - --cloud-provider=hcloud diff --git a/hetzner/deployments/hcloud-csi.yaml b/hetzner/deployments/hcloud-csi.yaml index 1dde3a2..991efd2 100644 --- a/hetzner/deployments/hcloud-csi.yaml +++ b/hetzner/deployments/hcloud-csi.yaml @@ -270,7 +270,7 @@ spec: secretKeyRef: key: token name: hcloud - image: hetznercloud/hcloud-csi-driver:2.2.0 + image: hetznercloud/hcloud-csi-driver:v2.3.2 imagePullPolicy: Always livenessProbe: failureThreshold: 5 @@ -341,7 +341,7 @@ spec: value: 0.0.0.0:9189 - name: ENABLE_METRICS value: "true" - image: hetznercloud/hcloud-csi-driver:2.2.0 + image: hetznercloud/hcloud-csi-driver:v2.3.2 imagePullPolicy: Always livenessProbe: failureThreshold: 5 diff --git a/hetzner/deployments/kubelet-serving-cert-approver.yaml b/hetzner/deployments/kubelet-serving-cert-approver.yaml deleted file mode 100644 index e35938c..0000000 --- a/hetzner/deployments/kubelet-serving-cert-approver.yaml +++ /dev/null @@ -1,231 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: kubelet-serving-cert-approver ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: kubelet-serving-cert-approver - namespace: kubelet-serving-cert-approver ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: certificates:kubelet-serving-cert-approver -rules: -- apiGroups: - - certificates.k8s.io - resources: - - certificatesigningrequests - verbs: - - get - - list - - watch -- apiGroups: - - certificates.k8s.io - resources: - - certificatesigningrequests/approval - verbs: - - update -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create -- apiGroups: - - certificates.k8s.io - resourceNames: - - kubernetes.io/kubelet-serving - resources: - - signers - verbs: - - approve ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: events:kubelet-serving-cert-approver -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: psp:kubelet-serving-cert-approver -rules: -- apiGroups: - - policy - resourceNames: - - kubelet-serving-cert-approver - resources: - - podsecuritypolicies - verbs: - - use ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: events:kubelet-serving-cert-approver - namespace: default -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: events:kubelet-serving-cert-approver -subjects: -- kind: ServiceAccount - name: kubelet-serving-cert-approver - namespace: kubelet-serving-cert-approver ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: psp:kubelet-serving-cert-approver - namespace: kubelet-serving-cert-approver -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:kubelet-serving-cert-approver -subjects: -- kind: ServiceAccount - name: kubelet-serving-cert-approver - namespace: kubelet-serving-cert-approver ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: kubelet-serving-cert-approver -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: certificates:kubelet-serving-cert-approver -subjects: -- kind: ServiceAccount - name: kubelet-serving-cert-approver - namespace: kubelet-serving-cert-approver ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: kubelet-serving-cert-approver - namespace: kubelet-serving-cert-approver -spec: - ports: - - name: metrics - port: 9090 - protocol: TCP - targetPort: metrics - selector: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - name: kubelet-serving-cert-approver - namespace: kubelet-serving-cert-approver -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - template: - metadata: - labels: - app.kubernetes.io/instance: kubelet-serving-cert-approver - app.kubernetes.io/name: kubelet-serving-cert-approver - spec: - nodeSelector: - node-role.kubernetes.io/control-plane: "" - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - operator: Exists - - effect: NoSchedule - key: node.cloudprovider.kubernetes.io/uninitialized - operator: Exists - containers: - - args: - - serve - env: - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: ghcr.io/alex1989hu/kubelet-serving-cert-approver:main - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - path: /healthz - port: health - initialDelaySeconds: 6 - name: cert-approver - ports: - - containerPort: 8080 - name: health - - containerPort: 9090 - name: metrics - readinessProbe: - httpGet: - path: /readyz - port: health - initialDelaySeconds: 3 - resources: - limits: - cpu: 250m - memory: 32Mi - requests: - cpu: 10m - memory: 16Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - privileged: false - readOnlyRootFilesystem: true - runAsNonRoot: true - priorityClassName: system-cluster-critical - securityContext: - fsGroup: 65534 - runAsGroup: 65534 - runAsUser: 65534 - serviceAccountName: kubelet-serving-cert-approver diff --git a/hetzner/images/variables.pkr.hcl b/hetzner/images/variables.pkr.hcl index 192aae6..ae155fa 100644 --- a/hetzner/images/variables.pkr.hcl +++ b/hetzner/images/variables.pkr.hcl @@ -17,7 +17,7 @@ variable "hcloud_type" { variable "talos_version" { type = string - default = "v1.4.0-beta.1" + default = "v1.4.1" } locals { diff --git a/hetzner/templates/controlplane.yaml.tpl b/hetzner/templates/controlplane.yaml.tpl index 132bcb2..36e8bea 100644 --- a/hetzner/templates/controlplane.yaml.tpl +++ b/hetzner/templates/controlplane.yaml.tpl @@ -114,10 +114,9 @@ cluster: externalCloudProvider: enabled: true manifests: - - https://raw.githubusercontent.com/siderolabs/talos-cloud-controller-manager/main/docs/deploy/cloud-controller-manager.yml + - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/talos-cloud-controller-manager-result.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/hcloud-cloud-controller-manager.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/hcloud-csi.yaml - - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/kubelet-serving-cert-approver.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/metrics-server-result.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/local-path-storage-ns.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/local-path-storage-result.yaml