From 259243a0c653f0f2776de1cd2ef8cbe6219a8bc0 Mon Sep 17 00:00:00 2001 From: Serge Logvinov Date: Sat, 29 Jan 2022 16:32:00 +0200 Subject: [PATCH] Switch to cilium network --- scaleway/Makefile | 5 + scaleway/deployments/cilium.yaml | 35 +- scaleway/deployments/cilium_result.yaml | 782 +++++++++++++++++++++++ scaleway/instances-master.tf | 23 +- scaleway/instances-web.tf | 37 ++ scaleway/network.tf | 11 +- scaleway/templates/controlplane.yaml.tpl | 10 +- scaleway/templates/web.yaml.tpl | 63 ++ scaleway/variables.tf | 2 + 9 files changed, 940 insertions(+), 28 deletions(-) create mode 100644 scaleway/deployments/cilium_result.yaml create mode 100644 scaleway/instances-web.tf create mode 100644 scaleway/templates/web.yaml.tpl diff --git a/scaleway/Makefile b/scaleway/Makefile index 81b3b3e..20e5d77 100644 --- a/scaleway/Makefile +++ b/scaleway/Makefile @@ -20,6 +20,8 @@ create-templates: @echo 'apiDomain: api.cluster.local' >> _cfgs/tfstate.vars @yq eval '.cluster.network.dnsDomain' _cfgs/controlplane.yaml | awk '{ print "domain: "$$1}' >> _cfgs/tfstate.vars @yq eval '.cluster.clusterName' _cfgs/controlplane.yaml | awk '{ print "clusterName: "$$1}' >> _cfgs/tfstate.vars + @yq eval '.cluster.id' _cfgs/controlplane.yaml | awk '{ print "clusterId: "$$1}' >> _cfgs/tfstate.vars + @yq eval '.cluster.secret' _cfgs/controlplane.yaml | awk '{ print "clusterSecret: "$$1}'>> _cfgs/tfstate.vars @yq eval '.machine.token' _cfgs/controlplane.yaml | awk '{ print "tokenMachine: "$$1}' >> _cfgs/tfstate.vars @yq eval '.machine.ca.crt' _cfgs/controlplane.yaml | awk '{ print "caMachine: "$$1}' >> _cfgs/tfstate.vars @yq eval '.cluster.token' _cfgs/controlplane.yaml | awk '{ print "token: "$$1}' >> _cfgs/tfstate.vars @@ -35,3 +37,6 @@ create-infrastructure: ## Bootstrap all nodes create-kubeconfig: ## Prepare kubeconfig talosctl --talosconfig _cfgs/talosconfig --nodes 172.16.0.11 kubeconfig . + +create-deployments: + helm template --namespace=kube-system --version=1.11.1 -f deployments/cilium.yaml cilium cilium/cilium > deployments/cilium_result.yaml diff --git a/scaleway/deployments/cilium.yaml b/scaleway/deployments/cilium.yaml index 642b54a..40722ce 100644 --- a/scaleway/deployments/cilium.yaml +++ b/scaleway/deployments/cilium.yaml @@ -1,6 +1,6 @@ --- -k8sServiceHost: "172.16.0.10" +k8sServiceHost: "172.16.0.5" k8sServicePort: "6443" agent: @@ -13,24 +13,27 @@ operator: enabled: false identityAllocationMode: crd +kubeProxyReplacement: strict +enableK8sEndpointSlice: true +localRedirectPolicy: true -bpf: - masquerade: false +healthChecking: true + +tunnel: "vxlan" +autoDirectNodeRoutes: false +# devices: [eth0] cni: install: true ipam: mode: "kubernetes" +k8s: + requireIPv4PodCIDR: true + requireIPv6PodCIDR: true -tunnel: "vxlan" -autoDirectNodeRoutes: false - -hostFirewall: true -kubeProxyReplacement: strict - -healthChecking: true - +bpf: + masquerade: false ipv4: enabled: true ipv6: @@ -43,17 +46,15 @@ nodePort: enabled: false externalIPs: enabled: true +hostFirewall: + enabled: true -k8s: - requireIPv4PodCIDR: true - requireIPv6PodCIDR: true +hubble: + enabled: false prometheus: enabled: true -encryption: - enabled: false - cgroup: autoMount: enabled: false diff --git a/scaleway/deployments/cilium_result.yaml b/scaleway/deployments/cilium_result.yaml new file mode 100644 index 0000000..b7b07aa --- /dev/null +++ b/scaleway/deployments/cilium_result.yaml @@ -0,0 +1,782 @@ +--- +# Source: cilium/templates/cilium-agent/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: cilium-config + namespace: kube-system +data: + + # Identity allocation mode selects how identities are shared between cilium + # nodes by setting how they are stored. The options are "crd" or "kvstore". + # - "crd" stores identities in kubernetes as CRDs (custom resource definition). + # These can be queried with: + # kubectl get ciliumid + # - "kvstore" stores identities in an etcd kvstore, that is + # configured below. Cilium versions before 1.6 supported only the kvstore + # backend. Upgrades from these older cilium versions should continue using + # the kvstore by commenting out the identity-allocation-mode below, or + # setting it to "kvstore". + identity-allocation-mode: crd + cilium-endpoint-gc-interval: "5m0s" + # Disable the usage of CiliumEndpoint CRD + disable-endpoint-crd: "false" + + # If you want to run cilium in debug mode change this value to true + debug: "false" + # The agent can be put into the following three policy enforcement modes + # default, always and never. + # https://docs.cilium.io/en/latest/policy/intro/#policy-enforcement-modes + enable-policy: "default" + # If you want metrics enabled in all of your Cilium agents, set the port for + # which the Cilium agents will have their metrics exposed. + # This option deprecates the "prometheus-serve-addr" in the + # "cilium-metrics-config" ConfigMap + # NOTE that this will open the port on ALL nodes where Cilium pods are + # scheduled. + prometheus-serve-addr: ":9090" + # Port to expose Envoy metrics (e.g. "9095"). Envoy metrics listener will be disabled if this + # field is not set. + proxy-prometheus-port: "9095" + + # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 + # address. + enable-ipv4: "true" + + # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6 + # address. + enable-ipv6: "true" + # Users who wish to specify their own custom CNI configuration file must set + # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration. + custom-cni-conf: "false" + enable-bpf-clock-probe: "true" + # If you want cilium monitor to aggregate tracing for packets, set this level + # to "low", "medium", or "maximum". The higher the level, the less packets + # that will be seen in monitor output. + monitor-aggregation: medium + + # The monitor aggregation interval governs the typical time between monitor + # notification events for each allowed connection. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-interval: 5s + + # The monitor aggregation flags determine which TCP flags which, upon the + # first observation, cause monitor notifications to be generated. + # + # Only effective when monitor aggregation is set to "medium" or higher. + monitor-aggregation-flags: all + # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic + # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps. + bpf-map-dynamic-size-ratio: "0.0025" + # bpf-policy-map-max specifies the maximum number of entries in endpoint + # policy map (per endpoint) + bpf-policy-map-max: "16384" + # bpf-lb-map-max specifies the maximum number of entries in bpf lb service, + # backend and affinity maps. + bpf-lb-map-max: "65536" + # bpf-lb-bypass-fib-lookup instructs Cilium to enable the FIB lookup bypass + # optimization for nodeport reverse NAT handling. + bpf-lb-external-clusterip: "false" + + # Pre-allocation of map entries allows per-packet latency to be reduced, at + # the expense of up-front memory allocation for the entries in the maps. The + # default value below will minimize memory usage in the default installation; + # users who are sensitive to latency may consider setting this to "true". + # + # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore + # this option and behave as though it is set to "true". + # + # If this value is modified, then during the next Cilium startup the restore + # of existing endpoints and tracking of ongoing connections may be disrupted. + # As a result, reply packets may be dropped and the load-balancing decisions + # for established connections may change. + # + # If this option is set to "false" during an upgrade from 1.3 or earlier to + # 1.4 or later, then it may cause one-time disruptions during the upgrade. + preallocate-bpf-maps: "false" + + # Regular expression matching compatible Istio sidecar istio-proxy + # container image names + sidecar-istio-proxy-image: "cilium/istio_proxy" + + # Name of the cluster. Only relevant when building a mesh of clusters. + cluster-name: default + # Unique ID of the cluster. Must be unique across all conneted clusters and + # in the range of 1 and 255. Only relevant when building a mesh of clusters. + cluster-id: "" + + # Encapsulation mode for communication between nodes + # Possible values: + # - disabled + # - vxlan (default) + # - geneve + tunnel: vxlan + # Enables L7 proxy for L7 policy enforcement and visibility + enable-l7-proxy: "true" + + enable-ipv4-masquerade: "true" + enable-ipv6-masquerade: "true" + enable-bpf-masquerade: "false" + + enable-xt-socket-fallback: "true" + install-iptables-rules: "true" + install-no-conntrack-iptables-rules: "false" + + auto-direct-node-routes: "false" + enable-bandwidth-manager: "false" + enable-local-redirect-policy: "true" + enable-host-firewall: "true" + + kube-proxy-replacement: "strict" + kube-proxy-replacement-healthz-bind-address: "" + enable-health-check-nodeport: "true" + node-port-bind-protection: "true" + enable-auto-protect-node-port-range: "true" + enable-session-affinity: "true" + enable-l2-neigh-discovery: "true" + k8s-require-ipv4-pod-cidr: "true" + k8s-require-ipv6-pod-cidr: "true" + enable-endpoint-health-checking: "true" + enable-health-checking: "true" + enable-well-known-identities: "false" + enable-remote-node-identity: "true" + operator-api-serve-addr: "127.0.0.1:9234" + ipam: "kubernetes" + disable-cnp-status-updates: "true" + enable-k8s-endpoint-slice: "true" + cgroup-root: "/sys/fs/cgroup" + enable-k8s-terminating-endpoint: "true" +--- +# Source: cilium/templates/cilium-agent/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium +rules: +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + - services + - nodes + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods + - pods/finalizers + verbs: + - get + - list + - watch + - update + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update +- apiGroups: + - "" + resources: + - nodes + - nodes/status + verbs: + - patch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + # Deprecated for removal in v1.10 + - create + - list + - watch + - update + + # This is used when validating policies in preflight. This will need to stay + # until we figure out how to avoid "get" inside the preflight, and then + # should be removed ideally. + - get +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumnetworkpolicies/status + - ciliumnetworkpolicies/finalizers + - ciliumclusterwidenetworkpolicies + - ciliumclusterwidenetworkpolicies/status + - ciliumclusterwidenetworkpolicies/finalizers + - ciliumendpoints + - ciliumendpoints/status + - ciliumendpoints/finalizers + - ciliumnodes + - ciliumnodes/status + - ciliumnodes/finalizers + - ciliumidentities + - ciliumidentities/finalizers + - ciliumlocalredirectpolicies + - ciliumlocalredirectpolicies/status + - ciliumlocalredirectpolicies/finalizers + - ciliumegressnatpolicies + - ciliumendpointslices + verbs: + - '*' +--- +# Source: cilium/templates/cilium-operator/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cilium-operator +rules: +- apiGroups: + - "" + resources: + # to automatically delete [core|kube]dns pods so that are starting to being + # managed by Cilium + - pods + verbs: + - get + - list + - watch + - delete +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + # to perform LB IP allocation for BGP + - services/status + verbs: + - update +- apiGroups: + - "" + resources: + # to perform the translation of a CNP that contains `ToGroup` to its endpoints + - services + - endpoints + # to check apiserver connectivity + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - cilium.io + resources: + - ciliumnetworkpolicies + - ciliumnetworkpolicies/status + - ciliumnetworkpolicies/finalizers + - ciliumclusterwidenetworkpolicies + - ciliumclusterwidenetworkpolicies/status + - ciliumclusterwidenetworkpolicies/finalizers + - ciliumendpoints + - ciliumendpoints/status + - ciliumendpoints/finalizers + - ciliumnodes + - ciliumnodes/status + - ciliumnodes/finalizers + - ciliumidentities + - ciliumendpointslices + - ciliumidentities/status + - ciliumidentities/finalizers + - ciliumlocalredirectpolicies + - ciliumlocalredirectpolicies/status + - ciliumlocalredirectpolicies/finalizers + verbs: + - '*' +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - update + - watch +# For cilium-operator running in HA mode. +# +# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election +# between multiple running instances. +# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less +# common and fewer objects in the cluster watch "all Leases". +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +--- +# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium +subjects: +- kind: ServiceAccount + name: "cilium" + namespace: kube-system +--- +# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cilium-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cilium-operator +subjects: +- kind: ServiceAccount + name: "cilium-operator" + namespace: kube-system +--- +# Source: cilium/templates/cilium-agent/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: cilium-agent + namespace: kube-system + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9095" + labels: + k8s-app: cilium +spec: + clusterIP: None + type: ClusterIP + selector: + k8s-app: cilium + ports: + - name: envoy-metrics + port: 9095 + protocol: TCP + targetPort: envoy-metrics +--- +# Source: cilium/templates/cilium-agent/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: kube-system + labels: + k8s-app: cilium +spec: + selector: + matchLabels: + k8s-app: cilium + updateStrategy: + rollingUpdate: + maxUnavailable: 2 + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "9090" + prometheus.io/scrape: "true" + # This annotation plus the CriticalAddonsOnly toleration makes + # cilium to be a critical pod in the cluster, which ensures cilium + # gets priority scheduling. + # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ + scheduler.alpha.kubernetes.io/critical-pod: "" + labels: + k8s-app: cilium + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + - matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - linux + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: + - cilium + topologyKey: kubernetes.io/hostname + containers: + - name: cilium-agent + image: "quay.io/cilium/cilium:v1.11.1@sha256:251ff274acf22fd2067b29a31e9fda94253d2961c061577203621583d7e85bd2" + imagePullPolicy: IfNotPresent + command: + - cilium-agent + args: + - --config-dir=/tmp/cilium/config-map + startupProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9876 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + failureThreshold: 105 + periodSeconds: 2 + successThreshold: 1 + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9876 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9876 + scheme: HTTP + httpHeaders: + - name: "brief" + value: "true" + periodSeconds: 30 + successThreshold: 1 + failureThreshold: 3 + timeoutSeconds: 5 + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_CLUSTERMESH_CONFIG + value: /var/lib/cilium/clustermesh/ + - name: CILIUM_CNI_CHAINING_MODE + valueFrom: + configMapKeyRef: + name: cilium-config + key: cni-chaining-mode + optional: true + - name: CILIUM_CUSTOM_CNI_CONF + valueFrom: + configMapKeyRef: + name: cilium-config + key: custom-cni-conf + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "172.16.0.5" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + lifecycle: + postStart: + exec: + command: + - "/cni-install.sh" + - "--enable-debug=false" + - "--cni-exclusive=true" + preStop: + exec: + command: + - /cni-uninstall.sh + resources: + requests: + cpu: 100m + memory: 128Mi + ports: + - name: prometheus + containerPort: 9090 + hostPort: 9090 + protocol: TCP + - name: envoy-metrics + containerPort: 9095 + hostPort: 9095 + protocol: TCP + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + mountPropagation: Bidirectional + # Check for duplicate mounts before mounting + - name: cilium-cgroup + mountPath: /sys/fs/cgroup + - name: cilium-run + mountPath: /var/run/cilium + - name: cni-path + mountPath: /host/opt/cni/bin + - name: etc-cni-netd + mountPath: /host/etc/cni/net.d + - name: clustermesh-secrets + mountPath: /var/lib/cilium/clustermesh + readOnly: true + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + # Needed to be able to load kernel modules + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: xtables-lock + mountPath: /run/xtables.lock + hostNetwork: true + initContainers: + - name: clean-cilium-state + image: "quay.io/cilium/cilium:v1.11.1@sha256:251ff274acf22fd2067b29a31e9fda94253d2961c061577203621583d7e85bd2" + imagePullPolicy: IfNotPresent + command: + - /init-container.sh + env: + - name: CILIUM_ALL_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-state + optional: true + - name: CILIUM_BPF_STATE + valueFrom: + configMapKeyRef: + name: cilium-config + key: clean-cilium-bpf-state + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "172.16.0.5" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + securityContext: + privileged: true + volumeMounts: + - name: bpf-maps + mountPath: /sys/fs/bpf + # Required to mount cgroup filesystem from the host to cilium agent pod + - name: cilium-cgroup + mountPath: /sys/fs/cgroup + mountPropagation: HostToContainer + - name: cilium-run + mountPath: /var/run/cilium + resources: + requests: + cpu: 100m + memory: 100Mi + restartPolicy: Always + priorityClassName: system-node-critical + serviceAccount: "cilium" + serviceAccountName: "cilium" + terminationGracePeriodSeconds: 1 + tolerations: + - operator: Exists + volumes: + # To keep state between restarts / upgrades + - name: cilium-run + hostPath: + path: /var/run/cilium + type: DirectoryOrCreate + # To keep state between restarts / upgrades for bpf maps + - name: bpf-maps + hostPath: + path: /sys/fs/bpf + type: DirectoryOrCreate + # To keep state between restarts / upgrades for cgroup2 filesystem + - name: cilium-cgroup + hostPath: + path: /sys/fs/cgroup + type: DirectoryOrCreate + # To install cilium cni plugin in the host + - name: cni-path + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + # To install cilium cni configuration in the host + - name: etc-cni-netd + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + # To be able to load kernel modules + - name: lib-modules + hostPath: + path: /lib/modules + # To access iptables concurrently with other processes (e.g. kube-proxy) + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # To read the clustermesh configuration + - name: clustermesh-secrets + secret: + secretName: cilium-clustermesh + # note: the leading zero means this number is in octal representation: do not remove it + defaultMode: 0400 + optional: true + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config +--- +# Source: cilium/templates/cilium-operator/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system + labels: + io.cilium/app: operator + name: cilium-operator +spec: + # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go + # for more details. + replicas: 1 + selector: + matchLabels: + io.cilium/app: operator + name: cilium-operator + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + labels: + io.cilium/app: operator + name: cilium-operator + spec: + # In HA mode, cilium-operator pods must not be scheduled on the same + # node as they will clash with each other. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: io.cilium/app + operator: In + values: + - operator + topologyKey: kubernetes.io/hostname + containers: + - name: cilium-operator + image: quay.io/cilium/operator-generic:v1.11.1@sha256:977240a4783c7be821e215ead515da3093a10f4a7baea9f803511a2c2b44a235 + imagePullPolicy: IfNotPresent + command: + - cilium-operator-generic + args: + - --config-dir=/tmp/cilium/config-map + - --debug=$(CILIUM_DEBUG) + env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: CILIUM_K8S_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CILIUM_DEBUG + valueFrom: + configMapKeyRef: + key: debug + name: cilium-config + optional: true + - name: KUBERNETES_SERVICE_HOST + value: "172.16.0.5" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + livenessProbe: + httpGet: + host: "127.0.0.1" + path: /healthz + port: 9234 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 3 + volumeMounts: + - name: cilium-config-path + mountPath: /tmp/cilium/config-map + readOnly: true + hostNetwork: true + restartPolicy: Always + priorityClassName: system-cluster-critical + serviceAccount: "cilium-operator" + serviceAccountName: "cilium-operator" + tolerations: + - operator: Exists + volumes: + # To read the configuration from the config map + - name: cilium-config-path + configMap: + name: cilium-config diff --git a/scaleway/instances-master.tf b/scaleway/instances-master.tf index f7d17fc..efa6ca5 100644 --- a/scaleway/instances-master.tf +++ b/scaleway/instances-master.tf @@ -4,14 +4,15 @@ resource "scaleway_instance_ip" "controlplane" { } resource "scaleway_instance_server" "controlplane" { - count = lookup(var.controlplane, "count", 0) - name = "master-${count.index + 1}" - image = data.scaleway_instance_image.talos.id - type = lookup(var.controlplane, "type", "DEV1-M") - enable_ipv6 = true - ip_id = scaleway_instance_ip.controlplane[count.index].id - security_group_id = scaleway_instance_security_group.controlplane.id - tags = concat(var.tags, ["infra", "master"]) + count = lookup(var.controlplane, "count", 0) + name = "master-${count.index + 1}" + image = data.scaleway_instance_image.talos.id + type = lookup(var.controlplane, "type", "DEV1-M") + enable_ipv6 = true + ip_id = scaleway_instance_ip.controlplane[count.index].id + security_group_id = scaleway_instance_security_group.controlplane.id + placement_group_id = scaleway_instance_placement_group.controlplane.id + tags = concat(var.tags, ["infra", "master"]) private_network { pn_id = scaleway_vpc_private_network.main.id @@ -39,3 +40,9 @@ resource "scaleway_instance_server" "controlplane" { ] } } + +resource "scaleway_instance_placement_group" "controlplane" { + name = "controlplane" + policy_type = "max_availability" + policy_mode = "enforced" +} diff --git a/scaleway/instances-web.tf b/scaleway/instances-web.tf new file mode 100644 index 0000000..44eb7ad --- /dev/null +++ b/scaleway/instances-web.tf @@ -0,0 +1,37 @@ + +# FIXME: does not work without enable_dynamic_ip + +resource "scaleway_instance_server" "web" { + count = lookup(var.instances, "web_count", 0) + name = "web-${count.index + 1}" + image = data.scaleway_instance_image.talos.id + type = lookup(var.instances, "web_instance_type", "DEV1-M") + enable_ipv6 = true + enable_dynamic_ip = true + tags = concat(var.tags, ["web"]) + + private_network { + pn_id = scaleway_vpc_private_network.main.id + } + + user_data = { + cloud-init = templatefile("${path.module}/templates/web.yaml.tpl", + merge(var.kubernetes, { + name = "web-${count.index + 1}" + type = "worker" + ipv4_vip = local.ipv4_vip + clusterDns = cidrhost(split(",", var.kubernetes["serviceSubnets"])[0], 10) + nodeSubnets = local.main_subnet + labels = "topology.kubernetes.io/region=fr-par" + }) + ) + } + + lifecycle { + ignore_changes = [ + image, + type, + user_data, + ] + } +} diff --git a/scaleway/network.tf b/scaleway/network.tf index 49af76f..23069b7 100644 --- a/scaleway/network.tf +++ b/scaleway/network.tf @@ -16,8 +16,15 @@ resource "scaleway_vpc_public_gateway" "main" { } resource "scaleway_vpc_public_gateway_dhcp" "main" { - subnet = local.main_subnet - pool_low = cidrhost(local.main_subnet, 16) + subnet = local.main_subnet + push_default_route = true + pool_low = cidrhost(local.main_subnet, 16) + + lifecycle { + ignore_changes = [ + dns_server_override + ] + } } resource "scaleway_vpc_private_network" "main" { diff --git a/scaleway/templates/controlplane.yaml.tpl b/scaleway/templates/controlplane.yaml.tpl index bc407d8..f9b0dd2 100644 --- a/scaleway/templates/controlplane.yaml.tpl +++ b/scaleway/templates/controlplane.yaml.tpl @@ -30,6 +30,8 @@ machine: addresses: - 169.254.2.53/32 - fd00::169:254:2:53/128 + kubespan: + enabled: true install: wipe: false sysctls: @@ -52,12 +54,18 @@ machine: cluster: controlPlane: endpoint: https://${ipv4_vip}:6443 + discovery: + enabled: true network: dnsDomain: ${domain} podSubnets: ${format("%#v",split(",",podSubnets))} serviceSubnets: ${format("%#v",split(",",serviceSubnets))} + cni: + name: custom + urls: + - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/scaleway/deployments/cilium_result.yaml proxy: - mode: ipvs + disabled: true apiServer: certSANs: - "${lbv4}" diff --git a/scaleway/templates/web.yaml.tpl b/scaleway/templates/web.yaml.tpl new file mode 100644 index 0000000..e91877d --- /dev/null +++ b/scaleway/templates/web.yaml.tpl @@ -0,0 +1,63 @@ +version: v1alpha1 +debug: false +persist: true +machine: + type: worker + token: ${tokenMachine} + ca: + crt: ${caMachine} + certSANs: [] + kubelet: + extraArgs: + cloud-provider: external + rotate-server-certificates: true + node-labels: ${labels} + clusterDNS: + - 169.254.2.53 + - ${clusterDns} + nodeIP: + validSubnets: ${format("%#v",split(",",nodeSubnets))} + network: + interfaces: + - interface: eth0 + dhcp: true + - interface: eth1 + dhcp: true + dhcpOptions: + routeMetric: 2048 + - interface: dummy0 + addresses: + - 169.254.2.53/32 + - fd00::169:254:2:53/128 + kubespan: + enabled: true + install: + wipe: true + sysctls: + net.core.somaxconn: 65535 + net.core.netdev_max_backlog: 4096 + net.ipv4.tcp_keepalive_time: 600 + net.ipv4.tcp_keepalive_intvl: 60 + fs.inotify.max_user_instances: 256 + systemDiskEncryption: + state: + provider: luks2 + keys: + - nodeID: {} + slot: 0 +cluster: + id: ${clusterID} + secret: ${clusterSecret} + controlPlane: + endpoint: https://${ipv4_vip}:6443 + clusterName: ${clusterName} + discovery: + enabled: true + network: + dnsDomain: ${domain} + serviceSubnets: ${format("%#v",split(",",serviceSubnets))} + proxy: + disabled: true + token: ${token} + ca: + crt: ${ca} diff --git a/scaleway/variables.tf b/scaleway/variables.tf index 5bb64bc..47adb67 100644 --- a/scaleway/variables.tf +++ b/scaleway/variables.tf @@ -32,6 +32,8 @@ variable "kubernetes" { domain = "cluster.local" apiDomain = "api.cluster.local" clusterName = "talos-k8s-hetzner" + clusterID = "" + clusterSecret = "" tokenMachine = "" caMachine = "" token = ""