From 0db27a7335b05ebc19e5c1caf019e0bf28cdd26a Mon Sep 17 00:00:00 2001 From: Adriano Pezzuto <2222461+bsctl@users.noreply.github.com> Date: Sun, 27 Aug 2023 10:23:10 +0200 Subject: [PATCH] docs: improve the nodes joining procedure (#362) --- deploy/nodes-prerequisites.sh | 47 ------------------- deploy/tenant-cloudinit.yaml | 32 ------------- docs/content/getting-started.md | 38 +++++++-------- .../content/guides/kamaji-azure-deployment.md | 22 +++++---- 4 files changed, 32 insertions(+), 107 deletions(-) delete mode 100755 deploy/nodes-prerequisites.sh delete mode 100644 deploy/tenant-cloudinit.yaml diff --git a/deploy/nodes-prerequisites.sh b/deploy/nodes-prerequisites.sh deleted file mode 100755 index c65d464..0000000 --- a/deploy/nodes-prerequisites.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -KUBERNETES_VERSION=$1; shift -HOSTS=("$@") - -# Install `containerd` as container runtime. -cat << EOF | tee containerd.conf -overlay -br_netfilter -EOF - -cat << EOF | tee 99-kubernetes-cri.conf -net.bridge.bridge-nf-call-iptables = 1 -net.ipv4.ip_forward = 1 -net.bridge.bridge-nf-call-ip6tables = 1 -EOF - -for i in "${!HOSTS[@]}"; do - HOST=${HOSTS[$i]} - ssh ${USER}@${HOST} -t 'sudo apt update && sudo apt install -y containerd' - ssh ${USER}@${HOST} -t 'sudo mkdir -p /etc/containerd' - ssh ${USER}@${HOST} -t 'containerd config default | sed -e "s#SystemdCgroup = false#SystemdCgroup = true#g" | sudo tee -a /etc/containerd/config.toml' - ssh ${USER}@${HOST} -t 'sudo systemctl restart containerd && sudo systemctl enable containerd' - scp containerd.conf ${USER}@${HOST}: - ssh ${USER}@${HOST} -t 'sudo chown -R root:root containerd.conf && sudo mv containerd.conf /etc/modules-load.d/containerd.conf' - ssh ${USER}@${HOST} -t 'sudo modprobe overlay && sudo modprobe br_netfilter' - scp 99-kubernetes-cri.conf ${USER}@${HOST}: - ssh ${USER}@${HOST} -t 'sudo chown -R root:root 99-kubernetes-cri.conf && sudo mv 99-kubernetes-cri.conf /etc/sysctl.d/99-kubernetes-cri.conf' - ssh ${USER}@${HOST} -t 'sudo sysctl --system' -done - -rm -f containerd.conf 99-kubernetes-cri.conf - -# Install `kubectl`, `kubelet`, and `kubeadm` in the desired version. - -INSTALL_KUBERNETES="sudo apt install -y kubelet=${KUBERNETES_VERSION}-00 kubeadm=${KUBERNETES_VERSION}-00 kubectl=${KUBERNETES_VERSION}-00 --allow-downgrades --allow-change-held-packages" - -for i in "${!HOSTS[@]}"; do - HOST=${HOSTS[$i]} - ssh ${USER}@${HOST} -t 'sudo apt update' - ssh ${USER}@${HOST} -t 'sudo apt install -y apt-transport-https ca-certificates curl' - ssh ${USER}@${HOST} -t 'sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg' - ssh ${USER}@${HOST} -t 'echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list' - ssh ${USER}@${HOST} -t 'sudo apt update' - ssh ${USER}@${HOST} -t ${INSTALL_KUBERNETES} - ssh ${USER}@${HOST} -t 'sudo apt-mark hold kubelet kubeadm kubectl' -done \ No newline at end of file diff --git a/deploy/tenant-cloudinit.yaml b/deploy/tenant-cloudinit.yaml deleted file mode 100644 index ae10bcc..0000000 --- a/deploy/tenant-cloudinit.yaml +++ /dev/null @@ -1,32 +0,0 @@ -#cloud-config -package_upgrade: true -packages: - - containerd - - apt-transport-https - - ca-certificates - - curl -write_files: - - owner: root:root - path: /etc/modules-load.d/containerd.conf - content: | - overlay - br_netfilter - - owner: root:root - path: /etc/sysctl.d/99-kubernetes-cri.conf - content: | - net.bridge.bridge-nf-call-iptables = 1 - net.ipv4.ip_forward = 1 - net.bridge.bridge-nf-call-ip6tables = 1 -runcmd: - - sudo modprobe overlay - - sudo modprobe br_netfilter - - sudo sysctl --system - - sudo mkdir -p /etc/containerd - - containerd config default | sed -e 's#SystemdCgroup = false#SystemdCgroup = true#g' | sudo tee -a /etc/containerd/config.toml - - sudo systemctl restart containerd - - sudo systemctl enable containerd - - sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg - - echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list - - sudo apt update - - sudo apt install -y kubelet=1.25.0-00 kubeadm=1.25.0-00 kubectl=1.25.0-00 - - sudo apt-mark hold kubelet kubeadm kubectl containerd \ No newline at end of file diff --git a/docs/content/getting-started.md b/docs/content/getting-started.md index 1095638..597cdb2 100644 --- a/docs/content/getting-started.md +++ b/docs/content/getting-started.md @@ -222,7 +222,7 @@ Kubernetes control plane is running at https://192.168.32.240:6443 CoreDNS is running at https://192.168.32.240:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy ``` -Check out how the Tenant control Plane advertises itself to workloads: +Check out how the Tenant Control Plane advertises itself to workloads: ```bash kubectl --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig get svc @@ -240,38 +240,34 @@ kubernetes 192.168.32.240:6443 18m And make sure it is `${TENANT_ADDR}:${TENANT_PORT}`. -### Prepare worker nodes to join +### Join worker nodes -Currently, Kamaji does not provide any helper for creation of tenant worker nodes. -You should get a set of machines from your infrastructure provider, turn them into worker nodes, and then join to the tenant control plane with the `kubeadm`. +The Tenant Control Plane is made of pods running in the Kamaji Admin Cluster. At this point, the tenant cluster has no worker nodes. So, the next step is to join some worker nodes to the Tenant Control Plane. -Kamaji is sticking to the [Cluster Management API](https://github.com/kubernetes-sigs/cluster-api) project contracts by providing a `ControlPlane` provider. -Please, refer to the [official repository](https://github.com/clastix/cluster-api-control-plane-provider-kamaji) to learn more about it. +Kamaji does not provide any helper for creation of tenant worker nodes, insteat it leverages the [Cluster Management API](https://github.com/kubernetes-sigs/cluster-api) project by providing a `ControlPlane` provider. This allows you to create the tenant clusters, including worker nodes, in a completely declarative way. Refer to the [Cluster API guide](guides/cluster-api.md) to learn more about supported providers. -You can use the provided helper script `/deploy/nodes-prerequisites.sh`, in order to install the dependencies on all the worker nodes: +An alternative approach for joining nodes is to use the `kubeadm` command on each node. Follow the related [documentation](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/) in order to: -- Install `containerd` as container runtime -- Install `crictl`, the command line for working with `containerd` -- Install `kubectl`, `kubelet`, and `kubeadm` in the desired version +- install `containerd` as container runtime +- install `crictl`, the command line for working with `containerd` +- install `kubectl`, `kubelet`, and `kubeadm` in the desired version -!!! warning "" - The provided script is just a facility: it assumes all worker nodes are running `Ubuntu 20.04`. Make sure to adapt the script if you're using a different distribution. - -Run the script: +After the installation is complete on all the nodes, open the command line on your Linux workstation and store the IP address of each node in an environment variable: ```bash -HOSTS=(${WORKER0} ${WORKER1} ${WORKER2}) -./nodes-prerequisites.sh ${TENANT_VERSION:1} ${HOSTS[@]} +WORKER0=
+WORKER1=
+WORKER2=
``` -### Join worker nodes -The current approach for joining nodes is to use `kubeadm` and therefore, we will create a bootstrap token to perform the action. In order to facilitate the step, we will store the entire command of joining in a variable: +Store the join command in a variable: ```bash JOIN_CMD=$(echo "sudo ")$(kubeadm --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig token create --print-join-command) + ``` -A bash loop will be used to join all the available nodes. +Use a loop to log in to and run the join command on each node: ```bash HOSTS=(${WORKER0} ${WORKER1} ${WORKER2}) @@ -281,6 +277,10 @@ for i in "${!HOSTS[@]}"; do done ``` +!!! tip "yaki" + This manual process can be further automated to handle the node prerequisites and joining. See [yaki](https://github.com/clastix/yaki) script, which you could modify for your preferred operating system and version. The provided script is just a facility: it assumes all worker nodes are running `Ubuntu 22.04`. Make sure to adapt the script if you're using a different distribution. + + Checking the nodes: ```bash diff --git a/docs/content/guides/kamaji-azure-deployment.md b/docs/content/guides/kamaji-azure-deployment.md index a8c7335..608f98b 100644 --- a/docs/content/guides/kamaji-azure-deployment.md +++ b/docs/content/guides/kamaji-azure-deployment.md @@ -274,13 +274,13 @@ NAME ENDPOINTS AGE kubernetes 10.240.0.100:6443 57m ``` -### Prepare worker nodes to join +### Join worker nodes -Currently, Kamaji does not provide any helper for creation of tenant worker nodes. -You should get a set of machines from your infrastructure provider, turn them into worker nodes, and then join to the tenant control plane with the `kubeadm`. +The Tenant Control Plane is made of pods running in the Kamaji Admin Cluster. At this point, the tenant cluster has no worker nodes. So, the next step is to join some worker nodes to the Tenant Control Plane. -Kamaji is sticking to the [Cluster Management API](https://github.com/kubernetes-sigs/cluster-api) project contracts by providing a `ControlPlane` provider. -An Azure-based cluster is not yet available: the available road-map is available on the [official repository](https://github.com/clastix/cluster-api-control-plane-provider-kamaji). +Kamaji does not provide any helper for creation of tenant worker nodes, insteat it leverages the [Cluster Management API](https://github.com/kubernetes-sigs/cluster-api) project by providing a `ControlPlane` provider. This allows you to create the tenant clusters, including worker nodes, in a completely declarative way. Currently, a Cluster API `ControlPlane` provider for Azure is not yet available: check the road-map on the [official repository](https://github.com/clastix/cluster-api-control-plane-provider-kamaji). + +An alternative approach to create and join worker nodes in Azure is to manually create the VMs, turn them into Kubernetes worker nodes and then join through the `kubeadm` command. Create an Azure VM Stateful Set to host worker nodes @@ -298,7 +298,6 @@ az vmss create \ --vnet-name $KAMAJI_VNET_NAME \ --subnet $TENANT_SUBNET_NAME \ --computer-name-prefix $TENANT_NAME- \ - --custom-data ./tenant-cloudinit.yaml \ --load-balancer "" \ --instance-count 0 @@ -313,15 +312,20 @@ az vmss scale \ --new-capacity 3 ``` -### Join worker nodes -The current approach for joining nodes is to use `kubeadm` and therefore, we will create a bootstrap token to perform the action. In order to facilitate the step, we will store the entire command of joining in a variable: +Once all the machines are ready, follow the related [documentation](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/) in order to: + +- install `containerd` as container runtime +- install `crictl`, the command line for working with `containerd` +- install `kubectl`, `kubelet`, and `kubeadm` in the desired version + +After the installation is complete on all the nodes, store the entire command of joining in a variable: ```bash TENANT_ADDR=$(kubectl -n ${TENANT_NAMESPACE} get svc ${TENANT_NAME} -o json | jq -r ."spec.loadBalancerIP") JOIN_CMD=$(echo "sudo kubeadm join ${TENANT_ADDR}:6443 ")$(kubeadm --kubeconfig=${TENANT_NAMESPACE}-${TENANT_NAME}.kubeconfig token create --print-join-command |cut -d" " -f4-) ``` -A bash loop will be used to join all the available nodes. +Use a loop to log in to and run the join command on each node: ```bash VMIDS=($(az vmss list-instances \