diff --git a/CHANGES.md b/CHANGES.md index 36712fea..aa3e607c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,12 +14,12 @@ Notable changes between versions. ### AWS -* Allow configuring controller and worker disks ([#1482](https://github.com/poseidon/typhoon/pull/1482)) +* Configure controller and worker disks ([#1482](https://github.com/poseidon/typhoon/pull/1482)) * Add `controller_disk_type`, `controller_disk_size`, and `controller_disk_iops` variables * Add `worker_disk_type`, `worker_disk_size`, and `worker_disk_iops` variables * Remove `disk_type`, `disk_size`, and `disk_iops` variables * Fix propagating settings to worker disks, previously ignored -* Allow configuring CPU pricing model for burstable instance types ([#1482](https://github.com/poseidon/typhoon/pull/1482)) +* Configure CPU pricing model for burstable instance types ([#1482](https://github.com/poseidon/typhoon/pull/1482)) * Add `controller_cpu_credits` and `worker_cpu_credits` variables (`standard` or `unlimited`) * Configure controller or worker instance architecture ([#1485](https://github.com/poseidon/typhoon/pull/1485)) * Add `controller_arch` and `worker_arch` variables (`amd64` or `arm64`) diff --git a/docs/advanced/arm64.md b/docs/advanced/arm64.md index fa6a72f4..5f3b123a 100644 --- a/docs/advanced/arm64.md +++ b/docs/advanced/arm64.md @@ -1,13 +1,11 @@ # ARM64 -Typhoon supports ARM64 Kubernetes clusters with ARM64 controller and worker nodes (full-cluster) or adding worker pools of ARM64 nodes to clusters with an x86/amd64 control plane for a hybdrid (mixed-arch) cluster. - -Typhoon ARM64 clusters (full-cluster or mixed-arch) are available on: +Typhoon supports Kubernetes clusters with ARM64 controller or worker nodes on several platforms: * AWS with Fedora CoreOS or Flatcar Linux * Azure with Flatcar Linux -## Cluster +## AWS Create a cluster on AWS with ARM64 controller and worker nodes. Container workloads must be `arm64` compatible and use `arm64` (or multi-arch) container images. @@ -22,17 +20,16 @@ Create a cluster on AWS with ARM64 controller and worker nodes. Container worklo dns_zone = "aws.example.com" dns_zone_id = "Z3PAABBCFAKEC0" + # instances + controller_type = "t4g.small" + controller_arch = "arm64" + worker_count = 2 + worker_type = "t4g.small" + worker_arch = "arm64" + worker_price = "0.0168" + # configuration ssh_authorized_key = "ssh-ed25519 AAAAB3Nz..." - - # optional - arch = "arm64" - networking = "cilium" - worker_count = 2 - worker_price = "0.0168" - - controller_type = "t4g.small" - worker_type = "t4g.small" } ``` @@ -47,17 +44,16 @@ Create a cluster on AWS with ARM64 controller and worker nodes. Container worklo dns_zone = "aws.example.com" dns_zone_id = "Z3PAABBCFAKEC0" + # instances + controller_type = "t4g.small" + controller_arch = "arm64" + worker_count = 2 + worker_type = "t4g.small" + worker_arch = "arm64" + worker_price = "0.0168" + # configuration ssh_authorized_key = "ssh-ed25519 AAAAB3Nz..." - - # optional - arch = "arm64" - networking = "cilium" - worker_count = 2 - worker_price = "0.0168" - - controller_type = "t4g.small" - worker_type = "t4g.small" } ``` @@ -71,115 +67,6 @@ ip-10-0-32-166 Ready 80s v1.30.3 10.0.32.166 F ip-10-0-5-79 Ready 77s v1.30.3 10.0.5.79 Fedora CoreOS 35.20211215.3.0 5.15.7-200.fc35.aarch64 containerd://1.5.8 ``` -## Hybrid - -Create a hybrid/mixed arch cluster by defining an AWS cluster. Then define a [worker pool](worker-pools.md#aws) with ARM64 workers. Optional taints are added to aid in scheduling. - -=== "FCOS Cluster" - - ```tf - module "gravitas" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.30.3" - - # AWS - cluster_name = "gravitas" - dns_zone = "aws.example.com" - dns_zone_id = "Z3PAABBCFAKEC0" - - # configuration - ssh_authorized_key = "ssh-ed25519 AAAAB3Nz..." - - # optional - networking = "cilium" - worker_count = 2 - worker_price = "0.021" - - daemonset_tolerations = ["arch"] # important - } - ``` - -=== "Flatcar Cluster" - - ```tf - module "gravitas" { - source = "git::https://github.com/poseidon/typhoon//aws/flatcar-linux/kubernetes?ref=v1.30.3" - - # AWS - cluster_name = "gravitas" - dns_zone = "aws.example.com" - dns_zone_id = "Z3PAABBCFAKEC0" - - # configuration - ssh_authorized_key = "ssh-ed25519 AAAAB3Nz..." - - # optional - networking = "cilium" - worker_count = 2 - worker_price = "0.021" - - daemonset_tolerations = ["arch"] # important - } - ``` - -=== "FCOS ARM64 Workers" - - ```tf - module "gravitas-arm64" { - source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes/workers?ref=v1.30.3" - - # AWS - vpc_id = module.gravitas.vpc_id - subnet_ids = module.gravitas.subnet_ids - security_groups = module.gravitas.worker_security_groups - - # configuration - name = "gravitas-arm64" - kubeconfig = module.gravitas.kubeconfig - ssh_authorized_key = var.ssh_authorized_key - - # optional - arch = "arm64" - instance_type = "t4g.small" - spot_price = "0.0168" - node_taints = ["arch=arm64:NoSchedule"] - } - ``` - -=== "Flatcar ARM64 Workers" - - ```tf - module "gravitas-arm64" { - source = "git::https://github.com/poseidon/typhoon//aws/flatcar-linux/kubernetes/workers?ref=v1.30.3" - - # AWS - vpc_id = module.gravitas.vpc_id - subnet_ids = module.gravitas.subnet_ids - security_groups = module.gravitas.worker_security_groups - - # configuration - name = "gravitas-arm64" - kubeconfig = module.gravitas.kubeconfig - ssh_authorized_key = var.ssh_authorized_key - - # optional - arch = "arm64" - instance_type = "t4g.small" - spot_price = "0.0168" - node_taints = ["arch=arm64:NoSchedule"] - } - ``` - -Verify amd64 (x86_64) and arm64 (aarch64) nodes are present. - -``` -$ kubectl get nodes -o wide -NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME -ip-10-0-1-73 Ready 111m v1.30.3 10.0.1.73 Fedora CoreOS 35.20211215.3.0 5.15.7-200.fc35.x86_64 containerd://1.5.8 -ip-10-0-22-79... Ready 111m v1.30.3 10.0.22.79 Flatcar Container Linux by Kinvolk 3033.2.0 (Oklo) 5.10.84-flatcar containerd://1.5.8 -ip-10-0-24-130 Ready 111m v1.30.3 10.0.24.130 Fedora CoreOS 35.20211215.3.0 5.15.7-200.fc35.x86_64 containerd://1.5.8 -ip-10-0-39-19 Ready 111m v1.30.3 10.0.39.19 Fedora CoreOS 35.20211215.3.0 5.15.7-200.fc35.x86_64 containerd://1.5.8 -``` - ## Azure Create a cluster on Azure with ARM64 controller and worker nodes. Container workloads must be `arm64` compatible and use `arm64` (or multi-arch) container images. @@ -194,13 +81,128 @@ module "ramius" { dns_zone = "azure.example.com" dns_zone_group = "example-group" + # instances + controller_arch = "arm64" + controller_type = "Standard_B2pls_v5" + worker_count = 2 + controller_arch = "arm64" + worker_type = "Standard_D2pls_v5" + # configuration ssh_authorized_key = "ssh-rsa AAAAB3Nz..." - - # optional - arch = "arm64" - controller_type = "Standard_D2pls_v5" - worker_type = "Standard_D2pls_v5" - worker_count = 2 } ``` + +## Hybrid + +Create a hybrid/mixed arch cluster by defining a cluster where [worker pool(s)](worker-pools.md#aws) have a different instance type architecture than controllers or other workers. Taints are added to aid in scheduling. + +Here's an AWS example, + +=== "FCOS Cluster" + + ```tf + module "gravitas" { + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes?ref=v1.30.3" + + # AWS + cluster_name = "gravitas" + dns_zone = "aws.example.com" + dns_zone_id = "Z3PAABBCFAKEC0" + + # instances + worker_count = 2 + worker_arch = "arm64" + worker_type = "t4g.medium" + worker_price = "0.021" + + # configuration + daemonset_tolerations = ["arch"] # important + networking = "cilium" + ssh_authorized_key = "ssh-ed25519 AAAAB3Nz..." + } + ``` + +=== "Flatcar Cluster" + + ```tf + module "gravitas" { + source = "git::https://github.com/poseidon/typhoon//aws/flatcar-linux/kubernetes?ref=v1.30.3" + + # AWS + cluster_name = "gravitas" + dns_zone = "aws.example.com" + dns_zone_id = "Z3PAABBCFAKEC0" + + # instances + worker_count = 2 + worker_arch = "arm64" + worker_type = "t4g.medium" + worker_price = "0.021" + + # configuration + daemonset_tolerations = ["arch"] # important + networking = "cilium" + ssh_authorized_key = "ssh-ed25519 AAAAB3Nz..." + } + ``` + +=== "FCOS ARM64 Workers" + + ```tf + module "gravitas-arm64" { + source = "git::https://github.com/poseidon/typhoon//aws/fedora-coreos/kubernetes/workers?ref=v1.30.3" + + # AWS + vpc_id = module.gravitas.vpc_id + subnet_ids = module.gravitas.subnet_ids + security_groups = module.gravitas.worker_security_groups + + # instances + arch = "arm64" + instance_type = "t4g.small" + spot_price = "0.0168" + + # configuration + name = "gravitas-arm64" + kubeconfig = module.gravitas.kubeconfig + node_taints = ["arch=arm64:NoSchedule"] + ssh_authorized_key = var.ssh_authorized_key + } + ``` + +=== "Flatcar ARM64 Workers" + + ```tf + module "gravitas-arm64" { + source = "git::https://github.com/poseidon/typhoon//aws/flatcar-linux/kubernetes/workers?ref=v1.30.3" + + # AWS + vpc_id = module.gravitas.vpc_id + subnet_ids = module.gravitas.subnet_ids + security_groups = module.gravitas.worker_security_groups + + # instances + arch = "arm64" + instance_type = "t4g.small" + spot_price = "0.0168" + + # configuration + name = "gravitas-arm64" + kubeconfig = module.gravitas.kubeconfig + node_taints = ["arch=arm64:NoSchedule"] + ssh_authorized_key = var.ssh_authorized_key + } + ``` + +Verify amd64 (x86_64) and arm64 (aarch64) nodes are present. + +``` +$ kubectl get nodes -o wide +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +ip-10-0-1-73 Ready 111m v1.30.3 10.0.1.73 Fedora CoreOS 35.20211215.3.0 5.15.7-200.fc35.x86_64 containerd://1.5.8 +ip-10-0-22-79... Ready 111m v1.30.3 10.0.22.79 Flatcar Container Linux by Kinvolk 3033.2.0 (Oklo) 5.10.84-flatcar containerd://1.5.8 +ip-10-0-24-130 Ready 111m v1.30.3 10.0.24.130 Fedora CoreOS 35.20211215.3.0 5.15.7-200.fc35.x86_64 containerd://1.5.8 +ip-10-0-39-19 Ready 111m v1.30.3 10.0.39.19 Fedora CoreOS 35.20211215.3.0 5.15.7-200.fc35.x86_64 containerd://1.5.8 +``` +