diff --git a/examples/terraform/modules/bootkube/bootkube.tf b/examples/terraform/modules/bootkube/bootkube.tf deleted file mode 100644 index e2566ff5..00000000 --- a/examples/terraform/modules/bootkube/bootkube.tf +++ /dev/null @@ -1,12 +0,0 @@ -# Self-hosted Kubernetes assets (kubeconfig, manifests) -module "bootkube" { - source = "git::https://github.com/poseidon/bootkube-terraform.git?ref=v0.6.1" - - cluster_name = "${var.cluster_name}" - api_servers = ["${var.k8s_domain_name}"] - etcd_servers = ["${var.controller_domains}"] - asset_dir = "${var.asset_dir}" - pod_cidr = "${var.pod_cidr}" - service_cidr = "${var.service_cidr}" - experimental_self_hosted_etcd = "${var.experimental_self_hosted_etcd}" -} diff --git a/examples/terraform/modules/bootkube/groups.tf b/examples/terraform/modules/bootkube/groups.tf deleted file mode 100644 index 8c6af7ae..00000000 --- a/examples/terraform/modules/bootkube/groups.tf +++ /dev/null @@ -1,53 +0,0 @@ -// Install Container Linux to disk -resource "matchbox_group" "container-linux-install" { - count = "${length(var.controller_names) + length(var.worker_names)}" - - name = "${format("container-linux-install-%s", element(concat(var.controller_names, var.worker_names), count.index))}" - profile = "${var.cached_install == "true" ? module.profiles.cached-container-linux-install : module.profiles.container-linux-install}" - - selector { - mac = "${element(concat(var.controller_macs, var.worker_macs), count.index)}" - } - - metadata { - ssh_authorized_key = "${var.ssh_authorized_key}" - } -} - -resource "matchbox_group" "controller" { - count = "${length(var.controller_names)}" - name = "${format("%s-%s", var.cluster_name, element(var.controller_names, count.index))}" - profile = "${module.profiles.bootkube-controller}" - - selector { - mac = "${element(var.controller_macs, count.index)}" - os = "installed" - } - - metadata { - domain_name = "${element(var.controller_domains, count.index)}" - etcd_name = "${element(var.controller_names, count.index)}" - etcd_initial_cluster = "${join(",", formatlist("%s=https://%s:2380", var.controller_names, var.controller_domains))}" - etcd_on_host = "${var.experimental_self_hosted_etcd ? "false" : "true"}" - k8s_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" - ssh_authorized_key = "${var.ssh_authorized_key}" - } -} - -resource "matchbox_group" "worker" { - count = "${length(var.worker_names)}" - name = "${format("%s-%s", var.cluster_name, element(var.worker_names, count.index))}" - profile = "${module.profiles.bootkube-worker}" - - selector { - mac = "${element(var.worker_macs, count.index)}" - os = "installed" - } - - metadata { - domain_name = "${element(var.worker_domains, count.index)}" - etcd_on_host = "${var.experimental_self_hosted_etcd ? "false" : "true"}" - k8s_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" - ssh_authorized_key = "${var.ssh_authorized_key}" - } -} diff --git a/examples/terraform/modules/bootkube/outputs.tf b/examples/terraform/modules/bootkube/outputs.tf deleted file mode 100644 index 41bd79f2..00000000 --- a/examples/terraform/modules/bootkube/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "kubeconfig" { - value = "${module.bootkube.kubeconfig}" -} diff --git a/examples/terraform/modules/bootkube/profiles.tf b/examples/terraform/modules/bootkube/profiles.tf deleted file mode 100644 index 6a2ce73c..00000000 --- a/examples/terraform/modules/bootkube/profiles.tf +++ /dev/null @@ -1,9 +0,0 @@ -// Create common profiles -module "profiles" { - source = "../profiles" - matchbox_http_endpoint = "${var.matchbox_http_endpoint}" - container_linux_version = "${var.container_linux_version}" - container_linux_channel = "${var.container_linux_channel}" - install_disk = "${var.install_disk}" - container_linux_oem = "${var.container_linux_oem}" -} diff --git a/examples/terraform/modules/bootkube/ssh.tf b/examples/terraform/modules/bootkube/ssh.tf deleted file mode 100644 index a9c4654a..00000000 --- a/examples/terraform/modules/bootkube/ssh.tf +++ /dev/null @@ -1,96 +0,0 @@ -# Secure copy etcd TLS assets and kubeconfig to all nodes. Activates kubelet.service -resource "null_resource" "copy-secrets" { - count = "${length(var.controller_names) + length(var.worker_names)}" - - connection { - type = "ssh" - host = "${element(concat(var.controller_domains, var.worker_domains), count.index)}" - user = "core" - timeout = "60m" - } - - provisioner "file" { - content = "${module.bootkube.kubeconfig}" - destination = "$HOME/kubeconfig" - } - - provisioner "file" { - content = "${module.bootkube.etcd_ca_cert}" - destination = "$HOME/etcd-client-ca.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_client_cert}" - destination = "$HOME/etcd-client.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_client_key}" - destination = "$HOME/etcd-client.key" - } - - provisioner "file" { - content = "${module.bootkube.etcd_server_cert}" - destination = "$HOME/etcd-server.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_server_key}" - destination = "$HOME/etcd-server.key" - } - - provisioner "file" { - content = "${module.bootkube.etcd_peer_cert}" - destination = "$HOME/etcd-peer.crt" - } - - provisioner "file" { - content = "${module.bootkube.etcd_peer_key}" - destination = "$HOME/etcd-peer.key" - } - - provisioner "remote-exec" { - inline = [ - "sudo mkdir -p /etc/ssl/etcd/etcd", - "sudo mv etcd-client* /etc/ssl/etcd/", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/server-ca.crt", - "sudo mv etcd-server.crt /etc/ssl/etcd/etcd/server.crt", - "sudo mv etcd-server.key /etc/ssl/etcd/etcd/server.key", - "sudo cp /etc/ssl/etcd/etcd-client-ca.crt /etc/ssl/etcd/etcd/peer-ca.crt", - "sudo mv etcd-peer.crt /etc/ssl/etcd/etcd/peer.crt", - "sudo mv etcd-peer.key /etc/ssl/etcd/etcd/peer.key", - "sudo chown -R etcd:etcd /etc/ssl/etcd", - "sudo chmod -R 500 /etc/ssl/etcd", - "sudo mv /home/core/kubeconfig /etc/kubernetes/kubeconfig", - ] - } -} - -# Secure copy bootkube assets to ONE controller and start bootkube to perform -# one-time self-hosted cluster bootstrapping. -resource "null_resource" "bootkube-start" { - # Without depends_on, this remote-exec may start before the kubeconfig copy. - # Terraform only does one task at a time, so it would try to bootstrap - # Kubernetes and Tectonic while no Kubelets are running. Ensure all nodes - # receive a kubeconfig before proceeding with bootkube and tectonic. - depends_on = ["null_resource.copy-secrets"] - - connection { - type = "ssh" - host = "${element(var.controller_domains, 0)}" - user = "core" - timeout = "60m" - } - - provisioner "file" { - source = "${var.asset_dir}" - destination = "$HOME/assets" - } - - provisioner "remote-exec" { - inline = [ - "sudo mv /home/core/assets /opt/bootkube", - "sudo systemctl start bootkube", - ] - } -} diff --git a/examples/terraform/modules/bootkube/variables.tf b/examples/terraform/modules/bootkube/variables.tf deleted file mode 100644 index 35473aae..00000000 --- a/examples/terraform/modules/bootkube/variables.tf +++ /dev/null @@ -1,104 +0,0 @@ -variable "matchbox_http_endpoint" { - type = "string" - description = "Matchbox HTTP read-only endpoint (e.g. http://matchbox.example.com:8080)" -} - -variable "container_linux_channel" { - type = "string" - description = "Container Linux channel corresponding to the container_linux_version" -} - -variable "container_linux_version" { - type = "string" - description = "Container Linux version of the kernel/initrd to PXE or the image to install" -} - -variable "cluster_name" { - type = "string" - description = "Cluster name" -} - -variable "ssh_authorized_key" { - type = "string" - description = "SSH public key to set as an authorized_key on machines" -} - -# Machines -# Terraform's crude "type system" does properly support lists of maps so we do this. - -variable "controller_names" { - type = "list" -} - -variable "controller_macs" { - type = "list" -} - -variable "controller_domains" { - type = "list" -} - -variable "worker_names" { - type = "list" -} - -variable "worker_macs" { - type = "list" -} - -variable "worker_domains" { - type = "list" -} - -# bootkube assets - -variable "k8s_domain_name" { - description = "Controller DNS name which resolves to a controller instance. Workers and kubeconfig's will communicate with this endpoint (e.g. cluster.example.com)" - type = "string" -} - -variable "asset_dir" { - description = "Path to a directory where generated assets should be placed (contains secrets)" - type = "string" -} - -variable "pod_cidr" { - description = "CIDR IP range to assign Kubernetes pods" - type = "string" - default = "10.2.0.0/16" -} - -variable "service_cidr" { - description = < /dev/null; do sleep 1; done' - [Install] - RequiredBy=kubelet.service - - name: kubelet.service - contents: | - [Unit] - Description=Kubelet via Hyperkube ACI - [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log" - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests - ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d - ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets - ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests - ExecStartPre=/bin/mkdir -p /var/lib/cni - ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --require-kubeconfig \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --anonymous-auth=false \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --network-plugin=cni \ - --lock-file=/var/run/lock/kubelet.lock \ - --exit-on-lock-contention \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --allow-privileged \ - --hostname-override={{.domain_name}} \ - --node-labels=node-role.kubernetes.io/master \ - --register-with-taints=node-role.kubernetes.io/master=:NoSchedule \ - --cluster_dns={{.k8s_dns_service_ip}} \ - --cluster_domain=cluster.local - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid - Restart=always - RestartSec=10 - [Install] - WantedBy=multi-user.target - - name: bootkube.service - contents: | - [Unit] - Description=Bootstrap a Kubernetes control plane with a temp api-server - ConditionPathExists=!/opt/bootkube/init_bootkube.done - [Service] - Type=oneshot - RemainAfterExit=true - WorkingDirectory=/opt/bootkube - ExecStart=/opt/bootkube/bootkube-start - ExecStartPost=/bin/touch /opt/bootkube/init_bootkube.done -storage: - {{ if index . "pxe" }} - disks: - - device: /dev/sda - wipe_table: true - partitions: - - label: ROOT - filesystems: - - name: root - mount: - device: "/dev/sda1" - format: "ext4" - create: - force: true - options: - - "-LROOT" - {{end}} - files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=quay.io/coreos/hyperkube - KUBELET_IMAGE_TAG=v1.7.3_coreos.0 - - path: /etc/hostname - filesystem: root - mode: 0644 - contents: - inline: - {{.domain_name}} - - path: /etc/sysctl.d/max-user-watches.conf - filesystem: root - contents: - inline: | - fs.inotify.max_user_watches=16184 - - path: /opt/bootkube/bootkube-start - filesystem: root - mode: 0544 - user: - id: 500 - group: - id: 500 - contents: - inline: | - #!/bin/bash - # Wrapper for bootkube start - set -e - # Move experimental manifests - [ -d /opt/bootkube/assets/experimental/manifests ] && mv /opt/bootkube/assets/experimental/manifests/* /opt/bootkube/assets/manifests && rm -r /opt/bootkube/assets/experimental/manifests - [ -d /opt/bootkube/assets/experimental/bootstrap-manifests ] && mv /opt/bootkube/assets/experimental/bootstrap-manifests/* /opt/bootkube/assets/bootstrap-manifests && rm -r /opt/bootkube/assets/experimental/bootstrap-manifests - BOOTKUBE_ACI="${BOOTKUBE_ACI:-quay.io/coreos/bootkube}" - BOOTKUBE_VERSION="${BOOTKUBE_VERSION:-v0.6.1}" - BOOTKUBE_ASSETS="${BOOTKUBE_ASSETS:-/opt/bootkube/assets}" - exec /usr/bin/rkt run \ - --trust-keys-from-https \ - --volume assets,kind=host,source=$BOOTKUBE_ASSETS \ - --mount volume=assets,target=/assets \ - --volume bootstrap,kind=host,source=/etc/kubernetes \ - --mount volume=bootstrap,target=/etc/kubernetes \ - $RKT_OPTS \ - ${BOOTKUBE_ACI}:${BOOTKUBE_VERSION} \ - --net=host \ - --dns=host \ - --exec=/bootkube -- start --asset-dir=/assets "$@" -passwd: - users: - - name: core - ssh_authorized_keys: - - {{.ssh_authorized_key}} diff --git a/examples/terraform/modules/profiles/cl/bootkube-worker.yaml.tmpl b/examples/terraform/modules/profiles/cl/bootkube-worker.yaml.tmpl deleted file mode 100644 index 7847e878..00000000 --- a/examples/terraform/modules/profiles/cl/bootkube-worker.yaml.tmpl +++ /dev/null @@ -1,111 +0,0 @@ ---- -systemd: - units: - - name: docker.service - enable: true - - name: locksmithd.service - mask: true - - name: kubelet.path - enable: true - contents: | - [Unit] - Description=Watch for kubeconfig - [Path] - PathExists=/etc/kubernetes/kubeconfig - [Install] - WantedBy=multi-user.target - - name: wait-for-dns.service - enable: true - contents: | - [Unit] - Description=Wait for DNS entries - Wants=systemd-resolved.service - Before=kubelet.service - [Service] - Type=oneshot - RemainAfterExit=true - ExecStart=/bin/sh -c 'while ! /usr/bin/grep '^[^#[:space:]]' /etc/resolv.conf > /dev/null; do sleep 1; done' - [Install] - RequiredBy=kubelet.service - - name: kubelet.service - contents: | - [Unit] - Description=Kubelet via Hyperkube ACI - [Service] - EnvironmentFile=/etc/kubernetes/kubelet.env - Environment="RKT_RUN_ARGS=--uuid-file-save=/var/run/kubelet-pod.uuid \ - --volume=resolv,kind=host,source=/etc/resolv.conf \ - --mount volume=resolv,target=/etc/resolv.conf \ - --volume var-lib-cni,kind=host,source=/var/lib/cni \ - --mount volume=var-lib-cni,target=/var/lib/cni \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log" - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests - ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d - ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets - ExecStartPre=/bin/mkdir -p /etc/kubernetes/inactive-manifests - ExecStartPre=/bin/mkdir -p /var/lib/cni - ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid - ExecStart=/usr/lib/coreos/kubelet-wrapper \ - --kubeconfig=/etc/kubernetes/kubeconfig \ - --require-kubeconfig \ - --client-ca-file=/etc/kubernetes/ca.crt \ - --anonymous-auth=false \ - --cni-conf-dir=/etc/kubernetes/cni/net.d \ - --network-plugin=cni \ - --lock-file=/var/run/lock/kubelet.lock \ - --exit-on-lock-contention \ - --pod-manifest-path=/etc/kubernetes/manifests \ - --allow-privileged \ - --hostname-override={{.domain_name}} \ - --node-labels=node-role.kubernetes.io/node \ - --cluster_dns={{.k8s_dns_service_ip}} \ - --cluster_domain=cluster.local - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/run/kubelet-pod.uuid - Restart=always - RestartSec=5 - [Install] - WantedBy=multi-user.target - -storage: - {{ if index . "pxe" }} - disks: - - device: /dev/sda - wipe_table: true - partitions: - - label: ROOT - filesystems: - - name: root - mount: - device: "/dev/sda1" - format: "ext4" - create: - force: true - options: - - "-LROOT" - {{end}} - files: - - path: /etc/kubernetes/kubelet.env - filesystem: root - mode: 0644 - contents: - inline: | - KUBELET_IMAGE_URL=quay.io/coreos/hyperkube - KUBELET_IMAGE_TAG=v1.7.3_coreos.0 - - path: /etc/hostname - filesystem: root - mode: 0644 - contents: - inline: - {{.domain_name}} - - path: /etc/sysctl.d/max-user-watches.conf - filesystem: root - contents: - inline: | - fs.inotify.max_user_watches=16184 -passwd: - users: - - name: core - ssh_authorized_keys: - - {{.ssh_authorized_key}} diff --git a/examples/terraform/modules/profiles/outputs.tf b/examples/terraform/modules/profiles/outputs.tf index 4c00256e..86762986 100644 --- a/examples/terraform/modules/profiles/outputs.tf +++ b/examples/terraform/modules/profiles/outputs.tf @@ -13,11 +13,3 @@ output "etcd3" { output "etcd3-gateway" { value = "${matchbox_profile.etcd3-gateway.name}" } - -output "bootkube-controller" { - value = "${matchbox_profile.bootkube-controller.name}" -} - -output "bootkube-worker" { - value = "${matchbox_profile.bootkube-worker.name}" -} diff --git a/examples/terraform/modules/profiles/profiles.tf b/examples/terraform/modules/profiles/profiles.tf index 5d50b043..52b94d74 100644 --- a/examples/terraform/modules/profiles/profiles.tf +++ b/examples/terraform/modules/profiles/profiles.tf @@ -78,15 +78,3 @@ resource "matchbox_profile" "etcd3-gateway" { name = "etcd3-gateway" container_linux_config = "${file("${path.module}/cl/etcd3-gateway.yaml.tmpl")}" } - -// Self-hosted Kubernetes (bootkube) Controller profile -resource "matchbox_profile" "bootkube-controller" { - name = "bootkube-controller" - container_linux_config = "${file("${path.module}/cl/bootkube-controller.yaml.tmpl")}" -} - -// Self-hosted Kubernetes (bootkube) Worker profile -resource "matchbox_profile" "bootkube-worker" { - name = "bootkube-worker" - container_linux_config = "${file("${path.module}/cl/bootkube-worker.yaml.tmpl")}" -}