From 909d33e123cf482d772e19e0aacf4d5a0b53cf1f Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Thu, 11 May 2017 12:40:27 -0700 Subject: [PATCH] resources: Add experimental self-hosted etcd manifests --- .gitignore | 2 + README.md | 28 ++++++----- assets.tf | 5 +- etcd-assets.tf | 41 ++++++++++++++++ .../bootstrap-manifests/bootstrap-etcd.yaml | 29 +++++++++++ .../experimental/manifests/etcd-operator.yaml | 30 ++++++++++++ .../experimental/manifests/etcd-service.yaml | 14 ++++++ .../kube-etcd-network-checkpointer.yaml | 48 +++++++++++++++++++ terraform.tfvars.example | 1 + variables.tf | 12 +++++ 10 files changed, 197 insertions(+), 13 deletions(-) create mode 100644 .gitignore create mode 100644 etcd-assets.tf create mode 100644 resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml create mode 100644 resources/experimental/manifests/etcd-operator.yaml create mode 100644 resources/experimental/manifests/etcd-service.yaml create mode 100644 resources/manifests/kube-etcd-network-checkpointer.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3356d6c --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.tfvars +*.tfstate* diff --git a/README.md b/README.md index 9b250b6..6ce63c6 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,9 @@ `bootkube-terraform` is a Terraform module that renders [bootkube](https://github.com/kubernetes-incubator/bootkube) assets, just like running the binary `bootkube render`. It aims to provide the same variable names, defaults, features, and outputs. -## Status - -Warning: This project may move. - -TODO: - -* Experimental manifests -* etcd TLS -* Self-hosted etcd - ## Usage -Use the `bootkube-terraform` module within your existing Terraform configs. See the input `variables.tf` of example `terraform.tfvars.example`. +Use the `bootkube-terraform` module within your existing Terraform configs. Provide the variables listed in `variables.tf` or check `terraform.tfvars.example` for examples. ```hcl module "bootkube" { @@ -24,6 +14,7 @@ module "bootkube" { api_servers = ["node1.example.com"] etcd_servers = ["http://127.0.0.1:2379"] output_path = "/home/core/clusters/mycluster" + experimental_self_hosted_etcd = false } ``` @@ -41,6 +32,8 @@ terraform apply Render bootkube assets directly with bootkube v0.4.2. +#### On-host etcd + ```sh bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --etcd-servers=http://127.0.0.1:2379 ``` @@ -50,3 +43,16 @@ Compare assets. The only diffs you should see are TLS credentials. ```sh diff -rw assets /home/core/cluster/mycluster ``` + +#### Self-hosted etcd + +```sh +bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --experimental-self-hosted-etcd +``` + +Compare assets. Note that experimental must be generated to a separate directory for terraform applies to sync. Move the experimental `bootstrap-manifests` and `manifests` files during deployment. + +```sh +diff -rw assets /home/core/cluster/mycluster +``` + diff --git a/assets.tf b/assets.tf index 2658466..d94fc92 100644 --- a/assets.tf +++ b/assets.tf @@ -5,7 +5,7 @@ resource "template_dir" "bootstrap-manifests" { vars { hyperkube_image = "${var.container_images["hyperkube"]}" - etcd_servers = "${join(",", var.etcd_servers)}" + etcd_servers = "${var.experimental_self_hosted_etcd ? format("http://%s:2379", var.kube_etcd_service_ip) : join(",", var.etcd_servers)}" cloud_provider = "${var.cloud_provider}" pod_cidr = "${var.pod_cidr}" @@ -20,7 +20,7 @@ resource "template_dir" "manifests" { vars { hyperkube_image = "${var.container_images["hyperkube"]}" - etcd_servers = "${join(",", var.etcd_servers)}" + etcd_servers = "${var.experimental_self_hosted_etcd ? format("http://%s:2379", var.kube_etcd_service_ip) : join(",", var.etcd_servers)}" cloud_provider = "${var.cloud_provider}" pod_cidr = "${var.pod_cidr}" @@ -36,6 +36,7 @@ resource "template_dir" "manifests" { } } + # Generated kubeconfig (auth/kubeconfig) data "template_file" "kubeconfig" { template = "${file("${path.module}/resources/kubeconfig")}" diff --git a/etcd-assets.tf b/etcd-assets.tf new file mode 100644 index 0000000..489ab2c --- /dev/null +++ b/etcd-assets.tf @@ -0,0 +1,41 @@ +# Experimental self-hosted etcd + +# Bootstrap etcd pod + +data "template_file" "bootstrap-etcd" { + template = "${file("${path.module}/resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml")}" + vars { + etcd_image = "${var.container_images["etcd"]}" + } +} + +resource "local_file" "bootstrap-etcd" { + count = "${var.experimental_self_hosted_etcd ? 1 : 0}" + content = "${data.template_file.bootstrap-etcd.rendered}" + filename = "${var.output_path}/experimental/bootstrap-manifests/bootstrap-etcd.yaml" +} + +# etcd operator deployment and etcd service + +resource "local_file" "etcd-operator" { + count = "${var.experimental_self_hosted_etcd ? 1 : 0}" + depends_on = ["template_dir.manifests"] + + content = "${file("${path.module}/resources/experimental/manifests/etcd-operator.yaml")}" + filename = "${var.output_path}/experimental/manifests/etcd-operator.yaml" +} + +data "template_file" "etcd-service" { + template = "${file("${path.module}/resources/experimental/manifests/etcd-service.yaml")}" + vars { + etcd_service_ip = "${var.kube_etcd_service_ip}" + } +} + +resource "local_file" "etcd-service" { + count = "${var.experimental_self_hosted_etcd ? 1 : 0}" + depends_on = ["template_dir.manifests"] + + content = "${data.template_file.etcd-service.rendered}" + filename = "${var.output_path}/experimental/manifests/etcd-service.yaml" +} diff --git a/resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml b/resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml new file mode 100644 index 0000000..58001fe --- /dev/null +++ b/resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Pod +metadata: + name: bootstrap-etcd + namespace: kube-system + labels: + k8s-app: boot-etcd +spec: + containers: + - name: etcd + image: ${etcd_image} + command: + - /usr/local/bin/etcd + - --name=boot-etcd + - --listen-client-urls=http://0.0.0.0:12379 + - --listen-peer-urls=http://0.0.0.0:12380 + - --advertise-client-urls=http://$(MY_POD_IP):12379 + - --initial-advertise-peer-urls=http://$(MY_POD_IP):12380 + - --initial-cluster=boot-etcd=http://$(MY_POD_IP):12380 + - --initial-cluster-token=bootkube + - --initial-cluster-state=new + - --data-dir=/var/etcd/data + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + hostNetwork: true + restartPolicy: Never diff --git a/resources/experimental/manifests/etcd-operator.yaml b/resources/experimental/manifests/etcd-operator.yaml new file mode 100644 index 0000000..534531f --- /dev/null +++ b/resources/experimental/manifests/etcd-operator.yaml @@ -0,0 +1,30 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: etcd-operator + namespace: kube-system + labels: + k8s-app: etcd-operator +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: etcd-operator + spec: + containers: + - name: etcd-operator + image: quay.io/coreos/etcd-operator:v0.2.5 + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule diff --git a/resources/experimental/manifests/etcd-service.yaml b/resources/experimental/manifests/etcd-service.yaml new file mode 100644 index 0000000..bb1bf94 --- /dev/null +++ b/resources/experimental/manifests/etcd-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: etcd-service + namespace: kube-system +spec: + selector: + app: etcd + etcd_cluster: kube-etcd + clusterIP: ${etcd_service_ip} + ports: + - name: client + port: 2379 + protocol: TCP diff --git a/resources/manifests/kube-etcd-network-checkpointer.yaml b/resources/manifests/kube-etcd-network-checkpointer.yaml new file mode 100644 index 0000000..c3c823c --- /dev/null +++ b/resources/manifests/kube-etcd-network-checkpointer.yaml @@ -0,0 +1,48 @@ +apiVersion: "extensions/v1beta1" +kind: DaemonSet +metadata: + name: kube-etcd-network-checkpointer + namespace: kube-system + labels: + tier: control-plane + k8s-app: kube-etcd-network-checkpointer +spec: + template: + metadata: + labels: + tier: control-plane + k8s-app: kube-etcd-network-checkpointer + annotations: + checkpointer.alpha.coreos.com/checkpoint: "true" + spec: + containers: + - image: quay.io/coreos/kenc:48b6feceeee56c657ea9263f47b6ea091e8d3035 + name: kube-etcd-network-checkpointer + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/kubernetes/selfhosted-etcd + name: checkpoint-dir + readOnly: false + - mountPath: /var/lock + name: var-lock + readOnly: false + command: + - /usr/bin/flock + - /var/lock/kenc.lock + - -c + - "kenc -r -m iptables && kenc -m iptables" + hostNetwork: true + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + volumes: + - name: checkpoint-dir + hostPath: + path: /etc/kubernetes/checkpoint-iptables + - name: var-lock + hostPath: + path: /var/lock diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 975d45a..4d88ca9 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -2,3 +2,4 @@ cluster_name = "example" api_servers = ["node1.example.com"] etcd_servers = ["http://127.0.0.1:2379"] output_path = "/home/core/clusters/mycluster" +experimental_self_hosted_etcd = false diff --git a/variables.tf b/variables.tf index 91c26bc..601b6d4 100644 --- a/variables.tf +++ b/variables.tf @@ -13,6 +13,11 @@ variable "etcd_servers" { type = "list" } +variable "experimental_self_hosted_etcd" { + description = "(Experimental) Create self-hosted etcd assets" + default = false +} + variable "output_path" { description = "Path to a directory where generated assets should be placed (contains secrets)" type = "string" @@ -42,6 +47,7 @@ variable "container_images" { default = { hyperkube = "quay.io/coreos/hyperkube:v1.6.2_coreos.0" + etcd = "quay.io/coreos/etcd:v3.1.6" } } @@ -57,6 +63,12 @@ variable "kube_dns_service_ip" { default = "10.3.0.10" } +variable "kube_etcd_service_ip" { + description = "Kubernetes service IP for self-hosted etcd (must be within server_cidr)" + type = "string" + default = "10.3.0.15" +} + variable "ca_certificate" { description = "Existing PEM-encoded CA certificate (generated if blank)" type = "string"