resources: Add experimental self-hosted etcd manifests

.gitignore

@@ -0,0 +1,2 @@
+*.tfvars
+*.tfstate*

README.md

@@ -2,19 +2,9 @@
 
 `bootkube-terraform` is a Terraform module that renders [bootkube](https://github.com/kubernetes-incubator/bootkube) assets, just like running the binary `bootkube render`. It aims to provide the same variable names, defaults, features, and outputs.
 
-## Status
-
-Warning: This project may move.
-
-TODO:
-
-* Experimental manifests
-* etcd TLS
-* Self-hosted etcd
-
 ## Usage
 
-Use the `bootkube-terraform` module within your existing Terraform configs. See the input `variables.tf` of example `terraform.tfvars.example`.
+Use the `bootkube-terraform` module within your existing Terraform configs. Provide the variables listed in `variables.tf` or check `terraform.tfvars.example` for examples.
 
 ```hcl
 module "bootkube" {
@@ -24,6 +14,7 @@ module "bootkube" {
   api_servers = ["node1.example.com"]
   etcd_servers = ["http://127.0.0.1:2379"]
   output_path = "/home/core/clusters/mycluster"
+  experimental_self_hosted_etcd = false
 }
 ```
 
@@ -41,6 +32,8 @@ terraform apply
 
 Render bootkube assets directly with bootkube v0.4.2.
 
+#### On-host etcd
+
 ```sh
 bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --etcd-servers=http://127.0.0.1:2379
 ```
@@ -50,3 +43,16 @@ Compare assets. The only diffs you should see are TLS credentials.
 ```sh
 diff -rw assets /home/core/cluster/mycluster
 ```
+
+#### Self-hosted etcd
+
+```sh
+bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --experimental-self-hosted-etcd
+```
+
+Compare assets. Note that experimental must be generated to a separate directory for terraform applies to sync. Move the experimental `bootstrap-manifests` and `manifests` files during deployment.
+
+```sh
+diff -rw assets /home/core/cluster/mycluster
+```
+

assets.tf

@@ -5,7 +5,7 @@ resource "template_dir" "bootstrap-manifests" {
 
   vars {
     hyperkube_image = "${var.container_images["hyperkube"]}"
-    etcd_servers    = "${join(",", var.etcd_servers)}"
+    etcd_servers    = "${var.experimental_self_hosted_etcd ? format("http://%s:2379", var.kube_etcd_service_ip) : join(",", var.etcd_servers)}"
 
     cloud_provider = "${var.cloud_provider}"
     pod_cidr       = "${var.pod_cidr}"
@@ -20,7 +20,7 @@ resource "template_dir" "manifests" {
 
   vars {
     hyperkube_image = "${var.container_images["hyperkube"]}"
-    etcd_servers    = "${join(",", var.etcd_servers)}"
+    etcd_servers    = "${var.experimental_self_hosted_etcd ? format("http://%s:2379", var.kube_etcd_service_ip) : join(",", var.etcd_servers)}"
 
     cloud_provider = "${var.cloud_provider}"
     pod_cidr       = "${var.pod_cidr}"
@@ -36,6 +36,7 @@ resource "template_dir" "manifests" {
   }
 }
 
+
 # Generated kubeconfig (auth/kubeconfig)
 data "template_file" "kubeconfig" {
   template = "${file("${path.module}/resources/kubeconfig")}"

etcd-assets.tf

@@ -0,0 +1,41 @@
+# Experimental self-hosted etcd
+
+# Bootstrap etcd pod
+
+data "template_file" "bootstrap-etcd" {
+  template = "${file("${path.module}/resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml")}"
+  vars {
+    etcd_image = "${var.container_images["etcd"]}"
+  }
+}
+
+resource "local_file" "bootstrap-etcd" {
+  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
+  content  = "${data.template_file.bootstrap-etcd.rendered}"
+  filename = "${var.output_path}/experimental/bootstrap-manifests/bootstrap-etcd.yaml"
+}
+
+# etcd operator deployment and etcd service
+
+resource "local_file" "etcd-operator" {
+  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
+  depends_on = ["template_dir.manifests"]
+
+  content  = "${file("${path.module}/resources/experimental/manifests/etcd-operator.yaml")}"
+  filename = "${var.output_path}/experimental/manifests/etcd-operator.yaml"
+}
+
+data "template_file" "etcd-service" {
+  template = "${file("${path.module}/resources/experimental/manifests/etcd-service.yaml")}"
+  vars {
+    etcd_service_ip = "${var.kube_etcd_service_ip}"
+  }
+}
+
+resource "local_file" "etcd-service" {
+  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
+  depends_on = ["template_dir.manifests"]
+
+  content  = "${data.template_file.etcd-service.rendered}"
+  filename = "${var.output_path}/experimental/manifests/etcd-service.yaml"
+}

resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: bootstrap-etcd
+  namespace: kube-system
+  labels:
+    k8s-app: boot-etcd
+spec:
+  containers:
+  - name: etcd
+    image: ${etcd_image}
+    command:
+    - /usr/local/bin/etcd
+    - --name=boot-etcd
+    - --listen-client-urls=http://0.0.0.0:12379
+    - --listen-peer-urls=http://0.0.0.0:12380
+    - --advertise-client-urls=http://$(MY_POD_IP):12379
+    - --initial-advertise-peer-urls=http://$(MY_POD_IP):12380
+    - --initial-cluster=boot-etcd=http://$(MY_POD_IP):12380
+    - --initial-cluster-token=bootkube
+    - --initial-cluster-state=new
+    - --data-dir=/var/etcd/data
+    env:
+      - name: MY_POD_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.podIP
+  hostNetwork: true
+  restartPolicy: Never

resources/experimental/manifests/etcd-operator.yaml

@@ -0,0 +1,30 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: etcd-operator
+  namespace: kube-system
+  labels:
+    k8s-app: etcd-operator
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: etcd-operator
+    spec:
+      containers:
+      - name: etcd-operator
+        image: quay.io/coreos/etcd-operator:v0.2.5 
+        env:
+        - name: MY_POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: MY_POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule

resources/experimental/manifests/etcd-service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: etcd-service
+  namespace: kube-system
+spec:
+  selector:
+    app: etcd
+    etcd_cluster: kube-etcd
+  clusterIP: ${etcd_service_ip}
+  ports:
+  - name: client
+    port: 2379
+    protocol: TCP

resources/manifests/kube-etcd-network-checkpointer.yaml

@@ -0,0 +1,48 @@
+apiVersion: "extensions/v1beta1"
+kind: DaemonSet
+metadata:
+  name: kube-etcd-network-checkpointer
+  namespace: kube-system
+  labels:
+    tier: control-plane
+    k8s-app: kube-etcd-network-checkpointer
+spec:
+  template:
+    metadata:
+      labels:
+        tier: control-plane
+        k8s-app: kube-etcd-network-checkpointer
+      annotations:
+        checkpointer.alpha.coreos.com/checkpoint: "true"
+    spec:
+      containers:
+      - image: quay.io/coreos/kenc:48b6feceeee56c657ea9263f47b6ea091e8d3035
+        name: kube-etcd-network-checkpointer
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /etc/kubernetes/selfhosted-etcd
+          name: checkpoint-dir
+          readOnly: false
+        - mountPath: /var/lock
+          name: var-lock
+          readOnly: false
+        command:
+        - /usr/bin/flock
+        - /var/lock/kenc.lock
+        - -c
+        - "kenc -r -m iptables && kenc -m iptables"
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      volumes:
+      - name: checkpoint-dir
+        hostPath:
+          path: /etc/kubernetes/checkpoint-iptables
+      - name: var-lock
+        hostPath:
+          path: /var/lock

terraform.tfvars.example

@@ -2,3 +2,4 @@ cluster_name = "example"
 api_servers = ["node1.example.com"]
 etcd_servers = ["http://127.0.0.1:2379"]
 output_path = "/home/core/clusters/mycluster"
+experimental_self_hosted_etcd = false

variables.tf

@@ -13,6 +13,11 @@ variable "etcd_servers" {
   type        = "list"
 }
 
+variable "experimental_self_hosted_etcd" {
+  description = "(Experimental) Create self-hosted etcd assets"
+  default = false
+}
+
 variable "output_path" {
   description = "Path to a directory where generated assets should be placed (contains secrets)"
   type        = "string"
@@ -42,6 +47,7 @@ variable "container_images" {
 
   default = {
     hyperkube = "quay.io/coreos/hyperkube:v1.6.2_coreos.0"
+    etcd = "quay.io/coreos/etcd:v3.1.6"
   }
 }
 
@@ -57,6 +63,12 @@ variable "kube_dns_service_ip" {
   default     = "10.3.0.10"
 }
 
+variable "kube_etcd_service_ip" {
+  description = "Kubernetes service IP for self-hosted etcd (must be within server_cidr)"
+  type = "string"
+  default = "10.3.0.15"
+}
+
 variable "ca_certificate" {
   description = "Existing PEM-encoded CA certificate (generated if blank)"
   type        = "string"