Add support for calico networking

* Add support for using Calico pod networking instead of flannel
* Add variable "networking" which may be "calico" or "flannel"
* Users MUST move the contents of assets_dir/manifests-networking
into the assets_dir/manifests directory before running bootkube
start. This is needed because Terraform cannot generate conditional
files into a template_dir because other resources write to the same
directory and delete.
https://github.com/terraform-providers/terraform-provider-template/issues/10

.gitignore

@@ -1,2 +1,4 @@
 *.tfvars
+.terraform
 *.tfstate*
+assets

assets.tf

@@ -22,7 +22,7 @@ resource "template_dir" "manifests" {
     hyperkube_image = "${var.container_images["hyperkube"]}"
     etcd_servers    = "${var.experimental_self_hosted_etcd ? format("https://%s:2379", cidrhost(var.service_cidr, 15)) : join(",", formatlist("https://%s:2379", var.etcd_servers))}"
 
-    cloud_provider = "${var.cloud_provider}"
+    cloud_provider      = "${var.cloud_provider}"
     pod_cidr            = "${var.pod_cidr}"
     service_cidr        = "${var.service_cidr}"
     kube_dns_service_ip = "${cidrhost(var.service_cidr, 10)}"
@@ -33,9 +33,9 @@ resource "template_dir" "manifests" {
     serviceaccount_pub = "${base64encode(tls_private_key.service-account.public_key_pem)}"
     serviceaccount_key = "${base64encode(tls_private_key.service-account.private_key_pem)}"
 
-    etcd_ca_cert = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
+    etcd_ca_cert     = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
     etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}"
-    etcd_client_key = "${base64encode(tls_private_key.client.private_key_pem)}"
+    etcd_client_key  = "${base64encode(tls_private_key.client.private_key_pem)}"
   }
 }
 
@@ -73,4 +73,3 @@ data "template_file" "user-kubeconfig" {
     server       = "${format("https://%s:443", element(var.api_servers, 0))}"
   }
 }
-

conditional.tf

@@ -1,8 +1,28 @@
 # Assets generated only when experimental self-hosted etcd is enabled
 
+resource "template_dir" "flannel-manifests" {
+  count           = "${var.networking == "flannel" ? 1 : 0}"
+  source_dir      = "${path.module}/resources/flannel"
+  destination_dir = "${var.asset_dir}/manifests-networking"
+
+  vars {
+    pod_cidr = "${var.pod_cidr}"
+  }
+}
+
+resource "template_dir" "calico-manifests" {
+  count           = "${var.networking == "calico" ? 1 : 0}"
+  source_dir      = "${path.module}/resources/calico"
+  destination_dir = "${var.asset_dir}/manifests-networking"
+
+  vars {
+    pod_cidr = "${var.pod_cidr}"
+  }
+}
+
 # bootstrap-etcd.yaml pod bootstrap-manifest
 resource "template_dir" "experimental-bootstrap-manifests" {
-  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
+  count           = "${var.experimental_self_hosted_etcd ? 1 : 0}"
   source_dir      = "${path.module}/resources/experimental/bootstrap-manifests"
   destination_dir = "${var.asset_dir}/experimental/bootstrap-manifests"
 
@@ -14,7 +34,7 @@ resource "template_dir" "experimental-bootstrap-manifests" {
 
 # etcd subfolder - bootstrap-etcd-service.json and migrate-etcd-cluster.json TPR
 resource "template_dir" "etcd-subfolder" {
-  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
+  count           = "${var.experimental_self_hosted_etcd ? 1 : 0}"
   source_dir      = "${path.module}/resources/etcd"
   destination_dir = "${var.asset_dir}/etcd"
 
@@ -26,7 +46,7 @@ resource "template_dir" "etcd-subfolder" {
 # etcd-operator deployment and etcd-service manifests
 # etcd client, server, and peer tls secrets
 resource "template_dir" "experimental-manifests" {
-  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
+  count           = "${var.experimental_self_hosted_etcd ? 1 : 0}"
   source_dir      = "${path.module}/resources/experimental/manifests"
   destination_dir = "${var.asset_dir}/experimental/manifests"
 
@@ -34,12 +54,12 @@ resource "template_dir" "experimental-manifests" {
     etcd_service_ip = "${cidrhost(var.service_cidr, 15)}"
 
     # Self-hosted etcd TLS certs / keys
-    etcd_ca_cert = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
+    etcd_ca_cert     = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
     etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}"
-    etcd_client_key = "${base64encode(tls_private_key.client.private_key_pem)}"
+    etcd_client_key  = "${base64encode(tls_private_key.client.private_key_pem)}"
     etcd_server_cert = "${base64encode(tls_locally_signed_cert.server.cert_pem)}"
-    etcd_server_key = "${base64encode(tls_private_key.server.private_key_pem)}"
-    etcd_peer_cert = "${base64encode(tls_locally_signed_cert.peer.cert_pem)}"
-    etcd_peer_key = "${base64encode(tls_private_key.peer.private_key_pem)}"
+    etcd_server_key  = "${base64encode(tls_private_key.server.private_key_pem)}"
+    etcd_peer_cert   = "${base64encode(tls_locally_signed_cert.peer.cert_pem)}"
+    etcd_peer_key    = "${base64encode(tls_private_key.peer.private_key_pem)}"
   }
 }

resources/calico/calico-bgp-peers.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico BGP Peers
+kind: CustomResourceDefinition
+metadata:
+  name: bgppeers.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: BGPPeer
+    plural: bgppeers
+    singular: bgppeer

resources/calico/calico-cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-node
+subjects:
+- kind: ServiceAccount
+  name: calico-node
+  namespace: kube-system

resources/calico/calico-cluster-role.yaml

@@ -0,0 +1,53 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: calico-node
+  namespace: kube-system
+rules:
+  - apiGroups: [""]
+    resources:
+      - namespaces
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: [""]
+    resources:
+      - pods/status
+    verbs:
+      - update
+  - apiGroups: [""]
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: [""]
+    resources:
+      - nodes
+    verbs:
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups: ["extensions"]
+    resources:
+      - networkpolicies
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - globalfelixconfigs
+      - bgppeers
+      - globalbgpconfigs
+      - ippools
+      - globalnetworkpolicies
+    verbs:
+      - create
+      - get
+      - list
+      - update
+      - watch

resources/calico/calico-config.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: calico-config
+  namespace: kube-system
+data:
+  # The CNI network configuration to install on each node.
+  cni_network_config: |-
+    {
+        "name": "k8s-pod-network",
+        "cniVersion": "0.3.0",
+        "type": "calico",
+        "log_level": "debug",
+        "datastore_type": "kubernetes",
+        "nodename": "__KUBERNETES_NODE_NAME__",
+        "ipam": {
+            "type": "host-local",
+            "subnet": "usePodCidr"
+        },
+        "policy": {
+            "type": "k8s",
+            "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
+        },
+        "kubernetes": {
+            "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
+            "kubeconfig": "__KUBECONFIG_FILEPATH__"
+        }
+    }

resources/calico/calico-gloabl-felix-configs.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Global Felix Configuration
+kind: CustomResourceDefinition
+metadata:
+   name: globalfelixconfigs.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: GlobalFelixConfig
+    plural: globalfelixconfigs
+    singular: globalfelixconfig

resources/calico/calico-global-bgp-configs.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Global BGP Configuration
+kind: CustomResourceDefinition
+metadata:
+  name: globalbgpconfigs.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: GlobalBGPConfig
+    plural: globalbgpconfigs
+    singular: globalbgpconfig

resources/calico/calico-ip-pools.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico IP Pools
+kind: CustomResourceDefinition
+metadata:
+  name: ippools.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: IPPool
+    plural: ippools
+    singular: ippool

resources/calico/calico-network-policies.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Global Network Policies
+kind: CustomResourceDefinition
+metadata:
+  name: globalnetworkpolicies.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: GlobalNetworkPolicy
+    plural: globalnetworkpolicies
+    singular: globalnetworkpolicy

resources/calico/calico-service-account.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-node
+  namespace: kube-system

resources/calico/calico.yaml

@@ -0,0 +1,117 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: calico-node
+  namespace: kube-system
+  labels:
+    k8s-app: calico-node
+spec:
+  selector:
+    matchLabels:
+      k8s-app: calico-node
+  template:
+    metadata:
+      labels:
+        k8s-app: calico-node
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      hostNetwork: true
+      serviceAccountName: calico-node
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+      containers:
+        - name: calico-node
+          image: quay.io/calico/node:v2.5.1
+          env:
+            - name: DATASTORE_TYPE
+              value: "kubernetes"
+            - name: FELIX_LOGSEVERITYSCREEN
+              value: "info"
+            - name: CLUSTER_TYPE
+              value: "k8s,bgp"
+            - name: CALICO_DISABLE_FILE_LOGGING
+              value: "true"
+            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
+              value: "ACCEPT"
+            - name: FELIX_IPV6SUPPORT
+              value: "false"
+            - name: WAIT_FOR_DATASTORE
+              value: "true"
+            - name: CALICO_IPV4POOL_CIDR
+              value: "${pod_cidr}"
+            - name: CALICO_IPV4POOL_IPIP
+              value: "always"
+            - name: NODENAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: FELIX_HEALTHENABLED
+              value: "true"
+          securityContext:
+            privileged: true
+          resources:
+            requests:
+              cpu: 250m
+          livenessProbe:
+            httpGet:
+              path: /liveness
+              port: 9099
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            failureThreshold: 6
+          readinessProbe:
+            httpGet:
+              path: /readiness
+              port: 9099
+            periodSeconds: 10
+          volumeMounts:
+            - mountPath: /lib/modules
+              name: lib-modules
+            - mountPath: /var/run/calico
+              name: var-run-calico
+              readOnly: false
+        - name: install-cni
+          image: quay.io/calico/cni:v1.10.0
+          command: ["/install-cni.sh"]
+          env:
+            - name: CNI_NETWORK_CONFIG
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: cni_network_config
+            - name: CNI_NET_DIR
+              value: "/etc/kubernetes/cni/net.d"
+            - name: KUBERNETES_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+      volumes:
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: var-run-calico
+          hostPath:
+            path: /var/run/calico
+        - name: cni-bin-dir
+          hostPath:
+            path: /opt/cni/bin
+        - name: cni-net-dir
+          hostPath:
+            path: /etc/kubernetes/cni/net.d
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

tls-etcd.tf

@@ -139,7 +139,7 @@ resource "tls_cert_request" "server" {
     common_name  = "etcd-server"
     organization = "etcd"
   }
-  
+
   ip_addresses = [
     "127.0.0.1",
     "${cidrhost(var.service_cidr, 15)}",
@@ -185,11 +185,11 @@ resource "tls_cert_request" "peer" {
     common_name  = "etcd-peer"
     organization = "etcd"
   }
-  
+
   ip_addresses = [
-    "${cidrhost(var.service_cidr, 20)}"
+    "${cidrhost(var.service_cidr, 20)}",
   ]
-  
+
   dns_names = "${concat(
     var.etcd_servers,
     list(

variables.tf

@@ -29,6 +29,12 @@ variable "cloud_provider" {
   default     = ""
 }
 
+variable "networking" {
+  description = "Choice of networking provider (flannel or calico)"
+  type        = "string"
+  default     = "flannel"
+}
+
 variable "pod_cidr" {
   description = "CIDR IP range to assign Kubernetes pods"
   type        = "string"