diff --git a/.gitignore b/.gitignore index 3356d6c..2061135 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.tfvars +.terraform *.tfstate* +assets diff --git a/assets.tf b/assets.tf index 982d265..a1cc936 100644 --- a/assets.tf +++ b/assets.tf @@ -22,7 +22,7 @@ resource "template_dir" "manifests" { hyperkube_image = "${var.container_images["hyperkube"]}" etcd_servers = "${var.experimental_self_hosted_etcd ? format("https://%s:2379", cidrhost(var.service_cidr, 15)) : join(",", formatlist("https://%s:2379", var.etcd_servers))}" - cloud_provider = "${var.cloud_provider}" + cloud_provider = "${var.cloud_provider}" pod_cidr = "${var.pod_cidr}" service_cidr = "${var.service_cidr}" kube_dns_service_ip = "${cidrhost(var.service_cidr, 10)}" @@ -33,9 +33,9 @@ resource "template_dir" "manifests" { serviceaccount_pub = "${base64encode(tls_private_key.service-account.public_key_pem)}" serviceaccount_key = "${base64encode(tls_private_key.service-account.private_key_pem)}" - etcd_ca_cert = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}" + etcd_ca_cert = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}" etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}" - etcd_client_key = "${base64encode(tls_private_key.client.private_key_pem)}" + etcd_client_key = "${base64encode(tls_private_key.client.private_key_pem)}" } } @@ -73,4 +73,3 @@ data "template_file" "user-kubeconfig" { server = "${format("https://%s:443", element(var.api_servers, 0))}" } } - diff --git a/assets-etcd.tf b/conditional.tf similarity index 54% rename from assets-etcd.tf rename to conditional.tf index 0397820..600bc06 100644 --- a/assets-etcd.tf +++ b/conditional.tf @@ -1,8 +1,28 @@ # Assets generated only when experimental self-hosted etcd is enabled +resource "template_dir" "flannel-manifests" { + count = "${var.networking == "flannel" ? 1 : 0}" + source_dir = "${path.module}/resources/flannel" + destination_dir = "${var.asset_dir}/manifests-networking" + + vars { + pod_cidr = "${var.pod_cidr}" + } +} + +resource "template_dir" "calico-manifests" { + count = "${var.networking == "calico" ? 1 : 0}" + source_dir = "${path.module}/resources/calico" + destination_dir = "${var.asset_dir}/manifests-networking" + + vars { + pod_cidr = "${var.pod_cidr}" + } +} + # bootstrap-etcd.yaml pod bootstrap-manifest resource "template_dir" "experimental-bootstrap-manifests" { - count = "${var.experimental_self_hosted_etcd ? 1 : 0}" + count = "${var.experimental_self_hosted_etcd ? 1 : 0}" source_dir = "${path.module}/resources/experimental/bootstrap-manifests" destination_dir = "${var.asset_dir}/experimental/bootstrap-manifests" @@ -14,7 +34,7 @@ resource "template_dir" "experimental-bootstrap-manifests" { # etcd subfolder - bootstrap-etcd-service.json and migrate-etcd-cluster.json TPR resource "template_dir" "etcd-subfolder" { - count = "${var.experimental_self_hosted_etcd ? 1 : 0}" + count = "${var.experimental_self_hosted_etcd ? 1 : 0}" source_dir = "${path.module}/resources/etcd" destination_dir = "${var.asset_dir}/etcd" @@ -26,7 +46,7 @@ resource "template_dir" "etcd-subfolder" { # etcd-operator deployment and etcd-service manifests # etcd client, server, and peer tls secrets resource "template_dir" "experimental-manifests" { - count = "${var.experimental_self_hosted_etcd ? 1 : 0}" + count = "${var.experimental_self_hosted_etcd ? 1 : 0}" source_dir = "${path.module}/resources/experimental/manifests" destination_dir = "${var.asset_dir}/experimental/manifests" @@ -34,12 +54,12 @@ resource "template_dir" "experimental-manifests" { etcd_service_ip = "${cidrhost(var.service_cidr, 15)}" # Self-hosted etcd TLS certs / keys - etcd_ca_cert = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}" + etcd_ca_cert = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}" etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}" - etcd_client_key = "${base64encode(tls_private_key.client.private_key_pem)}" + etcd_client_key = "${base64encode(tls_private_key.client.private_key_pem)}" etcd_server_cert = "${base64encode(tls_locally_signed_cert.server.cert_pem)}" - etcd_server_key = "${base64encode(tls_private_key.server.private_key_pem)}" - etcd_peer_cert = "${base64encode(tls_locally_signed_cert.peer.cert_pem)}" - etcd_peer_key = "${base64encode(tls_private_key.peer.private_key_pem)}" + etcd_server_key = "${base64encode(tls_private_key.server.private_key_pem)}" + etcd_peer_cert = "${base64encode(tls_locally_signed_cert.peer.cert_pem)}" + etcd_peer_key = "${base64encode(tls_private_key.peer.private_key_pem)}" } } diff --git a/resources/calico/calico-bgp-peers.yaml b/resources/calico/calico-bgp-peers.yaml new file mode 100644 index 0000000..d10f528 --- /dev/null +++ b/resources/calico/calico-bgp-peers.yaml @@ -0,0 +1,13 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico BGP Peers +kind: CustomResourceDefinition +metadata: + name: bgppeers.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BGPPeer + plural: bgppeers + singular: bgppeer diff --git a/resources/calico/calico-cluster-role-binding.yaml b/resources/calico/calico-cluster-role-binding.yaml new file mode 100644 index 0000000..fa0f42e --- /dev/null +++ b/resources/calico/calico-cluster-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: calico-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system diff --git a/resources/calico/calico-cluster-role.yaml b/resources/calico/calico-cluster-role.yaml new file mode 100644 index 0000000..8cdc303 --- /dev/null +++ b/resources/calico/calico-cluster-role.yaml @@ -0,0 +1,53 @@ +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: calico-node + namespace: kube-system +rules: + - apiGroups: [""] + resources: + - namespaces + verbs: + - get + - list + - watch + - apiGroups: [""] + resources: + - pods/status + verbs: + - update + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - update + - watch + - apiGroups: ["extensions"] + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - bgppeers + - globalbgpconfigs + - ippools + - globalnetworkpolicies + verbs: + - create + - get + - list + - update + - watch diff --git a/resources/calico/calico-config.yaml b/resources/calico/calico-config.yaml new file mode 100644 index 0000000..0a17dc0 --- /dev/null +++ b/resources/calico/calico-config.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: calico-config + namespace: kube-system +data: + # The CNI network configuration to install on each node. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.0", + "type": "calico", + "log_level": "debug", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "ipam": { + "type": "host-local", + "subnet": "usePodCidr" + }, + "policy": { + "type": "k8s", + "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__" + }, + "kubernetes": { + "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__", + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + } diff --git a/resources/calico/calico-gloabl-felix-configs.yaml b/resources/calico/calico-gloabl-felix-configs.yaml new file mode 100644 index 0000000..97d8aa9 --- /dev/null +++ b/resources/calico/calico-gloabl-felix-configs.yaml @@ -0,0 +1,13 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico Global Felix Configuration +kind: CustomResourceDefinition +metadata: + name: globalfelixconfigs.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalFelixConfig + plural: globalfelixconfigs + singular: globalfelixconfig diff --git a/resources/calico/calico-global-bgp-configs.yaml b/resources/calico/calico-global-bgp-configs.yaml new file mode 100644 index 0000000..f33f69b --- /dev/null +++ b/resources/calico/calico-global-bgp-configs.yaml @@ -0,0 +1,13 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico Global BGP Configuration +kind: CustomResourceDefinition +metadata: + name: globalbgpconfigs.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalBGPConfig + plural: globalbgpconfigs + singular: globalbgpconfig diff --git a/resources/calico/calico-ip-pools.yaml b/resources/calico/calico-ip-pools.yaml new file mode 100644 index 0000000..3bb6804 --- /dev/null +++ b/resources/calico/calico-ip-pools.yaml @@ -0,0 +1,13 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico IP Pools +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPPool + plural: ippools + singular: ippool diff --git a/resources/calico/calico-network-policies.yaml b/resources/calico/calico-network-policies.yaml new file mode 100644 index 0000000..b28cc35 --- /dev/null +++ b/resources/calico/calico-network-policies.yaml @@ -0,0 +1,13 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +description: Calico Global Network Policies +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkPolicy + plural: globalnetworkpolicies + singular: globalnetworkpolicy diff --git a/resources/calico/calico-service-account.yaml b/resources/calico/calico-service-account.yaml new file mode 100644 index 0000000..f16b4b0 --- /dev/null +++ b/resources/calico/calico-service-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system diff --git a/resources/calico/calico.yaml b/resources/calico/calico.yaml new file mode 100644 index 0000000..30a107d --- /dev/null +++ b/resources/calico/calico.yaml @@ -0,0 +1,117 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node +spec: + selector: + matchLabels: + k8s-app: calico-node + template: + metadata: + labels: + k8s-app: calico-node + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + hostNetwork: true + serviceAccountName: calico-node + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + - name: calico-node + image: quay.io/calico/node:v2.5.1 + env: + - name: DATASTORE_TYPE + value: "kubernetes" + - name: FELIX_LOGSEVERITYSCREEN + value: "info" + - name: CLUSTER_TYPE + value: "k8s,bgp" + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + - name: FELIX_IPV6SUPPORT + value: "false" + - name: WAIT_FOR_DATASTORE + value: "true" + - name: CALICO_IPV4POOL_CIDR + value: "${pod_cidr}" + - name: CALICO_IPV4POOL_IPIP + value: "always" + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: FELIX_HEALTHENABLED + value: "true" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + livenessProbe: + httpGet: + path: /liveness + port: 9099 + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + httpGet: + path: /readiness + port: 9099 + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - name: install-cni + image: quay.io/calico/cni:v1.10.0 + command: ["/install-cni.sh"] + env: + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + - name: CNI_NET_DIR + value: "/etc/kubernetes/cni/net.d" + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + volumes: + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/kubernetes/cni/net.d + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate diff --git a/resources/manifests/kube-flannel-cfg.yaml b/resources/flannel/kube-flannel-cfg.yaml similarity index 100% rename from resources/manifests/kube-flannel-cfg.yaml rename to resources/flannel/kube-flannel-cfg.yaml diff --git a/resources/manifests/kube-flannel.yaml b/resources/flannel/kube-flannel.yaml similarity index 100% rename from resources/manifests/kube-flannel.yaml rename to resources/flannel/kube-flannel.yaml diff --git a/tls-etcd.tf b/tls-etcd.tf index e021be5..35807fc 100644 --- a/tls-etcd.tf +++ b/tls-etcd.tf @@ -139,7 +139,7 @@ resource "tls_cert_request" "server" { common_name = "etcd-server" organization = "etcd" } - + ip_addresses = [ "127.0.0.1", "${cidrhost(var.service_cidr, 15)}", @@ -185,11 +185,11 @@ resource "tls_cert_request" "peer" { common_name = "etcd-peer" organization = "etcd" } - + ip_addresses = [ - "${cidrhost(var.service_cidr, 20)}" + "${cidrhost(var.service_cidr, 20)}", ] - + dns_names = "${concat( var.etcd_servers, list( diff --git a/variables.tf b/variables.tf index f74f1f7..4d1752f 100644 --- a/variables.tf +++ b/variables.tf @@ -29,6 +29,12 @@ variable "cloud_provider" { default = "" } +variable "networking" { + description = "Choice of networking provider (flannel or calico)" + type = "string" + default = "flannel" +} + variable "pod_cidr" { description = "CIDR IP range to assign Kubernetes pods" type = "string"