Configure the Calico MTU

* Add a network_mtu input variable (default 1500)
* Set the Calico CNI config (i.e. workload network interfaces)
* Set the Calico IP in IP MTU (for tunnel network interfaces)

conditional.tf

@@ -16,7 +16,8 @@ resource "template_dir" "calico-manifests" {
   destination_dir = "${var.asset_dir}/manifests-networking"
 
   vars {
-    pod_cidr = "${var.pod_cidr}"
+    network_mtu = "${var.network_mtu}"
+    pod_cidr    = "${var.pod_cidr}"
   }
 }
 

resources/calico/calico-config.yaml

@@ -13,6 +13,7 @@ data:
         "log_level": "debug",
         "datastore_type": "kubernetes",
         "nodename": "__KUBERNETES_NODE_NAME__",
+        "mtu": ${network_mtu},
         "ipam": {
             "type": "host-local",
             "subnet": "usePodCidr"

resources/calico/calico.yaml

@@ -19,32 +19,50 @@ spec:
       hostNetwork: true
       serviceAccountName: calico-node
       tolerations:
+        # Allow the pod to run on master nodes
         - key: node-role.kubernetes.io/master
           effect: NoSchedule
+        # Mark the pod as a critical add-on for rescheduling
         - key: "CriticalAddonsOnly"
           operator: "Exists"
       containers:
         - name: calico-node
           image: quay.io/calico/node:v2.5.1
           env:
+            # Use Kubernetes API as the backing datastore.
             - name: DATASTORE_TYPE
               value: "kubernetes"
+            # Enable felix info logging.
             - name: FELIX_LOGSEVERITYSCREEN
               value: "info"
+            # Cluster type to identify the deployment type
             - name: CLUSTER_TYPE
               value: "k8s,bgp"
+            # Disable file logging so `kubectl logs` works.
             - name: CALICO_DISABLE_FILE_LOGGING
               value: "true"
+            # Set Felix endpoint to host default action to ACCEPT.
             - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
               value: "ACCEPT"
+            # Disable IPV6 on Kubernetes.
             - name: FELIX_IPV6SUPPORT
               value: "false"
+            # Set MTU for tunnel device used if ipip is enabled
+            - name: FELIX_IPINIPMTU
+              value: "${network_mtu}"
+            # Wait for the datastore.
             - name: WAIT_FOR_DATASTORE
               value: "true"
+            # The Calico IPv4 pool CIDR (should match `--cluster-cidr`).
             - name: CALICO_IPV4POOL_CIDR
               value: "${pod_cidr}"
+            # Enable IPIP
             - name: CALICO_IPV4POOL_IPIP
               value: "always"
+            # Enable IP-in-IP within Felix.
+            - name: FELIX_IPINIPENABLED
+              value: "true"
+            # Set node name based on k8s nodeName.
             - name: NODENAME
               valueFrom:
                 fieldRef:
@@ -75,9 +93,11 @@ spec:
           volumeMounts:
             - mountPath: /lib/modules
               name: lib-modules
+              readOnly: true
             - mountPath: /var/run/calico
               name: var-run-calico
               readOnly: false
+        # Install Calico CNI binaries and CNI network config file on nodes
         - name: install-cni
           image: quay.io/calico/cni:v1.10.0
           command: ["/install-cni.sh"]
@@ -89,6 +109,7 @@ spec:
                   key: cni_network_config
             - name: CNI_NET_DIR
               value: "/etc/kubernetes/cni/net.d"
+            # Set node name based on k8s nodeName
             - name: KUBERNETES_NODE_NAME
               valueFrom:
                 fieldRef:

variables.tf

@@ -35,6 +35,12 @@ variable "networking" {
   default     = "flannel"
 }
 
+variable "network_mtu" {
+  description = "CNI interface MTU (applies to calico only)"
+  type        = "string"
+  default     = "1500"
+}
+
 variable "pod_cidr" {
   description = "CIDR IP range to assign Kubernetes pods"
   type        = "string"