Add KUBE_GCE_ENABLE_IP_ALIASES flag to the cluster turn up scripts.

KUBE_GCE_ENABLE_IP_ALIASES=true will enable allocation of PodCIDR ips
using the ip alias mechanism rather than using routes.

NODE_IP_RANGE will control the node instance IP cidr
KUBE_GCE_IP_ALIAS_SIZE controls the size of each podCIDR
IP_ALIAS_SUBNETWORK controls the name of the subnet created for the cluster

cluster/common.sh

@@ -734,6 +734,17 @@ EOF
 FEATURE_GATES: $(yaml-quote ${FEATURE_GATES})
 EOF
   fi
+
+  if [ -n "${PROVIDER_VARS:-}" ]; then
+    local var_name
+    local var_value
+
+    for var_name in ${PROVIDER_VARS}; do
+      eval "local var_value=\$(yaml-quote \${${var_name}})"
+      echo "${var_name}: ${var_value}" >>$file
+    done
+  fi
+
   if [[ "${master}" == "true" ]]; then
     # Master-only env vars.
     cat >>$file <<EOF

cluster/gce/config-default.sh

@@ -77,15 +77,16 @@ INITIAL_ETCD_CLUSTER="${MASTER_NAME}"
 ETCD_QUORUM_READ="${ENABLE_ETCD_QUORUM_READ:-false}"
 MASTER_TAG="${INSTANCE_PREFIX}-master"
 NODE_TAG="${INSTANCE_PREFIX}-minion"
-MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
+
 CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.244.0.0/14}"
+MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
+
 if [[ "${FEDERATION:-}" == true ]]; then
     NODE_SCOPES="${NODE_SCOPES:-compute-rw,monitoring,logging-write,storage-ro,https://www.googleapis.com/auth/ndev.clouddns.readwrite}"
 else
     NODE_SCOPES="${NODE_SCOPES:-compute-rw,monitoring,logging-write,storage-ro}"
 fi
 
-
 # Extra docker options for nodes.
 EXTRA_DOCKER_OPTS="${EXTRA_DOCKER_OPTS:-}"
 
@@ -173,6 +174,25 @@ fi
 # Optional: Enable Rescheduler
 ENABLE_RESCHEDULER="${KUBE_ENABLE_RESCHEDULER:-true}"
 
+# Optional: Enable allocation of pod IPs using IP aliases.
+#
+# ALPHA FEATURE.
+#
+# IP_ALIAS_SIZE is the size of the podCIDR allocated to a node.
+# IP_ALIAS_SUBNETWORK is the subnetwork to allocate from. If empty, a
+#   new subnetwork will be created for the cluster.
+ENABLE_IP_ALIASES=${KUBE_GCE_ENABLE_IP_ALIASES:-false}
+if [ ${ENABLE_IP_ALIASES} = true ]; then
+  # Size of ranges allocated to each node. gcloud alpha supports only /32 and /24.
+  IP_ALIAS_SIZE=${KUBE_GCE_IP_ALIAS_SIZE:-/24}
+  IP_ALIAS_SUBNETWORK=${KUBE_GCE_IP_ALIAS_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-default}
+  # NODE_IP_RANGE is used when ENABLE_IP_ALIASES=true. It is the primary range in
+  # the subnet and is the range used for node instance IPs.
+  NODE_IP_RANGE="${NODE_IP_RANGE:-10.40.0.0/22}"
+  # Add to the provider custom variables.
+  PROVIDER_VARS="${PROVIDER_VARS} ENABLE_IP_ALIASES"
+fi
+
 # Admission Controllers to invoke prior to persisting objects in cluster
 # If we included ResourceQuota, we should keep it at the end of the list to prevent incrementing quota usage prematurely.
 ADMISSION_CONTROL=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota

cluster/gce/config-test.sh

@@ -79,8 +79,13 @@ INITIAL_ETCD_CLUSTER="${MASTER_NAME}"
 ETCD_QUORUM_READ="${ENABLE_ETCD_QUORUM_READ:-false}"
 MASTER_TAG="${INSTANCE_PREFIX}-master"
 NODE_TAG="${INSTANCE_PREFIX}-minion"
+
 CLUSTER_IP_RANGE="${CLUSTER_IP_RANGE:-10.180.0.0/14}"
 MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}"
+# NODE_IP_RANGE is used when ENABLE_IP_ALIASES=true. It is the primary range in
+# the subnet and is the range used for node instance IPs.
+NODE_IP_RANGE="${NODE_IP_RANGE:-10.40.0.0/22}"
+
 RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}"
 
 # Optional: set feature gates
@@ -198,6 +203,25 @@ fi
 # Optional: Enable Rescheduler
 ENABLE_RESCHEDULER="${KUBE_ENABLE_RESCHEDULER:-true}"
 
+# Optional: Enable allocation of pod IPs using IP aliases.
+#
+# ALPHA FEATURE.
+#
+# IP_ALIAS_SIZE is the size of the podCIDR allocated to a node.
+# IP_ALIAS_SUBNETWORK is the subnetwork to allocate from. If empty, a
+#   new subnetwork will be created for the cluster.
+ENABLE_IP_ALIASES=${KUBE_GCE_ENABLE_IP_ALIASES:-false}
+if [ ${ENABLE_IP_ALIASES} = true ]; then
+  # Size of ranges allocated to each node. gcloud alpha supports only /32 and /24.
+  IP_ALIAS_SIZE=${KUBE_GCE_IP_ALIAS_SIZE:-/24}
+  IP_ALIAS_SUBNETWORK=${KUBE_GCE_IP_ALIAS_SUBNETWORK:-${INSTANCE_PREFIX}-subnet-default}
+  # NODE_IP_RANGE is used when ENABLE_IP_ALIASES=true. It is the primary range in
+  # the subnet and is the range used for node instance IPs.
+  NODE_IP_RANGE="${NODE_IP_RANGE:-10.40.0.0/22}"
+  # Add to the provider custom variables.
+  PROVIDER_VARS="${PROVIDER_VARS} ENABLE_IP_ALIASES"
+fi
+
 # If we included ResourceQuota, we should keep it at the end of the list to prevent incrementing quota usage prematurely.
 ADMISSION_CONTROL="${KUBE_ADMISSION_CONTROL:-NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,PodPreset,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota}"
 

cluster/gce/configure-vm.sh

@@ -585,6 +585,11 @@ EOF
     if [ -n "${SCHEDULING_ALGORITHM_PROVIDER:-}" ]; then
       cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
 scheduling_algorithm_provider: '$(echo "${SCHEDULING_ALGORITHM_PROVIDER}" | sed -e "s/'/''/g")'
+EOF
+    fi
+    if [ -n "${ENABLE_IP_ALIASES:-}" ]; then
+      cat <<EOF >>/srv/salt-overlay/pillar/cluster-params.sls
+enable_ip_aliases: '$(echo "$ENABLE_IP_ALIASES" | sed -e "s/'/''/g")'
 EOF
     fi
 }

cluster/gce/container-linux/configure-helper.sh

@@ -997,6 +997,10 @@ function start-kube-controller-manager {
   if [[ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]]; then
     params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
   fi
+  if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
+    params+=" --cidr-allocator-type=CloudAllocator"
+    params+=" --configure-cloud-routes=false"
+  fi
   if [[ -n "${FEATURE_GATES:-}" ]]; then
     params+=" --feature-gates=${FEATURE_GATES}"
   fi

cluster/gce/container-linux/master-helper.sh

@@ -31,11 +31,11 @@ source "${KUBE_ROOT}/cluster/gce/container-linux/helper.sh"
 #   detect-project
 #   get-bearer-token
 function create-master-instance {
-  local address_opt=""
+  local address=""
-  [[ -n ${1:-} ]] && address_opt="--address ${1}"
+  [[ -n ${1:-} ]] && address="${1}"
 
   write-master-env
-  create-master-instance-internal "${MASTER_NAME}" "${address_opt}"
+  create-master-instance-internal "${MASTER_NAME}" "${address}"
 }
 
 function replicate-master-instance() {
@@ -65,38 +65,58 @@ function replicate-master-instance() {
 
 
 function create-master-instance-internal() {
+  local gcloud="gcloud"
+  if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
+    gcloud="gcloud alpha"
+  fi
+
   local -r master_name="${1}"
-  local -r address_option="${2:-}"
+  local -r address="${2:-}"
 
   local preemptible_master=""
   if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]]; then
     preemptible_master="--preemptible --maintenance-policy TERMINATE"
   fi
 
-  gcloud compute instances create "${master_name}" \
+  local network=$(make-gcloud-network-argument \
-    ${address_option} \
+    "${NETWORK}" "${address:-}" \
+    "${ENABLE_IP_ALIASES:-}" "${IP_ALIAS_SUBNETWORK:-}" "${IP_ALIAS_SIZE:-}")
+
+  local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml"
+  metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/container-linux/master.yaml"
+  metadata="${metadata},configure-sh=${KUBE_ROOT}/cluster/gce/container-linux/configure.sh"
+  metadata="${metadata},cluster-name=${KUBE_TEMP}/cluster-name.txt"
+
+  local disk="name=${master_name}-pd"
+  disk="${disk},device-name=master-pd"
+  disk="${disk},mode=rw"
+  disk="${disk},boot=no"
+  disk="${disk},auto-delete=no"
+
+  ${gcloud} compute instances create "${master_name}" \
     --project "${PROJECT}" \
     --zone "${ZONE}" \
     --machine-type "${MASTER_SIZE}" \
     --image-project="${MASTER_IMAGE_PROJECT}" \
     --image "${MASTER_IMAGE}" \
     --tags "${MASTER_TAG}" \
-    --network "${NETWORK}" \
     --scopes "storage-ro,compute-rw,monitoring,logging-write" \
-    --can-ip-forward \
+    --metadata-from-file "${metadata}" \
-    --metadata-from-file \
+    --disk "${disk}" \
-      "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/container-linux/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/container-linux/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt" \
-    --disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \
     --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-30}" \
-    ${preemptible_master}
+    ${preemptible_master} \
+    ${network}
 }
 
 function get-metadata() {
   local zone="${1}"
   local name="${2}"
   local key="${3}"
+
+  local metadata_url="http://metadata.google.internal/computeMetadata/v1/instance/attributes/${key}"
+
   gcloud compute ssh "${name}" \
     --project "${PROJECT}" \
     --zone "${zone}" \
-    --command "curl \"http://metadata.google.internal/computeMetadata/v1/instance/attributes/${key}\" -H \"Metadata-Flavor: Google\"" 2>/dev/null
+    --command "curl '${metadata_url}' -H 'Metadata-Flavor: Google'" 2>/dev/null
 }

cluster/gce/gci/configure-helper.sh

@@ -1205,6 +1205,10 @@ function start-kube-controller-manager {
   if [[ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]]; then
     params+=" --terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}"
   fi
+  if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
+    params+=" --cidr-allocator-type=CloudAllocator"
+    params+=" --configure-cloud-routes=false"
+  fi
   if [[ -n "${FEATURE_GATES:-}" ]]; then
     params+=" --feature-gates=${FEATURE_GATES}"
   fi

cluster/gce/gci/master-helper.sh

@@ -31,12 +31,12 @@ source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
 #   detect-project
 #   get-bearer-token
 function create-master-instance {
-  local address_opt=""
+  local address=""
-  [[ -n ${1:-} ]] && address_opt="--address ${1}"
+  [[ -n ${1:-} ]] && address="${1}"
 
   write-master-env
   ensure-gci-metadata-files
-  create-master-instance-internal "${MASTER_NAME}" "${address_opt}"
+  create-master-instance-internal "${MASTER_NAME}" "${address}"
 }
 
 function replicate-master-instance() {
@@ -74,30 +74,51 @@ function replicate-master-instance() {
 
 
 function create-master-instance-internal() {
+  local gcloud="gcloud"
+  if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
+    gcloud="gcloud alpha"
+  fi
+
   local -r master_name="${1}"
-  local -r address_option="${2:-}"
+  local -r address="${2:-}"
 
   local preemptible_master=""
   if [[ "${PREEMPTIBLE_MASTER:-}" == "true" ]]; then
     preemptible_master="--preemptible --maintenance-policy TERMINATE"
   fi
 
-  gcloud compute instances create "${master_name}" \
+  local network=$(make-gcloud-network-argument \
-    ${address_option} \
+    "${NETWORK}" "${address:-}" \
+    "${ENABLE_IP_ALIASES:-}" "${IP_ALIAS_SUBNETWORK:-}" "${IP_ALIAS_SIZE:-}")
+
+  local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml"
+  metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml"
+  metadata="${metadata},configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh"
+  metadata="${metadata},cluster-name=${KUBE_TEMP}/cluster-name.txt"
+  metadata="${metadata},gci-update-strategy=${KUBE_TEMP}/gci-update.txt"
+  metadata="${metadata},gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt"
+  metadata="${metadata},gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt"
+  metadata="${metadata},kube-master-certs=${KUBE_TEMP}/kube-master-certs.yaml"
+
+  local disk="name=${master_name}-pd"
+  disk="${disk},device-name=master-pd"
+  disk="${disk},mode=rw"
+  disk="${disk},boot=no"
+  disk="${disk},auto-delete=no"
+
+  ${gcloud} compute instances create "${master_name}" \
     --project "${PROJECT}" \
     --zone "${ZONE}" \
     --machine-type "${MASTER_SIZE}" \
     --image-project="${MASTER_IMAGE_PROJECT}" \
     --image "${MASTER_IMAGE}" \
     --tags "${MASTER_TAG}" \
-    --network "${NETWORK}" \
     --scopes "storage-ro,compute-rw,monitoring,logging-write" \
-    --can-ip-forward \
+    --metadata-from-file "${metadata}" \
-    --metadata-from-file \
+    --disk "${disk}" \
-      "kube-env=${KUBE_TEMP}/master-kube-env.yaml,user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml,configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,cluster-name=${KUBE_TEMP}/cluster-name.txt,gci-update-strategy=${KUBE_TEMP}/gci-update.txt,gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt,kube-master-certs=${KUBE_TEMP}/kube-master-certs.yaml" \
-    --disk "name=${master_name}-pd,device-name=master-pd,mode=rw,boot=no,auto-delete=no" \
     --boot-disk-size "${MASTER_ROOT_DISK_SIZE:-10}" \
-    ${preemptible_master}
+    ${preemptible_master} \
+    ${network}
 }
 
 function get-metadata() {

cluster/gce/util.sh

@@ -449,6 +449,35 @@ function create-firewall-rule() {
   done
 }
 
+# Format the string argument for gcloud network.
+function make-gcloud-network-argument() {
+  local network="$1"
+  local address="$2"          # optional
+  local enable_ip_alias="$3"  # optional
+  local alias_subnetwork="$4" # optional
+  local alias_size="$5"       # optional
+
+  local ret=""
+
+  if [[ "${enable_ip_alias}" == 'true' ]]; then
+    ret="--network-interface"
+    ret="${ret} network=${network}"
+    # If address is omitted, instance will not receive an external IP.
+    ret="${ret},address=${address:-}"
+    ret="${ret},subnet=${alias_subnetwork}"
+    ret="${ret},aliases=pods-default:${alias_size}"
+    ret="${ret} --no-can-ip-forward"
+  else
+    ret="--network ${network}"
+    ret="${ret} --can-ip-forward"
+    if [[ -n ${address:-} ]]; then
+      ret="${ret} --address ${address}"
+    fi
+  fi
+
+  echo "${ret}"
+}
+
 # $1: version (required)
 function get-template-name-from-version() {
   # trim template name to pass gce name validation
@@ -475,20 +504,34 @@ function create-node-template() {
     fi
   fi
 
-  local attempt=1
+  local gcloud="gcloud"
+  if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
+    gcloud="gcloud alpha"
+  fi
+
   local preemptible_minions=""
   if [[ "${PREEMPTIBLE_NODE}" == "true" ]]; then
     preemptible_minions="--preemptible --maintenance-policy TERMINATE"
   fi
+
   local local_ssds=""
   if [ ! -z ${NODE_LOCAL_SSDS+x} ]; then
       for i in $(seq ${NODE_LOCAL_SSDS}); do
           local_ssds="$local_ssds--local-ssd=interface=SCSI "
       done
   fi
+
+  local network=$(make-gcloud-network-argument \
+    "${NETWORK}" "" \
+    "${ENABLE_IP_ALIASES:-}" \
+    "${IP_ALIAS_SUBNETWORK:-}" \
+    "${IP_ALIAS_SIZE:-}")
+
+  local attempt=1
   while true; do
     echo "Attempt ${attempt} to create ${1}" >&2
-    if ! gcloud compute instance-templates create "$template_name" \
+    if ! ${gcloud} compute instance-templates create \
+      "$template_name" \
       --project "${PROJECT}" \
       --machine-type "${NODE_SIZE}" \
       --boot-disk-type "${NODE_DISK_TYPE}" \
@@ -496,11 +539,11 @@ function create-node-template() {
       --image-project="${NODE_IMAGE_PROJECT}" \
       --image "${NODE_IMAGE}" \
       --tags "${NODE_TAG}" \
-      --network "${NETWORK}" \
       ${local_ssds} \
+      --region "${REGION}" \
+      ${network} \
       ${preemptible_minions} \
       $2 \
-      --can-ip-forward \
       --metadata-from-file $(echo ${@:3} | tr ' ' ',') >&2; then
         if (( attempt > 5 )); then
           echo -e "${color_red}Failed to create instance template $template_name ${color_norm}" >&2
@@ -597,6 +640,7 @@ function kube-up() {
   if [[ ${KUBE_USE_EXISTING_MASTER:-} == "true" ]]; then
     detect-master
     parse-master-env
+    create-subnetwork
     create-nodes
   elif [[ ${KUBE_REPLICATE_EXISTING_MASTER:-} == "true" ]]; then
     if  [[ "${MASTER_OS_DISTRIBUTION}" != "gci" && "${MASTER_OS_DISTRIBUTION}" != "debian" ]]; then
@@ -612,6 +656,7 @@ function kube-up() {
   else
     check-existing
     create-network
+    create-subnetwork
     write-cluster-name
     create-autoscaler-config
     create-master
@@ -680,6 +725,48 @@ function create-network() {
   fi
 }
 
+function create-subnetwork() {
+  case ${ENABLE_IP_ALIASES} in
+    true) ;;
+    false) return;;
+    *) echo "${color_red}Invalid argument to ENABLE_IP_ALIASES${color_norm}"
+       exit 1;;
+  esac
+
+  # Look for the subnet, it must exist and have a secondary range
+  # configured.
+  local subnet=$(gcloud alpha compute networks subnets describe \
+    --region ${REGION} ${IP_ALIAS_SUBNETWORK} 2>/dev/null)
+  if [[ -z ${subnet} ]]; then
+    # Only allow auto-creation for default subnets
+    if [[ ${IP_ALIAS_SUBNETWORK} != ${INSTANCE_PREFIX}-subnet-default ]]; then
+      echo "${color_red}Subnetwork ${NETWORK}:${IP_ALIAS_SUBNETWORK} does not exist${color_norm}"
+      exit 1
+    fi
+
+    if [ -z ${NODE_IP_RANGE:-} ]; then
+      echo "${color_red}NODE_IP_RANGE must be specified{color_norm}"
+      exit 1
+    fi
+
+    echo "Creating subnet ${NETWORK}:${IP_ALIAS_SUBNETWORK}"
+    gcloud alpha compute networks subnets create \
+      ${IP_ALIAS_SUBNETWORK} \
+      --description "Automatically generated subnet for ${INSTANCE_PREFIX} cluster. This will be removed on cluster teardown." \
+      --network ${NETWORK} \
+      --region ${REGION} \
+      --range ${NODE_IP_RANGE} \
+      --secondary-range "name=pods-default,range=${CLUSTER_IP_RANGE}"
+
+    echo "Created subnetwork ${IP_ALIAS_SUBNETWORK}"
+  else
+    if ! echo ${subnet} | grep --quiet secondaryIpRanges ${subnet}; then
+      echo "${color_red}Subnet ${IP_ALIAS_SUBNETWORK} does not have a secondary range${color_norm}"
+      exit 1
+    fi
+  fi
+}
+
 function delete-firewall-rules() {
   for fw in $@; do
     if [[ -n $(gcloud compute firewall-rules --project "${PROJECT}" describe "${fw}" --format='value(name)' 2>/dev/null || true) ]]; then
@@ -701,6 +788,24 @@ function delete-network() {
   fi
 }
 
+function delete-subnetwork() {
+  if [[ ${ENABLE_IP_ALIASES:-} != "true" ]]; then
+    return
+  fi
+
+  # Only delete automatically created subnets.
+  if [[ ${IP_ALIAS_SUBNETWORK} != ${INSTANCE_PREFIX}-subnet-default ]]; then
+    return
+  fi
+
+  echo "Removing auto-created subnet ${NETWORK}:${IP_ALIAS_SUBNETWORK}"
+  if [[ -n $(gcloud alpha compute networks subnets describe \
+        --region ${REGION} ${IP_ALIAS_SUBNETWORK} 2>/dev/null) ]]; then
+    gcloud alpha --quiet compute networks subnets delete \
+      --region ${REGION} ${IP_ALIAS_SUBNETWORK}
+  fi
+}
+
 # Assumes:
 #   NUM_NODES
 # Sets:
@@ -1414,6 +1519,9 @@ function kube-down() {
       "${CLUSTER_NAME}-default-internal-node" \
       "${NETWORK}-default-ssh" \
       "${NETWORK}-default-internal"  # Pre-1.5 clusters
+
+    delete-subnetwork
+
     if [[ "${KUBE_DELETE_NETWORK}" == "true" ]]; then
       delete-network || true  # might fail if there are leaked firewall rules
     fi

cluster/kube-up.sh

@@ -32,7 +32,6 @@ fi
 
 source "${KUBE_ROOT}/cluster/kube-util.sh"
 
-
 if [ -z "${ZONE-}" ]; then
   echo "... Starting cluster using provider: ${KUBERNETES_PROVIDER}" >&2
 else

cluster/kube-util.sh

@@ -28,6 +28,12 @@ else
     KUBERNETES_PROVIDER="${KUBERNETES_PROVIDER:-gce}"
 fi
 
+# PROVIDER_VARS is a list of cloud provider specific variables. Note:
+# this is a list of the _names_ of the variables, not the value of the
+# variables. Providers can add variables to be appended to kube-env.
+# (see `build-kube-env`).
+PROVIDER_VARS=""
+
 PROVIDER_UTILS="${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh"
 if [ -f ${PROVIDER_UTILS} ]; then
     source "${PROVIDER_UTILS}"