Merge pull request #18436 from gmarek/1000-kube-up

Allow creation of clusters larger than 500 nodes
2025-11-03 19:58:17 +00:00 · 2015-12-10 11:43:56 -08:00
parent 1b70a403bb 0c6126911b
commit b9aa71089e
2 changed files with 111 additions and 57 deletions
--- a/cluster/gce/upgrade.sh
+++ b/cluster/gce/upgrade.sh
@@ -174,6 +174,7 @@ function upgrade-nodes() {
 #
 # Vars set:
 #   SANITIZED_VERSION
+#   INSTANCE_GROUPS
 #   KUBELET_TOKEN
 #   KUBE_PROXY_TOKEN
 #   CA_CERT_BASE64
@@ -184,7 +185,7 @@ function prepare-node-upgrade() {
  echo "== Preparing node upgrade (to ${KUBE_VERSION}). ==" >&2
  SANITIZED_VERSION=$(echo ${KUBE_VERSION} | sed 's/[\.\+]/-/g')

-  detect-node-names
+  detect-node-names # sets INSTANCE_GROUPS

  # TODO(zmerlynn): Refactor setting scope flags.
  local scope_flags=
@@ -231,16 +232,18 @@ function do-node-upgrade() {
    subgroup="alpha compute"
  fi
  local template_name=$(get-template-name-from-version ${SANITIZED_VERSION})
+  for group in ${INSTANCE_GROUPS[@]}; do
    gcloud ${subgroup} rolling-updates \
        --project="${PROJECT}" \
        --zone="${ZONE}" \
        start \
-      --group="${NODE_INSTANCE_PREFIX}-group" \
+        --group="${group}" \
        --template="${template_name}" \
        --instance-startup-timeout=300s \
        --max-num-concurrent-instances=1 \
        --max-num-failed-instances=0 \
        --min-instance-update-time=0s
+  done

  # TODO(zmerlynn): Wait for the rolling-update to finish.

--- a/cluster/gce/util.sh
+++ b/cluster/gce/util.sh
@@ -253,12 +253,24 @@ function upload-server-tars() {
 #   NODE_INSTANCE_PREFIX
 # Vars set:
 #   NODE_NAMES
+#   INSTANCE_GROUPS
 function detect-node-names {
  detect-project
-  NODE_NAMES=($(gcloud compute instance-groups managed list-instances \
-    "${NODE_INSTANCE_PREFIX}-group" --zone "${ZONE}" --project "${PROJECT}" \
+  INSTANCE_GROUPS=()
+  INSTANCE_GROUPS+=($(gcloud compute instance-groups managed list --zone "${ZONE}" --project "${PROJECT}" | grep ${NODE_INSTANCE_PREFIX} | cut -f1 -d" "))
+  NODE_NAMES=()
+  if [[ -n "${INSTANCE_GROUPS[@]:-}" ]]; then
+    for group in "${INSTANCE_GROUPS[@]}"; do
+      NODE_NAMES+=($(gcloud compute instance-groups managed list-instances \
+        "${group}" --zone "${ZONE}" --project "${PROJECT}" \
        --format=yaml | grep instance: | cut -d ' ' -f 2))
+    done
+    echo "INSTANCE_GROUPS=${INSTANCE_GROUPS[*]}" >&2
    echo "NODE_NAMES=${NODE_NAMES[*]}" >&2
+  else 
+    echo "INSTANCE_GROUPS=" >&2
+    echo "NODE_NAMES=" >&2
+  fi
 }

 # Detect the information about the minions
@@ -713,17 +725,43 @@ function kube-up {
  
  create-node-instance-template $template_name

+  local defaulted_max_instances_per_mig=${MAX_INSTANCES_PER_MIG:-500}
+
+  if [[ ${defaulted_max_instances_per_mig} -le "0" ]]; then
+    echo "MAX_INSTANCES_PER_MIG cannot be negative. Assuming default 500"
+    defaulted_max_instances_per_mig=500
+  fi
+  local num_migs=$(((${NUM_NODES} + ${defaulted_max_instances_per_mig} - 1) / ${defaulted_max_instances_per_mig}))
+  local instances_per_mig=$(((${NUM_NODES} + ${num_migs} - 1) / ${num_migs}))
+  local last_mig_size=$((${NUM_NODES} - (${num_migs} - 1) * ${instances_per_mig}))
+
+  #TODO: parallelize this loop to speed up the process
+  for i in $(seq $((${num_migs} - 1))); do
    gcloud compute instance-groups managed \
-      create "${NODE_INSTANCE_PREFIX}-group" \
+        create "${NODE_INSTANCE_PREFIX}-group-$i" \
        --project "${PROJECT}" \
        --zone "${ZONE}" \
        --base-instance-name "${NODE_INSTANCE_PREFIX}" \
-      --size "${NUM_NODES}" \
+        --size "${instances_per_mig}" \
        --template "$template_name" || true;
    gcloud compute instance-groups managed wait-until-stable \
-      "${NODE_INSTANCE_PREFIX}-group" \
+        "${NODE_INSTANCE_PREFIX}-group-$i" \
 	      --zone "${ZONE}" \
 	      --project "${PROJECT}" || true;
+  done
+
+  gcloud compute instance-groups managed \
+      create "${NODE_INSTANCE_PREFIX}-group-${num_migs}" \
+      --project "${PROJECT}" \
+      --zone "${ZONE}" \
+      --base-instance-name "${NODE_INSTANCE_PREFIX}" \
+      --size "${last_mig_size}" \
+      --template "$template_name" || true;
+  gcloud compute instance-groups managed wait-until-stable \
+      "${NODE_INSTANCE_PREFIX}-group-${num_migs}" \
+      --zone "${ZONE}" \
+      --project "${PROJECT}" || true;
+
  detect-node-names
  detect-master

@@ -742,9 +780,12 @@ function kube-up {
    METRICS+="--custom-metric-utilization metric=custom.cloudmonitoring.googleapis.com/kubernetes.io/memory/node_reservation,"
    METRICS+="utilization-target=${TARGET_NODE_UTILIZATION},utilization-target-type=GAUGE "

-    echo "Creating node autoscaler."
-    gcloud compute instance-groups managed set-autoscaling "${NODE_INSTANCE_PREFIX}-group" --zone "${ZONE}" --project "${PROJECT}" \
+    echo "Creating node autoscalers."
+
+    for i in $(seq ${num_migs}); do
+      gcloud compute instance-groups managed set-autoscaling "${NODE_INSTANCE_PREFIX}-group-$i" --zone "${ZONE}" --project "${PROJECT}" \
          --min-num-replicas "${AUTOSCALER_MIN_NODES}" --max-num-replicas "${AUTOSCALER_MAX_NODES}" ${METRICS} || true
+    done
  fi

  echo "Waiting up to ${KUBE_CLUSTER_INITIALIZATION_TIMEOUT} seconds for cluster initialization."
@@ -810,31 +851,35 @@ function kube-up {
 # down the firewall rules and routes.
 function kube-down {
  detect-project
+  detect-node-names # For INSTANCE_GROUPS

  echo "Bringing down cluster"
  set +e  # Do not stop on error

-  # Delete autoscaler for nodes if present.
+  # Delete autoscaler for nodes if present. We assume that all or none instance groups have an autoscaler
  local autoscaler
  autoscaler=( $(gcloud compute instance-groups managed list --zone "${ZONE}" --project "${PROJECT}" \
-                 | grep "${NODE_INSTANCE_PREFIX}-group" \
+                 | grep "${NODE_INSTANCE_PREFIX}-group-1" \
                 | awk '{print $7}') )
  if [[ "${autoscaler:-}" == "yes" ]]; then
-    gcloud compute instance-groups managed stop-autoscaling "${NODE_INSTANCE_PREFIX}-group" --zone "${ZONE}" --project "${PROJECT}"
+    for group in ${INSTANCE_GROUPS[@]}; do
+      gcloud compute instance-groups managed stop-autoscaling "${group}" --zone "${ZONE}" --project "${PROJECT}"
+    done
  fi

  # Get the name of the managed instance group template before we delete the
  # managed instange group. (The name of the managed instnace group template may
  # change during a cluster upgrade.)
-  local template=$(get-template "${PROJECT}" "${ZONE}" "${NODE_INSTANCE_PREFIX}-group")
+  local template=$(get-template "${PROJECT}" "${ZONE}" "${NODE_INSTANCE_PREFIX}-group-1")

  # The gcloud APIs don't return machine parseable error codes/retry information. Therefore the best we can
  # do is parse the output and special case particular responses we are interested in.
-  if gcloud compute instance-groups managed describe "${NODE_INSTANCE_PREFIX}-group" --project "${PROJECT}" --zone "${ZONE}" &>/dev/null; then
+  for group in ${INSTANCE_GROUPS[@]}; do 
+    if gcloud compute instance-groups managed describe "${group}" --project "${PROJECT}" --zone "${ZONE}" &>/dev/null; then
      deleteCmdOutput=$(gcloud compute instance-groups managed delete --zone "${ZONE}" \
        --project "${PROJECT}" \
        --quiet \
-      "${NODE_INSTANCE_PREFIX}-group")
+        "${group}")
      if [[ "$deleteCmdOutput" != ""  ]]; then
        # Managed instance group deletion is done asynchronously, we must wait for it to complete, or subsequent steps fail
        deleteCmdOperationId=$(echo $deleteCmdOutput | grep "Operation:" | sed "s/.*Operation:[[:space:]]*\([^[:space:]]*\).*/\1/g")
@@ -850,6 +895,7 @@ function kube-down {
        fi
      fi
    fi
+  done

  if gcloud compute instance-templates describe --project "${PROJECT}" "${template}" &>/dev/null; then
    gcloud compute instance-templates delete \
@@ -982,12 +1028,13 @@ function get-template {
 #   KUBE_RESOURCE_FOUND
 function check-resources {
  detect-project
+  detect-node-names

  echo "Looking for already existing resources"
  KUBE_RESOURCE_FOUND=""

-  if gcloud compute instance-groups managed describe --project "${PROJECT}" --zone "${ZONE}" "${NODE_INSTANCE_PREFIX}-group" &>/dev/null; then
-    KUBE_RESOURCE_FOUND="Managed instance group ${NODE_INSTANCE_PREFIX}-group"
+  if [[ -n "${INSTANCE_GROUPS[@]:-}" ]]; then
+    KUBE_RESOURCE_FOUND="Managed instance groups ${INSTANCE_GROUPS[@]}"
    return 1
  fi

@@ -1090,11 +1137,13 @@ function prepare-push() {
    create-node-instance-template $tmp_template_name

    local template_name="${NODE_INSTANCE_PREFIX}-template"
+    for group in ${INSTANCE_GROUPS[@]}; do
      gcloud compute instance-groups managed \
-      set-instance-template "${NODE_INSTANCE_PREFIX}-group" \
+        set-instance-template "${group}" \
        --template "$tmp_template_name" \
        --zone "${ZONE}" \
        --project "${PROJECT}" || true;
+    done

    gcloud compute instance-templates delete \
      --project "${PROJECT}" \
@@ -1103,11 +1152,13 @@ function prepare-push() {

    create-node-instance-template "$template_name"

+    for group in ${INSTANCE_GROUPS[@]}; do
      gcloud compute instance-groups managed \
-      set-instance-template "${NODE_INSTANCE_PREFIX}-group" \
+        set-instance-template "${group}" \
        --template "$template_name" \
        --zone "${ZONE}" \
        --project "${PROJECT}" || true;
+    done

    gcloud compute instance-templates delete \
      --project "${PROJECT}" \