Merge pull request #46193 from cblecker/staging-godeps-dockerized

Automatic merge from submit-queue (batch tested with PRs 43443, 46193, 49071, 47252)

Run the update-staging-godeps script inside a docker container

**What this PR does / why we need it**:
This PR moves the update-staging-godeps script to run inside a docker container.

**Which issue this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)*: fixes #45757 

**Special notes for your reviewer**:

**Release note**:

```release-note
NONE
```

/assign @ixdy @sttts

build/build-image/Dockerfile

@@ -36,6 +36,13 @@ ENV KUBE_OUTPUT_SUBPATH _output/dockerized
 # Pick up version stuff here as we don't copy our .git over.
 ENV KUBE_GIT_VERSION_FILE ${HOME}/.dockerized-kube-version-defs
 
+# Add system-wide git user information
+RUN git config --system user.email "nobody@k8s.io" \
+  && git config --system user.name "kube-build-image"
+
+# Fix permissions on gopath
+RUN chmod -R a+rwx $GOPATH
+
 # Make log messages use the right timezone
 ADD localtime /etc/localtime
 RUN chmod a+r /etc/localtime

build/build-image/cross/Dockerfile

@@ -37,7 +37,7 @@ RUN for platform in ${KUBE_CROSSPLATFORMS}; do GOOS=${platform%/*} GOARCH=${plat
 
 # Install g++, then download and install protoc for generating protobuf output
 RUN apt-get update \
-  && apt-get install -y g++ rsync apt-utils file patch \
+  && apt-get install -y g++ rsync jq apt-utils file patch \
   && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /usr/local/src/protobuf \
@@ -62,14 +62,13 @@ RUN echo "deb http://archive.ubuntu.com/ubuntu xenial main universe" > /etc/apt/
 
 # work around 64MB tmpfs size in Docker 1.6
 ENV TMPDIR /tmp.k8s
-
-# Get the code coverage tool, goimports, and godep
 RUN mkdir $TMPDIR \
   && chmod a+rwx $TMPDIR \
-  && chmod o+t $TMPDIR \
+  && chmod o+t $TMPDIR
-  && go get golang.org/x/tools/cmd/cover \
+
-            golang.org/x/tools/cmd/goimports \
+# Get the code coverage tool and goimports
-            github.com/tools/godep
+RUN go get golang.org/x/tools/cmd/cover \
+            golang.org/x/tools/cmd/goimports
 
 # Download and symlink etcd. We need this for our integration tests.
 RUN export ETCD_VERSION=v3.0.17; \

build/build-image/cross/VERSION

@@ -1 +1 @@
-v1.8.3-1
+v1.8.3-2

build/build-image/rsyncd.sh

@@ -77,7 +77,7 @@ port = 8730
   secrets file = ${SECRETS}
   read only = false
   path = ${VOLUME}
-  filter = - /.make/ - /.git/ - /_tmp/
+  filter = - /.make/ - /_tmp/
 EOF
 
 exec /usr/bin/rsync --no-detach --daemon --config="${CONFFILE}" "$@"

build/common.sh

@@ -682,14 +682,10 @@ function kube::build::sync_to_container() {
   # they will not be re-generated by 'make'.
   kube::build::rsync \
     --delete \
-    --filter='+ /staging/**' \
-    --filter='- /.git/' \
     --filter='- /.make/' \
     --filter='- /_tmp/' \
     --filter='- /_output/' \
     --filter='- /' \
-    --filter='- zz_generated.*' \
-    --filter='- generated.proto' \
     "${KUBE_ROOT}/" "rsync://k8s@${KUBE_RSYNC_ADDR}/k8s/"
 
   kube::build::stop_rsyncd_container
@@ -715,8 +711,10 @@ function kube::build::copy_output() {
   # We are looking to copy out all of the built binaries along with various
   # generated files.
   kube::build::rsync \
-    --filter='- /vendor/' \
     --filter='- /_temp/' \
+    --filter='+ /vendor/' \
+    --filter='+ /Godeps/' \
+    --filter='+ /staging/***/Godeps/**' \
     --filter='+ /_output/dockerized/bin/**' \
     --filter='+ zz_generated.*' \
     --filter='+ generated.proto' \

hack/godep-restore.sh

@@ -24,6 +24,6 @@ source "${KUBE_ROOT}/hack/lib/util.sh"
 
 kube::util::ensure_godep_version v79
 
-echo "Starting to download all kubernetes godeps. This takes a while"
+kube::log::status "Starting to download all kubernetes godeps. This takes a while"
 GOPATH=${GOPATH}:${KUBE_ROOT}/staging godep restore "$@"
-echo "Download finished"
+kube::log::status "Download finished"

hack/lib/golang.sh

@@ -334,11 +334,13 @@ EOF
 
   local go_version
   go_version=($(go version))
-  if [[ "${go_version[2]}" < "go1.8" && "${go_version[2]}" != "devel" ]]; then
+  local minimum_go_version
+  minimum_go_version=go1.8.3
+  if [[ "${go_version[2]}" < "${minimum_go_version}" && "${go_version[2]}" != "devel" ]]; then
     kube::log::usage_from_stdin <<EOF
 Detected go version: ${go_version[*]}.
-Kubernetes requires go version 1.8 or greater.
+Kubernetes requires ${minimum_go_version} or greater.
-Please install Go version 1.8 or later.
+Please install ${minimum_go_version} or later.
 EOF
     return 2
   fi

hack/lib/util.sh

@@ -496,7 +496,7 @@ kube::util::ensure_clean_working_dir() {
 # Ensure that the given godep version is installed and in the path
 kube::util::ensure_godep_version() {
   GODEP_VERSION=${1:-"v79"}
-  if [[ "$(godep version)" == *"godep ${GODEP_VERSION}"* ]]; then
+  if [[ "$(godep version 2>/dev/null)" == *"godep ${GODEP_VERSION}"* ]]; then
     return
   fi
 
@@ -505,7 +505,7 @@ kube::util::ensure_godep_version() {
 
   GOPATH="${KUBE_TEMP}/go" go get -d -u github.com/tools/godep 2>/dev/null
   pushd "${KUBE_TEMP}/go/src/github.com/tools/godep" >/dev/null
-    git checkout "${GODEP_VERSION}"
+    git checkout -q "${GODEP_VERSION}"
     GOPATH="${KUBE_TEMP}/go" go install .
   popd >/dev/null
 
@@ -825,6 +825,18 @@ function kube::util::ensure-cfssl {
   popd > /dev/null
 }
 
+# kube::util::ensure_dockerized
+# Confirms that the script is being run inside a kube-build image
+#
+function kube::util::ensure_dockerized {
+  if [[ -f /kube-build-image ]]; then
+    return 0
+  else
+    echo "ERROR: This script is designed to be run inside a kube-build container"
+    exit 1
+  fi
+}
+
 # Some useful colors.
 if [[ -z "${color_start-}" ]]; then
   declare -r color_start="\033["

hack/update-staging-godeps-dockerized.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# updates the godeps.json file in the staging folders to allow clean vendoring
+# based on kubernetes levels.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+# Sets default values
+DRY_RUN=false
+FAIL_ON_DIFF=false
+GODEP_OPTS=""
+
+# Controls verbosity of the script output and logging.
+KUBE_VERBOSE="${KUBE_VERBOSE:-5}"
+if (( ${KUBE_VERBOSE} >= 6 )); then
+  GODEP_OPTS+=("-v")
+fi
+
+while getopts ":df" opt; do
+  case $opt in
+    d) # do not godep-restore into a temporary directory, but use the existing GOPATH
+      DRY_RUN=true
+      ;;
+    f) # fail if something in the Godeps.json files changed
+      FAIL_ON_DIFF=true
+      ;;
+    \?)
+      echo "Invalid option: -$OPTARG" >&2
+      exit 1
+      ;;
+  esac
+done
+
+kube::util::ensure_dockerized
+kube::golang::setup_env
+kube::util::ensure_single_dir_gopath
+kube::util::ensure_no_staging_repos_in_gopath
+kube::util::ensure_godep_version v79
+
+kube::log::status "Checking whether godeps are restored"
+if ! kube::util::godep_restored 2>&1 | sed 's/^/  /'; then
+  ${KUBE_ROOT}/hack/godep-restore.sh
+fi
+
+kube::util::ensure-temp-dir
+TMP_GOPATH="${KUBE_TEMP}/go"
+
+function updateGodepManifest() {
+  pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null
+    kube::log::status "Updating godeps for k8s.io/${repo}"
+    rm -rf Godeps # remove the current Godeps.json so we always rebuild it
+    GOPATH="${TMP_GOPATH}:${GOPATH}:${GOPATH}/src/k8s.io/kubernetes/staging" godep save ${GODEP_OPTS} ./... 2>&1 | sed 's/^/  /'
+
+    # Rewriting Godeps.json to remove commits that don't really exist because we haven't pushed the prereqs yet
+    go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" --override-import-path="k8s.io/${repo}"
+
+    # commit so that following repos do not see this repo as dirty
+    git add vendor >/dev/null
+    git commit -a -m "Updated Godeps.json" >/dev/null
+  popd >/dev/null
+}
+
+function diffGodepManifest() {
+  local ret=0
+  diff --ignore-matching-lines='^\s*\"GoVersion\":' --ignore-matching-line='^\s*\"GodepVersion\":' --ignore-matching-lines='^\s*\"Comment\"' -u "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" || ret=$?
+  if [[ "${ret}" != "0" && "${FAIL_ON_DIFF}" == true ]]; then
+    exit ${ret}
+  fi
+}
+
+# move into staging and save the dependencies for everything in order
+mkdir -p "${TMP_GOPATH}/src/k8s.io"
+for repo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do
+  cp -a "${KUBE_ROOT}/staging/src/k8s.io/${repo}" "${TMP_GOPATH}/src/k8s.io/"
+
+  pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null
+    git init >/dev/null
+    git config --local user.email "nobody@k8s.io"
+    git config --local user.name "$0"
+    git add . >/dev/null
+    git commit -q -m "Snapshot" >/dev/null
+  popd >/dev/null
+
+  updateGodepManifest
+  diffGodepManifest
+
+  if [ "${DRY_RUN}" != true ]; then
+    cp "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json"
+    # Assume Godeps.json is not updated, as the working tree needs to be clean
+    # Without this, the script would pause after each staging repo to prompt the
+    # user to commit all changes (difficult inside a container). It's safe to
+    # ignore this file, as we know it's going to be changing, and we don't copy
+    # the git tree back out from the container.
+    git update-index --assume-unchanged "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json"
+  fi
+done

hack/update-staging-godeps.sh

@@ -14,91 +14,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# updates the godeps.json file in the staging folders to allow clean vendoring
-# based on kubernetes levels.
-# TODO this does not address client-go, since it takes a different approach to vendoring
-# TODO client-go should probably be made consistent
-
 set -o errexit
 set -o nounset
 set -o pipefail
 
-V=""
-DRY_RUN=false
-FAIL_ON_DIFF=false
-while getopts ":vdf" opt; do
-  case $opt in
-    v) # increase verbosity
-      V="-v"
-      ;;
-    d) # do not godep-restore into a temporary directory, but use the existing GOPATH
-      DRY_RUN=true
-      ;;
-    f) # fail if something in the Godeps.json files changed
-      FAIL_ON_DIFF=true
-      ;;
-    \?)
-      echo "Invalid option: -$OPTARG" >&2
-      exit 1
-      ;;
-  esac
-done
-readonly V IN_PLACE
-
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
-source "${KUBE_ROOT}/hack/lib/util.sh"
 
-echo "Checking whether godeps are restored"
+# Ensure you have a clean working directory before starting
-if ! kube::util::godep_restored 2>&1 | sed 's/^/  /'; then
+kube::util::ensure_clean_working_dir
-  echo -e '\nExecute script 'hack/godep-restore.sh' to download dependencies.' 1>&2
-  exit 1
-fi
 
-kube::util::ensure-temp-dir
+# NOTE: All output from this script needs to be copied back to the calling
-kube::util::ensure_single_dir_gopath
+# source tree.  This is managed in kube::build::copy_output in build/common.sh.
-kube::util::ensure_godep_version v79
+# If the output set is changed update that function.
-kube::util::ensure_no_staging_repos_in_gopath
 
-TMP_GOPATH="${KUBE_TEMP}/go"
+${KUBE_ROOT}/build/run.sh hack/update-staging-godeps-dockerized.sh "$@"
 
-function updateGodepManifest() {
+# ex: ts=2 sw=2 et filetype=sh
-  local repo="${1}"
-  pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null
-    echo "Updating godeps for k8s.io/${repo}"
-    rm -rf Godeps # remove the current Godeps.json so we always rebuild it
-    GOPATH="${TMP_GOPATH}:${GOPATH}:${KUBE_ROOT}/staging" godep save ${V} ./... 2>&1 | sed 's/^/  /'
-
-    echo "Rewriting Godeps.json to remove commits that don't really exist because we haven't pushed the prereqs yet"
-    go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" --override-import-path="k8s.io/${repo}"
-
-    # commit so that following repos do not see this repo as dirty
-    git add vendor >/dev/null
-    git commit -a -m "Updated Godeps.json" >/dev/null
-  popd >/dev/null
-}
-
-# move into staging and save the dependencies for everything in order
-mkdir -p "${TMP_GOPATH}/src/k8s.io"
-for repo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do
-  kube::util::ensure_clean_working_dir
-
-  cp -a "${KUBE_ROOT}/staging/src/k8s.io/${repo}" "${TMP_GOPATH}/src/k8s.io/"
-
-  pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null
-    git init >/dev/null
-    git config --local user.email "nobody@k8s.io"
-    git config --local user.name "$0"
-    git add . >/dev/null
-    git commit -q -m "Snapshot" >/dev/null
-  popd >/dev/null
-
-  updateGodepManifest "${repo}"
-
-  if [ "${FAIL_ON_DIFF}" == true ]; then
-    diff --ignore-matching-lines='^\s*\"GoVersion\":' --ignore-matching-line='^\s*\"GodepVersion\":' --ignore-matching-lines='^\s*\"Comment\"' -u "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json" "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json"
-  fi
-  if [ "${DRY_RUN}" != true ]; then
-    cp "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps/Godeps.json"
-  fi
-done

hack/verify-staging-godeps.sh

@@ -19,4 +19,4 @@ set -o nounset
 set -o pipefail
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
-${KUBE_ROOT}/hack/update-staging-godeps.sh -d -f "$@"
+KUBE_VERBOSE=3 KUBE_RUN_COPY_OUTPUT=N ${KUBE_ROOT}/hack/update-staging-godeps.sh -d -f "$@"