Merge pull request #41987 from sttts/sttts-verify-staging-client-go-unify-with-copy-sh

Automatic merge from submit-queue (batch tested with PRs 42044, 41694, 41927, 42050, 41987)

Simplify and fix hack/{verify,update}-staging-{client-go,godeps}.sh

- merge `hack/{verify,update}-staging-client-go.sh`
- pin godep with shared code
- remove godep-restore completely from the process and replace with a simple check that godeps are restored
- add safety check in `staging/copy.sh` that there is no lingering `k8s.io/apimachinery` in the GOPATH which would lead to inconsistent client-go builds (!)
- check that all these scripts only operate in a clean working dir.

hack/lib/util.sh

@@ -411,6 +411,76 @@ kube::util::git_upstream_remote_name() {
     head -n 1 | awk '{print $1}'
 }
 
+# Checks whether godep restore was run in the current GOPATH, i.e. that all referenced repos exist
+# and are checked out to the referenced rev.
+kube::util::godep_restored() {
+  local -r godeps_json=${1:-Godeps/Godeps.json}
+  local -r gopath=${2:-${GOPATH%:*}}
+  if ! which jq &>/dev/null; then
+    echo "jq not found. Please install." 1>&2
+    return 1
+  fi
+  local root
+  local old_rev=""
+  while read path rev; do
+    rev=$(echo "${rev}" | sed "s/['\"]//g") # remove quotes which are around revs sometimes
+
+    if [[ "${rev}" == "${old_rev}" ]] && [[ "${path}" == "${root}"* ]]; then
+      # avoid checking the same git/hg root again
+      continue
+    fi
+
+    root="${path}"
+    while [ "${root}" != "." -a ! -d "${gopath}/src/${root}/.git" -a ! -d "${gopath}/src/${root}/.hg" ]; do
+      root=$(dirname "${root}")
+    done
+    if [ "${root}" == "." ]; then
+      echo "No checkout of ${path} found in GOPATH \"${gopath}\"." 1>&2
+      return 1
+    fi
+    local head
+    if [ -d "${gopath}/src/${root}/.git" ]; then
+      head="$(cd "${gopath}/src/${root}" && git rev-parse HEAD)"
+    else
+      head="$(cd "${gopath}/src/${root}" && hg parent --template '{node}')"
+    fi
+    if [ "${head}" != "${rev}" ]; then
+      echo "Unexpected HEAD '${head}' at ${gopath}/src/${root}, expected '${rev}'." 1>&2
+      return 1
+    fi
+    old_rev="${rev}"
+  done < <(jq '.Deps|.[]|.ImportPath + " " + .Rev' -r < "${godeps_json}")
+  return 0
+}
+
+# Exits script if working directory is dirty.
+kube::util::ensure_clean_working_dir() {
+  if ! git diff --exit-code; then
+    echo -e "\nUnexpected dirty working directory."
+    exit 1
+  fi
+}
+
+# Ensure that the given godep version is installed and in the path
+kube::util::ensure_godep_version() {
+  if [[ "$(godep version)" == *"godep ${1}"* ]]; then
+    return
+  fi
+
+  kube::util::ensure-temp-dir
+  mkdir -p "${KUBE_TEMP}/go/src"
+
+  GOPATH="${KUBE_TEMP}/go" go get -u github.com/tools/godep 2>/dev/null
+  pushd "${KUBE_TEMP}/go/src/github.com/tools/godep" >/dev/null
+    git checkout "${1:-v74}"
+    GOPATH="${KUBE_TEMP}/go" go install .
+  popd >/dev/null
+
+  PATH="${KUBE_TEMP}/go/bin:${PATH}"
+  hash -r # force bash to clear PATH cache
+  godep version
+}
+
 # Checks whether there are any files matching pattern $2 changed between the
 # current branch and upstream branch named by $1.
 # Returns 1 (false) if there are no changes, 0 (true) if there are changes

hack/update-staging-client-go.sh

@@ -19,4 +19,18 @@ set -o nounset
 set -o pipefail
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
-"${KUBE_ROOT}"/staging/copy.sh
+source "${KUBE_ROOT}/hack/lib/util.sh"
+
+kube::util::ensure_clean_working_dir
+kube::util::ensure_godep_version v74
+
+cd ${KUBE_ROOT}
+
+echo "Checking whether godeps are restored"
+if ! kube::util::godep_restored 2>&1 | sed 's/^/  /'; then
+  echo -e '\nRun 'godep restore' to download dependencies.' 1>&2
+  exit 1
+fi
+
+echo "Running staging/copy.sh"
+staging/copy.sh -u "$@" 2>&1 | sed 's/^/  /'

hack/update-staging-godeps.sh

@@ -23,102 +23,84 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+V=""
+DRY_RUN=false
+FAIL_ON_DIFF=false
+while getopts ":vdf" opt; do
+  case $opt in
+    v) # increase verbosity
+      V="-v"
+      ;;
+    d) # do not godep-restore into a temporary directory, but use the existing GOPATH
+      DRY_RUN=true
+      ;;
+    f) # fail if something in the Godeps.json files changed
+      FAIL_ON_DIFF=true
+      ;;
+    \?)
+      echo "Invalid option: -$OPTARG" >&2
+      exit 1
+      ;;
+  esac
+done
+readonly V IN_PLACE
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
+source "${KUBE_ROOT}/hack/lib/util.sh"
 
-kube::golang::setup_env
+echo "Checking whether godeps are restored"
-
+if ! kube::util::godep_restored 2>&1 | sed 's/^/  /'; then
-ORIGINAL_GOPATH="${GOPATH}"
+  echo -e '\nRun 'godep restore' to download dependencies.' 1>&2
-
+  exit 1
-godepBinDir=${TMPDIR:-/tmp/}/kube-godep-bin
-mkdir -p "${godepBinDir}"
-godep=${godepBinDir}/bin/godep
-pushd "${godepBinDir}" 2>&1 > /dev/null
-	# Build the godep tool
-	GOPATH="${godepBinDir}"
-	rm -rf *
-	go get -u github.com/tools/godep 2>/dev/null
-	export GODEP="${GOPATH}/bin/godep"
-	pin-godep() {
-		pushd "${GOPATH}/src/github.com/tools/godep" > /dev/null
-			git checkout "$1"
-			"${GODEP}" go install
-		popd > /dev/null
-	}
-	# Use to following if we ever need to pin godep to a specific version again
-	pin-godep 'v74'
-	"${godep}" version
-popd 2>&1 > /dev/null
-
-# keep the godep restore path reasonably stable to avoid unnecessary restores
-godepRestoreDir=${TMPDIR:-/tmp/}/kube-godep-restore
-
-TARGET_DIR=${TARGET_DIR:-${KUBE_ROOT}/staging}
-echo "working in ${TARGET_DIR}"
-
-SKIP_RESTORE=${SKIP_RESTORE:-}
-if [ "${SKIP_RESTORE}" != "true" ]; then
-	echo "starting godep restore"
-	mkdir -p "${godepRestoreDir}"
-
-	# add the vendor folder so that we don't redownload things during restore
-	GOPATH="${godepRestoreDir}:${KUBE_ROOT}/staging:${ORIGINAL_GOPATH}"
-	# restore from kubernetes godeps to ensure we get the correct levels
-	# you get errors about the staging repos not using a known version control system
-	${godep} restore > ${godepRestoreDir}/godep-restore.log
-	echo "finished godep restore"
 fi
 
-echo "forcing fake godep folders to match the current state of master in tmp"
+kube::util::ensure_clean_working_dir
-for stagingRepo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do
+kube::util::ensure-temp-dir
-	echo "    creating ${stagingRepo}"
+kube::util::ensure_godep_version v74
-	rm -rf ${godepRestoreDir}/src/k8s.io/${stagingRepo}
-	cp -a ${KUBE_ROOT}/staging/src/k8s.io/${stagingRepo} ${godepRestoreDir}/src/k8s.io
 
-	# we need a git commit in the godep folder, otherwise godep won't save
+TMP_GOPATH="${KUBE_TEMP}/go"
-	pushd ${godepRestoreDir}/src/k8s.io/${stagingRepo}
-	git init > /dev/null
-	# we need this so later commands work, but nothing should ever actually include these commits
-	# these are local only, not global
-	git config user.email "you@example.com"
-	git config user.name "Your Name"
-	git add . > /dev/null
-	git commit -qm "fake commit"
-	popd
-done
 
 function updateGodepManifest() {
-	local repo=${1}
+  local repo="${1}"
+  pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null
+    echo "Updating godeps for k8s.io/${repo}"
+    rm -rf Godeps # remove the current Godeps.json so we always rebuild it
+    GOPATH="${TMP_GOPATH}:${GOPATH}:${KUBE_ROOT}/staging" godep save ${V} ./... 2>&1 | sed 's/^/  /'
 
-	echo "starting ${repo} save"
+    echo "Rewriting Godeps.json to remove commits that don't really exist because we haven't pushed the prereqs yet"
-	mkdir -p ${TARGET_DIR}/src/k8s.io
+    go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" --client-go-import-path="k8s.io/${repo}"
-	# if target_dir isn't the same as source dir, you need copy
-	test "${KUBE_ROOT}/staging" = "${TARGET_DIR}" || cp -a ${KUBE_ROOT}/staging/src/${repo} ${TARGET_DIR}/src/k8s.io
-	# remove the current Godeps.json so we always rebuild it
-	rm -rf ${TARGET_DIR}/src/${repo}/Godeps
-	GOPATH="${godepRestoreDir}:${TARGET_DIR}"
-	pushd ${TARGET_DIR}/src/${repo}
-	${godep} save ./...
 
-	# now remove all the go files.	We'll re-run a restore, go get, godep save cycle in the sync scripts
+    # commit so that following repos do not see this repo as dirty
-	# to get the commits for other staging k8s.io repos anyway, so we don't need the added files
+    git add vendor >/dev/null
-	rm -rf vendor
+    git commit -a -m "Updated Godeps.json" >/dev/null
-
+  popd >/dev/null
-	echo "rewriting Godeps.json to remove commits that don't really exist because we haven't pushed the prereqs yet"
-	GOPATH="${ORIGINAL_GOPATH}"
-	go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${TARGET_DIR}/src/${repo}/Godeps/Godeps.json" --client-go-import-path="${repo}"
-
-	popd
-	echo "finished ${repo} save"
 }
 
 # move into staging and save the dependencies for everything in order
-for stagingRepo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do
+mkdir -p "${TMP_GOPATH}/src/k8s.io"
+for repo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do
   # we have to skip client-go because it does unusual manipulation of its godeps
-	if [ "${stagingRepo}" == "client-go" ]; then
+  if [ "${repo}" == "client-go" ]; then
     continue
   fi
 
-	updateGodepManifest "k8s.io/${stagingRepo}"
+  cp -a "${KUBE_ROOT}/staging/src/k8s.io/${repo}" "${TMP_GOPATH}/src/k8s.io/"
+
+  pushd "${TMP_GOPATH}/src/k8s.io/${repo}" >/dev/null
+    git init >/dev/null
+    git config --local user.email "nobody@k8s.io"
+    git config --local user.name "$0"
+    git add . >/dev/null
+    git commit -q -m "Snapshot" >/dev/null
+  popd >/dev/null
+
+  updateGodepManifest "${repo}"
+
+  if [ "${FAIL_ON_DIFF}" == true ]; then
+    diff --ignore-matching-lines='^\s*\"Comment\"' -u "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps" "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json"
+  fi
+  if [ "${DRY_RUN}" != true ]; then
+    cp "${TMP_GOPATH}/src/k8s.io/${repo}/Godeps/Godeps.json" "${KUBE_ROOT}/staging/src/k8s.io/${repo}/Godeps"
+  fi
 done

hack/verify-staging-client-go.sh

@@ -18,65 +18,12 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-V=""
-while getopts ":v" opt; do
-  case $opt in
-    v) # increase verbosity
-      V="-v"
-      ;;
-    \?)
-      echo "Invalid option: -$OPTARG" >&2
-      ;;
-  esac
-done
-readonly V
-
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 cd ${KUBE_ROOT}
 
-if ! git diff --exit-code; then
-    echo "ERROR: $0 cannot be run on a dirty working directory."
-    exit 1
-fi
-
 # Smoke test client-go examples
-go install ./staging/src/k8s.io/client-go/examples/...
+echo "Smoke testing client-go examples"
+go install ./staging/src/k8s.io/client-go/examples/... 2>&1 | sed 's/^/  /'
 
-# Create a temporary GOPATH for apimachinery and client-go, copy the current HEAD into each and turn them
+# Run update-staging-client-go.sh in dry-run mode, copy nothing into the staging dir
-# into a git repo. Then we can run copy.sh with this additional GOPATH. The Godeps.json will
+hack/update-staging-client-go.sh -d "$@"
-# have invalid git SHA1s, but it's good enough as a smoke test of copy.sh.
-if [ "${USE_TEMP_DIR:-1}" = 1 ]; then
-    TEMP_STAGING_GOPATH=$(mktemp -d -t verify-staging-client-go.XXXXX)
-    echo "Creating the temporary staging GOPATH directory: ${TEMP_STAGING_GOPATH}"
-    cleanup() {
-        if [ "${KEEP_TEMP_DIR:-0}" != 1 ]; then
-            rm -rf "${TEMP_STAGING_GOPATH}"
-        fi
-    }
-    trap cleanup EXIT SIGINT
-    mkdir -p "${TEMP_STAGING_GOPATH}/src/k8s.io"
-    ln -s "${PWD}" "${TEMP_STAGING_GOPATH}/src/k8s.io"
-else
-    TEMP_STAGING_GOPATH="${GOPATH}"
-fi
-for PACKAGE in apimachinery client-go; do
-    PACKAGE_PATH="${TEMP_STAGING_GOPATH}/src/k8s.io/${PACKAGE}"
-    echo "Creating a temporary ${PACKAGE} repo with a snapshot of HEAD"
-    rm -rf "${PACKAGE_PATH}"
-    mkdir -p "${PACKAGE_PATH}"
-    git archive --format=tar "HEAD:staging/src/k8s.io/${PACKAGE}" | tar -xf - -C "${PACKAGE_PATH}"
-    pushd "${PACKAGE_PATH}" >/dev/null
-    git init >/dev/null
-    git add *
-    git -c user.email="nobody@k8s.io" -c user.name="verify-staging-client-go.sh" commit -q -m "Snapshot"
-    popd >/dev/null
-done
-
-echo "Running godep restore"
-pushd "${TEMP_STAGING_GOPATH}/src/k8s.io/kubernetes" >/dev/null
-export GOPATH="${TEMP_STAGING_GOPATH}"
-godep restore ${V} 2>&1 | sed 's/^/  /'
-
-echo "Testing staging/copy.sh"
-staging/copy.sh -d 2>&1 | sed 's/^/  /'
-popd

hack/verify-staging-godeps.sh

@@ -14,36 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO this does not address client-go, since it takes a different approach to vendoring
-# TODO client-go should probably be made consistent
-
 set -o errexit
 set -o nounset
 set -o pipefail
 
-
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
-source "${KUBE_ROOT}/hack/lib/init.sh"
+${KUBE_ROOT}/hack/update-staging-godeps.sh -d -f "$@"
-
-
-TARGET_DIR=$(mktemp -d "${TMPDIR:-/tmp/}$(basename 0).XXXXXXXXXXXX")
-# Register function to be called on EXIT to remove folder.
-function cleanup {
-	SKIP_CLEANUP=${SKIP_CLEANUP:-}
-	if [ "${SKIP_CLEANUP}" != "true" ]; then
-		rm -rf "${TARGET_DIR}"
-	fi
-}
-trap cleanup EXIT
-
-TARGET_DIR=${TARGET_DIR} ${KUBE_ROOT}/hack/update-staging-godeps.sh
-
-# check each staging repo to make sure its Godeps.json is correct
-for stagingRepo in $(ls ${KUBE_ROOT}/staging/src/k8s.io); do
-	# we have to skip client-go because it does unusual manipulation of its godeps
-	if [ "${stagingRepo}" == "client-go" ]; then
-		continue
-	fi
-
-	diff --ignore-matching-lines='^\s*\"Comment\"' ${KUBE_ROOT}/staging/src/k8s.io/${stagingRepo}/Godeps/Godeps.json ${TARGET_DIR}/src/k8s.io/${stagingRepo}/Godeps/Godeps.json
-done

staging/copy.sh

@@ -18,9 +18,15 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+
 FAIL_ON_CHANGES=false
 DRY_RUN=false
-while getopts ":fd" opt; do
+RUN_FROM_UPDATE_SCRIPT=false
+while getopts ":fdu" opt; do
   case $opt in
     f)
       FAIL_ON_CHANGES=true
@@ -28,25 +34,31 @@ while getopts ":fd" opt; do
     d)
       DRY_RUN=true
       ;;
+    u)
+      RUN_FROM_UPDATE_SCRIPT=true
+      ;;
     \?)
       echo "Invalid option: -$OPTARG" >&2
+      exit 1
       ;;
   esac
 done
 readonly FAIL_ON_CHANGES DRY_RUN
 
-echo "**PLEASE** run \"godep restore\" before running this script"
+if [ "${RUN_FROM_UPDATE_SCRIPT}" != true ]; then
+  echo "Do not run this script directly, but via hack/update-staging-client-go.sh."
+  exit 1
+fi
+
 # PREREQUISITES: run `godep restore` in the main repo before calling this script.
 CLIENTSET="clientset"
 MAIN_REPO_FROM_SRC="k8s.io/kubernetes"
-MAIN_REPO="${GOPATH%:*}/src/${MAIN_REPO_FROM_SRC}"
+MAIN_REPO="$(cd "${KUBE_ROOT}"; pwd)" # absolute path
 CLIENT_REPO_FROM_SRC="k8s.io/client-go"
 CLIENT_REPO_TEMP_FROM_SRC="k8s.io/_tmp"
 CLIENT_REPO="${MAIN_REPO}/staging/src/${CLIENT_REPO_FROM_SRC}"
 CLIENT_REPO_TEMP="${MAIN_REPO}/staging/src/${CLIENT_REPO_TEMP_FROM_SRC}"
 
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
 if LANG=C sed --help 2>&1 | grep -q GNU; then
   SED="sed"
 elif which gsed &>/dev/null; then
@@ -108,6 +120,12 @@ find "${MAIN_REPO}/pkg/version" -maxdepth 1 -type f | xargs -I{} cp {} "${CLIENT
 mkcp "pkg/client/clientset_generated/${CLIENTSET}" "pkg/client/clientset_generated"
 mkcp "pkg/client/informers/informers_generated/externalversions" "pkg/client/informers/informers_generated"
 
+# safety check that we don't have another apimachinery in the GOPATH
+if go list -f '{{.Dir}}' k8s.io/apimachinery/pkg/runtime &>/dev/null; then
+  echo "ERROR: unexpected k8s.io/apimachinery in GOPATH '$GOPATH'" 1>&2
+  exit 1
+fi
+
 pushd "${CLIENT_REPO_TEMP}" > /dev/null
 echo "generating vendor/"
 # client-go depends on some apimachinery packages. Adding staging/ to the GOPATH
@@ -127,7 +145,7 @@ rm -rf "${CLIENT_REPO_TEMP}"/vendor/k8s.io/kubernetes
 mv "${CLIENT_REPO_TEMP}"/vendor "${CLIENT_REPO_TEMP}"/_vendor
 
 echo "rewriting Godeps.json"
-go run "${DIR}/godeps-json-updater.go" --godeps-file="${CLIENT_REPO_TEMP}/Godeps/Godeps.json" --client-go-import-path="${CLIENT_REPO_FROM_SRC}"
+go run "${KUBE_ROOT}/staging/godeps-json-updater.go" --godeps-file="${CLIENT_REPO_TEMP}/Godeps/Godeps.json" --client-go-import-path="${CLIENT_REPO_FROM_SRC}"
 
 echo "rewriting imports"
 grep -Rl "\"${MAIN_REPO_FROM_SRC}" "${CLIENT_REPO_TEMP}" | \
@@ -206,12 +224,12 @@ rm -rf "${CLIENT_REPO_TEMP}/staging"
 if [ "${FAIL_ON_CHANGES}" = true ]; then
   echo "running FAIL_ON_CHANGES"
   ret=0
-    if diff -NauprB -I "GoVersion.*\|GodepVersion.*" "${CLIENT_REPO}" "${CLIENT_REPO_TEMP}"; then
+  if diff --ignore-matching-lines='^\s*\"Comment\"' -NauprB -I "GoVersion.*\|GodepVersion.*" "${CLIENT_REPO}" "${CLIENT_REPO_TEMP}"; then
     echo "${CLIENT_REPO} up to date."
     cleanup
     exit 0
   else
-      echo "${CLIENT_REPO} is out of date. Please run hack/update-client-go.sh"
+    echo "${CLIENT_REPO} is out of date. Please run hack/update-staging-client-go.sh"
     cleanup
     exit 1
   fi