Merge pull request #115243 from liggitt/host-protoc

Use host protoc if it is the correct version

.gitignore

@@ -64,6 +64,9 @@ network_closure.sh
 /third_party/etcd*
 /default.etcd
 
+# Also ignore protoc installed by hack/install-protoc.sh
+/third_party/protoc*
+
 # User cluster configs
 .kubeconfig
 

build/dependencies.yaml

@@ -53,6 +53,13 @@ dependencies:
     - path: cluster/gce/gci/configure.sh
       match: DEFAULT_CRICTL_VERSION=
 
+  # protoc
+  - name: "protoc"
+    version: 3.19.4
+    refPaths:
+    - path: hack/lib/protoc.sh
+      match: PROTOC_VERSION=
+
   # etcd
   - name: "etcd"
     version: 3.5.6

hack/install-protoc.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is convenience to download and install protoc in third_party.
+# Usage: `hack/install-protoc.sh`.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${KUBE_ROOT}/hack/lib/protoc.sh"
+
+kube::protoc::install

hack/lib/init.sh

@@ -18,6 +18,9 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+# Short-circuit if init.sh has already been sourced
+[[ $(type -t kube::init::loaded) == function ]] && return 0
+
 # Unset CDPATH so that path interpolation can work correctly
 # https://github.com/kubernetes/kubernetes/issues/52255
 unset CDPATH
@@ -211,3 +214,8 @@ kube::realpath() {
   fi
   kube::readlinkdashf "${1}"
 }
+
+# Marker function to indicate init.sh has been fully sourced
+kube::init::loaded() {
+  return 0
+}

hack/lib/protoc.sh

@@ -18,10 +18,15 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+# Short-circuit if protoc.sh has already been sourced
+[[ $(type -t kube::protoc::loaded) == function ]] && return 0
+
 # The root of the build/dist directory
 KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
+PROTOC_VERSION=3.19.4
+
 # Generates $1/api.pb.go from the protobuf file $1/api.proto
 # and formats it correctly
 # $1: Full path to the directory where the api.proto file is
@@ -36,17 +41,15 @@ function kube::protoc::generate_proto() {
   kube::protoc::format "${package}"
 }
 
-# Checks that the current protoc version is at least version 3.0.0-beta1
+# Checks that the current protoc version matches the required version and
 # exit 1 if it's not the case
 function kube::protoc::check_protoc() {
-  if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3."* ]]; then
+  if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc ${PROTOC_VERSION}"* ]]; then
-    echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and"
+    echo "Generating protobuf requires protoc ${PROTOC_VERSION}."
-    echo "install the platform appropriate Protobuf package for your OS: "
+    echo "Run hack/install-protoc.sh or download and install the"
-    echo
+    echo "platform-appropriate Protobuf package for your OS from"
-    echo "  https://github.com/protocolbuffers/protobuf/releases"
+    echo "https://github.com/protocolbuffers/protobuf/releases"
-    echo
+    return 1
-    echo "WARNING: Protobuf changes are not being validated"
-    exit 1
   fi
 }
 
@@ -66,6 +69,7 @@ function kube::protoc::protoc() {
     PATH="${gogopath}:${PATH}" protoc \
       --proto_path="$(pwd -P)" \
       --proto_path="${KUBE_ROOT}/vendor" \
+      --proto_path="${KUBE_ROOT}/third_party/protobuf" \
       --gogo_out=paths=source_relative,plugins=grpc:. \
       api.proto
   )
@@ -94,3 +98,48 @@ function kube::protoc::diff() {
     exit 1
   fi
 }
+
+function kube::protoc::install() {
+  # run in a subshell to isolate caller from directory changes
+  (
+    local os
+    local arch
+    local download_folder
+    local download_file
+
+    os=$(kube::util::host_os)
+    arch=$(kube::util::host_arch)
+    download_folder="protoc-v${PROTOC_VERSION}-${os}-${arch}"
+    download_file="${download_folder}.zip"
+
+    cd "${KUBE_ROOT}/third_party" || return 1
+    if [[ $(readlink protoc) != "${download_folder}" ]]; then
+      local url
+      if [[ ${os} == "darwin" ]]; then
+        # TODO: switch to universal binary when updating to 3.20+
+        url="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-osx-x86_64.zip"
+      elif [[ ${os} == "linux" && ${arch} == "amd64" ]]; then
+        url="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip"
+      elif [[ ${os} == "linux" && ${arch} == "arm64" ]]; then
+        url="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-aarch_64.zip"
+      else
+        kube::log::info "This install script does not support ${os}/${arch}"
+        return 1
+      fi
+      kube::util::download_file "${url}" "${download_file}"
+      unzip -o "${download_file}" -d "${download_folder}"
+      ln -fns "${download_folder}" protoc
+      mv protoc/bin/protoc protoc/protoc
+      chmod -R +rX protoc/protoc
+      rm -fr protoc/include
+      rm "${download_file}"
+    fi
+    kube::log::info "protoc v${PROTOC_VERSION} installed. To use:"
+    kube::log::info "export PATH=\"$(pwd)/protoc:\${PATH}\""
+  )
+}
+
+# Marker function to indicate protoc.sh has been fully sourced
+kube::protoc::loaded() {
+  return 0
+}

hack/update-codegen.sh

@@ -24,6 +24,7 @@ KUBE_VERBOSE="${KUBE_VERBOSE:-1}"
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
+source "${KUBE_ROOT}/hack/lib/protoc.sh"
 cd "${KUBE_ROOT}"
 
 kube::golang::setup_env
@@ -115,7 +116,12 @@ function codegen::protobuf() {
         ':(glob)**/generated.pb.go' \
         | xargs -0 rm -f
 
-    build/run.sh hack/update-generated-protobuf-dockerized.sh "${apis[@]}"
+    if kube::protoc::check_protoc >/dev/null; then
+      hack/update-generated-protobuf-dockerized.sh "${apis[@]}"
+    else
+      kube::log::status "protoc ${PROTOC_VERSION} not found (can install with hack/install-protoc.sh); generating containerized..."
+      build/run.sh hack/update-generated-protobuf-dockerized.sh "${apis[@]}"
+    fi
 }
 
 # Generates types_swagger_doc_generated file for the given group version.
@@ -918,10 +924,15 @@ function codegen::protobindings() {
             | xargs -0 rm -f
     done
 
-    # NOTE: All output from this script needs to be copied back to the calling
+    if kube::protoc::check_protoc >/dev/null; then
-    # source tree.  This is managed in kube::build::copy_output in build/common.sh.
+      hack/update-generated-proto-bindings-dockerized.sh "${apis[@]}"
-    # If the output set is changed update that function.
+    else
-    build/run.sh hack/update-generated-proto-bindings-dockerized.sh "${apis[@]}"
+      kube::log::status "protoc ${PROTOC_VERSION} not found (can install with hack/install-protoc.sh); generating containerized..."
+      # NOTE: All output from this script needs to be copied back to the calling
+      # source tree.  This is managed in kube::build::copy_output in build/common.sh.
+      # If the output set is changed update that function.
+      build/run.sh hack/update-generated-proto-bindings-dockerized.sh "${apis[@]}"
+    fi
 }
 
 #

hack/update-generated-proto-bindings-dockerized.sh

@@ -22,6 +22,7 @@ set -o pipefail
 
 KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../" && pwd -P)"
 
+source "${KUBE_ROOT}/hack/lib/init.sh"
 source "${KUBE_ROOT}/hack/lib/protoc.sh"
 source "${KUBE_ROOT}/hack/lib/util.sh"
 
@@ -30,6 +31,8 @@ if [ "$#" == 0 ]; then
     exit 1
 fi
 
+kube::protoc::check_protoc
+
 for api; do
     # This can't use `git ls-files` because it runs in a container without the
     # .git dir synced.

hack/update-generated-protobuf-dockerized.sh

@@ -25,22 +25,14 @@ set -o pipefail
 
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
+source "${KUBE_ROOT}/hack/lib/protoc.sh"
 
+kube::protoc::check_protoc
 kube::golang::setup_env
 
 GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/go-to-protobuf
 GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo
 
-if [[ -z "$(which protoc)" || "$(protoc --version)" != "libprotoc 3."* ]]; then
-  echo "Generating protobuf requires protoc 3.0.0-beta1 or newer. Please download and"
-  echo "install the platform appropriate Protobuf package for your OS: "
-  echo
-  echo "  https://github.com/protocolbuffers/protobuf/releases"
-  echo
-  echo "WARNING: Protobuf changes are not being validated"
-  exit 1
-fi
-
 gotoprotobuf=$(kube::util::find-binary "go-to-protobuf")
 
 # requires the 'proto' tag to build (will remove when ready)