scottk/kubernetes
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/kubernetes.git synced 2025-11-03 19:58:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix Makefile deps to rebuild less often

Browse Source 
This should only rebuild when ACTUALLY needed.
This commit is contained in:
Tim Hockin
2016-06-02 21:28:31 -07:00
parent 9613e15801
commit a9f3ccdaa4
3 changed files with 98 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -102,4 +102,4 @@ kubernetes.tar.gz
zz_generated.* zz_generated.*
# make-related metadata # make-related metadata
.make.* /.make/

114
Makefile
View File

@@ -12,6 +12,12 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
DBG_MAKEFILE ?=
ifeq ($(DBG_MAKEFILE),1)
$(warning ***** starting makefile for goal(s) "$(MAKECMDGOALS)")
$(warning ***** $(shell date))
endif
# It's necessary to set this because some docker images don't make sh -> bash. # It's necessary to set this because some docker images don't make sh -> bash.
SHELL := /bin/bash SHELL := /bin/bash
@@ -31,7 +37,9 @@ OUT_DIR ?= _output
BIN_DIR := $(OUT_DIR)/bin BIN_DIR := $(OUT_DIR)/bin
PRJ_SRC_PATH := k8s.io/kubernetes PRJ_SRC_PATH := k8s.io/kubernetes
GENERATED_FILE_PREFIX := zz_generated. GENERATED_FILE_PREFIX := zz_generated.
MAKE_METAFILE_PREFIX := .make.
# Metadata for driving the build lives here.
META_DIR := .make
GOFLAGS ?= GOFLAGS ?=
KUBE_GOFLAGS = $(GOFLAGS) KUBE_GOFLAGS = $(GOFLAGS)
@@ -165,7 +173,7 @@ clean: clean_generated clean_meta
# Example: # Example:
# make clean_meta # make clean_meta
clean_meta: clean_meta:
find . -type f -name $(MAKE_METAFILE_PREFIX)\* | xargs rm -f rm -rf $(META_DIR)
.PHONE: clean_meta .PHONE: clean_meta
# Remove all auto-generated artifacts. # Remove all auto-generated artifacts.
@@ -221,6 +229,9 @@ cross:
# This variable holds a list of every directory that contains Go files in this # This variable holds a list of every directory that contains Go files in this
# project. Other rules and variables can use this as a starting point to # project. Other rules and variables can use this as a starting point to
# reduce filesystem accesses. # reduce filesystem accesses.
ifeq ($(DBG_MAKEFILE),1)
$(warning ***** finding all *.go dirs)
endif
ALL_GO_DIRS := $(shell \ ALL_GO_DIRS := $(shell \
find . \ find . \
-not \( \ -not \( \
@@ -237,21 +248,24 @@ ALL_GO_DIRS := $(shell \
) )
# The name of the make metadata file listing Go files. # The name of the make metadata file listing Go files.
GOFILES_METAFILE := $(MAKE_METAFILE_PREFIX)gofiles GOFILES_META := gofiles.mk
# Establish a dependency between the deps file and the dir. Whenever a dir # Establish a dependency between the deps file and the dir. Whenever a dir
# changes (files added or removed) the deps file will be considered stale. # changes (files added or removed) the deps file will be considered stale.
# #
# The variable value was set in $(GOFILES_META) and included as part of the
# dependency management logic.
#
# This is looser than we really need (e.g. we don't really care about non *.go # This is looser than we really need (e.g. we don't really care about non *.go
# files or even *_test.go files), but this is much easier to represent. # files or even *_test.go files), but this is much easier to represent.
# #
# Because we 'sinclude' the deps file, it is considered for rebuilding, as part # Because we 'sinclude' the deps file, it is considered for rebuilding, as part
# of make's normal evaluation. # of make's normal evaluation. If it gets rebuilt, make will restart.
# #
# The '$(eval)' is needed because this has a different RHS for each LHS, and # The '$(eval)' is needed because this has a different RHS for each LHS, and
# would otherwise produce results that make can't parse. # would otherwise produce results that make can't parse.
$(foreach dir, $(ALL_GO_DIRS), $(eval \ $(foreach dir, $(ALL_GO_DIRS), $(eval \
$(dir)/$(GOFILES_METAFILE): $(dir) \ $(META_DIR)/$(dir)/$(GOFILES_META): $(dir) \
)) ))
# How to rebuild a deps file. When make determines that the deps file is stale # How to rebuild a deps file. When make determines that the deps file is stale
@@ -259,18 +273,30 @@ $(foreach dir, $(ALL_GO_DIRS), $(eval \
# #
# This is looser than we really need (e.g. we don't really care about test # This is looser than we really need (e.g. we don't really care about test
# files), but this is MUCH faster than calling `go list`. # files), but this is MUCH faster than calling `go list`.
$(foreach dir, $(ALL_GO_DIRS), $(dir)/$(GOFILES_METAFILE)): #
@FILES=$$(ls $(@D)/*.go | grep -v $(GENERATED_FILE_PREFIX)); \ # We regenerate the output file in order to satisfy make's "newer than" rules,
echo "gofiles__$(@D) := $$(echo $${FILES})" >$@ # but we only need to rebuild targets if the contents actually changed. That
# is what the .stamp file represents.
$(foreach dir, $(ALL_GO_DIRS), $(META_DIR)/$(dir)/$(GOFILES_META)):
@FILES=$$(ls $</*.go | grep -v $(GENERATED_FILE_PREFIX)); \
mkdir -p $(@D); \
echo "gofiles__$< := $$(echo $${FILES})" >$@.tmp; \
cmp -s $@.tmp $@ || touch $@.stamp; \
mv $@.tmp $@
# Include any deps files as additional Makefile rules. This triggers make to # Include any deps files as additional Makefile rules. This triggers make to
# consider the deps files for rebuild, which makes the whole # consider the deps files for rebuild, which makes the whole
# dependency-management logic work. 'sinclude' is "silent include" which does # dependency-management logic work. 'sinclude' is "silent include" which does
# not fail if the file does not exist. # not fail if the file does not exist.
$(foreach dir, $(ALL_GO_DIRS), $(eval sinclude $(dir)/$(GOFILES_METAFILE))) $(foreach dir, $(ALL_GO_DIRS), $(eval \
sinclude $(META_DIR)/$(dir)/$(GOFILES_META) \
))
# Generate a list of all files that have a `+k8s:` comment-tag. This will be # Generate a list of all files that have a `+k8s:` comment-tag. This will be
# used to derive lists of files/dirs for generation tools. # used to derive lists of files/dirs for generation tools.
ifeq ($(DBG_MAKEFILE),1)
$(warning ***** finding all +k8s: tags)
endif
ALL_K8S_TAG_FILES := $(shell \ ALL_K8S_TAG_FILES := $(shell \
find $(ALL_GO_DIRS) -maxdepth 1 -type f -name \*.go \ find $(ALL_GO_DIRS) -maxdepth 1 -type f -name \*.go \
| xargs grep -l '^// *+k8s:' \ | xargs grep -l '^// *+k8s:' \
@@ -289,6 +315,9 @@ ALL_K8S_TAG_FILES := $(shell \
# scheme # scheme
# Find all the directories that request deep-copy generation. # Find all the directories that request deep-copy generation.
ifeq ($(DBG_MAKEFILE),1)
$(warning ***** finding all +k8s:deepcopy-gen tags)
endif
DEEP_COPY_DIRS := $(shell \ DEEP_COPY_DIRS := $(shell \
grep -l '+k8s:deepcopy-gen=' $(ALL_K8S_TAG_FILES) \ grep -l '+k8s:deepcopy-gen=' $(ALL_K8S_TAG_FILES) \
| xargs dirname \ | xargs dirname \
@@ -305,8 +334,8 @@ gen_deepcopy:
@$(MAKE) -s build WHAT=cmd/libs/go2idl/deepcopy-gen @$(MAKE) -s build WHAT=cmd/libs/go2idl/deepcopy-gen
@$(MAKE) -s $(addsuffix /$(DEEP_COPY_FILENAME), $(DEEP_COPY_DIRS)) @$(MAKE) -s $(addsuffix /$(DEEP_COPY_FILENAME), $(DEEP_COPY_DIRS))
# For each dir in DEEP_COPY_DIRS, this generates a statement of the form: # For each dir in DEEP_COPY_DIRS, this establishes a dependency between the
# path/to/dir/$(DEEP_COPY_FILENAME): <all files in dir, except generated ones> # output file and the input files that should trigger a rebuild.
# #
# Note that this is a deps-only statement, not a full rule (see below). # Note that this is a deps-only statement, not a full rule (see below).
# This has to be done in a distinct step because wildcards don't seem to work # This has to be done in a distinct step because wildcards don't seem to work
@@ -314,9 +343,13 @@ gen_deepcopy:
# #
# The '$(eval)' is needed because this has a different RHS for each LHS, and # The '$(eval)' is needed because this has a different RHS for each LHS, and
# would otherwise produce results that make can't parse. # would otherwise produce results that make can't parse.
#
# We depend on the $(GOFILES_META).stamp to detect when the set of input files
# has changed. This allows us to detect deleted input files.
$(foreach dir, $(DEEP_COPY_DIRS), $(eval \ $(foreach dir, $(DEEP_COPY_DIRS), $(eval \
$(dir)/$(DEEP_COPY_FILENAME): $(dir)/$(GOFILES_METAFILE) \ $(dir)/$(DEEP_COPY_FILENAME): $(META_DIR)/$(dir)/$(GOFILES_META).stamp \
$(gofiles__$(dir)) \ $(gofiles__$(dir)) \
$(BIN_DIR)/deepcopy-gen \
)) ))
# For each dir in DEEP_COPY_DIRS, handle deep-copy generation. # For each dir in DEEP_COPY_DIRS, handle deep-copy generation.
@@ -345,19 +378,22 @@ $(addsuffix /$(DEEP_COPY_FILENAME), $(DEEP_COPY_DIRS)):
# IDL. # IDL.
# All directories that request any form of conversion generation. # All directories that request any form of conversion generation.
ifeq ($(DBG_MAKEFILE),1)
$(warning ***** finding all +k8s:conversion-gen tags)
endif
CONVERSION_DIRS := $(shell \ CONVERSION_DIRS := $(shell \
grep '^// *+k8s:conversion-gen=' $(ALL_K8S_TAG_FILES) \ grep '^// *+k8s:conversion-gen=' $(ALL_K8S_TAG_FILES) \
| cut -f1 -d: \ | cut -f1 -d: \
| xargs dirname \ | xargs dirname \
| sort -u \ | sort -u \
) )
# The result file, in each pkg, of conversion generation. # The result file, in each pkg, of conversion generation.
CONVERSION_BASENAME := $(GENERATED_FILE_PREFIX)conversion CONVERSION_BASENAME := $(GENERATED_FILE_PREFIX)conversion
CONVERSION_FILENAME := $(CONVERSION_BASENAME).go CONVERSION_FILENAME := $(CONVERSION_BASENAME).go
# The name of the make metadata file controlling conversions. # The name of the make metadata file controlling conversions.
CONVERSIONS_METAFILE := $(MAKE_METAFILE_PREFIX)conversions CONVERSIONS_META := conversions.mk
# Unfortunately there's not a good way to use Go's build tools to check # Unfortunately there's not a good way to use Go's build tools to check
# if a binary needs to be rebuilt. We just have to try it. # if a binary needs to be rebuilt. We just have to try it.
@@ -372,33 +408,44 @@ gen_conversion:
# files or even *_test.go files), but this is much easier to represent. # files or even *_test.go files), but this is much easier to represent.
# #
# Because we 'sinclude' the deps file, it is considered for rebuilding, as part # Because we 'sinclude' the deps file, it is considered for rebuilding, as part
# of make's normal evaluation. # of make's normal evaluation. If it gets rebuilt, make will restart.
# #
# The '$(eval)' is needed because this has a different RHS for each LHS, and # The '$(eval)' is needed because this has a different RHS for each LHS, and
# would otherwise produce results that make can't parse. # would otherwise produce results that make can't parse.
$(foreach dir, $(CONVERSION_DIRS), $(eval $(dir)/$(CONVERSIONS_METAFILE): $(dir))) $(foreach dir, $(CONVERSION_DIRS), $(eval \
$(META_DIR)/$(dir)/$(CONVERSIONS_META): $(dir) \
))
# How to rebuild a deps file. When make determines that the deps file is stale # How to rebuild a deps file. When make determines that the deps file is stale
# (see above), it executes this rule, and then re-loads the deps file. # (see above), it executes this rule, and then re-loads the deps file.
# #
# This is looser than we really need (e.g. we don't really care about test # This is looser than we really need (e.g. we don't really care about test
# files), but this is MUCH faster than calling `go list`. # files), but this is MUCH faster than calling `go list`.
$(foreach dir, $(CONVERSION_DIRS), $(dir)/$(CONVERSIONS_METAFILE)): #
@TAGS=$$(grep -h '^// *+k8s:conversion-gen=' $(@D)/*.go \ # We regenerate the output file in order to satisfy make's "newer than" rules,
# but we only need to rebuild targets if the contents actually changed. That
# is what the .stamp file represents.
$(foreach dir, $(CONVERSION_DIRS), $(META_DIR)/$(dir)/$(CONVERSIONS_META)):
@TAGS=$$(grep -h '^// *+k8s:conversion-gen=' $</*.go \
| cut -f2- -d= \ | cut -f2- -d= \
| sed 's|$(PRJ_SRC_PATH)/||'); \ | sed 's|$(PRJ_SRC_PATH)/||'); \
echo "conversions__$(@D) := $$(echo $${TAGS})" >$@ mkdir -p $(@D); \
echo "conversions__$< := $$(echo $${TAGS})" >$@.tmp; \
cmp -s $@.tmp $@ || touch $@.stamp; \
mv $@.tmp $@
# Include any deps files as additional Makefile rules. This triggers make to # Include any deps files as additional Makefile rules. This triggers make to
# consider the deps files for rebuild, which makes the whole # consider the deps files for rebuild, which makes the whole
# dependency-management logic work. 'sinclude' is "silent include" which does # dependency-management logic work. 'sinclude' is "silent include" which does
# not fail if the file does not exist. # not fail if the file does not exist.
$(foreach dir, $(CONVERSION_DIRS), $(eval sinclude $(dir)/$(CONVERSIONS_METAFILE))) $(foreach dir, $(CONVERSION_DIRS), $(eval \
sinclude $(META_DIR)/$(dir)/$(CONVERSIONS_META) \
))
# For each dir in CONVERSION_DIRS, this generates a statement of the form: # For each dir in CONVERSION_DIRS, this establishes a dependency between the
# path/to/dir/$(CONVERSION_FILENAME): path/to/dir/$(GOFILES_METAFILE) \ # output file and the input files that should trigger a rebuild.
# $(gofiles__path/to/dir) #
# The variable value was set in $(GOFILES_METAFILE) and included as part of the # The variable value was set in $(GOFILES_META) and included as part of the
# dependency management logic. # dependency management logic.
# #
# Note that this is a deps-only statement, not a full rule (see below). # Note that this is a deps-only statement, not a full rule (see below).
@@ -407,16 +454,20 @@ $(foreach dir, $(CONVERSION_DIRS), $(eval sinclude $(dir)/$(CONVERSIONS_METAFILE
# #
# The '$(eval)' is needed because this has a different RHS for each LHS, and # The '$(eval)' is needed because this has a different RHS for each LHS, and
# would otherwise produce results that make can't parse. # would otherwise produce results that make can't parse.
#
# We depend on the $(GOFILES_META).stamp to detect when the set of input files
# has changed. This allows us to detect deleted input files.
$(foreach dir, $(CONVERSION_DIRS), $(eval \ $(foreach dir, $(CONVERSION_DIRS), $(eval \
$(dir)/$(CONVERSION_FILENAME): $(dir)/$(GOFILES_METAFILE) \ $(dir)/$(CONVERSION_FILENAME): $(META_DIR)/$(dir)/$(GOFILES_META).stamp \
$(gofiles__$(dir)) \ $(gofiles__$(dir)) \
$(BIN_DIR)/conversion-gen \
)) ))
# For each dir in CONVERSION_DIRS, for each target in $(conversions__$(dir)), # For each dir in CONVERSION_DIRS, for each target in $(conversions__$(dir)),
# this generates a statement of the form: # this establishes a dependency between the output file and the input files
# path/to/dir/$(CONVERSION_FILENAME): /path/to/target/$(GOFILES_METAFILE) \ # that should trigger a rebuild.
# $(gofiles__path/to/target/) #
# The variable value was set in $(GOFILES_METAFILE) and included as part of the # The variable value was set in $(GOFILES_META) and included as part of the
# dependency management logic. # dependency management logic.
# #
# Note that this is a deps-only statement, not a full rule (see below). # Note that this is a deps-only statement, not a full rule (see below).
@@ -425,9 +476,12 @@ $(foreach dir, $(CONVERSION_DIRS), $(eval \
# #
# The '$(eval)' is needed because this has a different RHS for each LHS, and # The '$(eval)' is needed because this has a different RHS for each LHS, and
# would otherwise produce results that make can't parse. # would otherwise produce results that make can't parse.
#
# We depend on the $(GOFILES_META).stamp to detect when the set of input files
# has changed. This allows us to detect deleted input files.
$(foreach dir, $(CONVERSION_DIRS), \ $(foreach dir, $(CONVERSION_DIRS), \
$(foreach tgt, $(conversions__$(dir)), $(eval \ $(foreach tgt, $(conversions__$(dir)), $(eval \
$(dir)/$(CONVERSION_FILENAME): $(tgt)/$(GOFILES_METAFILE) \ $(dir)/$(CONVERSION_FILENAME): $(META_DIR)/$(tgt)/$(GOFILES_META).stamp \
$(gofiles__$(tgt)) \ $(gofiles__$(tgt)) \
)) \ )) \
) )

2
hack/verify-flags-underscore.py
View File

@@ -63,6 +63,8 @@ def get_all_files(rootdir):
dirs.remove('third_party') dirs.remove('third_party')
if '.git' in dirs: if '.git' in dirs:
dirs.remove('.git') dirs.remove('.git')
if '.make' in dirs:
dirs.remove('.make')
if 'exceptions.txt' in files: if 'exceptions.txt' in files:
files.remove('exceptions.txt') files.remove('exceptions.txt')
if 'known-flags.txt' in files: if 'known-flags.txt' in files: