From cbacbe957ddc4e61ed1cbceecea745041b1bd72b Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Tue, 4 Apr 2017 12:36:29 -0700 Subject: [PATCH] build: cleanup working directories and mac build centralized all working directories at .work in the root of the tree. also centralize all download directories to .download at the root of the tree. cleaned up the rsync behavior between host and docker cross build when running on non-linux host. --- .gitignore | 5 ++-- Makefile | 60 ++++++++++++++++++++++++----------------- build/common.sh | 34 +++++++++++++---------- build/makelib/golang.mk | 3 ++- build/run | 47 ++++++++++++++++++++------------ external/.gitignore | 6 ++--- external/Makefile | 25 ++++++++++++----- 7 files changed, 110 insertions(+), 70 deletions(-) diff --git a/.gitignore b/.gitignore index c2e2d31a3..647e38655 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ -.vscode/ - +/.vscode/ /.work -/.tools +/.download /bin /vendor diff --git a/Makefile b/Makefile index bef098058..d9cdf7ffd 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,8 @@ GOOS ?= $(shell go env GOOS) GOARCH ?= $(shell go env GOARCH) # the working directory to store packages and intermediate build files -WORKDIR ?= .work +WORKDIR ?= $(abspath .work) +DOWNLOADDIR ?= $(abspath .download) # bin and relase dirs BIN_DIR=bin @@ -111,8 +112,8 @@ endif endif # Set the cgo flags to link externals -CGO_CFLAGS = -I$(abspath external/build/$(CROSS_TRIPLE)/include) -CGO_LDFLAGS = -L$(abspath external/build/$(CROSS_TRIPLE)/lib) +CGO_CFLAGS = -I$(abspath $(WORKDIR)/build/$(CROSS_TRIPLE)/include) +CGO_LDFLAGS = -L$(abspath $(WORKDIR)/build/$(CROSS_TRIPLE)/lib) endif @@ -138,6 +139,8 @@ GO_BUILDFLAGS=$(BUILDFLAGS) GO_LDFLAGS=$(LDFLAGS) GO_TAGS=$(TAGS) +GO_WORK_DIR ?= $(WORKDIR) +GO_TOOLS_DIR ?= $(DOWNLOADDIR)/tools GO_PKG_DIR ?= $(WORKDIR)/pkg include build/makelib/golang.mk @@ -154,14 +157,18 @@ include build/makelib/release.mk # ==================================================================================== # External Targets -export ALWAYS_BUILD DOWNLOADDIR +export WORKDIR DOWNLOADDIR CEPH_BRANCH ALLOCATOR + +ifneq ($(ALWAYS_BUILD),) +export ALWAYS_BUILD +endif external: ifeq ($(GOOS),linux) - @$(MAKE) -C external CEPH_BRANCH=$(CEPH_BRANCH) ALLOCATOR=$(ALLOCATOR) PLATFORMS=$(CROSS_TRIPLE) cross + @$(MAKE) -C external PLATFORMS=$(CROSS_TRIPLE) cross endif -external/build/$(CROSS_TRIPLE)/lib/libcephd.a: +$(WORKDIR)/build/$(CROSS_TRIPLE)/lib/libcephd.a: @$(MAKE) external external.clean: @@ -175,7 +182,7 @@ external.distclean: # ==================================================================================== # Targets -dev: external/build/$(CROSS_TRIPLE)/lib/libcephd.a +dev: $(WORKDIR)/build/$(CROSS_TRIPLE)/lib/libcephd.a @$(MAKE) go.init @$(MAKE) go.validate @$(MAKE) go.build @@ -213,6 +220,7 @@ clean: go.clean external.clean @rm -fr $(WORKDIR) $(RELEASE_DIR)/* $(BIN_DIR)/* distclean: go.distclean clean external.distclean + @rm -fr $(DOWNLOADDIR) cross.build: @$(MAKE) external @@ -283,20 +291,24 @@ help: @echo ' publish Publishes all packages from a release.' @echo '' @echo 'Options:' - @echo ' CHANNEL Sets the release channel. Can be set to master,' - @echo ' alpha, beta, or stable. Default is not set.' - @echo ' GOARCH The arch to build.' - @echo ' PIE Set to 1 to build build a position independent' - @echo ' executable. Can not be combined with LINKMODE' - @echo ' set to "static". The default is 0.' - @echo ' GOOS The OS to build for.' - @echo ' LINKMODE Set to "dynamic" to link all libraries dynamically.' - @echo ' Set to "stdlib" to link the standard library' - @echo ' dynamically and everything else statically. Set to' - @echo ' "static" to link everything statically. Default is' - @echo ' "dynamic".' - @echo ' VERSION The version information compiled into binaries.' - @echo ' The default is obtained from git.' - @echo ' V Set to 1 enable verbose build. Default is 0.' - @echo ' WORKDIR The working directory where most build files' - @echo ' are stored. The default is .work' + @echo ' CHANNEL Sets the release channel. Can be set to master,' + @echo ' alpha, beta, or stable. Default is not set.' + @echo ' DOWNLOADDIR A directory where downloaded files and other' + @echo ' files used during the build are cached. These' + @echo ' files help speedup the build by avoding network' + @echo ' transfers. Its safe to use these files across builds.' + @echo ' GOARCH The arch to build.' + @echo ' GOOS The OS to build for.' + @echo ' LINKMODE Set to "dynamic" to link all libraries dynamically.' + @echo ' Set to "stdlib" to link the standard library' + @echo ' dynamically and everything else statically. Set to' + @echo ' "static" to link everything statically. Default is' + @echo ' "dynamic".' + @echo ' PIE Set to 1 to build build a position independent' + @echo ' executable. Can not be combined with LINKMODE' + @echo ' set to "static". The default is 0.' + @echo ' VERSION The version information compiled into binaries.' + @echo ' The default is obtained from git.' + @echo ' V Set to 1 enable verbose build. Default is 0.' + @echo ' WORKDIR The working directory where most build files' + @echo ' are stored. The default is .work' diff --git a/build/common.sh b/build/common.sh index ab010d698..84b5879b0 100644 --- a/build/common.sh +++ b/build/common.sh @@ -71,6 +71,11 @@ function stop_rsync_container() { } function run_rsync() { + src=$1 + shift + + dst=$1 + shift # run the container as an rsyncd daemon so that we can copy the # source tree to the container volume. @@ -79,20 +84,21 @@ function run_rsync() { # wait for rsync to come up wait_for_rsync || stop_rsync_container ${id} - for pair in "$*" ; do - src="${pair%%-->*}" - dst="${pair##*-->}" - - rsync \ - --archive \ - --delete \ - --prune-empty-dirs \ - --filter='- /.work/' \ - --filter='- /.vscode/' \ - --filter='- index.lock' \ - $src $dst || { stop_rsync_container ${id}; return 1; } - done - + # NOTE: add --progress to show files being syncd + rsync \ + --archive \ + --delete \ + --prune-empty-dirs \ + "$@" \ + $src $dst || { stop_rsync_container ${id}; return 1; } stop_rsync_container ${id} } + +function rsync_host_to_container() { + run_rsync ${scriptdir}/.. rsync://localhost:${rsync_port}/volume/go/src/${source_repo} "$@" +} + +function rsync_container_to_host() { + run_rsync rsync://localhost:${rsync_port}/volume/go/src/${source_repo}/ ${scriptdir}/.. "$@" +} diff --git a/build/makelib/golang.mk b/build/makelib/golang.mk index c07f34a28..244667744 100644 --- a/build/makelib/golang.mk +++ b/build/makelib/golang.mk @@ -46,6 +46,7 @@ GO_SUBDIRS ?= cmd pkg # Optional directories (relative to CURDIR) GO_BIN_DIR ?= bin GO_TOOLS_DIR ?= .tools +GO_WORK_DIR ?= .work GO_VENDOR_DIR ?= vendor GO_PKG_DIR ?= @@ -83,7 +84,7 @@ export CGO_CFLAGS CGO_CPPFLAGS CGO_LDFLAGS # setup tools used during the build GO_TOOLS_HOST_DIR ?= $(abspath $(GO_TOOLS_DIR)/$(GOHOSTOS)_$(GOHOSTARCH)) -GLIDE_HOME := $(abspath $(GO_TOOLS_DIR)/glide) +GLIDE_HOME := $(abspath $(GO_WORK_DIR)/glide) GLIDE := $(GO_TOOLS_HOST_DIR)/glide GOLINT := $(GO_TOOLS_HOST_DIR)/golint export GLIDE_HOME diff --git a/build/run b/build/run index 500d48f92..ea6051bc2 100755 --- a/build/run +++ b/build/run @@ -60,19 +60,22 @@ if [ "`uname -s`" != "Linux" ]; then # /.netrc (bind mounted to host ${HOME}/.netrc) # /.aws (bind mounted to host ${HOME}/.aws) # /.docker (bind mounted to host ${HOME}/.docker) - # /.download + # /.download (bind mounted to host ${DOWNLOADDIR} if set) # /.ccache # /go/src/github.com/rook/rook (rsync'd from host ) - # now copy the source tree to the container volume - run_rsync "${scriptdir}/..-->rsync://localhost:${rsync_port}/volume/go/src/${source_repo}" + # now copy the source tree to the container volume. Note this also + # copies the .git directory but not the index.lock files which + # change frequently and sometimes cause rsync to fail. + rsync_host_to_container \ + --filter="- /.work/" \ + --filter="- /.download/" \ + --filter="- /.vscode/" \ + --filter="- index.lock" MOUNT_OPTS="${MOUNT_OPTS} -v ${container_volume}:${BUILDER_HOME}" CCACHE_ARGS="-e CCACHE_DIR=${BUILDER_HOME}/.ccache -e CCACHE_LOG" - - DOWNLOAD_ARGS="-e DOWNLOADDIR=${BUILDER_HOME}/.download" - else # On linux bind mounts are cheap so we dont need to rsync @@ -101,13 +104,13 @@ else CCACHE_ARGS="-e CCACHE_DIR=${BUILDER_HOME}/.ccache -e CCACHE_LOG \ -v ${CCACHE_DIR}:${BUILDER_HOME}/.ccache" fi +fi - # if DOWNLOADDIR is not set bind mount it - if [[ -n "${DOWNLOADDIR}" ]]; then - mkdir -p ${DOWNLOADDIR} - DOWNLOAD_ARGS="-e DOWNLOADDIR=${BUILDER_HOME}/.download \ - -v ${DOWNLOADDIR}:${BUILDER_HOME}/.download" - fi +# if DOWNLOADDIR is not set bind mount it +if [[ -n "${DOWNLOADDIR}" ]]; then + mkdir -p ${DOWNLOADDIR} + DOWNLOAD_ARGS="-e DOWNLOADDIR=${BUILDER_HOME}/go/src/${source_repo}/.download \ + -v ${DOWNLOADDIR}:${BUILDER_HOME}/go/src/${source_repo}/.download" fi # we copy credential files for github access @@ -133,6 +136,18 @@ AWS_ARGS="\ -e AWS_DEFAULT_REGION \ -v ${HOME}/.aws:${BUILDER_HOME}/.aws" +rsync_back() { + if [ "`uname -s`" != "Linux" ]; then + # rsync generated files back to the host. IMPORTANT: we never rsync the .git directory back + # since the host's version might have changed. + rsync_container_to_host \ + --filter="+ /bin/" \ + --filter="+ /release/" \ + --filter="+ /vendor/" \ + --filter="- /*" + fi +} + docker run \ --rm \ -e GOPATH="${BUILDER_HOME}/go" \ @@ -141,6 +156,7 @@ docker run \ -e VERSION \ -e LINKMODE=static \ -e CHANNEL \ + -e ALWAYS_BUILD \ ${TTY_ARGS} \ ${CCACHE_ARGS} \ ${DOWNLOAD_ARGS} \ @@ -152,9 +168,6 @@ docker run \ ${MOUNT_OPTS} \ -w ${BUILDER_HOME}/go/src/${source_repo} \ ${container_image} \ - "$@" + "$@" || { rsync_back; exit 1; } -# rsync the source tree back -if [ "`uname -s`" != "Linux" ]; then - run_rsync "rsync://localhost:${rsync_port}/volume/go/src/${source_repo}-->${scriptdir}/../.." -fi +rsync_back diff --git a/external/.gitignore b/external/.gitignore index 9cd050f84..59b873708 100644 --- a/external/.gitignore +++ b/external/.gitignore @@ -1,4 +1,2 @@ -install/ -build/ -download/ - +.work/ +.download/ diff --git a/external/Makefile b/external/Makefile index 534bb4d10..b61ce5879 100644 --- a/external/Makefile +++ b/external/Makefile @@ -25,7 +25,7 @@ ALLOCATOR ?= tcmalloc_minimal CEPH_BRANCH ?= kraken # Whether to build ceph everytime and ignore the timestamps -ALWAYS_BUILD ?= ON +ALWAYS_BUILD ?= 1 # The platforms supported for cross building PLATFORMS ?= x86_64-linux-gnu aarch64-linux-gnu @@ -51,10 +51,15 @@ ifeq ($(CCACHE),1) EXTERNAL_CCACHE=ON endif -# The build and download directories -ROOTDIR ?= $(CURDIR) -BUILDDIR ?= $(ROOTDIR)/build -DOWNLOADDIR ?= $(ROOTDIR)/download +# The working directory where build generated files are stored +WORKDIR ?= .work +BUILDDIR= $(WORKDIR)/build + +# A directory to hold files that are downloaded +# as part of the build process but are immutable and do not +# vary across build runs. These are can be cleaned with +# "make distclean" +DOWNLOADDIR ?= .download # The host platform HOST = $(shell gcc -dumpmachine) @@ -117,12 +122,18 @@ help: @echo ' tcmalloc_minimal (default).' @echo ' ALWAYS_BUILD Set to 1 to always build ceph. The' @echo ' default is 1.' - @echo ' CEPH_BRANCH The ceph branch to build from. Can be set' - @echo ' to kraken, or luminous.' @echo ' BUILD_TYPE The build type. Debug produces non-' @echo ' optimized builds. Release produces fully' @echo ' optimized builds. RelWithDebInfo produces' @echo ' optimized builds with debug information.' @echo ' The default is RelWithDebInfo.' @echo ' CCACHE Set to 1 to use ccache. The default is 1.' + @echo ' CEPH_BRANCH The ceph branch to build from. Can be set' + @echo ' to kraken, or luminous.' + @echo ' DOWNLOADDIR A directory where downloaded files and other' + @echo ' files used during the build are cached. These' + @echo ' files help speedup the build by avoding network' + @echo ' transfers. Its safe to use these files across builds.' @echo ' V Set to 1 enable verbose build. Default is 0.' + @echo ' WORKDIR The working directory where most build files' + @echo ' are stored. The default is .work'