feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

build: cleanup working directories and mac build

Browse Source 
centralized all working directories at .work in the root of the tree.  also centralize all download directories to .download at the root of the tree.

cleaned up the rsync behavior between host and docker cross build when running on non-linux host.
This commit is contained in:
Bassam Tabbara committed 2017-04-04 17:02:25 -07:00
1 parent 59f5815115
commit cbacbe957d
7 files changed
+110 -70
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+2 -3
View File
@@ -1,7 +1,6 @@
.vscode/
/.vscode/
/.work
/.tools
/.download
/bin
/vendor
Makefile
+36 -24
View File
@@ -67,7 +67,8 @@ GOOS ?= $(shell go env GOOS)
GOARCH ?= $(shell go env GOARCH)
# the working directory to store packages and intermediate build files
WORKDIR ?= .work
WORKDIR ?= $(abspath .work)
DOWNLOADDIR ?= $(abspath .download)
# bin and relase dirs
BIN_DIR=bin
@@ -111,8 +112,8 @@ endif
endif
# Set the cgo flags to link externals
CGO_CFLAGS = -I$(abspath external/build/$(CROSS_TRIPLE)/include)
CGO_LDFLAGS = -L$(abspath external/build/$(CROSS_TRIPLE)/lib)
CGO_CFLAGS = -I$(abspath $(WORKDIR)/build/$(CROSS_TRIPLE)/include)
CGO_LDFLAGS = -L$(abspath $(WORKDIR)/build/$(CROSS_TRIPLE)/lib)
endif
@@ -138,6 +139,8 @@ GO_BUILDFLAGS=$(BUILDFLAGS)
GO_LDFLAGS=$(LDFLAGS)
GO_TAGS=$(TAGS)
GO_WORK_DIR ?= $(WORKDIR)
GO_TOOLS_DIR ?= $(DOWNLOADDIR)/tools
GO_PKG_DIR ?= $(WORKDIR)/pkg
include build/makelib/golang.mk
@@ -154,14 +157,18 @@ include build/makelib/release.mk
# ====================================================================================
# External Targets
export ALWAYS_BUILD DOWNLOADDIR
export WORKDIR DOWNLOADDIR CEPH_BRANCH ALLOCATOR
ifneq ($(ALWAYS_BUILD),)
export ALWAYS_BUILD
endif
external:
ifeq ($(GOOS),linux)
@$(MAKE) -C external CEPH_BRANCH=$(CEPH_BRANCH) ALLOCATOR=$(ALLOCATOR) PLATFORMS=$(CROSS_TRIPLE) cross
@$(MAKE) -C external PLATFORMS=$(CROSS_TRIPLE) cross
endif
external/build/$(CROSS_TRIPLE)/lib/libcephd.a:
$(WORKDIR)/build/$(CROSS_TRIPLE)/lib/libcephd.a:
@$(MAKE) external
external.clean:
@@ -175,7 +182,7 @@ external.distclean:
# ====================================================================================
# Targets
dev: external/build/$(CROSS_TRIPLE)/lib/libcephd.a
dev: $(WORKDIR)/build/$(CROSS_TRIPLE)/lib/libcephd.a
@$(MAKE) go.init
@$(MAKE) go.validate
@$(MAKE) go.build
@@ -213,6 +220,7 @@ clean: go.clean external.clean
@rm -fr $(WORKDIR) $(RELEASE_DIR)/* $(BIN_DIR)/*
distclean: go.distclean clean external.distclean
@rm -fr $(DOWNLOADDIR)
cross.build:
@$(MAKE) external
@@ -283,20 +291,24 @@ help:
@echo ' publish Publishes all packages from a release.'
@echo ''
@echo 'Options:'
@echo ' CHANNEL Sets the release channel. Can be set to master,'
@echo ' alpha, beta, or stable. Default is not set.'
@echo ' GOARCH The arch to build.'
@echo ' PIE Set to 1 to build build a position independent'
@echo ' executable. Can not be combined with LINKMODE'
@echo ' set to "static". The default is 0.'
@echo ' GOOS The OS to build for.'
@echo ' LINKMODE Set to "dynamic" to link all libraries dynamically.'
@echo ' Set to "stdlib" to link the standard library'
@echo ' dynamically and everything else statically. Set to'
@echo ' "static" to link everything statically. Default is'
@echo ' "dynamic".'
@echo ' VERSION The version information compiled into binaries.'
@echo ' The default is obtained from git.'
@echo ' V Set to 1 enable verbose build. Default is 0.'
@echo ' WORKDIR The working directory where most build files'
@echo ' are stored. The default is .work'
@echo ' CHANNEL Sets the release channel. Can be set to master,'
@echo ' alpha, beta, or stable. Default is not set.'
@echo ' DOWNLOADDIR A directory where downloaded files and other'
@echo ' files used during the build are cached. These'
@echo ' files help speedup the build by avoding network'
@echo ' transfers. Its safe to use these files across builds.'
@echo ' GOARCH The arch to build.'
@echo ' GOOS The OS to build for.'
@echo ' LINKMODE Set to "dynamic" to link all libraries dynamically.'
@echo ' Set to "stdlib" to link the standard library'
@echo ' dynamically and everything else statically. Set to'
@echo ' "static" to link everything statically. Default is'
@echo ' "dynamic".'
@echo ' PIE Set to 1 to build build a position independent'
@echo ' executable. Can not be combined with LINKMODE'
@echo ' set to "static". The default is 0.'
@echo ' VERSION The version information compiled into binaries.'
@echo ' The default is obtained from git.'
@echo ' V Set to 1 enable verbose build. Default is 0.'
@echo ' WORKDIR The working directory where most build files'
@echo ' are stored. The default is .work'
build/common.sh
+20 -14
View File
@@ -71,6 +71,11 @@ function stop_rsync_container() {
}
function run_rsync() {
src=$1
shift
dst=$1
shift
# run the container as an rsyncd daemon so that we can copy the
# source tree to the container volume.
@@ -79,20 +84,21 @@ function run_rsync() {
# wait for rsync to come up
wait_for_rsync || stop_rsync_container ${id}
for pair in "$*" ; do
src="${pair%%-->*}"
dst="${pair##*-->}"
rsync \
--archive \
--delete \
--prune-empty-dirs \
--filter='- /.work/' \
--filter='- /.vscode/' \
--filter='- index.lock' \
$src $dst || { stop_rsync_container ${id}; return 1; }
done
# NOTE: add --progress to show files being syncd
rsync \
--archive \
--delete \
--prune-empty-dirs \
"$@" \
$src $dst || { stop_rsync_container ${id}; return 1; }
stop_rsync_container ${id}
}
function rsync_host_to_container() {
run_rsync ${scriptdir}/.. rsync://localhost:${rsync_port}/volume/go/src/${source_repo} "$@"
}
function rsync_container_to_host() {
run_rsync rsync://localhost:${rsync_port}/volume/go/src/${source_repo}/ ${scriptdir}/.. "$@"
}
build/makelib/golang.mk
+2 -1
View File
@@ -46,6 +46,7 @@ GO_SUBDIRS ?= cmd pkg
# Optional directories (relative to CURDIR)
GO_BIN_DIR ?= bin
GO_TOOLS_DIR ?= .tools
GO_WORK_DIR ?= .work
GO_VENDOR_DIR ?= vendor
GO_PKG_DIR ?=
@@ -83,7 +84,7 @@ export CGO_CFLAGS CGO_CPPFLAGS CGO_LDFLAGS
# setup tools used during the build
GO_TOOLS_HOST_DIR ?= $(abspath $(GO_TOOLS_DIR)/$(GOHOSTOS)_$(GOHOSTARCH))
GLIDE_HOME := $(abspath $(GO_TOOLS_DIR)/glide)
GLIDE_HOME := $(abspath $(GO_WORK_DIR)/glide)
GLIDE := $(GO_TOOLS_HOST_DIR)/glide
GOLINT := $(GO_TOOLS_HOST_DIR)/golint
export GLIDE_HOME
build/run
+30 -17
View File
@@ -60,19 +60,22 @@ if [ "`uname -s`" != "Linux" ]; then
# /.netrc (bind mounted to host ${HOME}/.netrc)
# /.aws (bind mounted to host ${HOME}/.aws)
# /.docker (bind mounted to host ${HOME}/.docker)
# /.download
# /.download (bind mounted to host ${DOWNLOADDIR} if set)
# /.ccache
# /go/src/github.com/rook/rook (rsync'd from host <source>)
# now copy the source tree to the container volume
run_rsync "${scriptdir}/..-->rsync://localhost:${rsync_port}/volume/go/src/${source_repo}"
# now copy the source tree to the container volume. Note this also
# copies the .git directory but not the index.lock files which
# change frequently and sometimes cause rsync to fail.
rsync_host_to_container \
--filter="- /.work/" \
--filter="- /.download/" \
--filter="- /.vscode/" \
--filter="- index.lock"
MOUNT_OPTS="${MOUNT_OPTS} -v ${container_volume}:${BUILDER_HOME}"
CCACHE_ARGS="-e CCACHE_DIR=${BUILDER_HOME}/.ccache -e CCACHE_LOG"
DOWNLOAD_ARGS="-e DOWNLOADDIR=${BUILDER_HOME}/.download"
else
# On linux bind mounts are cheap so we dont need to rsync
@@ -101,13 +104,13 @@ else
CCACHE_ARGS="-e CCACHE_DIR=${BUILDER_HOME}/.ccache -e CCACHE_LOG \
-v ${CCACHE_DIR}:${BUILDER_HOME}/.ccache"
fi
fi
# if DOWNLOADDIR is not set bind mount it
if [[ -n "${DOWNLOADDIR}" ]]; then
mkdir -p ${DOWNLOADDIR}
DOWNLOAD_ARGS="-e DOWNLOADDIR=${BUILDER_HOME}/.download \
-v ${DOWNLOADDIR}:${BUILDER_HOME}/.download"
fi
# if DOWNLOADDIR is not set bind mount it
if [[ -n "${DOWNLOADDIR}" ]]; then
mkdir -p ${DOWNLOADDIR}
DOWNLOAD_ARGS="-e DOWNLOADDIR=${BUILDER_HOME}/go/src/${source_repo}/.download \
-v ${DOWNLOADDIR}:${BUILDER_HOME}/go/src/${source_repo}/.download"
fi
# we copy credential files for github access
@@ -133,6 +136,18 @@ AWS_ARGS="\
-e AWS_DEFAULT_REGION \
-v ${HOME}/.aws:${BUILDER_HOME}/.aws"
rsync_back() {
if [ "`uname -s`" != "Linux" ]; then
# rsync generated files back to the host. IMPORTANT: we never rsync the .git directory back
# since the host's version might have changed.
rsync_container_to_host \
--filter="+ /bin/" \
--filter="+ /release/" \
--filter="+ /vendor/" \
--filter="- /*"
fi
}
docker run \
--rm \
-e GOPATH="${BUILDER_HOME}/go" \
@@ -141,6 +156,7 @@ docker run \
-e VERSION \
-e LINKMODE=static \
-e CHANNEL \
-e ALWAYS_BUILD \
${TTY_ARGS} \
${CCACHE_ARGS} \
${DOWNLOAD_ARGS} \
@@ -152,9 +168,6 @@ docker run \
${MOUNT_OPTS} \
-w ${BUILDER_HOME}/go/src/${source_repo} \
${container_image} \
"$@"
"$@" || { rsync_back; exit 1; }
# rsync the source tree back
if [ "`uname -s`" != "Linux" ]; then
run_rsync "rsync://localhost:${rsync_port}/volume/go/src/${source_repo}-->${scriptdir}/../.."
fi
rsync_back
external/.gitignore
+2 -4
View File
@@ -1,4 +1,2 @@
install/
build/
download/
.work/
.download/
external/Makefile
Vendored
+18 -7
View File
@@ -25,7 +25,7 @@ ALLOCATOR ?= tcmalloc_minimal
CEPH_BRANCH ?= kraken
# Whether to build ceph everytime and ignore the timestamps
ALWAYS_BUILD ?= ON
ALWAYS_BUILD ?= 1
# The platforms supported for cross building
PLATFORMS ?= x86_64-linux-gnu aarch64-linux-gnu
@@ -51,10 +51,15 @@ ifeq ($(CCACHE),1)
EXTERNAL_CCACHE=ON
endif
# The build and download directories
ROOTDIR ?= $(CURDIR)
BUILDDIR ?= $(ROOTDIR)/build
DOWNLOADDIR ?= $(ROOTDIR)/download
# The working directory where build generated files are stored
WORKDIR ?= .work
BUILDDIR= $(WORKDIR)/build
# A directory to hold files that are downloaded
# as part of the build process but are immutable and do not
# vary across build runs. These are can be cleaned with
# "make distclean"
DOWNLOADDIR ?= .download
# The host platform
HOST = $(shell gcc -dumpmachine)
@@ -117,12 +122,18 @@ help:
@echo ' tcmalloc_minimal (default).'
@echo ' ALWAYS_BUILD Set to 1 to always build ceph. The'
@echo ' default is 1.'
@echo ' CEPH_BRANCH The ceph branch to build from. Can be set'
@echo ' to kraken, or luminous.'
@echo ' BUILD_TYPE The build type. Debug produces non-'
@echo ' optimized builds. Release produces fully'
@echo ' optimized builds. RelWithDebInfo produces'
@echo ' optimized builds with debug information.'
@echo ' The default is RelWithDebInfo.'
@echo ' CCACHE Set to 1 to use ccache. The default is 1.'
@echo ' CEPH_BRANCH The ceph branch to build from. Can be set'
@echo ' to kraken, or luminous.'
@echo ' DOWNLOADDIR A directory where downloaded files and other'
@echo ' files used during the build are cached. These'
@echo ' files help speedup the build by avoding network'
@echo ' transfers. Its safe to use these files across builds.'
@echo ' V Set to 1 enable verbose build. Default is 0.'
@echo ' WORKDIR The working directory where most build files'
@echo ' are stored. The default is .work'