build: fix ccache when running in build container + misc

This commit is contained in:
Bassam Tabbara committed 2017-06-20 20:57:46 -07:00
1 parent b7c352fb25
commit bdf5887e13
7 files changed
+76 -52

No files matched your search

Vendored
+3 -9
View File
@@ -5,15 +5,9 @@ try {
stage('Checkout') {
checkout scm
sh 'git submodule sync --recursive'
sh 'git submodule update --init --recursive'
}
stage('Validation') {
sh 'external/ceph-submodule-check'
}
withEnv(["DOWNLOADDIR=${env.HOME}/.download", "ALWAYS_BUILD=0", "CHANNEL=${env.BRANCH_NAME}"]) {
withEnv(["DOWNLOADDIR=${env.HOME}/.download", "CHANNEL=${env.BRANCH_NAME}"]) {
stage('Build') {
sh 'build/run make -j\$(nproc) release'
@@ -55,7 +49,7 @@ try {
}
stage('Cleanup') {
sh 'build/run make -j\$(nproc) publish.cleanup'
sh 'build/run make -j\$(nproc) prune'
sh 'docker images'
deleteDir()
}
@@ -67,7 +61,7 @@ catch (Exception e) {
node("ec2-stateful") {
echo 'Cleaning up workspace'
sh 'build/run make -j\$(nproc) publish.cleanup'
sh 'build/run make -j\$(nproc) prune'
sh 'docker images'
deleteDir()
}
+33 -22
View File
@@ -63,11 +63,14 @@ endif
BIN_DIR ?= bin
RELEASE_DIR ?= release
# platforms for which we client and server bits
SERVER_PLATFORMS ?= linux_arm linux_amd64 linux_arm64
# platforms where we only build client bits
CLIENT_PLATFORMS ?= darwin_amd64 windows_amd64
# platforms where we build client and server bits
SERVER_PLATFORMS ?= linux_arm linux_amd64 linux_arm64
# the platforms to build
PLATFORMS ?= $(SERVER_PLATFORMS) $(CLIENT_PLATFORMS)
# the root go project
GO_PROJECT=github.com/rook/rook
@@ -117,7 +120,7 @@ include build/makelib/golang.mk
RELEASE_VERSION=$(VERSION)
RELEASE_CHANNEL=$(CHANNEL)
RELEASE_BIN_DIR=$(BIN_DIR)
RELEASE_PLATFORMS=$(CLIENT_PLATFORMS) $(SERVER_PLATFORMS)
RELEASE_PLATFORMS=$(PLATFORMS)
include build/makelib/release.mk
# ====================================================================================
@@ -127,7 +130,7 @@ dev:
@$(MAKE) go.init
@$(MAKE) go.validate
@$(MAKE) go.build
@$(MAKE) release.build.containers.$(GOOS)_$(GOARCH)
@$(MAKE) -C images
build.common:
@$(MAKE) go.init
@@ -168,7 +171,7 @@ cross.build:
cross.build.platform.%:
@$(MAKE) GOOS=$(word 1, $(subst _, ,$*)) GOARCH=$(word 2, $(subst _, ,$*)) PIE=$(PIE) cross.build
cross.parallel: $(foreach p,$(CLIENT_PLATFORMS) $(SERVER_PLATFORMS), cross.build.platform.$(p))
cross.parallel: $(foreach p,$(PLATFORMS), cross.build.platform.$(p))
cross:
@$(MAKE) go.init
@@ -176,11 +179,11 @@ cross:
@$(MAKE) cross.parallel
release: cross
@$(MAKE) release.build
@$(MAKE) -C images cross
publish:
ifneq ($(filter master alpha beta stable, $(CHANNEL)),)
@$(MAKE) release.publish
@$(MAKE) -C images push
else
@echo skipping publish. invalid channel "$(CHANNEL)"
endif
@@ -192,11 +195,11 @@ else
@echo skipping promote. invalid channel "$(CHANNEL)"
endif
publish.cleanup:
@$(MAKE) release.cleanup
prune:
@$(MAKE) -C images prune
.PHONY: build.common cross.build cross.parallel
.PHONY: dev build install test check vet fmt vendor clean distclean cross release publish
.PHONY: dev build install test check vet fmt vendor clean distclean cross images release publish promote prune
# ====================================================================================
# Help
@@ -206,38 +209,46 @@ help:
@echo 'Usage: make <OPTIONS> ... <TARGETS>'
@echo ''
@echo 'Targets:'
@echo ' build Build project for host platform.'
@echo ' cross Build project for all platforms.'
@echo ' build Build source code for host platform.'
@echo ' cross Build source code for all platforms.'
@echo ' Best done in the cross-build container'
@echo ' due to cross compiler dependencies.'
@echo ' check Runs unit tests.'
@echo ' clean Remove all files that are created '
@echo ' by building.'
@echo ' dev A quick build path for go projects'
@echo ' and containers.'
@echo ' images Builds container images for host platform.'
@echo ' distclean Remove all files that are created '
@echo ' by building or configuring.'
@echo ' fmt Check formatting of go sources.'
@echo ' lint Check syntax and styling of go sources.'
@echo ' help Show this help info.'
@echo ' prune Prune cached artifacts.'
@echo ' vendor Installs vendor dependencies.'
@echo ' vet Runs lint checks on go sources.'
@echo ''
@echo 'Distribution:'
@echo ' release Builds all packages.'
@echo ' publish Publishes all packages from a release.'
@echo 'Release Targets:'
@echo ' release Builds all release artifacts including'
@echo ' container images for all platforms.'
@echo ' publish Publishes all release artifacts to'
@echo ' appropriate public repositories.'
@echo ' promote Promotes a published release to a'
@echo ' release channel.'
@echo ''
@echo 'Options:'
@echo ' GOARCH The arch to build.'
@echo ' GOOS The OS to build for.'
@echo ' VERSION The version information compiled into binaries.'
@echo ' The default is obtained from git.'
@echo ' V Set to 1 enable verbose build. Default is 0.'
@echo ''
@echo 'Advanced Options:'
@echo ' CHANNEL Sets the release channel. Can be set to master,'
@echo ' alpha, beta, or stable. Default is not set.'
@echo ' DOWNLOADDIR A directory where downloaded files and other'
@echo ' files used during the build are cached. These'
@echo ' files help speedup the build by avoiding network'
@echo ' transfers. Its safe to use these files across builds.'
@echo ' GOARCH The arch to build.'
@echo ' GOOS The OS to build for.'
@echo ' PIE Set to 1 to build a position independent'
@echo ' executable. The default is 1.'
@echo ' VERSION The version information compiled into binaries.'
@echo ' The default is obtained from git.'
@echo ' V Set to 1 enable verbose build. Default is 0.'
@echo ' WORKDIR The working directory where most build files'
@echo ' are stored. The default is .work'
+7 -3
View File
@@ -27,7 +27,7 @@ fi
# build the the container if we've not done so already
if [[ -z "$(docker images -q ${container_image})" ]]; then
echo ==== building the cross-build container \(this could take minutes the first time\)
make_cmd="make --no-print-directory -C ${scriptdir}/../images cross-build-images PULL=0"
make_cmd="make --no-print-directory -C ${scriptdir}/../images cross-build PULL=0"
make_output=$($make_cmd 2>&1) || {
cat <<EOF >&2
=== cross build image failed for ${container_image}
@@ -82,7 +82,8 @@ if [ "`uname -s`" != "Linux" ]; then
MOUNT_OPTS="${MOUNT_OPTS} -v ${container_volume}:${BUILDER_HOME}"
CCACHE_ARGS="-e CCACHE_DIR=${BUILDER_HOME}/.ccache -e CCACHE_LOG"
CCACHE_ARGS="-e CCACHE_DIR=${BUILDER_HOME}/.ccache \
-e CCACHE_DIR_HOST=${BUILDER_HOME}/.ccache -e CCACHE_LOG"
else
# On linux bind mounts are cheap so we dont need to rsync
@@ -109,7 +110,9 @@ else
fi
if [[ -d "${CCACHE_DIR}" ]]; then
CCACHE_ARGS="-e CCACHE_DIR=${BUILDER_HOME}/.ccache -e CCACHE_LOG \
CCACHE_ARGS="-e CCACHE_DIR=${BUILDER_HOME}/.ccache \
-e CCACHE_DIR_HOST=${CCACHE_DIR} \
-e CCACHE_LOG \
-v ${CCACHE_DIR}:${BUILDER_HOME}/.ccache"
fi
fi
@@ -160,6 +163,7 @@ rsync_back() {
docker run \
--rm \
-h ${host} \
-e GOPATH="${BUILDER_HOME}/go" \
-e GO_PKG_DIR="" \
-e GITHUB_TOKEN \
+19 -15
View File
@@ -19,8 +19,8 @@ include image.mk
PLATFORMS ?= amd64 arm arm64
.PHONY: cross-build-images
cross-build-images:
.PHONY: cross-build
cross-build:
@$(MAKE) -C base GOARCH=amd64
@$(MAKE) -C cross-build-gnu
@$(MAKE) -C cross-build-go
@@ -30,7 +30,9 @@ build.%:
@$(MAKE) -C base build GOARCH=$*
@$(MAKE) -C rookd build GOARCH=$*
build: cross-build-images
build:
@$(MAKE) -C base GOARCH=amd64
@$(MAKE) -C cross-build-gnu
@$(MAKE) build.$(GOARCH)
push.%:
@@ -40,7 +42,9 @@ push: $(foreach p,$(PLATFORMS), push.$(p))
clean: clean.build
cross.all: $(foreach p,$(PLATFORMS), build.$(p))
cross: cross-build-images
cross:
@$(MAKE) -C base GOARCH=amd64
@$(MAKE) -C cross-build-gnu
@$(MAKE) cross.all
# ====================================================================================
@@ -51,19 +55,19 @@ help:
@echo 'Usage: make <OPTIONS> ... <TARGETS>'
@echo ''
@echo 'Targets:'
@echo ' build Build images for the host platform.'
@echo ' clean Remove all images created from the'
@echo ' current build.'
@echo ' cross Build images for all platforms.'
@echo ' prune Prunes orphaned and cached images at the.'
@echo ' host level.'
@echo ' push Pushes public images to registry.'
@echo ' build Build images for the host platform.'
@echo ' clean Remove all images created from the'
@echo ' current build.'
@echo ' cross Build images for all platforms.'
@echo ' prune Prunes orphaned and cached images at the.'
@echo ' host level.'
@echo ' push Pushes public images to registry.'
@echo ''
@echo 'Advanced Targets:'
@echo ' cross-build-images Build images used for cross-builds.'
@echo ' clean.all Remove all images including cached.'
@echo ' images at the host level.'
@echo ' prune.all Prune everything ignoring policy.'
@echo ' cross-build Build images used for cross-builds.'
@echo ' clean.all Remove all images including cached.'
@echo ' images at the host level.'
@echo ' prune.all Prune everything ignoring policy.'
@echo ''
@echo 'Options:'
@echo ' PLATFORMS The platforms to build when running the.'
+2 -2
View File
@@ -48,6 +48,7 @@ CLEAN_IMAGES = $(CEPH_BUILD_IMAGE)
CEPH_TARBALL = ceph-install.tar
TEMP := $(shell mktemp -d)
CCACHE_DIR_HOST ?= $(HOME)/.ccache
# ====================================================================================
# Targets
@@ -73,8 +74,7 @@ ceph:
@docker run \
-i $(RUN_ARGS) \
--cidfile $(TEMP)/cid \
-e CCACHE_DIR=/home/build/.ccache \
-v $(HOME)/.ccache:/home/build/.ccache \
-v $(CCACHE_DIR_HOST):/root/.ccache \
$(CEPH_BUILD_IMAGE)
@docker cp `cat $(TEMP)/cid`:/build/$(CEPH_TARBALL) $(TEMP)/$(CEPH_TARBALL)
@docker rm -f `cat $(TEMP)/cid`
+12
View File
@@ -149,3 +149,15 @@ prune.all:
echo removing orphaned image $$i; \
docker rmi $$i > /dev/null 2>&1; \
done
# nukes all images
nuke:
@for c in $$(docker ps -a -q); do \
echo stopping and removing container $${c}; \
docker stop $${c}; \
docker rm $${c}; \
done
@for i in $$(docker images -q); do \
echo removing image $$i; \
docker rmi -f $$i > /dev/null 2>&1; \
done
-1
View File
@@ -64,7 +64,6 @@ ROOKD = $(TEMP)/rookd
$(ROOKD):
@echo === building rookd for $(IMAGE)
@../../build/run make WORKDIR=/build/.workdir GO_PKG_DIR= GOOS=linux GOARCH=$(GOARCH)
@cp ../../bin/linux_$(GOARCH)/rookd $@
# ====================================================================================