From d7c9d41f040d2c0ed01cd3c0d0e00630a824fccb Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Tue, 18 Oct 2016 15:56:52 -0700 Subject: [PATCH 1/2] use rsync to speedup build on macOS we know use rsync to sync sources to the build container and avoid osxfs which is painfully slow. the macOS/docker build now is on par with linux builds --- Makefile | 4 +- build/clean | 8 ++ build/common.sh | 38 +++++++++ build/cross-image/Dockerfile | 3 +- build/cross-image/rsyncd.sh | 36 +++++++++ build/cross-image/version | 2 +- build/makelib/golang.mk | 12 ++- build/release/common.sh | 6 +- build/run | 145 +++++++++++++++++++++++++---------- 9 files changed, 202 insertions(+), 52 deletions(-) create mode 100755 build/clean create mode 100644 build/common.sh create mode 100755 build/cross-image/rsyncd.sh diff --git a/Makefile b/Makefile index 2b91ca639..3e593a6ef 100644 --- a/Makefile +++ b/Makefile @@ -157,12 +157,12 @@ fmt: go.fmt vendor: go.vendor clean: go.clean - @rm -fr $(WORKDIR) $(RELEASE_DIR) $(BIN_DIR) + @rm -fr $(WORKDIR) $(RELEASE_DIR)/* $(BIN_DIR)/* distclean: go.distclean clean build.platform.%: - @$(MAKE) GOOS=$(word 1, $(subst _, ,$*)) GOARCH=$(word 2, $(subst _, ,$*)) install + @$(MAKE) GOOS=$(word 1, $(subst _, ,$*)) GOARCH=$(word 2, $(subst _, ,$*)) build build.cross: $(foreach p,$(ALL_PLATFORMS), build.platform.$(p)) diff --git a/build/clean b/build/clean new file mode 100755 index 000000000..2589f5b34 --- /dev/null +++ b/build/clean @@ -0,0 +1,8 @@ +#!/bin/bash -e + +scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "${scriptdir}/common.sh" + +docker stop ${container_name} &> /dev/null || true +docker rm ${container_name} &> /dev/null || true +docker volume rm ${container_volume} &> /dev/null || true diff --git a/build/common.sh b/build/common.sh new file mode 100644 index 000000000..b12bd0395 --- /dev/null +++ b/build/common.sh @@ -0,0 +1,38 @@ +#!/bin/bash -e + +source_repo=github.com/quantum/castle + +container_version=$(cat ${scriptdir}/cross-image/version) +container_image=quantum/castle-cross-build:${container_version} +container_name=cross-build +container_volume=${container_name}-volume +rsync_port=10873 + +function ver() { + printf "%d%03d%03d%03d" $(echo "$1" | tr '.' ' ') +} + +function check_git() { + # git version 2.6.6+ through 2.8.3 had a bug with submodules. this makes it hard + # to share a cloned directory between host and container + # see https://github.com/git/git/blob/master/Documentation/RelNotes/2.8.3.txt#L33 + gitversion=$(git --version | cut -d" " -f3) + if (( $(ver ${gitversion}) > $(ver 2.6.6) && $(ver ${gitversion}) < $(ver 2.8.3) )); then + echo WARN: your running git version ${gitversion} which has a bug realted to relative + echo WARN: submodule paths. Please consider upgrading to 2.8.3 or later + fi +} + +function wait_for_rsync() { + # wait for rsync to come up + tries=100 + while (( ${tries} > 0 )) ; do + if rsync "rsync://localhost:${rsync_port}/" &> /dev/null ; then + return 0 + fi + tries=$(( ${tries} - 1 )) + sleep 0.1 + done + echo ERROR: rsyncd did not come up >&2 + exit 1 +} diff --git a/build/cross-image/Dockerfile b/build/cross-image/Dockerfile index 23c868929..97bdd405a 100644 --- a/build/cross-image/Dockerfile +++ b/build/cross-image/Dockerfile @@ -26,6 +26,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ llvm-dev \ mingw-w64 \ patch \ + rsync \ runit \ sudo \ uuid-dev \ @@ -154,6 +155,6 @@ RUN \ CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC=x86_64-linux-gnu-gcc go install -installsuffix race -a std WORKDIR /build -COPY entrypoint.sh /build +COPY entrypoint.sh rsyncd.sh /build/ ENTRYPOINT [ "/build/entrypoint.sh" ] diff --git a/build/cross-image/rsyncd.sh b/build/cross-image/rsyncd.sh new file mode 100755 index 000000000..b44b9836f --- /dev/null +++ b/build/cross-image/rsyncd.sh @@ -0,0 +1,36 @@ +#!/bin/bash -e + +VOLUME=${VOLUME:-/volume} +ALLOW=${ALLOW:-192.168.0.0/16 172.16.0.0/12 10.0.0.0/8} +OWNER=${OWNER:-nobody} +GROUP=${GROUP:-nogroup} + +if [ "${GROUP}" != "nogroup" ]; then + groupadd -g ${GROUP} rsync +fi +if [ "${OWNER}" != "nobody" ]; then + groupadd -u ${OWNER} -G rsync rsync +fi + +chown "${OWNER}:${GROUP}" "${VOLUME}" + +[ -f /etc/rsyncd.conf ] || cat < /etc/rsyncd.conf +uid = ${OWNER} +gid = ${GROUP} +use chroot = yes +log file = /dev/stdout +reverse lookup = no +[volume] + hosts deny = * + hosts allow = ${ALLOW} + read only = false + path = ${VOLUME} + comment = volume +EOF + +for dir in ${MKDIRS}; do + mkdir -p ${dir} + chown "${OWNER}:${GROUP}" ${dir} +done + +exec /usr/bin/rsync --no-detach --daemon --config /etc/rsyncd.conf "$@" diff --git a/build/cross-image/version b/build/cross-image/version index 446ba66e7..def9a0154 100644 --- a/build/cross-image/version +++ b/build/cross-image/version @@ -1 +1 @@ -0.1.4 \ No newline at end of file +0.1.5 \ No newline at end of file diff --git a/build/makelib/golang.mk b/build/makelib/golang.mk index cd1e09e57..a36f18186 100644 --- a/build/makelib/golang.mk +++ b/build/makelib/golang.mk @@ -71,6 +71,10 @@ GOHOST := GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) go GO_OUT_DIR := $(abspath $(GO_BIN_DIR)/$(GOOS)_$(GOARCH)) +ifeq ($(GOOS),windows) +GO_OUT_EXT := ".exe" +endif + ifneq ($(GO_PKG_DIR),) GO_PKG_FLAGS := -pkgdir $(abspath $(GO_PKG_DIR)/$(GOOS)_$(GOARCH)) GO_PKG_STATIC_FLAGS := -pkgdir $(abspath $(GO_PKG_DIR)/$(GOOS)_$(GOARCH)_static) -installsuffix static @@ -101,9 +105,9 @@ endif .PHONY: go.build go.build: go.vet go.fmt $(CGO_PREREQS) - $(foreach p,$(GO_NONSTATIC_PACKAGES),@CGO_ENABLED=1 $(GO) build -v -i $(GO_PKG_FLAGS) -o $(GO_OUT_DIR)/$(lastword $(subst /, ,$(p))) $(GO_TOOL_FLAGS) $(p)) - $(foreach p,$(GO_CGO_PACKAGES),@CGO_ENABLED=1 $(GO) build -v -i $(GO_PKG_FLAGS) -o $(GO_OUT_DIR)/$(lastword $(subst /, ,$(p))) $(GO_TOOL_FLAGS) $(p)) - $(foreach p,$(GO_STATIC_PACKAGES),@CGO_ENABLED=0 $(GO) build -v -i $(GO_PKG_STATIC_FLAGS) -o $(GO_OUT_DIR)/$(lastword $(subst /, ,$(p))) $(GO_TOOL_FLAGS) $(p)) + $(foreach p,$(GO_NONSTATIC_PACKAGES),@CGO_ENABLED=1 $(GO) build -v -i $(GO_PKG_FLAGS) -o $(GO_OUT_DIR)/$(lastword $(subst /, ,$(p)))$(GO_OUT_EXT) $(GO_TOOL_FLAGS) $(p)) + $(foreach p,$(GO_CGO_PACKAGES),@CGO_ENABLED=1 $(GO) build -v -i $(GO_PKG_FLAGS) -o $(GO_OUT_DIR)/$(lastword $(subst /, ,$(p)))$(GO_OUT_EXT) $(GO_TOOL_FLAGS) $(p)) + $(foreach p,$(GO_STATIC_PACKAGES),@CGO_ENABLED=0 $(GO) build -v -i $(GO_PKG_STATIC_FLAGS) -o $(GO_OUT_DIR)/$(lastword $(subst /, ,$(p)))$(GO_OUT_EXT) $(GO_TOOL_FLAGS) $(p)) .PHONY: go.install go.install: go.vet go.fmt $(CGO_PREREQS) @@ -140,7 +144,7 @@ $(GO_TOOLS_DIR)/glide: .PHONY: go.clean go.clean: ; - @rm -rf $(GO_BIN_DIR) + @rm -rf $(GO_BIN_DIR)/* ifneq ($(GO_PKG_DIR),) @rm -rf $(GO_PKG_DIR) endif diff --git a/build/release/common.sh b/build/release/common.sh index b995e6b64..35d60c470 100644 --- a/build/release/common.sh +++ b/build/release/common.sh @@ -4,11 +4,7 @@ get_bindir() { local os=$1 local arch=$2 - bindir=${RELEASE_BIN_DIR} - if [[ ${os}_${arch} != ${RELEASE_HOST_PLATFORM} ]]; then - bindir=${bindir}/${os}_${arch} - fi - + bindir=${RELEASE_BIN_DIR}/${os}_${arch} echo ${bindir} } diff --git a/build/run b/build/run index bbe937d59..6fb0fff9d 100755 --- a/build/run +++ b/build/run @@ -1,25 +1,7 @@ #!/bin/bash -e scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -friendly_name=cross-build -version=$(cat ${scriptdir}/cross-image/version) -container=quantum/castle-${friendly_name}:${version} -volume=cross-build - -function ver() { - printf "%d%03d%03d%03d" $(echo "$1" | tr '.' ' ') -} - -function check_git() { - # git version 2.6.6+ through 2.8.3 had a bug with submodules. this makes it hard - # to share a cloned directory between host and container - # see https://github.com/git/git/blob/master/Documentation/RelNotes/2.8.3.txt#L33 - gitversion=$(git --version | cut -d" " -f3) - if (( $(ver ${gitversion}) > $(ver 2.6.6) && $(ver ${gitversion}) < $(ver 2.8.3) )); then - echo WARN: your running git version ${gitversion} which has a bug realted to relative - echo WARN: submodule paths. Please consider upgrading to 2.8.3 or later - fi -} +source "${scriptdir}/common.sh" check_git @@ -28,17 +10,10 @@ if [ ! -z $DOCKER_HOST ]; then return 1 fi -# if our volume does not exist create one. -# NOTE: osxfs is really slow so we avoid using it -if [[ ! $(docker volume ls | grep $volume) ]]; then - echo ==== Creating docker volume ${volume} - docker volume create --name ${volume} -fi - # if the container does not exist the pull or build it -if [[ ! $(docker images -q ${container}) ]]; then - echo ==== Pulling ${container} - docker pull ${container} +if [[ ! $(docker images -q ${container_image}) ]]; then + echo ==== Pulling ${container_image} + docker pull ${container_image} fi # on Linux we run the container with the same uid and gid of the user to avoid @@ -50,8 +25,100 @@ else BUILDER_HOME=/root fi -if [[ `which ccache` && -n "${CCACHE_DIR}" ]]; then - CCACHE_ARGS="-e CCACHE_DIR=${BUILDER_HOME}/.ccache -e CCACHE_LOG -v ${CCACHE_DIR}:${BUILDER_HOME}/.ccache" +cleanup() { + docker stop ${container_name} &> /dev/null || true + docker rm ${container_name} &> /dev/null || true +} + +# mount arguments +MOUNT_OPTS="\ + -v ${scriptdir}/../bin:${BUILDER_HOME}/go/src/${source_repo}/bin \ + -v ${scriptdir}/../release:${BUILDER_HOME}/go/src/${source_repo}/release \ +" + +if [ "`uname -s`" != "Linux" ]; then + + # On non-linux hosts, its is EXTREMELY slow to mount the source + # tree inside the container. As such we use rsync to copy the source + # tree to a new docker volumer on each run. The first rsync take a + # little bit of time, but subsequent ones go by fast. This approach + # is MUCH faster than using osxfs, NFS or other approaches. + + # On non-linux the layout is as follows: + # + # /var/run/docker.sock (bind mounted to host /var/run/docker.sock) + # /${BUILDER_HOME} + # /.netrc (bind mounted to host ${HOME}/.netrc) + # /.docker/config.json (bind mounted to host ${HOME}/.docker/config.json) + # /go (bind mounted to a docker volume) + # /.work + # /.ccache + # /bin (bind mounted to host /bin) + # /src/github.com/quantum/castle (rsync'd from host ) + # + + # create a new volume to hold our go workspace + if [[ ! $(docker volume ls | grep ${container_volume}) ]]; then + echo ==== Syncing sources for first time. This could take a few seconds. + docker volume create --name ${container_volume} &> /dev/null + fi + + trap "cleanup" EXIT + cleanup + + # run the container as an rsyncd daemon so that we can copy the + # source tree to the container volume. + docker run \ + --name "${container_name}" \ + -d \ + -e MKDIRS="/volume/src/${source_repo}" \ + -p ${rsync_port}:873 \ + -v ${container_volume}:/volume \ + ${container_image} \ + /build/rsyncd.sh &> /dev/null + + # wait for rsync to come up + wait_for_rsync + + # now copy the source tree to the container volume + rsync \ + --archive \ + --delete \ + --prune-empty-dirs \ + --filter='- /.work/' \ + --filter='- /.glide/' \ + --filter='- /.vscode/' \ + --filter='- /bin/' \ + --filter='- /release/' \ + ${scriptdir}/.. \ + rsync://localhost:${rsync_port}/volume/src/${source_repo} + + MOUNT_OPTS="${MOUNT_OPTS} -v ${container_volume}:${BUILDER_HOME}/go" + +else + + # On linux bind mounts are cheap so we dont need to rsync + # the layout is as follows: + # + # /var/run/docker.sock (bind mounted to host /var/run/docker.sock) + # /${BUILDER_HOME} + # /.netrc (bind mounted to host ${HOME}/.netrc) + # /.docker/config.json (bind mounted to host ${HOME}/.docker/config.json) + # /go + # /.work (bind mounted to host /.work/cross) + # /.ccache (bind mounted to host CCACHE_DIR) + # /bin (bind mounted to host /bin) + # /src/github.com/quantum/castle (bind mounted to ) + # + MOUNT_OPTS="${MOUNT_OPTS} \ + -v ${scriptdir}/../.work/cross:${BUILDER_HOME}/go/.work \ + -v ${scriptdir}/..:${BUILDER_HOME}/go/src/${source_repo}" + + CCACHE_MOUNT_ARGS="-v ${CCACHE_DIR}:${BUILDER_HOME}/go/.ccache" +fi + +if [[ `which ccache` && -d "${CCACHE_DIR}" ]]; then + CCACHE_ARGS="-e CCACHE_DIR=${BUILDER_HOME}/go/.ccache -e CCACHE_LOG ${CCACHE_MOUNT_ARGS}" fi # we copy credential files for github access @@ -71,12 +138,14 @@ if [ -z "${DISABLE_NESTED_DOCKER}" ]; then -v /var/run/docker.sock:/var/run/docker.sock" fi +trap "cleanup" EXIT +cleanup + docker run \ --rm \ - --name "${friendly_name}" \ - --hostname "${friendly_name}" \ + --name "${container_name}" \ -e GOPATH="${BUILDER_HOME}/go" \ - -e WORKDIR="${BUILDER_HOME}/go/.cross" \ + -e WORKDIR="${BUILDER_HOME}/go/.work" \ -e GO_PKG_DIR="" \ -e GITHUB_TOKEN \ -e VERSION \ @@ -86,9 +155,7 @@ docker run \ ${USER_ARGS} \ ${DOCKER_NESTED_ARGS} \ ${NET_ARGS} \ - -v cross-build:${BUILDER_HOME}/go \ - -v ${scriptdir}/../bin:${BUILDER_HOME}/go/bin \ - -v ${scriptdir}/..:${BUILDER_HOME}/go/src/github.com/quantum/castle \ - -w ${BUILDER_HOME}/go/src/github.com/quantum/castle \ - ${container} \ + ${MOUNT_OPTS} \ + -w ${BUILDER_HOME}/go/src/${source_repo} \ + ${container_image} \ "$@" From de18ae8cdd3708eebd0e1c39f1ee983c9dfcc8b7 Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Tue, 18 Oct 2016 21:31:57 -0700 Subject: [PATCH 2/2] fix permissions for vendor directory --- build/clean | 13 ++++++++++--- build/cross-image/rsyncd.sh | 9 +++++---- build/cross-image/version | 2 +- build/run | 5 ++++- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/build/clean b/build/clean index 2589f5b34..c1b0fee7f 100755 --- a/build/clean +++ b/build/clean @@ -3,6 +3,13 @@ scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source "${scriptdir}/common.sh" -docker stop ${container_name} &> /dev/null || true -docker rm ${container_name} &> /dev/null || true -docker volume rm ${container_volume} &> /dev/null || true +if [[ $(docker ps -f name=${container_name} | grep -w ${container_name}) ]]; then + docker stop ${container_name} + docker rm ${container_name} +fi + +if [[ $(docker volume ls | grep ${container_volume}) ]]; then + docker volume rm ${container_volume} +fi + +echo SUCCESS \ No newline at end of file diff --git a/build/cross-image/rsyncd.sh b/build/cross-image/rsyncd.sh index b44b9836f..6790891c0 100755 --- a/build/cross-image/rsyncd.sh +++ b/build/cross-image/rsyncd.sh @@ -5,11 +5,12 @@ ALLOW=${ALLOW:-192.168.0.0/16 172.16.0.0/12 10.0.0.0/8} OWNER=${OWNER:-nobody} GROUP=${GROUP:-nogroup} -if [ "${GROUP}" != "nogroup" ]; then - groupadd -g ${GROUP} rsync +if [[ "${GROUP}" != "nogroup" && "${GROUP}" != "root" ]]; then + groupadd -g ${GROUP} rsync fi -if [ "${OWNER}" != "nobody" ]; then - groupadd -u ${OWNER} -G rsync rsync + +if [[ "${OWNER}" != "nobody" && "${OWNER}" != "root" ]]; then + groupadd -u ${OWNER} -G rsync rsync fi chown "${OWNER}:${GROUP}" "${VOLUME}" diff --git a/build/cross-image/version b/build/cross-image/version index def9a0154..a19223320 100644 --- a/build/cross-image/version +++ b/build/cross-image/version @@ -1 +1 @@ -0.1.5 \ No newline at end of file +0.1.6 \ No newline at end of file diff --git a/build/run b/build/run index 6fb0fff9d..f2a24e192 100755 --- a/build/run +++ b/build/run @@ -71,6 +71,8 @@ if [ "`uname -s`" != "Linux" ]; then docker run \ --name "${container_name}" \ -d \ + -e OWNER=root \ + -e GROUP=root \ -e MKDIRS="/volume/src/${source_repo}" \ -p ${rsync_port}:873 \ -v ${container_volume}:/volume \ @@ -93,7 +95,8 @@ if [ "`uname -s`" != "Linux" ]; then ${scriptdir}/.. \ rsync://localhost:${rsync_port}/volume/src/${source_repo} - MOUNT_OPTS="${MOUNT_OPTS} -v ${container_volume}:${BUILDER_HOME}/go" + MOUNT_OPTS="${MOUNT_OPTS} \ + -v ${container_volume}:${BUILDER_HOME}/go" else