From f3098d04f23a910073e20b8a8aa899bb6d55654f Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Sun, 11 Sep 2016 14:47:29 -0700 Subject: [PATCH 1/3] build castle in a container. added circle.yml --- .gitmodules | 2 +- Makefile | 1 + build/build-container/Dockerfile | 51 ++++++++++++++++++++++++++++++++ build/publish-build-container.sh | 7 +++++ build/run-build.sh | 21 +++++++++++++ circle.yml | 16 ++++++++++ dist/Dockerfile | 7 ----- 7 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 build/build-container/Dockerfile create mode 100755 build/publish-build-container.sh create mode 100755 build/run-build.sh create mode 100644 circle.yml delete mode 100644 dist/Dockerfile diff --git a/.gitmodules b/.gitmodules index eae2aa637..913725b85 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "ceph"] path = ceph - url = git@github.com:quantum/ceph + url = https://github.com/quantum/ceph.git branch = wip-embedded-ceph diff --git a/Makefile b/Makefile index 42a97a9c0..2a1837252 100644 --- a/Makefile +++ b/Makefile @@ -104,6 +104,7 @@ endif @echo "##### building castlectl" go build $(BUILDFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o bin/castlectl ./cmd/castlectl +.PHONY: ceph ceph: git submodule update --init --recursive diff --git a/build/build-container/Dockerfile b/build/build-container/Dockerfile new file mode 100644 index 000000000..1157d90bb --- /dev/null +++ b/build/build-container/Dockerfile @@ -0,0 +1,51 @@ +FROM ubuntu:xenial +MAINTAINER Bassam Tabbara + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + cython \ + cython3 \ + git \ + golang-1.6 \ + libaio-dev \ + libatomic-ops-dev \ + libblkid-dev \ + libboost-date-time-dev \ + libboost-dev \ + libboost-iostreams-dev \ + libboost-program-options-dev \ + libboost-random-dev \ + libboost-regex-dev \ + libboost-system-dev \ + libboost-thread-dev \ + libcrypto++-dev \ + libcurl4-openssl-dev \ + libexpat1-dev \ + libfcgi-dev \ + libgoogle-perftools-dev \ + libjemalloc-dev \ + libkeyutils-dev \ + libldap2-dev \ + libleveldb-dev \ + libsnappy-dev \ + libssl-dev \ + libudev-dev \ + python-sphinx \ + python-dev \ + python3-all-dev \ + xfslibs-dev \ + yasm \ + --no-install-recommends && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +ENV GOPATH /go +ENV PATH $GOPATH/bin:/usr/lib/go-1.6/bin:$PATH +RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" + +VOLUME $GOPATH/src/github.com/quantum/castle +WORKDIR $GOPATH/src/github.com/quantum/castle + +ENTRYPOINT [ "/bin/bash", "-c" ] diff --git a/build/publish-build-container.sh b/build/publish-build-container.sh new file mode 100755 index 000000000..58e375abc --- /dev/null +++ b/build/publish-build-container.sh @@ -0,0 +1,7 @@ +#!/bin/bash -e + +scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +cd $scriptdir/build-container +docker build -t quantum/castle-build . +docker push quantum/castle-build diff --git a/build/run-build.sh b/build/run-build.sh new file mode 100755 index 000000000..48bb6e82e --- /dev/null +++ b/build/run-build.sh @@ -0,0 +1,21 @@ +#!/bin/bash -e + +scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + + +# git version 2.6.6+ through 2.8.3 had a bug with submodules +# see https://github.com/git/git/blob/master/Documentation/RelNotes/2.8.3.txt#L33 +function ver { + printf "%d%03d%03d%03d" $(echo "$1" | tr '.' ' ') +} +gitversion=$(git --version | cut -d" " -f3) +if (( $(ver ${gitversion}) > $(ver 2.6.6) && $(ver ${gitversion}) < $(ver 2.8.3) )); then + echo WARN: your running git version ${gitversion} which has a realted to relative + echo WARN: submodule paths. Please consider upgrading to 2.8.3 or later +fi + +docker run \ + -v ${HOME}/.netrc:/root/.netrc \ + -v ${scriptdir}/..:/go/src/github.com/quantum/castle \ + quantum/castle-build \ + "$*" diff --git a/circle.yml b/circle.yml new file mode 100644 index 000000000..bea3cc1a8 --- /dev/null +++ b/circle.yml @@ -0,0 +1,16 @@ +machine: + environment: + NPROCS: $(grep -c ^processor /proc/cpuinfo) + services: + - docker + +dependencies: + override: + - docker info + - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS + - echo -e "machine github.com\n\tlogin quantumbuild\n\tpassword $GITHUB_TOKEN" > ~/.netrc + - ./build/run-build.sh make -j$NPROCS build + +test: + override: + - echo no tests diff --git a/dist/Dockerfile b/dist/Dockerfile deleted file mode 100644 index 73181ee45..000000000 --- a/dist/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM ubuntu:16.04 -MAINTAINER Bassam Tabbara - -VOLUME /ceph -VOLUME /build - -ENTRYPOINT ["/tmp/entrypoint.sh", "/tmp/micro-ceph"] From 1b8f4af27bea532e807670479a7595dfb3dbe53e Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Sun, 11 Sep 2016 20:10:18 -0700 Subject: [PATCH 2/3] add circle badges --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 89d118707..187dbc7db 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ # Castle - Software Defined Storage +[![Circle CI](https://circleci.com/gh/quantum/castle.svg?style=svg&circle-token=2b2ee7eb7f621e612a76f861353e141a8caaf8dd)](https://circleci.com/gh/quantum/castle) From a15df92546e794354eb6302f05de8d431fd25005 Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Sun, 11 Sep 2016 20:24:40 -0700 Subject: [PATCH 3/3] Performance improvements to circleci build --- Makefile | 2 +- build/build-container/Dockerfile | 8 +++++++- build/run-build.sh | 4 ++-- circle.yml | 3 ++- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 2a1837252..a61c87fa9 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ build: vendor ceph @echo "##### configuring ceph" cd ceph/build && cmake $(CEPHD_CMAKE) .. @echo "##### building ceph" - cd ceph/build && $(MAKE) $(MFLAGS) cephd + cd ceph/build && $(MAKE) $(MAKEFLAGS) $(MFLAGS) cephd ifeq ($(DEBUG),0) @echo "##### stripping libcephd.a" strip -S ceph/build/lib/libcephd.a diff --git a/build/build-container/Dockerfile b/build/build-container/Dockerfile index 1157d90bb..a143ac03f 100644 --- a/build/build-container/Dockerfile +++ b/build/build-container/Dockerfile @@ -1,7 +1,13 @@ FROM ubuntu:xenial MAINTAINER Bassam Tabbara -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq \ +# git version 2.6.6+ through 2.8.3 had a bug with submodules +# see https://github.com/git/git/blob/master/Documentation/RelNotes/2.8.3.txt#L33 +# this is why we are adding a newer verion of git than is available on xenial + +RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E1DF1F24 && \ + echo "deb http://ppa.launchpad.net/git-core/ppa/ubuntu xenial main" > /etc/apt/sources.list.d/git.list && \ + apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq \ build-essential \ ca-certificates \ cmake \ diff --git a/build/run-build.sh b/build/run-build.sh index 48bb6e82e..d74088d12 100755 --- a/build/run-build.sh +++ b/build/run-build.sh @@ -2,7 +2,6 @@ scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - # git version 2.6.6+ through 2.8.3 had a bug with submodules # see https://github.com/git/git/blob/master/Documentation/RelNotes/2.8.3.txt#L33 function ver { @@ -10,11 +9,12 @@ function ver { } gitversion=$(git --version | cut -d" " -f3) if (( $(ver ${gitversion}) > $(ver 2.6.6) && $(ver ${gitversion}) < $(ver 2.8.3) )); then - echo WARN: your running git version ${gitversion} which has a realted to relative + echo WARN: your running git version ${gitversion} which has a bug realted to relative echo WARN: submodule paths. Please consider upgrading to 2.8.3 or later fi docker run \ + -it \ -v ${HOME}/.netrc:/root/.netrc \ -v ${scriptdir}/..:/go/src/github.com/quantum/castle \ quantum/castle-build \ diff --git a/circle.yml b/circle.yml index bea3cc1a8..fae0cfbbf 100644 --- a/circle.yml +++ b/circle.yml @@ -1,6 +1,6 @@ machine: environment: - NPROCS: $(grep -c ^processor /proc/cpuinfo) + NPROCS: 16 services: - docker @@ -8,6 +8,7 @@ dependencies: override: - docker info - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS + - docker pull quantum/castle-build - echo -e "machine github.com\n\tlogin quantumbuild\n\tpassword $GITHUB_TOKEN" > ~/.netrc - ./build/run-build.sh make -j$NPROCS build