From 07c0245889fd75c53ca5e1405c51dac024479ad2 Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Fri, 23 Sep 2016 11:48:39 -0700 Subject: [PATCH 1/2] Makefile improvements Makefile now attempts to build only what is needed. Dependency checking has been improved but its not perfect yet. Also added support for building in a private gopath which would support building from forked repos. --- .gitignore | 1 + Documentation/build.md | 16 +++--- Makefile | 125 ++++++++++++++++++++++++----------------- circle.yml | 12 +++- 4 files changed, 90 insertions(+), 64 deletions(-) mode change 100644 => 100755 Makefile diff --git a/.gitignore b/.gitignore index 3a3b4e8c8..409cf4502 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ tools/ vendor/ .vscode/ .glide/ +.gopath/ coverage.out coverage.html debug.test diff --git a/Documentation/build.md b/Documentation/build.md index 5d8ee9b11..895bf359b 100644 --- a/Documentation/build.md +++ b/Documentation/build.md @@ -10,7 +10,7 @@ binaries can easily be used inside minimal or scratch containers, or run on mini linux distributions like CoreOS. ``` -make STATIC=1 build +make STATIC=1 ``` ## Dyanmic binary @@ -21,7 +21,7 @@ and the rest of the libraries continue to be linked statically. Ceph has a lot of dependencies and we take this approach to simplify the distribution. ``` -make STATIC=0 build +make STATIC=0 ``` ## Hardened binary (PIE) @@ -29,7 +29,7 @@ make STATIC=0 build You can build a Position Independent executable as follows: ``` -make STATIC=0 PIE=1 build +make STATIC=0 PIE=1 ``` ## Switching Allocators @@ -39,7 +39,7 @@ Three allocators are currently supported: jemalloc, tcmalloc and libc. To specif an allocator during the build run the following: ``` -make ALLOCATOR=jemalloc build +make ALLOCATOR=jemalloc ``` ## Debug Builds @@ -47,7 +47,7 @@ make ALLOCATOR=jemalloc build To build a binary with debug symbols run the following: ``` -make DEBUG=1 build +make DEBUG=1 ``` Note the binary will be significantly larger in size. @@ -57,7 +57,7 @@ Note the binary will be significantly larger in size. To turn on verbose build output run the following ``` -make DEBUG=1 build +make V=1 ``` ## Parallel Builds @@ -65,7 +65,7 @@ make DEBUG=1 build You can speed up the build significantly by passing the -j flag to make as follows: ``` -make -j4 build +make -j4 ``` ## Enabling ccache @@ -81,7 +81,7 @@ to run with ccache enable run the following: ``` export CCACHE_DIR=~/.ccache -make CCACHE=1 build +make ``` to check ccache stats run: diff --git a/Makefile b/Makefile old mode 100644 new mode 100755 index 739109dc1..99d259971 --- a/Makefile +++ b/Makefile @@ -1,12 +1,18 @@ -REPOPATH = github.com/quantum/castle +ORG = github.com/quantum +PROJ = castle +GOPATH = $(CURDIR)/.gopath +ORGPATH = $(GOPATH)/src/$(ORG) +REPOPATH = $(ORGPATH)/$(PROJ) GOOS = $(shell go env GOOS) GOARCH = $(shell go env GOARCH) -# TODO: support for building outside of a $GOPATH +export GOPATH + # TODO: support for profiling in ceph # TODO: support for tracing in ceph # TODO: jemalloc and static linking are currently broken due to https://github.com/jemalloc/jemalloc/issues/442 -# TODO: remove leveldb +# TODO: remove leveldb from ceph +# TODO: cross compile castlectl? # TODO: can we strip -s all binaries? # Can be used for additional go build flags @@ -85,74 +91,87 @@ ifeq ($(CCACHE),1) CEPHD_CMAKE += -DWITH_CCACHE=ON endif -# skip vendoring option, useful for dev scenarios where vendor dir may be dirty -SKIPVENDOR ?= 0 - # set the version number. ifeq ($(origin VERSION), undefined) VERSION = $(shell git describe --dirty --always) endif -LDFLAGS += -X $(REPOPATH)/pkg/version.Version=$(VERSION) +LDFLAGS += -X $(ORG)/$(PROJ)/pkg/version.Version=$(VERSION) + +GO_SOURCEDIRS=cmd pkg +GO_SOURCES := $(shell find $(GO_SOURCEDIRS) -name '*.go') +GO_PKGS=$(shell go list ./... | grep --invert-match vendor) .PHONY: all -all: build test +all: build .PHONY: build -build: vendor ceph - @mkdir -p ceph/build - @echo "##### configuring ceph" - cd ceph/build && cmake $(CEPHD_CMAKE) .. - @echo "##### building ceph" - cd ceph/build && $(MAKE) cephd -ifeq ($(DEBUG),0) - @echo "##### stripping libcephd.a" - strip -S ceph/build/lib/libcephd.a -endif - @echo "##### building castled" - go build $(BUILDFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o bin/castled ./cmd/castled - @echo "##### building castlectl" - go build $(BUILDFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o bin/castlectl ./cmd/castlectl - -.PHONY: ceph -ceph: - git submodule update --init --recursive +build: bin/castled bin/castlectl vet fmt .PHONY: release release: build -.PHONY: test -test: tools/glide - go test -cover -timeout 30s $(BUILDFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' $(shell ./tools/glide novendor) +.PHONY: check +check: test -.PHONY: vet -vet: tools/glide - go vet $(shell ./tools/glide novendor) +$(REPOPATH): + @mkdir -p $(ORGPATH) + @ln -s ../../../.. $(REPOPATH) -.PHONY: fmt -fmt: tools/glide - go fmt $(shell ./tools/glide novendor) +bin/castled: $(REPOPATH) vendor/vendor.stamp ceph/build $(GO_SOURCES) + @cd ceph/build && $(MAKE) cephd + @if test ceph/build/lib/libcephd.a -nt bin/castled -o ! -f bin/castled; then \ + if test $(DEBUG) -eq 0; then \ + strip -S ceph/build/lib/libcephd.a; \ + fi; \ + go build $(BUILDFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o bin/castled ./cmd/castled; \ + fi -.PHONY: clean -clean: - rm -rf bin - if [ -d ceph/build ]; then cd ceph/build && $(MAKE) clean; fi - if [ -d ceph/src/rocksdb ]; then cd ceph/src/rocksdb && $(MAKE) clean; fi +bin/castlectl: $(REPOPATH) vendor/vendor.stamp $(GO_SOURCES) + @go build $(BUILDFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o bin/castlectl ./cmd/castlectl -.PHONY: cleanall -cleanall: - rm -rf bin - rm -rf tools vendor - rm -rf ceph/build +# we take a dependency on this Makefile since the ceph cmake config is in it +ceph/build: Makefile + @mkdir -p ceph/build + @cd ceph/build && cmake $(CEPHD_CMAKE) .. .PHONY: vendor vendor: tools/glide -ifeq ($(SKIPVENDOR),0) - ./tools/glide install -endif + @./tools/glide install + +# the stamp file ensures that we run glide install once. every other time is manual +vendor/vendor.stamp: tools/glide + @./tools/glide install + @touch $@ + +.PHONY: test +test: build + @go test -cover -timeout 30s $(BUILDFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' $(GO_PKGS) + +.PHONY: vet +vet: $(REPOPATH) tools/glide $(GO_SOURCES) + @go vet $(shell ./tools/glide novendor) + +.PHONY: fmt +fmt: $(REPOPATH) tools/glide $(GO_SOURCES) + @go fmt $(shell ./tools/glide novendor) + +.PHONY: cleancommon +cleancommon: + @rm -rf bin + @rm -fr $(GOPATH) + @if [ -d ceph/src/rocksdb ]; then cd ceph/src/rocksdb && $(MAKE) clean > /dev/null; fi + +.PHONY: clean +clean: cleancommon + @if [ -d ceph/build ]; then cd ceph/build && $(MAKE) clean; fi + +.PHONY: cleanall +cleanall: cleancommon + @rm -rf tools vendor + @rm -rf ceph/build tools/glide: - @echo "Downloading glide" - mkdir -p tools - curl -L https://github.com/Masterminds/glide/releases/download/v0.12.1/glide-v0.12.1-$(GOOS)-$(GOARCH).tar.gz | tar -xz -C tools - mv tools/$(GOOS)-$(GOARCH)/glide tools/glide - rm -r tools/$(GOOS)-$(GOARCH) + @mkdir -p tools + @curl -sL https://github.com/Masterminds/glide/releases/download/v0.12.2/glide-v0.12.2-$(GOOS)-$(GOARCH).tar.gz | tar -xz -C tools + @mv tools/$(GOOS)-$(GOARCH)/glide tools/glide + @rm -r tools/$(GOOS)-$(GOARCH) diff --git a/circle.yml b/circle.yml index ee0f3c14e..f099d1159 100644 --- a/circle.yml +++ b/circle.yml @@ -4,16 +4,22 @@ machine: services: - docker +checkout: + pre: + - echo -e "machine github.com\n\tlogin quantumbuild\n\tpassword $GITHUB_TOKEN" > ~/.netrc + - rm ~/.gitconfig + post: + - git submodule sync --recursive + - git submodule update --recursive --init + dependencies: cache_directories: - ~/.ccache override: - - docker info - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS - docker pull quantum/castle-build - - echo -e "machine github.com\n\tlogin quantumbuild\n\tpassword $GITHUB_TOKEN" > ~/.netrc - - ./build/run-build.sh make -j$NPROCS build + - ./build/run-build.sh make -j$NPROCS check test: override: From 7a609765f4b168800bfe4e49160ff5bd6d5faac7 Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Fri, 23 Sep 2016 15:38:16 -0700 Subject: [PATCH 2/2] fix go fmt and vet issues --- pkg/castled/config.go | 4 ++-- pkg/etcdmgr/bootstrap/discovery.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/castled/config.go b/pkg/castled/config.go index 98b67ed6d..6938d00cc 100644 --- a/pkg/castled/config.go +++ b/pkg/castled/config.go @@ -73,11 +73,11 @@ func generateConfigFile(cluster *ClusterInfo, pathRoot, user, keyringPath string } if err := addClientConfigFileSection(configFile, getQualifiedUser(user), keyringPath); err != nil { - return "", fmt.Errorf("failed to add %s admin client config section, %+v", err) + return "", fmt.Errorf("failed to add admin client config section, %+v", err) } if err := addInitialMonitorsConfigFileSections(configFile, cluster); err != nil { - return "", fmt.Errorf("failed to add %s initial monitor config sections, %+v", err) + return "", fmt.Errorf("failed to add initial monitor config sections, %+v", err) } // write the entire config to disk diff --git a/pkg/etcdmgr/bootstrap/discovery.go b/pkg/etcdmgr/bootstrap/discovery.go index efbf77608..ff889a879 100644 --- a/pkg/etcdmgr/bootstrap/discovery.go +++ b/pkg/etcdmgr/bootstrap/discovery.go @@ -69,11 +69,11 @@ func GetCurrentNodes(token string) ([]string, error) { nodes := make([]string, 0, len(res.Node.Nodes)) for _, nn := range res.Node.Nodes { if nn.Value == nil { - log.Println("Skipping %q because no value exists", nn.Key) + log.Printf("Skipping %q because no value exists", nn.Key) } n, err := newDiscoveryNode(*nn.Value, 2379) if err != nil { - log.Println("invalid peer url %q in discovery service: %v", *nn.Value, err) + log.Printf("invalid peer url %q in discovery service: %v", *nn.Value, err) continue } for _, node := range n {