From f3ae0a2fa82ac9b6ed2597db3cf29c5deecc75a8 Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Tue, 13 Jun 2017 14:25:27 -0700 Subject: [PATCH] build: support for building on mac --- Makefile | 44 +++++++++++++--------------------- build/common.sh | 2 +- build/makelib/golang.mk | 31 +++++++++++++----------- build/release/release.sh | 6 +++-- images/Makefile | 7 +++--- images/image.mk | 16 +++++++++---- images/rookd/Makefile | 5 ++-- images/toolbox/Dockerfile | 21 ++-------------- images/toolbox/Dockerfile.base | 34 ++++++++++++++++++++++++++ images/toolbox/Makefile | 27 +++++++++++++++------ 10 files changed, 113 insertions(+), 80 deletions(-) create mode 100644 images/toolbox/Dockerfile.base diff --git a/Makefile b/Makefile index 7d36ab1c3..205bc00d4 100644 --- a/Makefile +++ b/Makefile @@ -91,16 +91,12 @@ CHANNEL ?= CLIENT_PACKAGES = $(GO_PROJECT) SERVER_PACKAGES = $(GO_PROJECT)/cmd/rookd +GO_PACKAGES = ifneq ($(filter $(GOOS)_$(GOARCH),$(CLIENT_PLATFORMS) $(SERVER_PLATFORMS)),) -GO_PACKAGES = $(CLIENT_PACKAGES) +GO_PACKAGES += $(CLIENT_PACKAGES) endif - ifneq ($(filter $(GOOS)_$(GOARCH),$(SERVER_PLATFORMS)),) -ifeq ($(GOOS)_$(PIE),linux_1) -GO_PIE_PACKAGES = $(SERVER_PACKAGES) -else -GO_PACKAGES = $(SERVER_PACKAGES) -endif +GO_PACKAGES += $(SERVER_PACKAGES) endif GO_BUILDFLAGS=$(BUILDFLAGS) @@ -111,6 +107,7 @@ GO_BIN_DIR = $(BIN_DIR) GO_WORK_DIR ?= $(WORKDIR) GO_TOOLS_DIR ?= $(DOWNLOADDIR)/tools GO_PKG_DIR ?= $(WORKDIR)/pkg +GO_PIE ?= $(PIE) include build/makelib/golang.mk @@ -126,18 +123,24 @@ include build/makelib/release.mk # ==================================================================================== # Targets -dev: - @$(MAKE) go.init - @$(MAKE) go.validate - @$(MAKE) go.build - @$(MAKE) -C images - build.common: @$(MAKE) go.init @$(MAKE) go.validate build: build.common @$(MAKE) go.build +ifneq ($(GOOS),linux) + @$(MAKE) go.build GOOS=linux PIE=0 +endif + @$(MAKE) -C images + +cross.build.platform.%: + @$(MAKE) GOOS=$(word 1, $(subst _, ,$*)) GOARCH=$(word 2, $(subst _, ,$*)) PIE=$(PIE) go.build + +cross.build.parallel: $(foreach p,$(PLATFORMS), cross.build.platform.$(p)) + +build.all: build.common + @$(MAKE) cross.build.parallel install: build.common @$(MAKE) go.install @@ -167,19 +170,6 @@ clean: go.clean distclean: go.distclean clean @rm -fr $(DOWNLOADDIR) -cross.build: - @$(MAKE) go.build - -cross.build.platform.%: - @$(MAKE) GOOS=$(word 1, $(subst _, ,$*)) GOARCH=$(word 2, $(subst _, ,$*)) PIE=$(PIE) cross.build - -cross.parallel: $(foreach p,$(PLATFORMS), cross.build.platform.$(p)) - -build.all: - @$(MAKE) go.init - @$(MAKE) go.validate - @$(MAKE) cross.parallel - release: build.all @$(MAKE) -C images build.all @$(MAKE) release.build @@ -201,7 +191,7 @@ endif prune: @$(MAKE) -C images prune -.PHONY: build.common cross.build cross.parallel +.PHONY: build.common cross.build.parallel .PHONY: dev build build.all install test check vet fmt vendor clean distclean release publish promote prune # ==================================================================================== diff --git a/build/common.sh b/build/common.sh index 1d4a184ae..a6d988488 100644 --- a/build/common.sh +++ b/build/common.sh @@ -18,7 +18,7 @@ source_repo=github.com/rook/rook host=$(hostname) rootdir=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd -P) -host_hash=`echo ${host}-${rootdir} | sha256sum | cut -c1-8` +host_hash=`echo ${host}-${rootdir} | shasum -a 256 | cut -c1-8` build_registry=build-${host_hash} container_image=${build_registry}/cross-amd64 container_volume=cross-volume diff --git a/build/makelib/golang.mk b/build/makelib/golang.mk index 954155a99..15b232dce 100644 --- a/build/makelib/golang.mk +++ b/build/makelib/golang.mk @@ -21,18 +21,15 @@ $(error the variable $$GO_PROJECT must be set prior to including golang.mk) endif # These targets will statically or dynamically linked depending on whether they -# import the standard net, os/user packages. +# import the standard net, os/user packages. This will use the standard golang +# settings. GO_PACKAGES ?= -# These targets will statically or dynamically linked depending on whether they -# import the standard net, os/user packages. Buildmode PIE will be enabled. -GO_PIE_PACKAGES ?= - # These targets will be statically linked. GO_STATIC_PACKAGES ?= -ifeq ($(GO_PACKAGES)$(GO_PIE_PACKAGES)$(GO_STATIC_PACKAGES),) -$(error please set GO_PACKAGES, GO_PIE_PACKAGES, and/or GO_STATIC_PACKAGES prior to including golang.mk) +ifeq ($(GO_PACKAGES)$(GO_STATIC_PACKAGES),) +$(error please set GO_PACKAGES, and/or GO_STATIC_PACKAGES prior to including golang.mk) endif # Optional. These are sudirs that we look for all go files to test, vet, and fmt @@ -46,6 +43,7 @@ GO_VENDOR_DIR ?= vendor GO_PKG_DIR ?= # Optional build flags passed to go tools +GO_PIE ?= 0 GO_BUILDFLAGS ?= GO_LDFLAGS ?= GO_TAGS ?= @@ -85,15 +83,21 @@ endif # into the system's root dir. using our own pkg dir avoid thats ifneq ($(GO_PKG_DIR),) GO_PKG_BASE_DIR := $(abspath $(GO_PKG_DIR)/$(GOOS)_$(GOARCH)) +ifeq ($(GOOS)_$(PIE),linux_1) GO_PKG_FLAGS := -pkgdir $(GO_PKG_BASE_DIR) -GO_PKG_PIE_FLAGS := -pkgdir $(GO_PKG_BASE_DIR)_pie_shared +else +GO_PKG_FLAGS := -pkgdir $(GO_PKG_BASE_DIR)_pie_shared +endif GO_PKG_STATIC_FLAGS := -pkgdir $(GO_PKG_BASE_DIR)_static endif -GO_FLAGS = $(GO_BUILDFLAGS) $(GO_PKG_FLAGS) -tags '$(GO_TAGS)' -ldflags '$(GO_LDFLAGS)' -GO_PIE_FLAGS = $(GO_BUILDFLAGS) $(GO_PKG_PIE_FLAGS) -installsuffix pie -buildmode pie -tags '$(GO_TAGS)' -ldflags '$(GO_LDFLAGS)' +GO_FLAGS = $(GO_BUILDFLAGS) $(GO_PKG_FLAGS) -tags '$(GO_TAGS)' -ldflags '$(GO_LDFLAGS)' GO_STATIC_FLAGS = $(GO_BUILDFLAGS) $(GO_PKG_STATIC_FLAGS) -installsuffix static -tags '$(GO_TAGS)' -ldflags '$(GO_LDFLAGS)' +ifeq ($(GOOS)_$(PIE),linux_1) +GO_FLAGS += -installsuffix pie -buildmode pie +endif + # ==================================================================================== # Targets @@ -117,7 +121,7 @@ go.init: $(GO_VENDOR_DIR)/vendor.stamp define go.project go.build.packages.$(1): @echo === go build $(1) $(GOOS)_$(GOARCH) - @CGO_ENABLED=$(3) $(GO) build -v -i -o $(GO_OUT_DIR)/$(1)$(GO_OUT_EXT) $(4) $(2) + @$(3) $(GO) build -v -i -o $(GO_OUT_DIR)/$(1)$(GO_OUT_EXT) $(4) $(2) go.build.packages: go.build.packages.$(1) @@ -127,9 +131,8 @@ go.install.packages.$(1): go.install.packages: go.install.packages.$(1) endef -$(foreach p,$(GO_PACKAGES),$(eval $(call go.project,$(lastword $(subst /, ,$(p))),$(p),1,$(GO_FLAGS)))) -$(foreach p,$(GO_PIE_PACKAGES),$(eval $(call go.project,$(lastword $(subst /, ,$(p))),$(p),1,$(GO_PIE_FLAGS)))) -$(foreach p,$(GO_STATIC_PACKAGES),$(eval $(call go.project,$(lastword $(subst /, ,$(p))),$(p),0,$(GO_STATIC_FLAGS)))) +$(foreach p,$(GO_PACKAGES),$(eval $(call go.project,$(lastword $(subst /, ,$(p))),$(p),CGO_ENABLED=1,$(GO_FLAGS)))) +$(foreach p,$(GO_STATIC_PACKAGES),$(eval $(call go.project,$(lastword $(subst /, ,$(p))),$(p),CGO_ENABLED=0,$(GO_STATIC_FLAGS)))) .PHONY: go.build go.build: diff --git a/build/release/release.sh b/build/release/release.sh index 3e157e354..96e3875de 100755 --- a/build/release/release.sh +++ b/build/release/release.sh @@ -6,14 +6,14 @@ source ${scriptdir}/common.sh action=$1 shift -mkdir -p ${RELEASE_DIR} - case ${action} in build.init) + mkdir -p ${RELEASE_DIR} write_version_file ;; publish.init) + mkdir -p ${RELEASE_DIR} publish_version_file ;; @@ -28,6 +28,8 @@ case ${action} in ;; promote.init) + mkdir -p ${RELEASE_DIR} + if check_release_version; then echo ${RELEASE_VERSION} can not be promoted. Must build from a tag like v0.4.0. exit 1 diff --git a/images/Makefile b/images/Makefile index b664bcc48..15af6d3f6 100644 --- a/images/Makefile +++ b/images/Makefile @@ -16,7 +16,6 @@ include image.mk PLATFORMS ?= amd64 arm arm64 PLATFORM_IMAGES ?= rookd toolbox -HOST_IMAGES ?= cross # ==================================================================================== # Image Builds @@ -33,6 +32,8 @@ cross-go.%: cross-gnu.% cross.%: cross-go.% @$(MAKE) -C cross GOARCH=$* +cross: cross.amd64 + ceph.%: cross-gnu.$(GOHOSTARCH) @$(MAKE) -C ceph GOARCH=$* @@ -42,9 +43,9 @@ rookd.%: ceph.% base.% toolbox.%: rookd.% @$(MAKE) -C toolbox GOARCH=$* -build.all: $(foreach p,$(PLATFORMS), $(foreach i,$(PLATFORM_IMAGES), $(i).$(p))) $(foreach i,$(HOST_IMAGES), $(i).$(GOHOSTARCH)) +build.all: $(foreach p,$(PLATFORMS), $(foreach i,$(PLATFORM_IMAGES), $(i).$(p))) -build.images: $(foreach i,$(PLATFORM_IMAGES) $(HOST_IMAGES), $(i).$(GOHOSTARCH)) +build.images: $(foreach i,$(PLATFORM_IMAGES), $(i).$(GOHOSTARCH)) # ==================================================================================== # Help diff --git a/images/image.mk b/images/image.mk index dcaf63682..99ffcce4f 100644 --- a/images/image.mk +++ b/images/image.mk @@ -14,6 +14,8 @@ SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +# container images are always on linux +override GOOS := linux include $(SELF_DIR)/../build/makelib/cross.mk CACHE_REGISTRY := cache @@ -22,15 +24,12 @@ CACHE_REGISTRY := cache ifeq ($(origin BUILD_REGISTRY), undefined) HOSTNAME := $(shell hostname) ROOTDIR := $(shell cd $(SELF_DIR)/.. && pwd -P) -BUILD_REGISTRY := build-$(shell echo $(HOSTNAME)-$(ROOTDIR) | sha256sum | cut -c1-8) +BUILD_REGISTRY := build-$(shell echo $(HOSTNAME)-$(ROOTDIR) | shasum -a 256 | cut -c1-8) endif # public registry used for images that are pushed REGISTRY ?= quay.io/rook -# if we are running inside the container get our own cid -SELF_CID := $(shell cat /proc/self/cgroup | grep docker | grep -o -E '[0-9a-f]{64}' | head -n 1) - # the base ubuntu image to use OSBASE ?= ubuntu:zesty @@ -50,6 +49,8 @@ SED_CMD?=sed -i "" endif ifeq ($(UNAME_S),Linux) SED_CMD?=sed -i +# if we are running inside the container get our own cid +SELF_CID := $(shell cat /proc/self/cgroup | grep docker | grep -o -E '[0-9a-f]{64}' | head -n 1) endif INTERACTIVE:=$(shell [ -t 0 ] && echo 1) @@ -133,8 +134,13 @@ PRUNE_KEEP ?= 24 PRUNE_DRYRUN ?= 0 -CACHE_DATE_FORMAT := "%Y-%m-%d.%H%M%S.%N" +CACHE_DATE_FORMAT := "%Y-%m-%d.%H%M%S" +ifeq ($(UNAME_S),Linux) CACHE_PRUNE_DATE := $(shell date -u --date="$(PRUNE_HOURS) hours ago" +"$(CACHE_DATE_FORMAT)") +endif +ifeq ($(UNAME_S),Darwin) +CACHE_PRUNE_DATE := $(shell date -u -v -$(PRUNE_HOURS)H +"$(CACHE_DATE_FORMAT)") +endif CACHE_TAG := $(shell date -u +"$(CACHE_DATE_FORMAT)") cache.lookup: diff --git a/images/rookd/Makefile b/images/rookd/Makefile index 7bf2cded6..9d8e3157e 100755 --- a/images/rookd/Makefile +++ b/images/rookd/Makefile @@ -58,9 +58,10 @@ rookd-base-image: $(CEPH_TARBALL) @cd $(TEMP) && $(SED_CMD) 's|BASEIMAGE|$(BASEIMAGE)|g' Dockerfile @docker build $(BUILD_ARGS) --build-arg EXTRALIBS=$(EXTRA_LIBS) -t $(ROOKD_BASE_IMAGE) $(TEMP) -# since this is a leaf image avoid leaving around a lot of dangling images +# since this is a leaf image we avoid leaving around a lot of dangling images # by removing the last build of the final rookd image OLD_IMAGE_ID := $(shell docker images -q $(ROOKD_IMAGE)) +CURRENT_IMAGE_ID := $$(docker images -q $(ROOKD_IMAGE)) build.images: rookd-base-image @echo === docker build $(ROOKD_IMAGE) @@ -68,5 +69,5 @@ build.images: rookd-base-image @cp ../../bin/linux_$(GOARCH)/rookd $(TEMP) @cd $(TEMP) && $(SED_CMD) 's|BASEIMAGE|$(ROOKD_BASE_IMAGE)|g' Dockerfile @docker build $(BUILD_ARGS) -t $(ROOKD_IMAGE) $(TEMP) - @[ -z "$(OLD_IMAGE_ID)" ] || docker rmi $(OLD_IMAGE_ID) + @[ "$(OLD_IMAGE_ID)" != "$(CURRENT_IMAGE_ID)" ] && [ -n "$(OLD_IMAGE_ID)" ] && docker rmi $(OLD_IMAGE_ID) || true @rm -fr $(TEMP) diff --git a/images/toolbox/Dockerfile b/images/toolbox/Dockerfile index 533a09968..d51a9671a 100644 --- a/images/toolbox/Dockerfile +++ b/images/toolbox/Dockerfile @@ -14,25 +14,8 @@ FROM BASEIMAGE -RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ - binutils \ - s3cmd \ - curl \ - fio \ - gdb \ - iperf3 \ - jq \ - kmod \ - less \ - man-db \ - sudo \ - vim && \ - DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \ - DEBIAN_FRONTEND=noninteractive apt-get autoremove -y && \ - DEBIAN_FRONTEND=noninteractive apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - COPY rook /usr/local/bin + COPY toolbox.sh /toolbox/ + CMD [ "/toolbox/toolbox.sh" ] diff --git a/images/toolbox/Dockerfile.base b/images/toolbox/Dockerfile.base new file mode 100644 index 000000000..97be585c6 --- /dev/null +++ b/images/toolbox/Dockerfile.base @@ -0,0 +1,34 @@ +# Copyright 2016 The Rook Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM BASEIMAGE + +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ + binutils \ + s3cmd \ + curl \ + fio \ + gdb \ + iperf3 \ + jq \ + kmod \ + less \ + man-db \ + sudo \ + vim && \ + DEBIAN_FRONTEND=noninteractive apt-get upgrade -y && \ + DEBIAN_FRONTEND=noninteractive apt-get autoremove -y && \ + DEBIAN_FRONTEND=noninteractive apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/images/toolbox/Makefile b/images/toolbox/Makefile index 7144bf7dd..eb6d1a05a 100644 --- a/images/toolbox/Makefile +++ b/images/toolbox/Makefile @@ -14,18 +14,31 @@ include ../image.mk -IMAGE = $(BUILD_REGISTRY)/toolbox-$(GOARCH) -IMAGES = $(IMAGE) -CACHE_IMAGES = $(IMAGE) +TOOLBOX_IMAGE = $(BUILD_REGISTRY)/toolbox-$(GOARCH) +TOOLBOX_BASE_IMAGE = $(BUILD_REGISTRY)/toolbox-base-$(GOARCH) +IMAGES = $(TOOLBOX_IMAGE) $(TOOLBOX_BASE_IMAGE) +CACHE_IMAGES = $(TOOLBOX_BASE_IMAGE) BASEIMAGE = $(BUILD_REGISTRY)/rookd-base-$(GOARCH) TEMP := $(shell mktemp -d) -build.images: - @echo === docker build $(IMAGE) +toolbox-base-image: + @echo === docker build $(TOOLBOX_BASE_IMAGE) + @cp Dockerfile.base $(TEMP)/Dockerfile + @cd $(TEMP) && $(SED_CMD) 's|BASEIMAGE|$(BASEIMAGE)|g' Dockerfile + @docker build $(BUILD_ARGS) -t $(TOOLBOX_BASE_IMAGE) $(TEMP) + +# since this is a leaf image we avoid leaving around a lot of dangling images +# by removing the last build of the final rookd image +OLD_IMAGE_ID := $(shell docker images -q $(TOOLBOX_IMAGE)) +CURRENT_IMAGE_ID := $$(docker images -q $(TOOLBOX_IMAGE)) + +build.images: toolbox-base-image + @echo === docker build $(TOOLBOX_IMAGE) @cp -a . $(TEMP) @cp ../../bin/linux_$(GOARCH)/rook $(TEMP) - @cd $(TEMP) && $(SED_CMD) 's|BASEIMAGE|$(BASEIMAGE)|g' Dockerfile - @docker build $(BUILD_ARGS) -t $(IMAGE) $(TEMP) + @cd $(TEMP) && $(SED_CMD) 's|BASEIMAGE|$(TOOLBOX_BASE_IMAGE)|g' Dockerfile + @docker build $(BUILD_ARGS) -t $(TOOLBOX_IMAGE) $(TEMP) + @[ "$(OLD_IMAGE_ID)" != "$(CURRENT_IMAGE_ID)" ] && [ -n "$(OLD_IMAGE_ID)" ] && docker rmi $(OLD_IMAGE_ID) || true @rm -fr $(TEMP)