Merge pull request #2671 from phlogistonjohn/jjm-build-vars

Add build variables for docker and sha sum calculation

build/common.sh

@@ -17,7 +17,10 @@
 BUILD_HOST=$(hostname)
 BUILD_REPO=github.com/rook/rook
 BUILD_ROOT=$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd -P)
-BUILD_REGISTRY=build-$(echo ${BUILD_HOST}-${BUILD_ROOT} | shasum -a 256 | cut -c1-8)
+SHA256CMD=${SHA256CMD:-shasum -a 256}
+BUILD_REGISTRY=build-$(echo ${BUILD_HOST}-${BUILD_ROOT} | ${SHA256CMD} | cut -c1-8)
+
+DOCKERCMD=${DOCKERCMD:-docker}
 
 OUTPUT_DIR=${BUILD_ROOT}/_output
 WORK_DIR=${BUILD_ROOT}/.work
@@ -45,7 +48,7 @@ function check_git() {
 }
 
 function start_rsync_container() {
-    docker run \
+    ${DOCKERCMD} run \
         -d \
         -e OWNER=root \
         -e GROUP=root \
@@ -74,8 +77,8 @@ function wait_for_rsync() {
 function stop_rsync_container() {
     local id=$1
 
-    docker stop ${id} &> /dev/null || true
-    docker rm ${id} &> /dev/null || true
+    ${DOCKERCMD} stop ${id} &> /dev/null || true
+    ${DOCKERCMD} rm ${id} &> /dev/null || true
 }
 
 function run_rsync() {

build/makelib/common.mk

@@ -16,6 +16,7 @@
 .SUFFIXES:
 
 SHELL := /bin/bash
+SHA256CMD := shasum -a 256
 
 ifeq ($(origin PLATFORM), undefined)
 ifeq ($(origin GOOS), undefined)
@@ -91,7 +92,10 @@ endif
 
 # a registry that is scoped to the current build tree on this host
 ifeq ($(origin BUILD_REGISTRY), undefined)
-BUILD_REGISTRY := build-$(shell echo $(HOSTNAME)-$(ROOT_DIR) | shasum -a 256 | cut -c1-8)
+BUILD_REGISTRY := build-$(shell echo $(HOSTNAME)-$(ROOT_DIR) | $(SHA256CMD) | cut -c1-8)
+endif
+ifeq ($(BUILD_REGISTRY),build-)
+$(error Failed to get unique ID for host+dir. Check that '$(SHA256CMD)' functions or override SHA256CMD)
 endif
 
 # Select which images (backends) to make; default to all possible images

build/run

@@ -30,7 +30,7 @@ if [ "`uname -m`" != "x86_64" ]; then
     exit 1
 fi
 echo ==== building the cross container \(this could take minutes the first time\)
-make_cmd="make --no-print-directory -C ${scriptdir}/../images cross PULL=0"
+make_cmd="make --no-print-directory -C ${scriptdir}/../images cross PULL=0 DOCKERCMD=${DOCKERCMD}"
 make_output=$($make_cmd 2>&1) || {
     cat <<EOF >&2
 === cross build image failed for ${CROSS_IMAGE}
@@ -65,10 +65,10 @@ if [ "`uname -s`" != "Linux" ]; then
     # runs of the build container are supported they will share the same volume
     # and we will be rsyncing to it at different times. This could lead to
     # undefined behavior but this should be a rare case on non-linux envs.
-    if [[ ! $(docker volume ls | grep ${CROSS_IMAGE_VOLUME}) ]]; then
+    if [[ ! $(${DOCKERCMD} volume ls | grep ${CROSS_IMAGE_VOLUME}) ]]; then
         echo ==== Creating docker volume "${CROSS_IMAGE_VOLUME}" and syncing sources
         echo ==== for first time. This could take a few seconds.
-        docker volume create --name ${CROSS_IMAGE_VOLUME} &> /dev/null
+        ${DOCKERCMD} volume create --name ${CROSS_IMAGE_VOLUME} &> /dev/null
     fi
 
     # On non-linux the layout is as follows:
@@ -154,7 +154,7 @@ rsync_back() {
     fi
 }
 
-docker run \
+${DOCKERCMD} run \
     --rm \
     -h ${BUILD_HOST} \
     -e BUILD_REGISTRY=${BUILD_REGISTRY} \

images/cassandra/Makefile

@@ -29,7 +29,7 @@ do.build:
 	@echo === docker build $(CASSANDRA_OPERATOR_IMAGE)
 	@cp Dockerfile $(TEMP)
 	@cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rook $(TEMP)
-	@docker build $(BUILD_ARGS) \
+	@$(DOCKERCMD) build $(BUILD_ARGS) \
 		--build-arg ARCH=$(GOARCH) \
     	--build-arg TINI_VERSION=$(TINI_VERSION) \
 		-t $(CASSANDRA_OPERATOR_IMAGE) \

images/ceph/Makefile

@@ -34,7 +34,7 @@ do.build:
 	@cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rook $(TEMP)
 	@cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rookflex $(TEMP)
 	@cd $(TEMP) && $(SED_CMD) 's|BASEIMAGE|$(BASEIMAGE)|g' Dockerfile
-	@docker build $(BUILD_ARGS) \
+	@$(DOCKERCMD) build $(BUILD_ARGS) \
 		--build-arg ARCH=$(GOARCH) \
 		--build-arg TINI_VERSION=$(TINI_VERSION) \
 		-t $(CEPH_IMAGE) \

images/cockroachdb/Makefile

@@ -28,7 +28,7 @@ do.build:
 	@echo === docker build $(COCKROACHDB_IMAGE)
 	@cp Dockerfile $(TEMP)
 	@cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rook $(TEMP)
-	@docker build $(BUILD_ARGS) \
+	@$(DOCKERCMD) build $(BUILD_ARGS) \
 		-t $(COCKROACHDB_IMAGE) \
 		$(TEMP)
 	@rm -fr $(TEMP)

images/cross/Makefile

@@ -22,7 +22,7 @@ TEMP := $(shell mktemp -d)
 do.build:
 	@echo === docker build $(IMAGE)
 	@cp -a . $(TEMP)
-	@docker build $(BUILD_ARGS) \
+	@$(DOCKERCMD) build $(BUILD_ARGS) \
 		--build-arg ARCH=$(GOARCH) \
 		--build-arg TINI_VERSION=$(TINI_VERSION) \
 		-t $(IMAGE) \

images/edgefs/Makefile

@@ -27,21 +27,21 @@ TEMP := $(shell mktemp -d)
 
 # since this is a leaf image we avoid leaving around a lot of dangling images
 # by removing the last build of the final cockroachdb image
-OLD_IMAGE_ID := $(shell docker images -q $(EDGEFS_IMAGE))
-CURRENT_IMAGE_ID := $$(docker images -q $(EDGEFS_IMAGE))
+OLD_IMAGE_ID := $(shell $(DOCKERCMD) images -q $(EDGEFS_IMAGE))
+CURRENT_IMAGE_ID := $$($(DOCKERCMD) images -q $(EDGEFS_IMAGE))
 IMAGE_FILENAME := $(IMAGE_OUTPUT_DIR)/edgefs.tar.gz
 
 do.build:
 	@echo === docker build $(EDGEFS_IMAGE)
 	@cp Dockerfile $(TEMP)
 	@cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rook $(TEMP)
-	@docker build $(BUILD_ARGS) \
+	@$(DOCKERCMD) build $(BUILD_ARGS) \
 		-t $(EDGEFS_IMAGE) \
 		$(TEMP)
-	@[ "$(OLD_IMAGE_ID)" != "$(CURRENT_IMAGE_ID)" ] && [ -n "$(OLD_IMAGE_ID)" ] && docker rmi $(OLD_IMAGE_ID) || true
+	@[ "$(OLD_IMAGE_ID)" != "$(CURRENT_IMAGE_ID)" ] && [ -n "$(OLD_IMAGE_ID)" ] && $(DOCKERCMD) rmi $(OLD_IMAGE_ID) || true
 	@if [ ! -e "$(IMAGE_FILENAME)" ] || [ "$(OLD_IMAGE_ID)" != "$(CURRENT_IMAGE_ID)" ] || [ -n "$(OLD_IMAGE_ID)" ]; then \
 		echo === saving image $(EDGEFS_IMAGE); \
 		mkdir -p $(IMAGE_OUTPUT_DIR); \
-		docker save $(EDGEFS_IMAGE) | gzip -c > $(IMAGE_FILENAME); \
+		$(DOCKERCMD) save $(EDGEFS_IMAGE) | gzip -c > $(IMAGE_FILENAME); \
 	fi
 	@rm -fr $(TEMP)

images/image.mk

@@ -17,6 +17,8 @@
 
 override GOOS=linux
 
+DOCKERCMD := docker
+
 # include the common make file
 SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
 include $(SELF_DIR)/../build/makelib/common.mk
@@ -82,23 +84,23 @@ prune: cache.prune
 
 clean.images:
 	@for i in $(CLEAN_IMAGES); do \
-		if [ -n "$$(docker images -q $$i)" ]; then \
-			for c in $$(docker ps -a -q --no-trunc --filter=ancestor=$$i); do \
+		if [ -n "$$($(DOCKERCMD) images -q $$i)" ]; then \
+			for c in $$($(DOCKERCMD) ps -a -q --no-trunc --filter=ancestor=$$i); do \
 				if [ "$$c" != "$(SELF_CID)" ]; then \
 					echo stopping and removing container $${c} referencing image $$i; \
-					docker stop $${c}; \
-					docker rm $${c}; \
+					$(DOCKERCMD) stop $${c}; \
+					$(DOCKERCMD) rm $${c}; \
 				fi; \
 			done; \
 			echo cleaning image $$i; \
-			docker rmi $$i > /dev/null 2>&1 || true; \
+			$(DOCKERCMD) rmi $$i > /dev/null 2>&1 || true; \
 		fi; \
 	done
 
 # this will clean everything for this build
 clean.build:
 	@echo === cleaning images for $(BUILD_REGISTRY)
-	@$(MAKE) clean.images CLEAN_IMAGES="$(shell docker images | grep -E '^$(BUILD_REGISTRY)/' | awk '{print $$1":"$$2}')"
+	@$(MAKE) clean.images CLEAN_IMAGES="$(shell $(DOCKERCMD) images | grep -E '^$(BUILD_REGISTRY)/' | awk '{print $$1":"$$2}')"
 
 # =====================================================================================
 # Caching
@@ -128,22 +130,22 @@ CACHE_TAG := $(shell date -u +"$(CACHE_DATE_FORMAT)")
 
 cache.lookup:
 	@IMAGE_NAME=$${LOOKUP_IMAGE#*/} ;\
-	if [ -n "$$(docker images -q $(LOOKUP_IMAGE))" ]; then exit 0; fi; \
-	if [ -z "$$(docker images -q $(CACHE_REGISTRY)/$${IMAGE_NAME})" ]; then \
+	if [ -n "$$($(DOCKERCMD) images -q $(LOOKUP_IMAGE))" ]; then exit 0; fi; \
+	if [ -z "$$($(DOCKERCMD) images -q $(CACHE_REGISTRY)/$${IMAGE_NAME})" ]; then \
 		$(MAKE) $(MISS_TARGET); \
 	else \
-		docker tag $$(docker images -q $(CACHE_REGISTRY)/$${IMAGE_NAME}) $(LOOKUP_IMAGE); \
+		$(DOCKERCMD) tag $$($(DOCKERCMD) images -q $(CACHE_REGISTRY)/$${IMAGE_NAME}) $(LOOKUP_IMAGE); \
 	fi;
 
 cache.images:
 	@for i in $(CACHE_IMAGES); do \
-		IMGID=$$(docker images -q $$i); \
+		IMGID=$$($(DOCKERCMD) images -q $$i); \
 		if [ -n "$$IMGID" ]; then \
 			echo === caching image $$i; \
 			CACHE_IMAGE=$(CACHE_REGISTRY)/$${i#*/}; \
-			docker tag $$i $${CACHE_IMAGE}:$(CACHE_TAG); \
-			for r in $$(docker images --format "{{.ID}}#{{.Repository}}:{{.Tag}}" | grep $$IMGID | grep $(CACHE_REGISTRY)/ | grep -v $${CACHE_IMAGE}:$(CACHE_TAG)); do \
-				docker rmi $${r#*#} > /dev/null 2>&1 || true; \
+			$(DOCKERCMD) tag $$i $${CACHE_IMAGE}:$(CACHE_TAG); \
+			for r in $$($(DOCKERCMD) images --format "{{.ID}}#{{.Repository}}:{{.Tag}}" | grep $$IMGID | grep $(CACHE_REGISTRY)/ | grep -v $${CACHE_IMAGE}:$(CACHE_TAG)); do \
+				$(DOCKERCMD) rmi $${r#*#} > /dev/null 2>&1 || true; \
 			done; \
 		fi; \
 	done
@@ -152,31 +154,31 @@ cache.images:
 cache.prune:
 	@echo === pruning images older than $(PRUNE_HOURS) hours
 	@echo === keeping a minimum of $(PRUNE_KEEP) images
-	@EXPIRED=$$(docker images --format "{{.Tag}}#{{.Repository}}:{{.Tag}}" \
+	@EXPIRED=$$($(DOCKERCMD) images --format "{{.Tag}}#{{.Repository}}:{{.Tag}}" \
 		| grep -E '$(CACHE_REGISTRY)/' \
 		| sort -r \
 		| awk -v i=0 -v cd="$(CACHE_PRUNE_DATE)" -F  "#" '{if ($$1 <= cd && i >= $(PRUNE_KEEP)) print $$2; i++ }') &&\
 	for i in $$EXPIRED; do \
 		echo removing expired cache image $$i; \
-		[ $(PRUNE_DRYRUN) = 1 ] || docker rmi $$i > /dev/null 2>&1 || true; \
+		[ $(PRUNE_DRYRUN) = 1 ] || $(DOCKERCMD) rmi $$i > /dev/null 2>&1 || true; \
 	done
-	@for i in $$(docker images -q -f dangling=true); do \
+	@for i in $$($(DOCKERCMD) images -q -f dangling=true); do \
 		echo removing dangling image $$i; \
-		docker rmi $$i > /dev/null 2>&1 || true; \
+		$(DOCKERCMD) rmi $$i > /dev/null 2>&1 || true; \
 	done
 
 # =====================================================================================
 # Debugging nukes all images
 #
 debug.nuke:
-	@for c in $$(docker ps -a -q --no-trunc); do \
+	@for c in $$($(DOCKERCMD) ps -a -q --no-trunc); do \
 		if [ "$$c" != "$(SELF_CID)" ]; then \
 			echo stopping and removing container $${c}; \
-			docker stop $${c}; \
-			docker rm $${c}; \
+			$(DOCKERCMD) stop $${c}; \
+			$(DOCKERCMD) rm $${c}; \
 		fi; \
 	done
-	@for i in $$(docker images -q); do \
+	@for i in $$($(DOCKERCMD) images -q); do \
 		echo removing image $$i; \
-		docker rmi -f $$i > /dev/null 2>&1; \
+		$(DOCKERCMD) rmi -f $$i > /dev/null 2>&1; \
 	done

images/minio/Makefile

@@ -28,7 +28,7 @@ do.build:
 	@echo === docker build $(MINIO_IMAGE)
 	@cp Dockerfile $(TEMP)
 	@cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rook $(TEMP)
-	@docker build $(BUILD_ARGS) \
+	@$(DOCKERCMD) build $(BUILD_ARGS) \
 		--build-arg ARCH=$(GOARCH) \
 		-t $(MINIO_IMAGE) \
 		$(TEMP)

images/nfs/Makefile

@@ -34,8 +34,8 @@ TEMP := $(shell mktemp -d)
 
 # since this is a leaf image we avoid leaving around a lot of dangling images
 # by removing the last build of the final nfs image
-OLD_IMAGE_ID := $(shell docker images -q $(NFS_IMAGE))
-CURRENT_IMAGE_ID := $$(docker images -q $(NFS_IMAGE))
+OLD_IMAGE_ID := $(shell $(DOCKERCMD) images -q $(NFS_IMAGE))
+CURRENT_IMAGE_ID := $$($(DOCKERCMD) images -q $(NFS_IMAGE))
 IMAGE_FILENAME := $(IMAGE_OUTPUT_DIR)/nfs.tar.gz
 
 do.build:
@@ -44,13 +44,13 @@ do.build:
 	@cp start.sh $(TEMP)
 	@cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rook $(TEMP)
 	@cd $(TEMP) && $(SED_CMD) 's|NFS_BASEIMAGE|$(NFS_BASEIMAGE)|g' Dockerfile
-	@docker build $(BUILD_ARGS) \
+	@$(DOCKERCMD) build $(BUILD_ARGS) \
 		-t $(NFS_IMAGE) \
 		$(TEMP)
-	@[ "$(OLD_IMAGE_ID)" != "$(CURRENT_IMAGE_ID)" ] && [ -n "$(OLD_IMAGE_ID)" ] && docker rmi $(OLD_IMAGE_ID) || true
+	@[ "$(OLD_IMAGE_ID)" != "$(CURRENT_IMAGE_ID)" ] && [ -n "$(OLD_IMAGE_ID)" ] && $(DOCKERCMD) rmi $(OLD_IMAGE_ID) || true
 	@if [ ! -e "$(IMAGE_FILENAME)" ] || [ "$(OLD_IMAGE_ID)" != "$(CURRENT_IMAGE_ID)" ] || [ -n "$(OLD_IMAGE_ID)" ]; then \
 		echo === saving image $(NFS_IMAGE); \
 		mkdir -p $(IMAGE_OUTPUT_DIR); \
-		docker save $(NFS_IMAGE) | gzip -c > $(IMAGE_FILENAME); \
+		$(DOCKERCMD) save $(NFS_IMAGE) | gzip -c > $(IMAGE_FILENAME); \
 	fi
 	@rm -fr $(TEMP)