feedc0de/my-rook-config
1
0
Fork 0
forked from rook/rook
Code Pull Requests Activity

build: rework release scripts and jenkinsfile

Browse Source 
inital commit to convert release scripts to makefiles for improved maintenance.

now publishing to docker hub including manifest list support.

initial attempt at reworking the jenkinsfile to pipeline syntax.
This commit is contained in:
Bassam Tabbara
2017-06-29 15:20:27 -07:00
parent 6188a9bf41
commit e55fd55e7d
13 changed files with 283 additions and 784 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1 -1
View File
@@ -1,6 +1,6 @@
/.vscode/
/.work
/.download
/.cache
/.idea
/bin
Jenkinsfile
Vendored
+42 -55
View File
@@ -1,69 +1,56 @@
// Rook build for Jenkins Pipelines
try {
node("ec2-stateful") {
pipeline {
agent { label 'ec2-stateful' }
stage('Checkout') {
checkout scm
}
options {
timeout(time: 1, unit: 'HOURS')
timestamps()
}
withEnv(["DOWNLOADDIR=${env.HOME}/.download", "CHANNEL=${env.BRANCH_NAME}"]) {
environment {
PRUNE_HOURS = 48
PRUNE_KEEP = 48
}
stage('Build') {
sh 'build/run make -j\$(nproc) release'
stages {
stage('Build') {
steps {
sh 'build/run make -j\$(nproc) build.all'
}
stage('Tests') {
}
stage('Unit Tests') {
steps {
sh 'build/run make -j\$(nproc) check'
}
stage('E2E') {
def exists = fileExists 'release/version'
if (!exists) {
error('The file release/version does not exist')
}
def rook_tag = readFile (file: 'release/version', encoding : 'utf-8').trim()
if (rook_tag == '') {
error('Failed to get rook_tag from version file')
}
echo 'Rook Tag is ' + rook_tag
sh "e2e/scripts/smoke_test.sh ${rook_tag} Kubernetes v1.6"
junit 'e2e/results/*.xml'
}
stage('Publish') {
withCredentials([
[$class: 'UsernamePasswordMultiBinding', credentialsId: 'rook-quay-io', usernameVariable: 'DOCKER_USER', passwordVariable: 'DOCKER_PASSWORD'],
[$class: 'UsernamePasswordMultiBinding', credentialsId: 'rook-jenkins-aws', usernameVariable: 'AWS_ACCESS_KEY_ID', passwordVariable: 'AWS_SECRET_ACCESS_KEY'],
[$class: 'StringBinding', credentialsId: 'quantumbuild-token', variable: 'GITHUB_TOKEN']
]) {
sh 'docker login -u="${DOCKER_USER}" -p="${DOCKER_PASSWORD}" quay.io'
sh 'build/run make -j\$(nproc) publish'
}
stage('Publish') {
when {
anyOf {
branch 'master'
branch 'release-*'
}
}
stage('Cleanup') {
sh 'make clean'
sh 'make prune PRUNE_KEEP_CACHED=48 PRUNE_KEEP_ORPHANS=48'
deleteDir()
environment {
DOCKER = credentials('rook-docker-hub')
QUAY = credentials('rook-quay-io')
AWS = credentials('rook-jenkins-aws')
}
steps {
sh 'docker login -u="${DOCKER_USR}" -p="${DOCKER_PSW}"'
sh 'docker login -u="${QUAY_USR}" -p="${QUAY_PSW}" quay.io'
sh 'build/run make -C build/release -j\$(nproc)'
sh 'build/run make -C build/release -j\$(nproc) publish'
}
}
}
post {
always {
sh 'make clean'
sh 'make prune'
deleteDir()
}
}
}
catch (Exception e) {
echo 'Failure encountered'
node("ec2-stateful") {
sh 'make clean'
sh 'make prune PRUNE_KEEP_CACHED=48 PRUNE_KEEP_ORPHANS=48'
deleteDir()
}
exit 1
}
Makefile
+15 -90
View File
@@ -12,15 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# remove default suffixes as we dont use them
.SUFFIXES:
include build/makelib/common.mk
# set the shell to bash in case some environments use sh
.PHONY: all
all: build
include build/makelib/common.mk
# ====================================================================================
# Build Options
@@ -49,30 +45,10 @@ ifeq ($(DEBUG),0)
LDFLAGS += -s -w
endif
# the working directory to store packages and intermediate build files
ifeq ($(origin WORKDIR), undefined)
WORKDIR := $(abspath .work)
endif
ifeq ($(origin DOWNLOADDIR), undefined)
DOWNLOADDIR := $(abspath .download)
endif
# bin and relase dirs
ifeq ($(origin BIN_DIR),undefined)
BIN_DIR := $(abspath bin)
endif
ifeq ($(origin RELEASE_DIR), undefined)
RELEASE_DIR := $(abspath release)
endif
# platforms where we only build client bits
CLIENT_PLATFORMS ?= darwin_amd64 windows_amd64
# platforms for which we client and server bits.
SERVER_PLATFORMS ?= linux_arm linux_amd64 linux_arm64
# the platforms to build
PLATFORMS ?= $(SERVER_PLATFORMS) $(CLIENT_PLATFORMS)
# platforms
PLATFORMS ?= $(ALL_PLATFORMS)
SERVER_PLATFORMS := $(filter linux_%,$(PLATFORMS))
CLIENT_PLATFORMS := $(filter-out linux_%,$(PLATFORMS))
# client projects that we build on all platforms
CLIENT_PACKAGES = $(GO_PROJECT)/cmd/rookctl
@@ -83,24 +59,16 @@ SERVER_PACKAGES = $(GO_PROJECT)/cmd/rook
# the root go project
GO_PROJECT=github.com/rook/rook
# set the version number. you should not need to do this
# for the majority of scenarios.
ifeq ($(origin VERSION), undefined)
VERSION := $(shell git describe --dirty --always --tags)
endif
LDFLAGS += -X $(GO_PROJECT)/pkg/version.Version=$(VERSION)
# the release channel. Can be set to master, alpha, beta or stable
CHANNEL ?=
# ====================================================================================
# Setup Go projects
GO_STATIC_PACKAGES=
ifneq ($(filter $(GOOS)_$(GOARCH),$(CLIENT_PLATFORMS) $(SERVER_PLATFORMS)),)
ifneq ($(filter $(PLATFORM),$(CLIENT_PLATFORMS) $(SERVER_PLATFORMS)),)
GO_STATIC_PACKAGES += $(CLIENT_PACKAGES)
endif
ifneq ($(filter $(GOOS)_$(GOARCH),$(SERVER_PLATFORMS)),)
ifneq ($(filter $(PLATFORM),$(SERVER_PLATFORMS)),)
GO_STATIC_PACKAGES += $(SERVER_PACKAGES)
endif
@@ -108,22 +76,8 @@ GO_BUILDFLAGS=$(BUILDFLAGS)
GO_LDFLAGS=$(LDFLAGS)
GO_TAGS=$(TAGS)
GO_BIN_DIR = $(BIN_DIR)
GO_WORK_DIR ?= $(WORKDIR)
GO_TOOLS_DIR ?= $(DOWNLOADDIR)/tools
GO_PKG_DIR ?= $(WORKDIR)/pkg
include build/makelib/golang.mk
# ====================================================================================
# Setup Distribution
RELEASE_VERSION=$(VERSION)
RELEASE_CHANNEL=$(CHANNEL)
RELEASE_BIN_DIR=$(BIN_DIR)
RELEASE_PLATFORMS=$(PLATFORMS)
include build/makelib/release.mk
# ====================================================================================
# Targets
@@ -168,36 +122,18 @@ fmt:
vendor: go.vendor
clean: go.clean
@rm -fr $(WORKDIR) $(RELEASE_DIR) $(BIN_DIR)
@$(MAKE) release.cleanup
clean:
@rm -fr $(BIN_DIR)
@$(MAKE) -C images clean
distclean: go.distclean clean
@rm -fr $(DOWNLOADDIR)
release: build.all
@$(MAKE) release.build
publish:
ifneq ($(filter master alpha beta stable, $(CHANNEL)),)
@$(MAKE) release.publish
else
@echo skipping publish. invalid channel "$(CHANNEL)"
endif
promote:
ifneq ($(filter master alpha beta stable, $(CHANNEL)),)
@$(MAKE) release.promote
else
@echo skipping promote. invalid channel "$(CHANNEL)"
endif
@rm -fr $(CACHE_DIR)
prune:
@$(MAKE) -C images prune
.PHONY: build.common cross.build.parallel
.PHONY: build build.all install test check vet fmt vendor clean distclean release publish promote prune
.PHONY: all build.common cross.build.parallel
.PHONY: build build.all install test check vet fmt vendor clean distclean prune
# ====================================================================================
# Help
@@ -223,28 +159,17 @@ help:
@echo ' vendor Installs vendor dependencies.'
@echo ' vet Runs lint checks on go sources.'
@echo ''
@echo 'Release Targets:'
@echo ' release Builds all release artifacts including'
@echo ' container images for all platforms.'
@echo ' publish Publishes all release artifacts to'
@echo ' appropriate public repositories.'
@echo ' promote Promotes a published release to a'
@echo ' release channel.'
@echo ''
@echo 'Options:'
@echo ' DEBUG Whether to generate debug symbols. Default is 0.'
@echo ' GOARCH The arch to build.'
@echo ' GOOS The OS to build for.'
@echo ' PLATFORM The platform to build.'
@echo ' VERSION The version information compiled into binaries.'
@echo ' The default is obtained from git.'
@echo ' V Set to 1 enable verbose build. Default is 0.'
@echo ''
@echo 'Advanced Options:'
@echo ' CHANNEL Sets the release channel. Can be set to master,'
@echo ' alpha, beta, or stable. Default is not set.'
@echo ' DOWNLOADDIR A directory where downloaded files and other'
@echo ' CACHE_DIR A directory where downloaded files and other'
@echo ' files used during the build are cached. These'
@echo ' files help speedup the build by avoiding network'
@echo ' transfers. Its safe to use these files across builds.'
@echo ' WORKDIR The working directory where most build files'
@echo ' WORK_DIR The working directory where most build files'
@echo ' are stored. The default is .work'
build/makelib/common.mk
+42 -12
View File
@@ -12,45 +12,56 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Optional OS and ARCH to build
ifeq ($(origin GOOS), undefined)
GOOS := $(shell go env GOOS)
endif
# remove default suffixes as we dont use them
.SUFFIXES:
ifeq ($(origin GOARCH), undefined)
ifeq ($(origin PLATFORM), undefined)
GOOS := $(shell go env GOOS)
GOARCH := $(shell go env GOARCH)
PLATFORM := $(GOOS)_$(GOARCH)
else
GOOS := $(word 1, $(subst _, ,$(PLATFORM)))
GOARCH := $(word 2, $(subst _, ,$(PLATFORM)))
endif
GOHOSTOS := $(shell go env GOHOSTOS)
GOHOSTARCH := $(shell go env GOHOSTARCH)
HOST_PLATFORM := $(shell go env GOHOSTOS)_$(shell go env GOHOSTARCH)
# set cross compile options
ifeq ($(GOOS)_$(GOARCH),linux_amd64)
ALL_PLATFORMS ?= darwin_amd64 windows_amd64 linux_arm linux_amd64 linux_arm64
ifeq ($(PLATFORM),linux_amd64)
CROSS_TRIPLE = x86_64-linux-gnu
DEBIAN_ARCH = amd64
endif
ifeq ($(GOOS)_$(GOARCH),linux_arm)
ifeq ($(PLATFORM),linux_arm)
GOARM=7
DEBIAN_ARCH = armhf
CROSS_TRIPLE = arm-linux-gnueabihf
endif
ifeq ($(GOOS)_$(GOARCH),linux_arm64)
ifeq ($(PLATFORM),linux_arm64)
DEBIAN_ARCH = arm64
CROSS_TRIPLE = aarch64-linux-gnu
endif
ifeq ($(GOOS)_$(GOARCH),darwin_amd64)
ifeq ($(PLATFORM),darwin_amd64)
CROSS_TRIPLE=x86_64-apple-darwin15
endif
ifeq ($(GOOS)_$(GOARCH),windows_amd64)
ifeq ($(PLATFORM),windows_amd64)
CROSS_TRIPLE=x86_64-w64-mingw32
endif
ifneq ($(GOOS)_$(GOARCH),$(GOHOSTOS)_$(GOHOSTARCH))
ifneq ($(PLATFORM),$(HOST_PLATFORM))
CC := $(CROSS_TRIPLE)-gcc
CXX := $(CROSS_TRIPLE)-g++
export CC CXX
endif
# set the version number. you should not need to do this
# for the majority of scenarios.
ifeq ($(origin VERSION), undefined)
VERSION := $(shell git describe --dirty --always --tags)
endif
# a registry that is scoped to the current build tree on this host
ifeq ($(origin BUILD_REGISTRY), undefined)
HOSTNAME := $(shell hostname)
@@ -58,3 +69,22 @@ SELFDIR := $(dir $(lastword $(MAKEFILE_LIST)))
ROOTDIR := $(shell cd $(SELFDIR)/../.. && pwd -P)
BUILD_REGISTRY := build-$(shell echo $(HOSTNAME)-$(ROOTDIR) | shasum -a 256 | cut -c1-8)
endif
# include the common make file
COMMON_SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
ifeq ($(origin BIN_DIR),undefined)
BIN_DIR := $(abspath $(COMMON_SELF_DIR)/../../bin)
endif
ifeq ($(origin WORK_DIR), undefined)
WORK_DIR := $(abspath $(COMMON_SELF_DIR)/../../.work)
endif
ifeq ($(origin CACHE_DIR), undefined)
CACHE_DIR := $(abspath $(COMMON_SELF_DIR)/../../.cache)
endif
TOOLS_DIR := $(CACHE_DIR)/tools
TOOLS_HOST_DIR := $(TOOLS_DIR)/$(HOST_PLATFORM)
COMMA := ,
SPACE :=
SPACE +=
build/makelib/golang.mk
+19 -31
View File
@@ -17,7 +17,7 @@
#
ifeq ($(GO_PROJECT),)
$(error the variable $$GO_PROJECT must be set prior to including golang.mk)
$(error the variable GO_PROJECT must be set prior to including golang.mk)
endif
# These targets will be statically linked.
@@ -31,14 +31,10 @@ endif
GO_SUBDIRS ?= cmd pkg
# Optional directories (relative to CURDIR)
GO_BIN_DIR ?= bin
GO_TOOLS_DIR ?= .tools
GO_WORK_DIR ?= .work
GO_VENDOR_DIR ?= vendor
GO_PKG_DIR ?=
GO_PKG_DIR ?= $(WORK_DIR)/pkg
# Optional build flags passed to go tools
GO_PIE ?= 0
GO_BUILDFLAGS ?=
GO_LDFLAGS ?=
GO_TAGS ?=
@@ -53,17 +49,16 @@ GO_ALL_PACKAGES := $(foreach t,$(GO_SUBDIRS),$(GO_PROJECT)/$(t)/...)
GOPATH := $(shell go env GOPATH)
# setup tools used during the build
GO_TOOLS_HOST_DIR := $(abspath $(GO_TOOLS_DIR)/$(GOHOSTOS)_$(GOHOSTARCH))
GLIDE_VERSION=v0.12.3
GLIDE_HOME := $(abspath $(GO_WORK_DIR)/glide)
GLIDE := $(GO_TOOLS_HOST_DIR)/glide-$(GLIDE_VERSION)
GOLINT := $(GO_TOOLS_HOST_DIR)/golint
GLIDE_HOME := $(abspath $(CACHE_DIR)/glide)
GLIDE := $(TOOLS_HOST_DIR)/glide-$(GLIDE_VERSION)
GOLINT := $(TOOLS_HOST_DIR)/golint
export GLIDE_HOME
GO := go
GOHOST := GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH) go
GO_OUT_DIR := $(abspath $(GO_BIN_DIR)/$(GOOS)_$(GOARCH))
GO_OUT_DIR := $(abspath $(BIN_DIR)/$(PLATFORM))
ifeq ($(GOOS),windows)
GO_OUT_EXT := .exe
@@ -75,7 +70,7 @@ endif
# we run go build with -i which on most system's would want to install packages
# into the system's root dir. using our own pkg dir avoid thats
ifneq ($(GO_PKG_DIR),)
GO_PKG_BASE_DIR := $(abspath $(GO_PKG_DIR)/$(GOOS)_$(GOARCH))
GO_PKG_BASE_DIR := $(abspath $(GO_PKG_DIR)/$(PLATFORM))
GO_PKG_STATIC_FLAGS := -pkgdir $(GO_PKG_BASE_DIR)_static
endif
@@ -84,7 +79,7 @@ GO_STATIC_FLAGS = $(GO_BUILDFLAGS) $(GO_PKG_STATIC_FLAGS) -installsuffix static
# ====================================================================================
# Targets
ifeq ($(filter help clean distclean go.clean, $(MAKECMDGOALS)),)
ifeq ($(filter help clean distclean prune go.clean, $(MAKECMDGOALS)),)
.PHONY: go.check
go.check:
ifneq ($(shell $(GO) version | grep -q -E '\bgo($(GO_SUPPORTED_VERSIONS))\b' && echo 0 || echo 1), 0)
@@ -103,13 +98,13 @@ go.init: $(GO_VENDOR_DIR)/vendor.stamp
define go.project
go.build.packages.$(1):
@echo === go build $(1) $(GOOS)_$(GOARCH)
@echo === go build $(1) $(PLATFORM)
@$(3) $(GO) build -v -i -o $(GO_OUT_DIR)/$(1)$(GO_OUT_EXT) $(4) $(2)
go.build.packages: go.build.packages.$(1)
go.install.packages.$(1):
@echo === go install $(1) $(GOOS)_$(GOARCH)
@echo === go install $(1) $(PLATFORM)
@$(3) $(GO) install -v $(4) $(2)
go.install.packages: go.install.packages.$(1)
endef
@@ -155,24 +150,17 @@ go.vendor $(GO_VENDOR_DIR)/vendor.stamp: $(GLIDE)
$(GLIDE):
@echo === installing glide
@mkdir -p $(GO_TOOLS_HOST_DIR)/tmp
@curl -sL https://github.com/Masterminds/glide/releases/download/$(GLIDE_VERSION)/glide-$(GLIDE_VERSION)-$(GOHOSTOS)-$(GOHOSTARCH).tar.gz | tar -xz -C $(GO_TOOLS_HOST_DIR)/tmp
@mv $(GO_TOOLS_HOST_DIR)/tmp/$(GOHOSTOS)-$(GOHOSTARCH)/glide $(GLIDE)
@rm -fr $(GO_TOOLS_HOST_DIR)/tmp
@mkdir -p $(TOOLS_HOST_DIR)/tmp
@curl -sL https://github.com/Masterminds/glide/releases/download/$(GLIDE_VERSION)/glide-$(GLIDE_VERSION)-$(GOHOSTOS)-$(GOHOSTARCH).tar.gz | tar -xz -C $(TOOLS_HOST_DIR)/tmp
@mv $(TOOLS_HOST_DIR)/tmp/$(GOHOSTOS)-$(GOHOSTARCH)/glide $(GLIDE)
@rm -fr $(TOOLS_HOST_DIR)/tmp
$(GOLINT):
@echo === installing golint
@mkdir -p $(GO_TOOLS_HOST_DIR)/tmp
@GOPATH=$(GO_TOOLS_HOST_DIR)/tmp GOBIN=$(GO_TOOLS_HOST_DIR) $(GOHOST) get github.com/golang/lint/golint
@rm -fr $(GO_TOOLS_HOST_DIR)/tmp
.PHONY: go.clean
go.clean: ;
@rm -rf $(GO_BIN_DIR)
ifneq ($(GO_PKG_DIR),)
@rm -rf $(GO_PKG_DIR)
endif
@mkdir -p $(TOOLS_HOST_DIR)/tmp
@GOPATH=$(TOOLS_HOST_DIR)/tmp GOBIN=$(TOOLS_HOST_DIR) $(GOHOST) get github.com/golang/lint/golint
@rm -fr $(TOOLS_HOST_DIR)/tmp
.PHONY: go.distclean
go.distclean: go.clean
@rm -rf $(GO_TOOLS_DIR) $(GO_VENDOR_DIR) $(GLIDE_HOME)
go.distclean:
@rm -rf $(GO_VENDOR_DIR)
build/makelib/release.mk
-115
View File
@@ -1,115 +0,0 @@
# Copyright 2016 The Rook Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ====================================================================================
# Makefile helper functions for distribution
# ====================================================================================
# Configuration
# the release version to use when publishing
RELEASE_VERSION ?=
ifndef RELEASE_VERSION
$(error RELEASE_VERSION must be set before including release.mk)
endif
RELEASE_CHANNEL ?=
ifndef RELEASE_CHANNEL
$(error RELEASE_CHANNEL must be set before including release.mk)
endif
ifndef BIN_DIR
$(error BIN_DIR must be set before including release.mk)
endif
ifndef RELEASE_DIR
$(error RELEASE_DIR must be set before including release.mk)
endif
# Optional. the platforms to release
RELEASE_PLATFORMS ?=
# Optional. the platforms we release only clients bits
RELEASE_CLIENT_ONLY_PLATFORMS ?=
# Optional. the flavors to release
RELEASE_FLAVORS := binaries containers
# Optional. Github token repo and user
GITHUB_TOKEN ?=
GITHUB_USER ?= rook
GITHUB_REPO ?= rook
# Optional. Docker registry to publish to
RELEASE_REGISTRY ?= rook
# Optional. S3 bucket to publish artifacts to
AWS_DEFAULT_REGION ?= us-east-1
RELEASE_S3_BUCKET ?= rook-release
export RELEASE_VERSION RELEASE_CHANNEL RELEASE_BIN_DIR RELEASE_DIR
export GITHUB_TOKEN GITHUB_USER GITHUB_REPO
export RELEASE_S3_BUCKET RELEASE_REGISTRY BUILD_REGISTRY
ifneq ($(AWS_ACCESS_KEY_ID),)
export AWS_ACCESS_KEY_ID
endif
ifneq ($(AWS_SECRET_ACCESS_KEY),)
export AWS_SECRET_ACCESS_KEY
endif
export AWS_DEFAULT_REGION
# ====================================================================================
# Targets
define release-target
release.build.$(1).$(2):
@build/release/release.sh build $(2) $(1)
release.build.all: release.build.$(1).$(2)
release.publish.$(1).$(2):
@build/release/release.sh publish $(2) $(1)
release.publish.all: release.publish.$(1).$(2)
release.promote.$(1).$(2):
@build/release/release.sh promote $(2) $(1)
release.promote.all: release.promote.$(1).$(2)
release.cleanup.$(1).$(2):
@build/release/release.sh cleanup $(2) $(1)
release.cleanup.all: release.cleanup.$(1).$(2)
endef
$(foreach f,$(RELEASE_FLAVORS),$(foreach p,$(RELEASE_PLATFORMS), $(eval $(call release-target,$(f),$(p)))))
release.build:
@build/release/release.sh build.init
@$(MAKE) release.build.all
release.publish:
@build/release/release.sh publish.init
@$(MAKE) release.publish.all
release.promote:
@build/release/release.sh promote.init
@$(MAKE) release.promote.all
@build/release/release.sh promote.complete
release.cleanup: release.cleanup.all
build/release/Makefile
+132
View File
@@ -0,0 +1,132 @@
# Copyright 2016 The Rook Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
all: build
include ../makelib/common.mk
# ====================================================================================
# Options
CHANNEL ?= master
ifeq ($(filter master alpha beta stable,$(CHANNEL)),)
$(error invalid channel $(CHANNEL))
endif
BIN_DIR ?= $(abspath ../../bin)
PLATFORMS ?= $(ALL_PLATFORMS)
FLAVORS ?= binaries images
TEMP_DIR := $(shell mktemp -d)
DOCKER_REGISTRY ?= rook
QUAY_REGISTRY ?= quay.io/rook
REGISTRIES ?= $(DOCKER_REGISTRY) $(QUAY_REGISTRY)
IMAGES ?= rook toolbox
IMAGE_ARCHS := $(subst linux_,,$(filter linux_%,$(PLATFORMS)))
IMAGE_PLATFORMS := $(subst _,/,$(subst $(SPACE),$(COMMA),$(filter linux_%,$(PLATFORMS))))
S3_BUCKET ?= rook-release
S3_CP := aws s3 cp --only-show-errors
S3_SYNC := aws s3 sync --only-show-errors --delete
S3_CURRENT_BUCKET := s3://$(S3_BUCKET)/$(CHANNEL)/$(VERSION)
# ====================================================================================
# tools
MANIFEST_TOOL_VERSION=v0.5.0
MANIFEST_TOOL := $(TOOLS_HOST_DIR)/manifest-tool-$(MANIFEST_TOOL_VERSION)
$(MANIFEST_TOOL):
@echo === installing manifest-tool
@curl -sL https://github.com/estesp/manifest-tool/releases/download/v0.5.0/manifest-tool-$(GOHOSTOS)-$(GOHOSTARCH) > $@
@chmod +x $@
# ====================================================================================
# Targets
build: $(addprefix build.,$(FLAVORS)) ;
publish: $(addprefix publish.,$(FLAVORS)) ;
promote: $(addprefix promote.,$(FLAVORS)) ;
clean: $(addprefix clean.,$(FLAVORS)) ;
# catch all for unimplemented targets / flavors
%: ; @:
# ====================================================================================
# binary files
publish.binaries.%:
@$(S3_CP) $(BIN_DIR)/$*/rookctl$(if $(filter windows_%,$*),.exe) $(S3_CURRENT_BUCKET)/$*/rookctl$(if $(filter windows_%,$*),.exe)
publish.binaries: $(addprefix publish.binaries.,$(PLATFORMS))
@echo "$(VERSION)" > $(TEMP_DIR)/version;
@$(S3_CP) "$(TEMP_DIR)/version" $(S3_CURRENT_BUCKET)/version
promote.binaries:
@$(S3_SYNC) s3://$(S3_BUCKET)/master/$(VERSION) s3://$(S3_BUCKET)/$(CHANNEL)/current
# ====================================================================================
# images
define repo.targets
build.image.$(1).$(2).$(3): ; @docker tag $(BUILD_REGISTRY)/$(2)-$(3) $(1)/$(2)-$(3):$(VERSION)
build.all.images: build.image.$(1).$(2).$(3)
publish.image.$(1).$(2).$(3): ; @docker push $(1)/$(2)-$(3):$(VERSION)
publish.all.images: publish.image.$(1).$(2).$(3)
promote.image.$(1).$(2).$(3):
@docker pull $(1)/$(2)-$(3):$(VERSION)
@[ "$(CHANNEL)" = "master" ] || docker tag $(1)/$(2)-$(3):$(VERSION) $(1)/$(2)-$(3):$(VERSION)-$(CHANNEL)
@docker tag $(1)/$(2)-$(3):$(VERSION) $(1)/$(2)-$(3):$(CHANNEL)
@[ "$(CHANNEL)" = "master" ] || docker push $(1)/$(2)-$(3):$(VERSION)-$(CHANNEL)
@docker push $(1)/$(2)-$(3):$(CHANNEL)
promote.all.images: promote.image.$(1).$(2).$(3)
clean.image.$(1).$(2).$(3):
@[ -z "$$$$(docker images -q $(1)/$(2)-$(3):$(VERSION))" ] || docker rmi $(1)/$(2)-$(3):$(VERSION)
@[ -z "$$$$(docker images -q $(1)/$(2)-$(3):$(VERSION)-$(CHANNEL))" ] || docker rmi $(1)/$(2)-$(3):$(VERSION)-$(CHANNEL)
@[ -z "$$$$(docker images -q $(1)/$(2)-$(3):$(CHANNEL))" ] || docker rmi $(1)/$(2)-$(3):$(CHANNEL)
clean.all.images: clean.image.$(1).$(2).$(3)
endef
$(foreach r,$(REGISTRIES), $(foreach i,$(IMAGES), $(foreach p,$(IMAGE_ARCHS),$(eval $(call repo.targets,$(r),$(i),$(p))))))
publish.manifest.image.%: publish.all.images
@$(MANIFEST_TOOL) push from-args --platforms $(IMAGE_PLATFORMS) --template $(DOCKER_REGISTRY)/$*-ARCH:$(VERSION) --target $(DOCKER_REGISTRY)/$*:$(VERSION)
promote.manifest.image.%: promote.all.images
@[ "$(CHANNEL)" = "master" ] || $(MANIFEST_TOOL) push from-args --platforms $(IMAGE_PLATFORMS) --template $(DOCKER_REGISTRY)/$*-ARCH:$(VERSION) --target $(DOCKER_REGISTRY)/$*:$(VERSION)-$(CHANNEL)
@$(MANIFEST_TOOL) push from-args --platforms $(IMAGE_PLATFORMS) --template $(DOCKER_REGISTRY)/$*-ARCH:$(VERSION) --target $(DOCKER_REGISTRY)/$*:$(CHANNEL)
build.images: build.all.images
publish.images: $(addprefix publish.manifest.image.,$(IMAGES))
promote.images: $(addprefix promote.manifest.image.,$(IMAGES))
clean.images: clean.all.images
# ====================================================================================
# Help
.PHONY: help
help:
@echo 'Usage: make <OPTIONS> ... <TARGETS>'
@echo ''
@echo 'Targets:'
@echo ' build Build all release artifacts.'
@echo ' clean Remove all release artifacts.'
@echo ' publish Publish all release artifacts.'
@echo ' promote Promote a build to a channel.'
@echo ' tag Tag a build for release.'
@echo ''
@echo 'Options:'
@echo ' VERSION Sets the release version.'
@echo ' CHANNEL Sets the release channel. Can be set to master,'
@echo ' alpha, beta, or stable. Default is not set.'
@echo ' PLATFORMS The supported platforms to build when running.'
@echo ' the build.all target. The default is'
@echo ' all supported platforms'
build/Releasing.md → build/release/README.md
+25 -25
View File
@@ -34,10 +34,10 @@ Jenkins would build the tagged release and publish its artifacts like any other
## Artifacts
Each build from master has the following release artifacts:
- binaries (rookctl) including debug symbols
- binaries (rookctl)
- containers (rook, toolbox)
Binaries go to an S3 bucket `rook-release` and have the following layout:
Binaries go to an S3 bucket `rook-release` (and https://release.rook.io) and have the following layout:
```
/releases
@@ -47,17 +47,17 @@ Binaries go to an S3 bucket `rook-release` and have the following layout:
/v0.3.0-2-g787822d
(binaries)
/v0.3.0-2-g770ebbc
rook-v0.3.0-darwin-amd64.zip
rook-v0.3.0-linux-amd64-debug.tar.gz
rook-v0.3.0-linux-amd64.tar.gz
rook-v0.3.0-linux-arm64-debug.tar.gz
rook-v0.3.0-linux-arm64.tar.gz
rook-v0.3.0-windows-amd64.zip
darwin_amd64/rookctl
linux_amd64/rookctl
linux_arm/rookctl
linux_arm64/rookctl
windows_amd64/rookctl
version
/current
(binaries)
```
Containers go to quay.io where we have the following repos:
Containers go to docker hub and quay.io where we have the following repos:
```
rook/rook
@@ -76,11 +76,11 @@ Once a tagged release from master passes integration test we can proceed with pr
To promote a release run the `promote` target as follows:
```
make promote CHANNEL=alpha VERSION=v0.4.0
make -C build/release promote CHANNEL=alpha VERSION=v0.4.0
```
**NOTE:** Promoting requires that you have AWS credentials (in `~/.aws` or in environment), github token (`export GITHUB_TOKEN`), and
quay.io write access (via `~/.docker/credentials` or `~/.docker/config.json`). See the [AWS config docs](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for help setting up AWS credentials.
**NOTE:** Promoting requires that you have AWS credentials (in `~/.aws` or in environment),
docker hub, and quay.io write access (via `~/.docker/credentials` or `~/.docker/config.json`). See the [AWS config docs](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for help setting up AWS credentials.
After promoting a release, on S3 there will be a path for each channel and release promoted as follows:
@@ -90,12 +90,12 @@ After promoting a release, on S3 there will be a path for each channel and relea
/alpha
/v0.2.2
/v0.3.0
rook-v0.3.0-darwin-amd64.zip
rook-v0.3.0-linux-amd64-debug.tar.gz
rook-v0.3.0-linux-amd64.tar.gz
rook-v0.3.0-linux-arm64-debug.tar.gz
rook-v0.3.0-linux-arm64.tar.gz
rook-v0.3.0-windows-amd64.zip
darwin_amd64/rookctl
linux_amd64/rookctl
linux_arm/rookctl
linux_arm64/rookctl
windows_amd64/rookctl
version
/v0.4.0
/current
/beta
@@ -107,18 +107,18 @@ After promoting a release, on S3 there will be a path for each channel and relea
/current
```
Similarly in quay.io we will tag containers as follows:
Similarly in docker hub and quay.io we will tag containers as follows:
```
alpha-<version> -- for each version in the alpha channel
alpha-latest -- for the latest
beta-<version> -- for each version in the beta channel
beta-latest -- for the latest
master -- for the latest from master
<version>-alpha -- for each version in the alpha channel
alpha -- for the latest alpha
<version>-beta -- for each version in the beta channel
beta -- for the latest beta
<version> -- for each version that is in the stable channel
latest -- for the latest stable version
```
Finally, the github release will be created in the rook repo as a draft and binaries pushed to it. Next, edit the release and
add the release notes then publish it.
Finally, create a github release, fill in the relase notes, and publish it.
You're done.
build/release/binaries.sh
-88
View File
@@ -1,88 +0,0 @@
#!/bin/bash -e
scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source ${scriptdir}/common.sh
get_archive_ext() {
local os=$1
local ext=tar.gz
if [[ ${os} != "linux" ]]; then
ext=zip
fi
echo ${ext}
}
get_archive_name() {
local os=$1
local arch=$2
local suffix=$3
local file=rookctl-${os}-${arch}
if [[ -n ${suffix} ]]; then
file=${file}-${suffix}
fi
local ext=$(get_archive_ext $os)
echo ${file}.${ext}
}
build() {
local os=$1
local arch=$2
local version=${RELEASE_VERSION}
local bindir=$(get_bindir ${os} ${arch})
if [[ ${os} == "windows" ]]; then
local ext=".exe"
fi
local files=( rookctl${ext} )
if [[ ${os} == "linux" ]]; then
local tarfile=$(get_archive_name $os $arch)
echo creating tar ${tarfile}
tar czf "${RELEASE_DIR}/${tarfile}" -C "${bindir}" ${files[*]}
else
local zipfile=$(get_archive_name $os $arch)
local zippath=${RELEASE_DIR}/${zipfile}
echo creating zip ${zipfile}
$(cd ${bindir} && zip -qr ${zippath} ${files[*]})
fi
}
publish() {
local os=$1
local arch=$2
local file=$(get_archive_name $os $arch)
s3_upload ${RELEASE_DIR}/$file
# we will always tag master builds as latest. i.e. auto-promote master
if [[ "${RELEASE_CHANNEL}" == "master" ]]; then
s3_promote_release
fi
}
promote() {
local os=$1
local arch=$2
local file=$(get_archive_name $os $arch)
if [[ ! -e ${RELEASE_DIR}/$file ]]; then
s3_download ${RELEASE_DIR}/$file
fi
github_upload ${RELEASE_DIR}/$file
}
cleanup() {
:
}
action=$1
shift
${action} "$@"
build/release/common.sh
-166
View File
@@ -1,166 +0,0 @@
#!/bin/bash -e
get_bindir() {
local os=$1
local arch=$2
bindir=${RELEASE_BIN_DIR}/${os}_${arch}
echo ${bindir}
}
layout_root() {
local os=$1
local arch=$2
local dir=$3
shift 3
local bins="$@"
local bindir=$(get_bindir ${os} ${arch})
mkdir -p $dir/root/usr/bin
mkdir -p $dir/root/etc/ssl/certs
for b in $bins; do
cp $bindir/$b $dir/root/usr/bin
done
cp /etc/ssl/certs/ca-certificates.crt $dir/root/etc/ssl/certs
}
check_release_version() {
if echo ${RELEASE_VERSION} | grep -q -E '^([[:digit:]]+)\.([[:digit:]]+)\.([[:digit:]]+)$'; then
return 0
else
return 1
fi
}
get_mediatype_from_extension() {
local file=$1
local ext=${file##*.}
local mediatype
case ${ext} in
gz) echo "application/gzip" ;;
zip) echo "application/zip" ;;
*) echo "UNSUPPORTED" ;;
esac
}
github_get_release_id() {
curl -4 -s -H "Authorization: token ${GITHUB_TOKEN}" \
"https://api.github.com/repos/${GITHUB_USER}/${GITHUB_REPO}/releases/tags/${RELEASE_VERSION}" | jq -r '.id'
}
github_get_upload_url() {
id=`cat ${RELEASE_DIR}/release_id`
curl -4 -s -H "Authorization: token ${GITHUB_TOKEN}" \
"https://api.github.com/repos/${GITHUB_USER}/${GITHUB_REPO}/releases/${id}" | jq -r '.upload_url'
}
github_create_release() {
id=$(github_get_release_id)
if [[ ${id} == "null" ]]; then
echo creating a new github release for ${RELEASE_VERSION}
id=$(curl -4 -s -H "Authorization: token ${GITHUB_TOKEN}" \
-H "Content-Type: application/json" \
-d "{\"tag_name\": \"${RELEASE_VERSION}\",\"target_commitish\": \"master\",\"name\": \"${RELEASE_VERSION}\",\"body\": \"TBD\",\"draft\": true,\"prerelease\": true}" \
"https://api.github.com/repos/${GITHUB_USER}/${GITHUB_REPO}/releases" | jq -r '.id')
if [[ ${id} == "null" ]]; then
echo error: failed to create github release
return 1
fi
cat <<EOF > ${RELEASE_DIR}/release_id
${id}
EOF
fi
}
github_delete_release() {
id=$(github_get_release_id)
if [[ ${id} != "null" ]]; then
echo deleting existing github release ${id} for ${RELEASE_VERSION}
curl -4 -s -X DELETE -H "Authorization: token ${GITHUB_TOKEN}" \
"https://api.github.com/repos/${GITHUB_USER}/${GITHUB_REPO}/releases/${id}"
fi
}
github_create_or_replace_release() {
github_delete_release
github_create_release
}
github_upload() {
local filepath=$1
local mediatype=$(get_mediatype_from_extension $filepath)
local filename=$(basename $filepath)
upload_url=$(github_get_upload_url)
if [[ ${upload_url} == "null" ]]; then
echo error: could not get upload url for github release
return 1
fi
if [[ ! -e ${filepath} ]]; then
echo error: could not find file ${filepath}
return 1
fi
upload_url=${upload_url%{*} # remove the trailing {?name,label}
echo uploading ${filename} github release ${RELEASE_VERSION}
id=$(curl -4 -s -H "Authorization: token ${GITHUB_TOKEN}" \
-H "Accept: application/vnd.github.manifold-preview" \
-H "Content-Type: ${mediatype}" \
--data-binary @${filepath} \
"${upload_url}?name=${filename}" | jq -r '.id')
if [[ ${id} == "null" ]]; then
echo error: failed to upload ${filename} to github
return 1
fi
}
github_release_complete() {
rm -fr ${RELEASE_DIR}/release_id
}
write_version_file() {
# upload a file with the version number
cat <<EOF > ${RELEASE_DIR}/version
${RELEASE_VERSION}
EOF
}
publish_version_file() {
s3_upload ${RELEASE_DIR}/version
}
# we assume that AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and possibly AWS_DEFAULT_REGION are already set
# or ~/.aws/credentials and ~/.aws/config are configured
s3_upload() {
local filepath=$1
local filename=$(basename $filepath)
echo uploading ${filename} to S3 bucket ${RELEASE_S3_BUCKET}
aws s3 cp --only-show-errors ${filepath} s3://${RELEASE_S3_BUCKET}/${RELEASE_CHANNEL}/${RELEASE_VERSION}/${filename}
}
s3_download() {
local filepath=$1
local filename=$(basename $filepath)
echo downloading ${filename} from S3 bucket ${RELEASE_S3_BUCKET}
aws s3 cp --only-show-errors s3://${RELEASE_S3_BUCKET}/${RELEASE_CHANNEL}/${RELEASE_VERSION}/${filename} ${filepath}
}
s3_promote_release() {
echo copying ${RELEASE_VERSION} to current for channel ${RELEASE_CHANNEL} in S3 bucket ${RELEASE_S3_BUCKET}
aws s3 sync --only-show-errors --delete s3://${RELEASE_S3_BUCKET}/master/${RELEASE_VERSION} s3://${RELEASE_S3_BUCKET}/${RELEASE_CHANNEL}/current
}
build/release/containers.sh
-142
View File
@@ -1,142 +0,0 @@
#!/bin/bash -e
scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source ${scriptdir}/common.sh
build_registry=${BUILD_REGISTRY}
registry=${RELEASE_REGISTRY}
get_build_image_name() {
local os=$1
local arch=$2
local repo=$3
local image=${repo}-${arch}:latest
echo ${build_registry}/${image}
}
get_image_name() {
local os=$1
local arch=$2
local repo=$3
local version=$4
local image=${repo}-${arch}:${version}
if [[ ${arch} == "amd64" ]]; then
image=${repo}:${version}
fi
echo ${registry}/${image}
}
build_image() {
local os=$1
local arch=$2
local repo=$3
local build_image=$(get_build_image_name $os $arch $repo)
local image=$(get_image_name $os $arch $repo ${RELEASE_VERSION})
docker tag ${build_image} ${image}
}
build() {
local os=$1
local arch=$2
[[ ${os} == "linux" ]] || return 0
build_image $os $arch rook
build_image $os $arch toolbox
}
publish_image() {
local os=$1
local arch=$2
local repo=$3
local name=$(get_image_name $os $arch $repo ${RELEASE_VERSION})
echo pushing docker container ${name}
docker push ${name}
# we will always tag master builds as latest. i.e. auto-promote master
if [[ "${RELEASE_CHANNEL}" == "master" ]]; then
local dst=$(get_image_name $os $arch $repo master-latest)
echo pushing docker container ${dst}
docker tag ${name} ${dst}
docker push ${dst}
fi
}
publish() {
local os=$1
local arch=$2
[[ ${os} == "linux" ]] || return 0
publish_image $os $arch rook
publish_image $os $arch toolbox
}
promote_image() {
local os=$1
local arch=$2
local repo=$3
local src=$(get_image_name $os $arch $repo ${RELEASE_VERSION})
local dst1=$(get_image_name $os $arch $repo ${RELEASE_CHANNEL}-latest)
local dst2=$(get_image_name $os $arch $repo ${RELEASE_CHANNEL}-${RELEASE_VERSION})
echo promoting container ${src} to ${dst1} and ${dst2}
docker pull ${src}
docker tag ${src} ${dst1}
docker tag ${src} ${dst2}
docker push ${dst1}
docker push ${dst2}
}
promote() {
local os=$1
local arch=$2
[[ ${os} == "linux" ]] || return 0
promote_image $os $arch rook
promote_image $os $arch toolbox
}
cleanup_image() {
local os=$1
local arch=$2
local repo=$3
local img
for t in \
${RELEASE_VERSION} \
${RELEASE_CHANNEL}-latest \
${RELEASE_CHANNEL}-${RELEASE_VERSION} \
; do
img=$(get_image_name $os $arch $repo ${t})
if [[ -n "$(docker images -q ${img} 2> /dev/null)" ]]; then
echo removing docker image ${img}
docker rmi ${img} || true
fi
done
}
cleanup() {
local os=$1
local arch=$2
[[ ${os} == "linux" ]] || return 0
cleanup_image $os $arch rook
cleanup_image $os $arch toolbox
}
action=$1
shift
${action} "$@"
build/release/release.sh
-52
View File
@@ -1,52 +0,0 @@
#!/bin/bash -e
scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source ${scriptdir}/common.sh
action=$1
shift
case ${action} in
build.init)
mkdir -p ${RELEASE_DIR}
write_version_file
;;
publish.init)
mkdir -p ${RELEASE_DIR}
publish_version_file
;;
build|publish|promote|cleanup)
platform=$1
shift
flavor=$1
shift
${scriptdir}/${flavor}.sh ${action} ${platform%_*} ${platform#*_}
;;
promote.init)
mkdir -p ${RELEASE_DIR}
if check_release_version; then
echo ${RELEASE_VERSION} can not be promoted. Must build from a tag like v0.4.0.
exit 1
fi
echo promoting release ${RELEASE_VERSION} to channel ${RELEASE_CHANNEL}
write_version_file
github_create_or_replace_release
;;
promote.complete)
s3_promote_release
github_release_complete
;;
*)
echo "unknown action ${action}"
exit 1
;;
esac
build/run
+7 -7
View File
@@ -65,7 +65,7 @@ if [ "`uname -s`" != "Linux" ]; then
# /.netrc (bind mounted to host ${HOME}/.netrc)
# /.aws (bind mounted to host ${HOME}/.aws)
# /.docker (bind mounted to host ${HOME}/.docker)
# /.download (bind mounted to host ${DOWNLOADDIR} if set)
# /.download (bind mounted to host ${CACHE_DIR} if set)
# /.ccache
# /go/src/github.com/rook/rook (rsync'd from host <source>)
@@ -93,7 +93,7 @@ else
# /.aws (bind mounted to host ${HOME}/.aws)
# /.docker (bind mounted to host ${HOME}/.docker)
# /.ccache (bind mounted to host ${CCACHE_DIR})
# /.download (bind mounted to host ${DOWNLOADDIR} if set)
# /.download (bind mounted to host ${CACHE_DIR} if set)
# /go
# /pkg (bind mounted to <source>/.work/cross_pkg)
# /src/github.com/rook/rook (bind mounted to <source>)
@@ -115,11 +115,11 @@ else
fi
fi
# if DOWNLOADDIR is not set bind mount it
if [[ -n "${DOWNLOADDIR}" ]]; then
mkdir -p ${DOWNLOADDIR}
DOWNLOAD_ARGS="-e DOWNLOADDIR=${BUILDER_HOME}/go/src/${source_repo}/.download \
-v ${DOWNLOADDIR}:${BUILDER_HOME}/go/src/${source_repo}/.download"
# if CACHE_DIR is not set bind mount it
if [[ -n "${CACHE_DIR}" ]]; then
mkdir -p ${CACHE_DIR}
DOWNLOAD_ARGS="-e CACHE_DIR=${BUILDER_HOME}/go/src/${source_repo}/.download \
-v ${CACHE_DIR}:${BUILDER_HOME}/go/src/${source_repo}/.download"
fi
# we copy credential files for github access