From 59f5815115dd2437fa18ee678a3c1da69bc2793a Mon Sep 17 00:00:00 2001 From: Bassam Tabbara Date: Sat, 1 Apr 2017 20:20:18 -0700 Subject: [PATCH] build: support for promoting a build support for build promotion. For example, make promote CHANNEL=alpha VERISON=v0.4.0, will push the v0.4.0 bits to the alpha channel. --- Jenkinsfile | 15 ++-- Makefile | 7 ++ build/common.sh | 4 +- build/makelib/release.mk | 22 +++++- build/release/binaries.sh | 81 +++++++++++++++------- build/release/common.sh | 23 ++++++- build/release/containers.sh | 134 ++++++++++++++++++++++++------------ build/release/metadata.sh | 28 ++++++++ build/release/release.sh | 10 ++- build/run | 7 +- 10 files changed, 244 insertions(+), 87 deletions(-) create mode 100755 build/release/metadata.sh diff --git a/Jenkinsfile b/Jenkinsfile index 827d1b90f..1cbb0be6b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,18 +24,19 @@ try { } stage('Publish') { - withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'rook-quay-io', - usernameVariable: 'DOCKER_USER', passwordVariable: 'DOCKER_PASSWORD']]) { + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: 'rook-quay-io', usernameVariable: 'DOCKER_USER', passwordVariable: 'DOCKER_PASSWORD'], + [$class: 'UsernamePasswordMultiBinding', credentialsId: 'rook-jenkins-aws', usernameVariable: 'AWS_ACCESS_KEY_ID', passwordVariable: 'AWS_SECRET_ACCESS_KEY'], + [$class: 'StringBinding', credentialsId: 'quantumbuild-token', variable: 'GITHUB_TOKEN'] + ]) { sh 'docker login -u="${DOCKER_USER}" -p="${DOCKER_PASSWORD}" quay.io' - } - - withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'rook-jenkins-aws', - usernameVariable: 'AWS_ACCESS_KEY_ID', passwordVariable: 'AWS_SECRET_ACCESS_KEY']]) { sh 'build/run make -j\$(nproc) publish' } } + stage('Cleanup') { sh 'build/run make -j\$(nproc) publish.cleanup' + sh 'docker images' deleteDir() } } @@ -46,6 +47,8 @@ catch (Exception e) { node("ec2-stateful") { echo 'Cleaning up workspace' + sh 'build/run make -j\$(nproc) publish.cleanup' + sh 'docker images' deleteDir() } diff --git a/Makefile b/Makefile index 75187eeac..bef098058 100644 --- a/Makefile +++ b/Makefile @@ -238,6 +238,13 @@ else @echo skipping publish. invalid channel "$(CHANNEL)" endif +promote: +ifneq ($(filter master alpha beta stable, $(CHANNEL)),) + @$(MAKE) release.promote +else + @echo skipping promote. invalid channel "$(CHANNEL)" +endif + publish.cleanup: ifneq ($(filter master alpha beta stable, $(CHANNEL)),) @$(MAKE) release.cleanup diff --git a/build/common.sh b/build/common.sh index 1430fde61..ab010d698 100644 --- a/build/common.sh +++ b/build/common.sh @@ -88,10 +88,8 @@ function run_rsync() { --delete \ --prune-empty-dirs \ --filter='- /.work/' \ - --filter='- /.glide/' \ --filter='- /.vscode/' \ - --filter='- /bin/' \ - --filter='- /release/' \ + --filter='- index.lock' \ $src $dst || { stop_rsync_container ${id}; return 1; } done diff --git a/build/makelib/release.mk b/build/makelib/release.mk index 53106a91c..77406b294 100644 --- a/build/makelib/release.mk +++ b/build/makelib/release.mk @@ -44,7 +44,7 @@ RELEASE_PLATFORMS ?= RELEASE_CLIENT_ONLY_PLATFORMS ?= # Optional. the flavors to release -RELEASE_FLAVORS := binaries containers +RELEASE_FLAVORS := metadata binaries containers # Optional. Github token repo and user GITHUB_TOKEN ?= @@ -61,7 +61,16 @@ RELEASE_S3_BUCKET ?= rook-release export RELEASE_VERSION RELEASE_CHANNEL RELEASE_BIN_DIR RELEASE_DIR export GITHUB_TOKEN GITHUB_USER GITHUB_REPO export RELEASE_S3_BUCKET RELEASE_REGISTRY -export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION + +ifneq ($(AWS_ACCESS_KEY_ID),) +export AWS_ACCESS_KEY_ID +endif + +ifneq ($(AWS_SECRET_ACCESS_KEY),) +export AWS_SECRET_ACCESS_KEY +endif + +export AWS_DEFAULT_REGION # ==================================================================================== # Targets @@ -77,6 +86,11 @@ release.publish.$(1).$(2): release.publish.all: release.publish.$(1).$(2) +release.promote.$(1).$(2): + @build/release/release.sh promote $(2) $(1) + +release.promote.all: release.promote.$(1).$(2) + release.cleanup.$(1).$(2): @build/release/release.sh cleanup $(2) $(1) @@ -89,4 +103,8 @@ release.build: release.build.all release.publish: release.publish.all +release.promote: + @build/release/release.sh init + @$(MAKE) release.promote.all + release.cleanup: release.cleanup.all diff --git a/build/release/binaries.sh b/build/release/binaries.sh index dad51e161..1e1010184 100755 --- a/build/release/binaries.sh +++ b/build/release/binaries.sh @@ -3,6 +3,34 @@ scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" source ${scriptdir}/common.sh +get_archive_ext() { + local os=$1 + + local ext=tar.gz + if [[ ${os} != "linux" ]]; then + ext=zip + fi + + echo ${ext} +} + +get_archive_name() { + local os=$1 + local arch=$2 + local suffix=$3 + local version=${RELEASE_VERSION} + + local file=rook-${version}-${os}-${arch} + + if [[ -n ${suffix} ]]; then + file=${file}-${suffix} + fi + + local ext=$(get_archive_ext $os) + + echo ${file}.${ext} +} + build() { local os=$1 local arch=$2 @@ -18,48 +46,53 @@ build() { if [[ ${os} == "linux" ]]; then files+=( rookd${ext} ) - local tarfile=${RELEASE_DIR}/rook-${version}-${os}-${arch}.tar.gz + local tarfile=$(get_archive_name $os $arch) echo creating tar ${tarfile} - tar czf "${tarfile}" -C "${bindir}" ${files[*]} + tar czf "${RELEASE_DIR}/${tarfile}" -C "${bindir}" ${files[*]} # create a package with debug symbols files=( rookd${ext}.debug ) - local tarfile=${RELEASE_DIR}/rook-${version}-${os}-${arch}-debug.tar.gz + local tarfile=$(get_archive_name $os $arch "debug") echo creating debug tar ${tarfile} - tar czf "${tarfile}" -C "${bindir}" ${files[*]} + tar czf "${RELEASE_DIR}/${tarfile}" -C "${bindir}" ${files[*]} else - local zipfile=$(realpath ${RELEASE_DIR}/rook-${version}-${os}-${arch}.zip) + local zipfile=$(get_archive_name $os $arch) + local zippath=$(realpath ${RELEASE_DIR}/${zipfile}) echo creating zip ${zipfile} - $(cd ${bindir} && zip -qr ${zipfile} ${files[*]}) + $(cd ${bindir} && zip -qr ${zippath} ${files[*]}) fi } publish() { local os=$1 local arch=$2 - local version=${RELEASE_VERSION} + local file=$(get_archive_name $os $arch) - local file=${RELEASE_DIR}/rook-${version}-${os}-${arch} - local ext=tar.gz - - if [[ ${os} != "linux" ]]; then - ext=zip - fi - - echo uploading $file.$ext to S3 - s3_upload $file.$ext + echo uploading $file to S3 + s3_upload ${RELEASE_DIR}/$file if [[ ${os} == "linux" ]]; then - echo uploading $file-debug.$ext to S3 - s3_upload $file-debug.$ext + file=$(get_archive_name $os $arch "debug") + echo uploading $file to S3 + s3_upload ${RELEASE_DIR}/$file fi +} - # upload a file with the version number - tmpdir=$(mktemp -d) - cat < $tmpdir/version -${version} -EOF - s3_upload $tmpdir/version +promote() { + local os=$1 + local arch=$2 + local file=$(get_archive_name $os $arch) + local ext=$(get_archive_ext $os) + + echo promoting $file to channel ${RELEASE_CHANNEL} + s3_promote_file $file + github_upload ${RELEASE_DIR}/$file $ext + + if [[ ${os} == "linux" ]]; then + file=$(get_archive_name $os $arch "debug") + s3_promote_file $file + github_upload ${RELEASE_DIR}/$file $ext + fi } cleanup() { diff --git a/build/release/common.sh b/build/release/common.sh index d08a2f6f5..60dc792c2 100644 --- a/build/release/common.sh +++ b/build/release/common.sh @@ -28,17 +28,27 @@ layout_root() { cp /etc/ssl/certs/ca-certificates.crt $dir/root/etc/ssl/certs } +check_release_version() { + if echo ${RELEASE_VERSION} | grep -q -E '^([[:digit:]]+)\.([[:digit:]]+)\.([[:digit:]]+)$'; then + return 0 + else + return 1 + fi +} + github_get_release_id() { curl -4 -s -H "Authorization: token ${GITHUB_TOKEN}" \ "https://api.github.com/repos/${GITHUB_USER}/${GITHUB_REPO}/releases/tags/${RELEASE_VERSION}" | jq -r '.id' } -github_check_release() { +github_create_release() { id=$(github_get_release_id) if [[ ${id} == "null" ]]; then - echo "ERROR: github release tag ${RELEASE_VERSION} was not found. Did you create the release?" 1>&2 - return 1; + curl -4 -s -H "Authorization: token ${GITHUB_TOKEN}" \ + -H "Content-Type: application/json" \ + -d "{\"tag_name\": \"${RELEASE_VERSION}\",\"target_commitish\": \"master\",\"name\": \"${RELEASE_VERSION}\",\"body\": \"TBD\",\"draft\": true,\"prerelease\": true}" \ + "https://api.github.com/repos/${GITHUB_USER}/${GITHUB_REPO}/releases" fi } @@ -64,3 +74,10 @@ s3_upload() { # or ~/.aws/credentials and ~/.aws/config are configured aws s3 cp --only-show-errors ${filepath} s3://${RELEASE_S3_BUCKET}/${RELEASE_CHANNEL}/${RELEASE_VERSION}/${filename} } + +s3_promote_file() { + local filename=$1 + + aws s3 cp --only-show-errors s3://${RELEASE_S3_BUCKET}/master/${RELEASE_VERSION}/${filename} s3://${RELEASE_S3_BUCKET}/${RELEASE_CHANNEL}/${RELEASE_VERSION}/${filename} + aws s3 cp --only-show-errors s3://${RELEASE_S3_BUCKET}/master/${RELEASE_VERSION}/${filename} s3://${RELEASE_S3_BUCKET}/${RELEASE_CHANNEL}/current/${filename} +} diff --git a/build/release/containers.sh b/build/release/containers.sh index 8bd3a9dee..0d58ddd9e 100755 --- a/build/release/containers.sh +++ b/build/release/containers.sh @@ -5,6 +5,47 @@ source ${scriptdir}/common.sh registry=${RELEASE_REGISTRY}/ +get_image_name() { + local os=$1 + local arch=$2 + local repo=$3 + local version=$4 + + local tag=${repo}-${arch}:${version} + if [[ ${arch} == "amd64" ]]; then + tag=${repo}:${version} + fi + + echo ${tag} +} + +build_artifact() { + local os=$1 + local arch=$2 + local tmpdir=$3 + local repo=$4 + + shift 4 + local bins="$@" + + layout_root $os $arch $tmpdir $bins + mkdir $tmpdir/root/tmp + + local name=$(get_image_name $os $arch $repo ${RELEASE_VERSION}) + + echo building docker container ${name} + docker build -t ${registry}${name} $tmpdir + + local file=${name/\//-} + local file=${file/:/-} + local dockerout=${file}.docker + echo ${file} + + echo generate ACIs from docker containers + (cd ${RELEASE_DIR} && docker save -o ${dockerout} ${registry}${name}) + (cd ${RELEASE_DIR} && docker2aci ${dockerout}) +} + build() { local os=$1 local arch=$2 @@ -23,37 +64,15 @@ EOF rm -fr $tmpdir } -build_artifact() { +publish_artifact() { local os=$1 local arch=$2 - local tmpdir=$3 - local repo=$4 - local version=${RELEASE_VERSION} + local repo=$3 - shift 4 - local bins="$@" + local name=${registry}$(get_image_name $os $arch $repo ${RELEASE_VERSION}) - layout_root $os $arch $tmpdir $bins - mkdir $tmpdir/root/tmp - - tag=${repo}-${arch}:${version} - if [[ ${arch} == "amd64" ]]; then - tag=${repo}:${version} - fi - - echo building docker container ${tag} - docker build -t ${registry}${tag} $tmpdir - - local file=${tag/\//-} - local file=${file/:/-} - local dockerout=${file}.docker - echo ${file} - - echo generate ACIs from docker containers - (cd ${RELEASE_DIR} && docker save -o ${dockerout} ${registry}${tag}) - (cd ${RELEASE_DIR} && docker2aci ${dockerout}) - - rm -fr $tmpdir + echo pushing docker container ${name} + docker push ${name} } publish() { @@ -65,28 +84,57 @@ publish() { publish_artifact $os $arch rook/rookd } +promote_artifact() { + local os=$1 + local arch=$2 + local repo=$3 + + local src=${registry}$(get_image_name $os $arch $repo ${RELEASE_VERSION}) + local dst1=${registry}$(get_image_name $os $arch $repo ${RELEASE_CHANNEL}-latest) + local dst2=${registry}$(get_image_name $os $arch $repo ${RELEASE_CHANNEL}-${RELEASE_VERSION}) + + echo promoting container ${src} to ${dst1} and ${dst2} + docker pull ${src} + docker tag ${src} ${dst1} + docker tag ${src} ${dst2} + docker push ${dst1} + docker push ${dst2} +} + +promote() { + local os=$1 + local arch=$2 + + [[ ${os} == "linux" ]] || return 0 + + promote_artifact $os $arch rook/rookd +} + +cleanup_artifact() { + local os=$1 + local arch=$2 + local repo=$3 + + local src=${registry}$(get_image_name $os $arch $repo ${RELEASE_VERSION}) + echo removing docker image ${src} + docker rmi ${src} || true + + local dst1=${registry}$(get_image_name $os $arch $repo ${RELEASE_CHANNEL}-latest) + echo removing docker image ${dst1} + docker rmi ${dst1} || true + + local dst2=${registry}$(get_image_name $os $arch $repo ${RELEASE_CHANNEL}-${RELEASE_VERSION}) + echo removing docker image ${dst2} + docker rmi ${dst2} || true +} + cleanup() { local os=$1 local arch=$2 [[ ${os} == "linux" ]] || return 0 - docker rmi ${registry}${tag} || true -} - -publish_artifact() { - local os=$1 - local arch=$2 - local repo=$3 - local version=${RELEASE_VERSION} - - tag=${repo}-${arch}:${version} - if [[ ${arch} == "amd64" ]]; then - tag=${repo}:${version} - fi - - echo pushing docker container ${tag} - docker push ${registry}${tag} + cleanup_artifact $os $arch rook/rookd } action=$1 diff --git a/build/release/metadata.sh b/build/release/metadata.sh new file mode 100755 index 000000000..000c40c3a --- /dev/null +++ b/build/release/metadata.sh @@ -0,0 +1,28 @@ +#!/bin/bash -e + +scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source ${scriptdir}/common.sh + +build() { + # upload a file with the version number + cat < ${RELEASE_DIR}/version +${version} +EOF +} + +publish() { + s3_upload ${RELEASE_DIR}/version +} + +promote() { + s3_promote_file version +} + +cleanup() { + : +} + +action=$1 +shift + +${action} "$@" diff --git a/build/release/release.sh b/build/release/release.sh index 2c8d7e22e..92a16a392 100755 --- a/build/release/release.sh +++ b/build/release/release.sh @@ -9,7 +9,7 @@ shift mkdir -p ${RELEASE_DIR} case ${action} in - build|publish|cleanup) + build|publish|promote|cleanup) platform=$1 shift @@ -19,8 +19,12 @@ case ${action} in ${scriptdir}/${flavor}.sh ${action} ${platform%_*} ${platform#*_} ;; - check) - github_check_release "$@" + init) + if check_release_version; then + echo ${RELEASE_VERSION} can not be promoted. Must build from a tag like v0.4.0. + exit 1 + fi + github_create_release ;; *) diff --git a/build/run b/build/run index 62014c63c..500d48f92 100755 --- a/build/run +++ b/build/run @@ -58,7 +58,8 @@ if [ "`uname -s`" != "Linux" ]; then # /var/run/docker.sock (bind mounted to host /var/run/docker.sock) # /${BUILDER_HOME} (bind mounted to an independent docker volume) # /.netrc (bind mounted to host ${HOME}/.netrc) - # /.docker/config.json (bind mounted to host ${HOME}/.docker/config.json) + # /.aws (bind mounted to host ${HOME}/.aws) + # /.docker (bind mounted to host ${HOME}/.docker) # /.download # /.ccache # /go/src/github.com/rook/rook (rsync'd from host ) @@ -80,7 +81,8 @@ else # /var/run/docker.sock (bind mounted to host /var/run/docker.sock) # /${BUILDER_HOME} # /.netrc (bind mounted to host ${HOME}/.netrc) - # /.docker/config.json (bind mounted to host ${HOME}/.docker/config.json) + # /.aws (bind mounted to host ${HOME}/.aws) + # /.docker (bind mounted to host ${HOME}/.docker) # /.ccache (bind mounted to host ${CCACHE_DIR}) # /.download (bind mounted to host ${DOWNLOADDIR} if set) # /go (bind mounted to /.work/cross ) @@ -152,7 +154,6 @@ docker run \ ${container_image} \ "$@" - # rsync the source tree back if [ "`uname -s`" != "Linux" ]; then run_rsync "rsync://localhost:${rsync_port}/volume/go/src/${source_repo}-->${scriptdir}/../.."