From 49f2f0b441a3b835b180bcd4ef8fc3cc9209067e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 29 Apr 2021 14:37:13 +0200 Subject: [PATCH 1/2] build: fix sed command for linux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The following command ended up being missinterpreted on linux, generating a wrong file named ceph.clusterserviceversion.yaml'\'' with broken quotes. Making it an array ensures the command and the args are parsed correctly. Tested on both linux and macos. Signed-off-by: Sébastien Han --- cluster/olm/ceph/generate-rook-csv.sh | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/cluster/olm/ceph/generate-rook-csv.sh b/cluster/olm/ceph/generate-rook-csv.sh index 460f6f2eb..2021cfc03 100755 --- a/cluster/olm/ceph/generate-rook-csv.sh +++ b/cluster/olm/ceph/generate-rook-csv.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e ################## @@ -76,7 +76,8 @@ ROOK_OP_VERSION=$3 ############# # VARIABLES # ############# -SED_I=${SED_CMD:-"sed -i'' -e"} +SED_I=(sed -i'' -e) +[ -n "$SED_CMD" ] || read -ra SED_CMD <<< "${SED_I[@]}" YQ_CMD_DELETE=($yq delete -i) YQ_CMD_MERGE_OVERWRITE=($yq merge --inplace --overwrite --prettyPrint) YQ_CMD_MERGE=($yq merge --inplace --append -P ) @@ -222,26 +223,26 @@ function hack_csv() { # rook-ceph-osd --> serviceAccountName # rook-ceph-osd --> rule - $SED_I 's/rook-ceph-global/rook-ceph-system/' "$CSV_FILE_NAME" - $SED_I 's/rook-ceph-object-bucket/rook-ceph-system/' "$CSV_FILE_NAME" - $SED_I 's/rook-ceph-cluster-mgmt/rook-ceph-system/' "$CSV_FILE_NAME" + "${SED_I[@]}" 's/rook-ceph-global/rook-ceph-system/' "$CSV_FILE_NAME" + "${SED_I[@]}" 's/rook-ceph-object-bucket/rook-ceph-system/' "$CSV_FILE_NAME" + "${SED_I[@]}" 's/rook-ceph-cluster-mgmt/rook-ceph-system/' "$CSV_FILE_NAME" - $SED_I 's/rook-ceph-mgr-cluster/rook-ceph-mgr/' "$CSV_FILE_NAME" - $SED_I 's/rook-ceph-mgr-system/rook-ceph-mgr/' "$CSV_FILE_NAME" + "${SED_I[@]}" 's/rook-ceph-mgr-cluster/rook-ceph-mgr/' "$CSV_FILE_NAME" + "${SED_I[@]}" 's/rook-ceph-mgr-system/rook-ceph-mgr/' "$CSV_FILE_NAME" - $SED_I 's/cephfs-csi-nodeplugin/rook-csi-cephfs-plugin-sa/' "$CSV_FILE_NAME" - $SED_I 's/cephfs-external-provisioner-runner/rook-csi-cephfs-provisioner-sa/' "$CSV_FILE_NAME" + "${SED_I[@]}" 's/cephfs-csi-nodeplugin/rook-csi-cephfs-plugin-sa/' "$CSV_FILE_NAME" + "${SED_I[@]}" 's/cephfs-external-provisioner-runner/rook-csi-cephfs-provisioner-sa/' "$CSV_FILE_NAME" - $SED_I 's/rbd-csi-nodeplugin/rook-csi-rbd-plugin-sa/' "$CSV_FILE_NAME" - $SED_I 's/rbd-external-provisioner-runner/rook-csi-rbd-provisioner-sa/' "$CSV_FILE_NAME" + "${SED_I[@]}" 's/rbd-csi-nodeplugin/rook-csi-rbd-plugin-sa/' "$CSV_FILE_NAME" + "${SED_I[@]}" 's/rbd-external-provisioner-runner/rook-csi-rbd-provisioner-sa/' "$CSV_FILE_NAME" # The operator-sdk also does not properly respect when # Roles differ from the Service Account name # The operator-sdk instead assumes the Role/ClusterRole is the ServiceAccount name # # To account for these mappings, we have to replace Role/ClusterRole names with # the corresponding ServiceAccount. - $SED_I 's/cephfs-external-provisioner-cfg/rook-csi-cephfs-provisioner-sa/' "$CSV_FILE_NAME" - $SED_I 's/rbd-external-provisioner-cfg/rook-csi-rbd-provisioner-sa/' "$CSV_FILE_NAME" + "${SED_I[@]}" 's/cephfs-external-provisioner-cfg/rook-csi-cephfs-provisioner-sa/' "$CSV_FILE_NAME" + "${SED_I[@]}" 's/rbd-external-provisioner-cfg/rook-csi-rbd-provisioner-sa/' "$CSV_FILE_NAME" } function generate_package() { From 0f7128ece9af055989f6e3ef240ac950f748c450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 29 Apr 2021 14:40:47 +0200 Subject: [PATCH 2/2] build: fix parallel build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We now the generate the CRDs as well as the CSV on a single GOARCH only: amd64. This helps us avoiding builds colliding with each others. Closes: https://github.com/rook/rook/issues/7655 Signed-off-by: Sébastien Han --- .../workflows/{macos-build.yml => build.yml} | 25 ++++++++++++++++++- images/ceph/Makefile | 18 +++++++------ tests/scripts/github-action-helper.sh | 14 +++++++++-- 3 files changed, 46 insertions(+), 11 deletions(-) rename .github/workflows/{macos-build.yml => build.yml} (76%) diff --git a/.github/workflows/macos-build.yml b/.github/workflows/build.yml similarity index 76% rename from .github/workflows/macos-build.yml rename to .github/workflows/build.yml index 2ccefea04..8adf4f708 100644 --- a/.github/workflows/macos-build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: MacOS build +name: Builds on: pull_request: @@ -44,6 +44,29 @@ jobs: working-directory: /Users/runner/go/src/github.com/rook/rook run: tests/scripts/validate_modified_files.sh crd + - name: setup tmate session for debugging + if: failure() + uses: mxschmitt/action-tmate@v3 + + linux-build-all: + runs-on: ubuntu-18.04 + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: setup golang + uses: actions/setup-go@v2 + with: + go-version: 1.16 + + - name: set up QEMU + uses: docker/setup-qemu-action@master + with: + platforms: all + + - name: build.all rook + run: tests/scripts/github-action-helper.sh build_rook_all + - name: setup tmate session for debugging if: failure() uses: mxschmitt/action-tmate@v3 \ No newline at end of file diff --git a/images/ceph/Makefile b/images/ceph/Makefile index 0b2e6db0f..7f33cee19 100755 --- a/images/ceph/Makefile +++ b/images/ceph/Makefile @@ -82,14 +82,16 @@ do.build: generate-csv-ceph-templates generate-csv-ceph-templates: $(OPERATOR_SDK) $(YQ) @if [ ! "$(INCLUDE_CSV_TEMPLATES)" = "" ]; then\ - BEFORE_GEN_CRD_SIZE=$$(wc -l < ../../cluster/examples/kubernetes/ceph/crds.yaml);\ - $(MAKE) -C ../.. NO_OB_OBC_VOL_GEN=true MAX_DESC_LEN=0 crds;\ - AFTER_GEN_CRD_SIZE=$$(wc -l < ../../cluster/examples/kubernetes/ceph/crds.yaml);\ - if [ "$$BEFORE_GEN_CRD_SIZE" -le "$$AFTER_GEN_CRD_SIZE" ]; then\ - echo "the new crd file must be smaller since the description fields were stripped!";\ - echo "length before $$BEFORE_GEN_CRD_SIZE";\ - echo "length after $$AFTER_GEN_CRD_SIZE";\ - exit 1;\ + if [ "$(GOARCH)" = amd64 ]; then\ + BEFORE_GEN_CRD_SIZE=$$(wc -l < ../../cluster/examples/kubernetes/ceph/crds.yaml);\ + $(MAKE) -C ../.. NO_OB_OBC_VOL_GEN=true MAX_DESC_LEN=0 crds;\ + AFTER_GEN_CRD_SIZE=$$(wc -l < ../../cluster/examples/kubernetes/ceph/crds.yaml);\ + if [ "$$BEFORE_GEN_CRD_SIZE" -le "$$AFTER_GEN_CRD_SIZE" ]; then\ + echo "the new crd file must be smaller since the description fields were stripped!";\ + echo "length before $$BEFORE_GEN_CRD_SIZE";\ + echo "length after $$AFTER_GEN_CRD_SIZE";\ + exit 1;\ + fi;\ fi;\ ../../cluster/olm/ceph/generate-rook-csv-templates.sh;\ fi diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 1447466c6..01e2c22aa 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -79,13 +79,23 @@ function create_bluestore_partitions_and_pvcs_for_wal(){ } function build_rook() { + build_type=build + if [ -n "$1" ]; then + build_type=$1 + fi GOPATH=$(go env GOPATH) make clean # set VERSION to a dummy value since Jenkins normally sets it for us. Do this to make Helm happy and not fail with "Error: Invalid Semantic Version" - make -j$nproc IMAGES='ceph' VERSION=0 build + make -j$nproc IMAGES='ceph' VERSION=0 "$build_type" # validate build tests/scripts/validate_modified_files.sh build docker images - docker tag $(docker images | awk '/build-/ {print $1}') rook/ceph:master + if [[ "$build_type" == "build" ]]; then + docker tag $(docker images | awk '/build-/ {print $1}') rook/ceph:master + fi +} + +function build_rook_all() { + build_rook build.all } function validate_yaml() {