forked from rook/rook
Sébastien Han
@@ -1,82 +0,0 @@
|
||||
name: Canary integration tests ARM64
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * *' # every day at midnight
|
||||
|
||||
defaults:
|
||||
run:
|
||||
# reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
|
||||
jobs:
|
||||
canary-arm64:
|
||||
runs-on: [self-hosted, ubuntu-20.04, ARM64]
|
||||
if: github.repository == 'rook/rook'
|
||||
env:
|
||||
BLOCK: /dev/sdb
|
||||
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: setup golang
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.16
|
||||
|
||||
- name: teardown minikube and docker
|
||||
run: |
|
||||
uptime
|
||||
minikube delete
|
||||
docker system prune -a
|
||||
|
||||
- name: setup minikube
|
||||
run: |
|
||||
# sudo apt-get install build-essential -y
|
||||
# curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-arm64
|
||||
# sudo install minikube-linux-arm64 /usr/local/bin/minikube
|
||||
# sudo rm -f minikube-linux-arm64
|
||||
# curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl"
|
||||
# sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
|
||||
minikube start --memory 28g --cpus=12 --driver=docker
|
||||
|
||||
- name: print k8s cluster status
|
||||
run: tests/scripts/github-action-helper.sh print_k8s_cluster_status
|
||||
|
||||
- name: use local disk and create partitions for osds
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh use_local_disk
|
||||
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 1
|
||||
|
||||
- name: validate-yaml
|
||||
run: tests/scripts/github-action-helper.sh validate_yaml
|
||||
|
||||
- name: deploy cluster
|
||||
run: |
|
||||
# removing liveness probes since the env is slow and the probe is killing the daemons
|
||||
yq write -d1 -i cluster/examples/kubernetes/ceph/cluster-test.yaml "spec.healthCheck.livenessProbe.mon.disabled" true
|
||||
yq write -d1 -i cluster/examples/kubernetes/ceph/cluster-test.yaml "spec.healthCheck.livenessProbe.mgr.disabled" true
|
||||
yq write -d1 -i cluster/examples/kubernetes/ceph/cluster-test.yaml "spec.healthCheck.livenessProbe.osd.disabled" true
|
||||
tests/scripts/github-action-helper.sh deploy_cluster
|
||||
# there are no package for arm64 nfs-ganesha
|
||||
kubectl delete -f cluster/examples/kubernetes/ceph/nfs-test.yaml
|
||||
|
||||
- name: wait for prepare pod
|
||||
run: timeout 900 sh -c 'until kubectl -n rook-ceph logs -f $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}'); do sleep 5; done' || kubectl -n rook-ceph get all && kubectl logs -n rook-ceph deploy/rook-ceph-operator
|
||||
|
||||
- name: wait for ceph to be ready
|
||||
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready all 1
|
||||
|
||||
- name: teardown minikube and docker
|
||||
run: |
|
||||
minikube delete
|
||||
docker system prune -a
|
||||
|
||||
- name: upload canary test result
|
||||
uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
with:
|
||||
name: canary-arm64
|
||||
path: test
|
||||
@@ -0,0 +1,334 @@
|
||||
name: Daily nightly jobs
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 0 * * *" # every day at midnight
|
||||
|
||||
defaults:
|
||||
run:
|
||||
# reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
|
||||
shell: bash --noprofile --norc -eo pipefail -x {0}
|
||||
|
||||
jobs:
|
||||
canary-arm64:
|
||||
runs-on: [self-hosted, ubuntu-20.04, ARM64]
|
||||
if: github.repository == 'rook/rook'
|
||||
env:
|
||||
BLOCK: /dev/sdb
|
||||
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: setup golang
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.16
|
||||
|
||||
- name: teardown minikube and docker
|
||||
run: |
|
||||
uptime
|
||||
minikube delete
|
||||
docker system prune -a
|
||||
|
||||
- name: setup minikube
|
||||
run: |
|
||||
# sudo apt-get install build-essential -y
|
||||
# curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-arm64
|
||||
# sudo install minikube-linux-arm64 /usr/local/bin/minikube
|
||||
# sudo rm -f minikube-linux-arm64
|
||||
# curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl"
|
||||
# sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
|
||||
minikube start --memory 28g --cpus=12 --driver=docker
|
||||
|
||||
- name: print k8s cluster status
|
||||
run: tests/scripts/github-action-helper.sh print_k8s_cluster_status
|
||||
|
||||
- name: use local disk and create partitions for osds
|
||||
run: |
|
||||
tests/scripts/github-action-helper.sh use_local_disk
|
||||
tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 1
|
||||
|
||||
- name: validate-yaml
|
||||
run: tests/scripts/github-action-helper.sh validate_yaml
|
||||
|
||||
- name: deploy cluster
|
||||
run: |
|
||||
# removing liveness probes since the env is slow and the probe is killing the daemons
|
||||
yq write -d1 -i cluster/examples/kubernetes/ceph/cluster-test.yaml "spec.healthCheck.livenessProbe.mon.disabled" true
|
||||
yq write -d1 -i cluster/examples/kubernetes/ceph/cluster-test.yaml "spec.healthCheck.livenessProbe.mgr.disabled" true
|
||||
yq write -d1 -i cluster/examples/kubernetes/ceph/cluster-test.yaml "spec.healthCheck.livenessProbe.osd.disabled" true
|
||||
tests/scripts/github-action-helper.sh deploy_cluster
|
||||
# there are no package for arm64 nfs-ganesha
|
||||
kubectl delete -f cluster/examples/kubernetes/ceph/nfs-test.yaml
|
||||
|
||||
- name: wait for prepare pod
|
||||
run: timeout 900 sh -c 'until kubectl -n rook-ceph logs -f $(kubectl -n rook-ceph get pod -l app=rook-ceph-osd-prepare -o jsonpath='{.items[*].metadata.name}'); do sleep 5; done' || kubectl -n rook-ceph get all && kubectl logs -n rook-ceph deploy/rook-ceph-operator
|
||||
|
||||
- name: wait for ceph to be ready
|
||||
run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready all 1
|
||||
|
||||
- name: teardown minikube and docker
|
||||
run: |
|
||||
minikube delete
|
||||
docker system prune -a
|
||||
|
||||
- name: upload canary test result
|
||||
uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
with:
|
||||
name: canary-arm64
|
||||
path: test
|
||||
|
||||
smoke-suite-pacific-devel:
|
||||
runs-on: ubuntu-18.04
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: setup golang
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.16
|
||||
|
||||
- name: setup minikube
|
||||
uses: manusa/actions-setup-minikube@v2.4.2
|
||||
with:
|
||||
minikube version: "v1.22.0"
|
||||
kubernetes version: "v1.22.0"
|
||||
start args: --memory 6g --cpus=2
|
||||
github token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: print k8s cluster status
|
||||
run: tests/scripts/github-action-helper.sh print_k8s_cluster_status
|
||||
|
||||
- name: use local disk
|
||||
run: tests/scripts/github-action-helper.sh use_local_disk_for_integration_test
|
||||
|
||||
- name: build rook
|
||||
run: tests/scripts/github-action-helper.sh build_rook
|
||||
|
||||
- name: TestCephSmokeSuite
|
||||
run: |
|
||||
export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
|
||||
SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="pacific-devel" go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration
|
||||
|
||||
- name: Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
if: failure()
|
||||
with:
|
||||
name: ceph-smoke-suite-artifact
|
||||
path: /home/runner/work/rook/rook/tests/integration/_output/tests/
|
||||
|
||||
smoke-suite-ceph-master:
|
||||
runs-on: ubuntu-18.04
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: setup golang
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.16
|
||||
|
||||
- name: setup minikube
|
||||
uses: manusa/actions-setup-minikube@v2.4.2
|
||||
with:
|
||||
minikube version: "v1.22.0"
|
||||
kubernetes version: "v1.22.0"
|
||||
start args: --memory 6g --cpus=2
|
||||
github token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: print k8s cluster status
|
||||
run: tests/scripts/github-action-helper.sh print_k8s_cluster_status
|
||||
|
||||
- name: use local disk
|
||||
run: tests/scripts/github-action-helper.sh use_local_disk_for_integration_test
|
||||
|
||||
- name: build rook
|
||||
run: tests/scripts/github-action-helper.sh build_rook
|
||||
|
||||
- name: TestCephSmokeSuite
|
||||
run: |
|
||||
export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
|
||||
SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION=master go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration
|
||||
|
||||
- name: Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
if: failure()
|
||||
with:
|
||||
name: ceph-smoke-suite-artifact
|
||||
path: /home/runner/work/rook/rook/tests/integration/_output/tests/
|
||||
|
||||
object-suite-pacific-devel:
|
||||
runs-on: ubuntu-18.04
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: setup golang
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.16
|
||||
|
||||
- name: setup minikube
|
||||
uses: manusa/actions-setup-minikube@v2.4.2
|
||||
with:
|
||||
minikube version: "v1.22.0"
|
||||
kubernetes version: "v1.22.0"
|
||||
start args: --memory 6g --cpus=2
|
||||
github token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: print k8s cluster status
|
||||
run: tests/scripts/github-action-helper.sh print_k8s_cluster_status
|
||||
|
||||
- name: use local disk
|
||||
run: tests/scripts/github-action-helper.sh use_local_disk_for_integration_test
|
||||
|
||||
- name: build rook
|
||||
run: tests/scripts/github-action-helper.sh build_rook
|
||||
|
||||
- name: TestCephObjectSuite
|
||||
run: |
|
||||
export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
|
||||
SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="pacific-devel" go test -v -timeout 1800s -failfast -run TestCephObjectSuite github.com/rook/rook/tests/integration
|
||||
|
||||
- name: Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
if: failure()
|
||||
with:
|
||||
name: ceph-object-suite-artifact
|
||||
path: /home/runner/work/rook/rook/tests/integration/_output/tests/
|
||||
|
||||
object-suite-master:
|
||||
runs-on: ubuntu-18.04
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: setup golang
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.16
|
||||
|
||||
- name: setup minikube
|
||||
uses: manusa/actions-setup-minikube@v2.4.2
|
||||
with:
|
||||
minikube version: "v1.22.0"
|
||||
kubernetes version: "v1.22.0"
|
||||
start args: --memory 6g --cpus=2
|
||||
github token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: print k8s cluster status
|
||||
run: tests/scripts/github-action-helper.sh print_k8s_cluster_status
|
||||
|
||||
- name: use local disk
|
||||
run: tests/scripts/github-action-helper.sh use_local_disk_for_integration_test
|
||||
|
||||
- name: build rook
|
||||
run: tests/scripts/github-action-helper.sh build_rook
|
||||
|
||||
- name: TestCephObjectSuite
|
||||
run: |
|
||||
export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
|
||||
SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION=master go test -v -timeout 1800s -failfast -run TestCephObjectSuite github.com/rook/rook/tests/integration
|
||||
|
||||
- name: Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
if: failure()
|
||||
with:
|
||||
name: ceph-object-suite-artifact
|
||||
path: /home/runner/work/rook/rook/tests/integration/_output/tests/
|
||||
|
||||
upgrade-from-pacific-stable-to-pacific-devel:
|
||||
runs-on: ubuntu-18.04
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: setup golang
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.16
|
||||
|
||||
- name: setup minikube
|
||||
uses: manusa/actions-setup-minikube@v2.4.2
|
||||
with:
|
||||
minikube version: "v1.22.0"
|
||||
kubernetes version: "v1.22.0"
|
||||
start args: --memory 6g --cpus=2
|
||||
github token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: print k8s cluster status
|
||||
run: tests/scripts/github-action-helper.sh print_k8s_cluster_status
|
||||
|
||||
- name: use local disk
|
||||
run: tests/scripts/github-action-helper.sh use_local_disk_for_integration_test
|
||||
|
||||
- name: build rook
|
||||
run: tests/scripts/github-action-helper.sh build_rook
|
||||
|
||||
- name: TestCephUpgradeSuite
|
||||
run: |
|
||||
export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
|
||||
go test -v -timeout 1800s -failfast -run TestCephUpgradeSuite/TestUpgradeCephToPacificDevel github.com/rook/rook/tests/integration
|
||||
|
||||
- name: Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
if: failure()
|
||||
with:
|
||||
name: ceph-upgrade-suite-artifact
|
||||
path: /home/runner/work/rook/rook/tests/integration/_output/tests/
|
||||
|
||||
upgrade-from-octopus-stable-to-octopus-devel:
|
||||
runs-on: ubuntu-18.04
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: setup golang
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.16
|
||||
|
||||
- name: setup minikube
|
||||
uses: manusa/actions-setup-minikube@v2.4.2
|
||||
with:
|
||||
minikube version: "v1.22.0"
|
||||
kubernetes version: "v1.22.0"
|
||||
start args: --memory 6g --cpus=2
|
||||
github token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: print k8s cluster status
|
||||
run: tests/scripts/github-action-helper.sh print_k8s_cluster_status
|
||||
|
||||
- name: use local disk
|
||||
run: tests/scripts/github-action-helper.sh use_local_disk_for_integration_test
|
||||
|
||||
- name: build rook
|
||||
run: tests/scripts/github-action-helper.sh build_rook
|
||||
|
||||
- name: TestCephUpgradeSuite
|
||||
run: |
|
||||
export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
|
||||
go test -v -timeout 1800s -failfast -run TestCephUpgradeSuite/TestUpgradeCephToOctopusDevel github.com/rook/rook/tests/integration
|
||||
|
||||
- name: Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
if: failure()
|
||||
with:
|
||||
name: ceph-upgrade-suite-artifact
|
||||
path: /home/runner/work/rook/rook/tests/integration/_output/tests/
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
- name: TestCephUpgradeSuite
|
||||
run: |
|
||||
export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1)
|
||||
go test -v -timeout 1800s -failfast -run CephUpgradeSuite github.com/rook/rook/tests/integration
|
||||
go test -v -timeout 1800s -failfast -run CephUpgradeSuite/TestUpgradeRookToMaster github.com/rook/rook/tests/integration
|
||||
|
||||
- name: Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
|
||||
@@ -51,8 +51,11 @@ const (
|
||||
octopusTestImage = "quay.io/ceph/ceph:v15"
|
||||
// test with the latest pacific build
|
||||
pacificTestImage = "quay.io/ceph/ceph:v16.2.6"
|
||||
// test with the current development version of Pacific
|
||||
pacificDevelTestImage = "quay.io/ceph/daemon-base:latest-pacific-devel"
|
||||
octopusDevelTestImage = "quay.io/ceph/daemon-base:latest-octopus-devel"
|
||||
// test with the latest master image
|
||||
masterTestImage = "ceph/daemon-base:latest-master-devel"
|
||||
masterTestImage = "quay.io/ceph/daemon-base:latest-master-devel"
|
||||
cephOperatorLabel = "app=rook-ceph-operator"
|
||||
defaultclusterName = "test-cluster"
|
||||
|
||||
@@ -67,7 +70,9 @@ var (
|
||||
NautilusVersion = cephv1.CephVersionSpec{Image: nautilusTestImage}
|
||||
NautilusPartitionVersion = cephv1.CephVersionSpec{Image: nautilusTestImagePartition}
|
||||
OctopusVersion = cephv1.CephVersionSpec{Image: octopusTestImage}
|
||||
OctopusDevelVersion = cephv1.CephVersionSpec{Image: octopusDevelTestImage}
|
||||
PacificVersion = cephv1.CephVersionSpec{Image: pacificTestImage}
|
||||
PacificDevelVersion = cephv1.CephVersionSpec{Image: pacificDevelTestImage}
|
||||
MasterVersion = cephv1.CephVersionSpec{Image: masterTestImage, AllowUnsupported: true}
|
||||
)
|
||||
|
||||
@@ -83,6 +88,17 @@ type CephInstaller struct {
|
||||
T func() *testing.T
|
||||
}
|
||||
|
||||
func ReturnCephVersion() cephv1.CephVersionSpec {
|
||||
switch os.Getenv("CEPH_SUITE_VERSION") {
|
||||
case "master":
|
||||
return MasterVersion
|
||||
case "pacific-devel":
|
||||
return PacificDevelVersion
|
||||
default:
|
||||
return PacificVersion
|
||||
}
|
||||
}
|
||||
|
||||
// CreateCephOperator creates rook-operator via kubectl
|
||||
func (h *CephInstaller) CreateCephOperator() (err error) {
|
||||
// creating rook resources
|
||||
|
||||
@@ -75,7 +75,7 @@ func (s *ObjectSuite) SetupSuite() {
|
||||
EnableAdmissionController: true,
|
||||
UseCrashPruner: true,
|
||||
RookVersion: installer.LocalBuildTag,
|
||||
CephVersion: installer.PacificVersion,
|
||||
CephVersion: installer.ReturnCephVersion(),
|
||||
}
|
||||
s.settings.ApplyEnvVars()
|
||||
s.installer, s.k8sh = StartTestCluster(s.T, s.settings)
|
||||
|
||||
@@ -94,7 +94,7 @@ func (s *SmokeSuite) SetupSuite() {
|
||||
EnableAdmissionController: true,
|
||||
UseCrashPruner: true,
|
||||
RookVersion: installer.LocalBuildTag,
|
||||
CephVersion: installer.PacificVersion,
|
||||
CephVersion: installer.ReturnCephVersion(),
|
||||
}
|
||||
s.settings.ApplyEnvVars()
|
||||
s.installer, s.k8sh = StartTestCluster(s.T, s.settings)
|
||||
|
||||
@@ -35,6 +35,10 @@ import (
|
||||
const (
|
||||
rbdPodName = "test-pod-upgrade"
|
||||
operatorContainer = "rook-ceph-operator"
|
||||
poolName = "upgradepool"
|
||||
storageClassName = "block-upgrade"
|
||||
blockName = "block-claim-upgrade"
|
||||
bucketPrefix = "generate-me" // use generated bucket name for this test
|
||||
)
|
||||
|
||||
// ************************************************
|
||||
@@ -90,86 +94,26 @@ func (s *UpgradeSuite) TearDownSuite() {
|
||||
s.installer.UninstallRook()
|
||||
}
|
||||
|
||||
func (s *UpgradeSuite) TestUpgradeToMaster() {
|
||||
//
|
||||
// Create block, object, and file storage before the upgrade
|
||||
//
|
||||
poolName := "upgradepool"
|
||||
storageClassName := "block-upgrade"
|
||||
blockName := "block-claim-upgrade"
|
||||
logger.Infof("Initializing block before the upgrade")
|
||||
func (s *UpgradeSuite) TestUpgradeRookToMaster() {
|
||||
message := "my simple message"
|
||||
objectStoreName := "upgraded-object"
|
||||
objectUserID := "upgraded-user"
|
||||
preFilename := "pre-upgrade-file"
|
||||
numOSDs, filesystemName, rbdFilesToRead, cephfsFilesToRead := s.deployClusterforUpgrade(objectStoreName, objectUserID, message, preFilename)
|
||||
s.settings.CephVersion = installer.NautilusVersion
|
||||
|
||||
clusterInfo := client.AdminClusterInfo(s.namespace)
|
||||
setupBlockLite(s.helper, s.k8sh, s.Suite, clusterInfo, poolName, storageClassName, blockName, rbdPodName)
|
||||
|
||||
createPodWithBlock(s.helper, s.k8sh, s.Suite, s.namespace, storageClassName, rbdPodName, blockName)
|
||||
|
||||
// FIX: We should require block images to be removed. See tracking issue:
|
||||
// <INSERT ISSUE>
|
||||
requireBlockImagesRemoved := false
|
||||
defer blockTestDataCleanUp(s.helper, s.k8sh, s.Suite, clusterInfo, poolName, storageClassName, blockName, rbdPodName, requireBlockImagesRemoved)
|
||||
|
||||
// Create the filesystem
|
||||
logger.Infof("Initializing file before the upgrade")
|
||||
filesystemName := "upgrade-test-fs"
|
||||
activeCount := 1
|
||||
createFilesystem(s.helper, s.k8sh, s.Suite, s.settings, filesystemName, activeCount)
|
||||
|
||||
// Start the file test client
|
||||
fsStorageClass := "file-upgrade"
|
||||
assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, s.settings.OperatorNamespace, s.namespace, fsStorageClass))
|
||||
createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.settings, filesystemName, fsStorageClass)
|
||||
defer func() {
|
||||
blockTestDataCleanUp(s.helper, s.k8sh, s.Suite, clusterInfo, poolName, storageClassName, blockName, rbdPodName, requireBlockImagesRemoved)
|
||||
cleanupFilesystemConsumer(s.helper, s.k8sh, s.Suite, s.namespace, filePodName)
|
||||
cleanupFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName)
|
||||
}()
|
||||
|
||||
logger.Infof("Initializing object before the upgrade")
|
||||
objectStoreName := "upgraded-object"
|
||||
runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.settings, objectStoreName, 1, false)
|
||||
|
||||
logger.Infof("Initializing object user before the upgrade")
|
||||
objectUserID := "upgraded-user"
|
||||
createCephObjectUser(s.Suite, s.helper, s.k8sh, s.namespace, objectStoreName, objectUserID, false, false)
|
||||
|
||||
logger.Info("Initializing object bucket claim before the upgrade")
|
||||
bucketStorageClassName := "rook-smoke-delete-bucket"
|
||||
bucketPrefix := "generate-me" // use generated bucket name for this test
|
||||
cobErr := s.helper.BucketClient.CreateBucketStorageClass(s.namespace, objectStoreName, bucketStorageClassName, "Delete", region)
|
||||
require.Nil(s.T(), cobErr)
|
||||
cobcErr := s.helper.BucketClient.CreateObc(obcName, bucketStorageClassName, bucketPrefix, maxObject, false)
|
||||
require.Nil(s.T(), cobcErr)
|
||||
defer func() {
|
||||
_ = s.helper.ObjectUserClient.Delete(s.namespace, objectUserID)
|
||||
_ = s.helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketPrefix, maxObject, false)
|
||||
_ = s.helper.BucketClient.DeleteBucketStorageClass(s.namespace, objectStoreName, bucketStorageClassName, "Delete", region)
|
||||
objectStoreCleanUp(s.Suite, s.helper, s.k8sh, s.settings.Namespace, objectStoreName)
|
||||
}()
|
||||
|
||||
created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool {
|
||||
// do not check if bound here b/c this fails in Rook v1.4
|
||||
return s.helper.BucketClient.CheckOBC(obcName, "created")
|
||||
})
|
||||
require.True(s.T(), created)
|
||||
|
||||
// verify that we're actually running the right pre-upgrade image
|
||||
s.verifyOperatorImage(installer.Version1_6)
|
||||
|
||||
message := "my simple message"
|
||||
preFilename := "pre-upgrade-file"
|
||||
assert.NoError(s.T(), s.k8sh.WriteToPod("", rbdPodName, preFilename, message))
|
||||
assert.NoError(s.T(), s.k8sh.ReadFromPod("", rbdPodName, preFilename, message))
|
||||
|
||||
// we will keep appending to this to continue verifying old files through the upgrades
|
||||
rbdFilesToRead := []string{preFilename}
|
||||
cephfsFilesToRead := []string{}
|
||||
|
||||
// Get some info about the currently deployed OSDs to determine later if they are all updated
|
||||
osdDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd")
|
||||
require.NoError(s.T(), err)
|
||||
osdDeps := osdDepList.Items
|
||||
numOSDs := len(osdDeps) // there should be this many upgraded OSDs
|
||||
require.NotEqual(s.T(), 0, numOSDs)
|
||||
|
||||
//
|
||||
// Upgrade Rook from v1.6 to master
|
||||
//
|
||||
@@ -179,7 +123,7 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
|
||||
|
||||
s.verifyOperatorImage(installer.LocalBuildTag)
|
||||
s.verifyRookUpgrade(numOSDs)
|
||||
err = s.installer.WaitForToolbox(s.namespace)
|
||||
err := s.installer.WaitForToolbox(s.namespace)
|
||||
assert.NoError(s.T(), err)
|
||||
|
||||
logger.Infof("Done with automatic upgrade from %s to master", installer.Version1_6)
|
||||
@@ -223,6 +167,131 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
|
||||
checkCephObjectUser(s.Suite, s.helper, s.k8sh, s.namespace, objectStoreName, objectUserID, true, false)
|
||||
}
|
||||
|
||||
func (s *UpgradeSuite) TestUpgradeCephToOctopusDevel() {
|
||||
message := "my simple message"
|
||||
objectStoreName := "upgraded-object"
|
||||
objectUserID := "upgraded-user"
|
||||
preFilename := "pre-upgrade-file"
|
||||
s.settings.CephVersion = installer.OctopusVersion
|
||||
numOSDs, filesystemName, rbdFilesToRead, cephfsFilesToRead := s.deployClusterforUpgrade(objectStoreName, objectUserID, message, preFilename)
|
||||
clusterInfo := client.AdminClusterInfo(s.namespace)
|
||||
requireBlockImagesRemoved := false
|
||||
defer func() {
|
||||
blockTestDataCleanUp(s.helper, s.k8sh, s.Suite, clusterInfo, poolName, storageClassName, blockName, rbdPodName, requireBlockImagesRemoved)
|
||||
cleanupFilesystemConsumer(s.helper, s.k8sh, s.Suite, s.namespace, filePodName)
|
||||
cleanupFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName)
|
||||
_ = s.helper.ObjectUserClient.Delete(s.namespace, objectUserID)
|
||||
_ = s.helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketPrefix, maxObject, false)
|
||||
_ = s.helper.BucketClient.DeleteBucketStorageClass(s.namespace, objectStoreName, bucketStorageClassName, "Delete", region)
|
||||
objectStoreCleanUp(s.Suite, s.helper, s.k8sh, s.settings.Namespace, objectStoreName)
|
||||
}()
|
||||
|
||||
//
|
||||
// Upgrade from octopus to octopus
|
||||
//
|
||||
logger.Infof("*** UPGRADING CEPH FROM OCTOPUS STABLE TO OCTOPUS DEVEL ***")
|
||||
s.gatherLogs(s.settings.OperatorNamespace, "_before_pacific_upgrade")
|
||||
s.upgradeCephVersion(installer.OctopusDevelVersion.Image, numOSDs)
|
||||
// Verify reading and writing to the test clients
|
||||
newFile := "post-octopus-upgrade-file"
|
||||
s.verifyFilesAfterUpgrade(filesystemName, newFile, message, rbdFilesToRead, cephfsFilesToRead)
|
||||
logger.Infof("Verified upgrade from octopus stable to octopus devel")
|
||||
|
||||
checkCephObjectUser(s.Suite, s.helper, s.k8sh, s.namespace, objectStoreName, objectUserID, true, false)
|
||||
}
|
||||
|
||||
func (s *UpgradeSuite) TestUpgradeCephToPacificDevel() {
|
||||
message := "my simple message"
|
||||
objectStoreName := "upgraded-object"
|
||||
objectUserID := "upgraded-user"
|
||||
preFilename := "pre-upgrade-file"
|
||||
s.settings.CephVersion = installer.PacificVersion
|
||||
numOSDs, filesystemName, rbdFilesToRead, cephfsFilesToRead := s.deployClusterforUpgrade(objectStoreName, objectUserID, message, preFilename)
|
||||
clusterInfo := client.AdminClusterInfo(s.namespace)
|
||||
requireBlockImagesRemoved := false
|
||||
defer func() {
|
||||
blockTestDataCleanUp(s.helper, s.k8sh, s.Suite, clusterInfo, poolName, storageClassName, blockName, rbdPodName, requireBlockImagesRemoved)
|
||||
cleanupFilesystemConsumer(s.helper, s.k8sh, s.Suite, s.namespace, filePodName)
|
||||
cleanupFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName)
|
||||
_ = s.helper.ObjectUserClient.Delete(s.namespace, objectUserID)
|
||||
_ = s.helper.BucketClient.DeleteObc(obcName, bucketStorageClassName, bucketPrefix, maxObject, false)
|
||||
_ = s.helper.BucketClient.DeleteBucketStorageClass(s.namespace, objectStoreName, bucketStorageClassName, "Delete", region)
|
||||
objectStoreCleanUp(s.Suite, s.helper, s.k8sh, s.settings.Namespace, objectStoreName)
|
||||
}()
|
||||
|
||||
//
|
||||
// Upgrade from octopus to pacific
|
||||
//
|
||||
logger.Infof("*** UPGRADING CEPH FROM PACIFIC STABLE TO PACIFIC DEVEL ***")
|
||||
s.gatherLogs(s.settings.OperatorNamespace, "_before_pacific_upgrade")
|
||||
s.upgradeCephVersion(installer.PacificDevelVersion.Image, numOSDs)
|
||||
// Verify reading and writing to the test clients
|
||||
newFile := "post-pacific-upgrade-file"
|
||||
s.verifyFilesAfterUpgrade(filesystemName, newFile, message, rbdFilesToRead, cephfsFilesToRead)
|
||||
logger.Infof("Verified upgrade from pacific stable to pacific devel")
|
||||
|
||||
checkCephObjectUser(s.Suite, s.helper, s.k8sh, s.namespace, objectStoreName, objectUserID, true, false)
|
||||
}
|
||||
|
||||
func (s *UpgradeSuite) deployClusterforUpgrade(objectStoreName, objectUserID, message, preFilename string) (int, string, []string, []string) {
|
||||
//
|
||||
// Create block, object, and file storage before the upgrade
|
||||
//
|
||||
logger.Infof("Initializing block before the upgrade")
|
||||
clusterInfo := client.AdminClusterInfo(s.namespace)
|
||||
setupBlockLite(s.helper, s.k8sh, s.Suite, clusterInfo, poolName, storageClassName, blockName, rbdPodName)
|
||||
|
||||
createPodWithBlock(s.helper, s.k8sh, s.Suite, s.namespace, storageClassName, rbdPodName, blockName)
|
||||
|
||||
// Create the filesystem
|
||||
logger.Infof("Initializing file before the upgrade")
|
||||
filesystemName := "upgrade-test-fs"
|
||||
activeCount := 1
|
||||
createFilesystem(s.helper, s.k8sh, s.Suite, s.settings, filesystemName, activeCount)
|
||||
|
||||
// Start the file test client
|
||||
fsStorageClass := "file-upgrade"
|
||||
assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, s.settings.OperatorNamespace, s.namespace, fsStorageClass))
|
||||
createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.settings, filesystemName, fsStorageClass)
|
||||
|
||||
logger.Infof("Initializing object before the upgrade")
|
||||
runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.settings, objectStoreName, 1, false)
|
||||
|
||||
logger.Infof("Initializing object user before the upgrade")
|
||||
createCephObjectUser(s.Suite, s.helper, s.k8sh, s.namespace, objectStoreName, objectUserID, false, false)
|
||||
|
||||
logger.Info("Initializing object bucket claim before the upgrade")
|
||||
cobErr := s.helper.BucketClient.CreateBucketStorageClass(s.namespace, objectStoreName, bucketStorageClassName, "Delete", region)
|
||||
require.Nil(s.T(), cobErr)
|
||||
cobcErr := s.helper.BucketClient.CreateObc(obcName, bucketStorageClassName, bucketPrefix, maxObject, false)
|
||||
require.Nil(s.T(), cobcErr)
|
||||
|
||||
created := utils.Retry(12, 2*time.Second, "OBC is created", func() bool {
|
||||
// do not check if bound here b/c this fails in Rook v1.4
|
||||
return s.helper.BucketClient.CheckOBC(obcName, "created")
|
||||
})
|
||||
require.True(s.T(), created)
|
||||
|
||||
// verify that we're actually running the right pre-upgrade image
|
||||
s.verifyOperatorImage(installer.Version1_6)
|
||||
|
||||
assert.NoError(s.T(), s.k8sh.WriteToPod("", rbdPodName, preFilename, message))
|
||||
assert.NoError(s.T(), s.k8sh.ReadFromPod("", rbdPodName, preFilename, message))
|
||||
|
||||
// we will keep appending to this to continue verifying old files through the upgrades
|
||||
rbdFilesToRead := []string{preFilename}
|
||||
cephfsFilesToRead := []string{}
|
||||
|
||||
// Get some info about the currently deployed OSDs to determine later if they are all updated
|
||||
osdDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd")
|
||||
require.NoError(s.T(), err)
|
||||
osdDeps := osdDepList.Items
|
||||
numOSDs := len(osdDeps) // there should be this many upgraded OSDs
|
||||
require.NotEqual(s.T(), 0, numOSDs)
|
||||
|
||||
return numOSDs, filesystemName, rbdFilesToRead, cephfsFilesToRead
|
||||
}
|
||||
|
||||
func (s *UpgradeSuite) gatherLogs(systemNamespace, testSuffix string) {
|
||||
// Gather logs before Ceph upgrade to help with debugging
|
||||
if installer.TestLogCollectionLevel() == "all" {
|
||||
|
||||
Reference in New Issue
Block a user