From 2e242ca886f3e59f48a078407221edd609dbca0f Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Fri, 3 Jan 2020 09:47:28 -0700 Subject: [PATCH] ceph: upgrade test versions v1.0->v1.1->v1.2->v1.3 In the Ceph upgrade integration test, upgrade from Rook v1.0 with Ceph Mimic v13 to install legacy disk-based OSDs. Then upgrade Ceph to Nautilus, then Rook to v1.1, then v1.2, then v1.3 (master currently) to make sure that Rook is able to run legacy OSDs created without ceph-volume. Signed-off-by: Blaine Gardner --- .../installer/ceph_manifests_v1.0.go | 13 +- .../installer/ceph_manifests_v1.1.go | 490 ++++++++++++++++++ .../installer/ceph_manifests_v1.2.go | 213 ++++++++ .../installer/ceph_manifests_v1.3.go | 52 ++ tests/framework/installer/installer.go | 5 +- tests/integration/ceph_upgrade_test.go | 253 +++++---- 6 files changed, 895 insertions(+), 131 deletions(-) create mode 100644 tests/framework/installer/ceph_manifests_v1.1.go create mode 100644 tests/framework/installer/ceph_manifests_v1.2.go create mode 100644 tests/framework/installer/ceph_manifests_v1.3.go diff --git a/tests/framework/installer/ceph_manifests_v1.0.go b/tests/framework/installer/ceph_manifests_v1.0.go index d38605cf9..30b15a457 100644 --- a/tests/framework/installer/ceph_manifests_v1.0.go +++ b/tests/framework/installer/ceph_manifests_v1.0.go @@ -17,16 +17,27 @@ limitations under the License. package installer import ( + "fmt" "strconv" "github.com/google/uuid" ) -// CephManifestsV1_0 wraps rook yaml definitions +const ( + // Version tag for Rook v1.0 + Version1_0 = "v1.0.6" +) + +// CephManifestsV1_0 wraps rook yaml definitions for Rook-Ceph v1.0 manifests type CephManifestsV1_0 struct { imageTag string } +// RookImage returns the rook image under test for v1.0 +func (m *CephManifestsV1_0) RookImage() string { + return fmt.Sprintf("rook/ceph:%s", Version1_0) +} + func (m *CephManifestsV1_0) GetRookCRDs() string { return ` apiVersion: apiextensions.k8s.io/v1beta1 diff --git a/tests/framework/installer/ceph_manifests_v1.1.go b/tests/framework/installer/ceph_manifests_v1.1.go new file mode 100644 index 000000000..3bda1fd0e --- /dev/null +++ b/tests/framework/installer/ceph_manifests_v1.1.go @@ -0,0 +1,490 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "fmt" + "testing" + + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/require" +) + +const ( + version1_1 = "v1.1.8" +) + +// CephManifestsV1_1 wraps rook yaml definitions for Rook-Ceph v1.1 manifests +type CephManifestsV1_1 struct { + K8sh *utils.K8sHelper + Namespace string + SystemNamespace string + OperatorContainer string + T func() *testing.T +} + +// UpgradeToV1_1 performs the steps necessary to upgrade a Rook v1.0 cluster to v1.1. It does not +// verify the upgrade but merely starts the upgrade process. +func (m *CephManifestsV1_1) UpgradeToV1_1() { + require.NoError(m.T(), m.K8sh.ResourceOperation("create", m.createManifest())) + require.NoError(m.T(), m.K8sh.ResourceOperation("apply", m.applyManifest())) + + require.NoError(m.T(), + m.K8sh.SetDeploymentVersion(m.SystemNamespace, m.OperatorContainer, m.OperatorContainer, version1_1)) +} + +// RookImage returns the rook image under test for v1.1 +func (m *CephManifestsV1_1) RookImage() string { + return fmt.Sprintf("rook/ceph:%s", version1_1) +} + +func (m *CephManifestsV1_1) createManifest() string { + return ` +# Service account, role, and role binding to allow cmd-reporter job to run in Ceph cluster namespace +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-cmd-reporter + namespace: ` + m.Namespace + ` +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cmd-reporter + namespace: ` + m.Namespace + ` +rules: +- apiGroups: + - "" + resources: + - pods + - configmaps + verbs: + - get + - list + - watch + - create + - update + - delete +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cmd-reporter + namespace: ` + m.Namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-cmd-reporter +subjects: +- kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: ` + m.Namespace + ` + + +# CRDs, role, and binding for Object Buckets +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: objectbuckets.objectbucket.io +spec: + group: objectbucket.io + versions: + - name: v1alpha1 + served: true + storage: true + names: + kind: ObjectBucket + listKind: ObjectBucketList + plural: objectbuckets + singular: objectbucket + shortNames: + - ob + - obs + scope: Cluster + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: objectbucketclaims.objectbucket.io +spec: + versions: + - name: v1alpha1 + served: true + storage: true + group: objectbucket.io + names: + kind: ObjectBucketClaim + listKind: ObjectBucketClaimList + plural: objectbucketclaims + singular: objectbucketclaim + shortNames: + - obc + - obcs + scope: Namespaced + subresources: + status: {} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-object-bucket + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" +rules: +- apiGroups: + - "" + verbs: + - "*" + resources: + - secrets + - configmaps +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "objectbucket.io" + verbs: + - "*" + resources: + - "*" +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-object-bucket +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-object-bucket +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: ` + m.SystemNamespace + ` +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: ` + m.SystemNamespace + ` + name: cephfs-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-provisioner-role-cfg + namespace: ` + m.SystemNamespace + ` +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-provisioner-sa + namespace: ` + m.SystemNamespace + ` +roleRef: + kind: Role + name: cephfs-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: ` + m.SystemNamespace + ` + name: rbd-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-role-cfg + namespace: ` + m.SystemNamespace + ` +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-provisioner-sa + namespace: ` + m.SystemNamespace + ` +roleRef: + kind: Role + name: rbd-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io +` +} + +func (m *CephManifestsV1_1) applyManifest() string { + return ` +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-global-rules + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" +rules: +- apiGroups: + - "" + resources: + # Pod access is needed for fencing + - pods + # Node access is needed for determining nodes where mons should run + - nodes + - nodes/proxy + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + # PVs and PVCs are managed by the Rook provisioner + - persistentvolumes + - persistentvolumeclaims + - endpoints + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - ceph.rook.io + resources: + - "*" + verbs: + - "*" +- apiGroups: + - rook.io + resources: + - "*" + verbs: + - "*" +- apiGroups: + - policy + - apps + resources: + #this is for the clusterdisruption controller + - poddisruptionbudgets + #this is for both clusterdisruption and nodedrain controllers + - deployments + verbs: + - "*" +- apiGroups: + - healthchecking.openshift.io + resources: + - machinedisruptionbudgets + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - machine.openshift.io + resources: + - machines + verbs: + - get + - list + - watch + - create + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: rook-ceph-system + namespace: ` + m.SystemNamespace + ` + labels: + operator: rook + storage-backend: ceph +rules: + - apiGroups: + - "" + resources: + - pods + - configmaps + - services + verbs: + - get + - list + - watch + - patch + - create + - update + - delete + - apiGroups: + - apps + resources: + - daemonsets + - statefulsets + - deployments + verbs: + - get + - list + - watch + - create + - update + - delete +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-external-provisioner-runner-rules + labels: + rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true" +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin-rules + labels: + rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true" +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-external-provisioner-runner-rules + labels: + rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true" +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete", "get", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] +` +} diff --git a/tests/framework/installer/ceph_manifests_v1.2.go b/tests/framework/installer/ceph_manifests_v1.2.go new file mode 100644 index 000000000..27547bd80 --- /dev/null +++ b/tests/framework/installer/ceph_manifests_v1.2.go @@ -0,0 +1,213 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "fmt" + "testing" + + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/require" +) + +const ( + version1_2 = "v1.2.1" +) + +// CephManifestsV1_2 wraps rook yaml definitions for Rook-Ceph v1.2 manifests +type CephManifestsV1_2 struct { + K8sh *utils.K8sHelper + Namespace string + SystemNamespace string + OperatorContainer string + T func() *testing.T +} + +// UpgradeToV1_2 performs the steps necessary to upgrade a Rook v1.1 cluster to v1.2. It does not +// verify the upgrade but merely starts the upgrade process. +func (m *CephManifestsV1_2) UpgradeToV1_2() { + require.NoError(m.T(), m.K8sh.ResourceOperation("apply", m.applyManifest())) + + require.NoError(m.T(), + m.K8sh.SetDeploymentVersion(m.SystemNamespace, m.OperatorContainer, m.OperatorContainer, version1_2)) +} + +// RookImage returns the rook image under test for v1.2 +func (m *CephManifestsV1_2) RookImage() string { + return fmt.Sprintf("rook/ceph:%s", version1_2) +} + +func (m *CephManifestsV1_2) applyManifest() string { + return ` +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-global-rules + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" +rules: +- apiGroups: + - "" + resources: + # Pod access is needed for fencing + - pods + # Node access is needed for determining nodes where mons should run + - nodes + - nodes/proxy + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + # PVs and PVCs are managed by the Rook provisioner + - persistentvolumes + - persistentvolumeclaims + - endpoints + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - ceph.rook.io + resources: + - "*" + verbs: + - "*" +- apiGroups: + - rook.io + resources: + - "*" + verbs: + - "*" +- apiGroups: + - policy + - apps + resources: + # This is for the clusterdisruption controller + - poddisruptionbudgets + # This is for both clusterdisruption and nodedrain controllers + - deployments + - replicasets + verbs: + - "*" +- apiGroups: + - healthchecking.openshift.io + resources: + - machinedisruptionbudgets + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - machine.openshift.io + resources: + - machines + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - storage.k8s.io + resources: + - csidrivers + verbs: + - create +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephclients.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephClient + listKind: CephClientList + plural: cephclients + singular: cephclient + scope: Namespaced + version: v1 + validation: + openAPIV3Schema: + properties: + spec: + properties: + caps: + type: object +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-osd + namespace: ` + m.Namespace + ` +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list +--- +# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-osd +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-osd +subjects: +- kind: ServiceAccount + name: rook-ceph-osd + namespace: ` + m.Namespace + ` +` +} diff --git a/tests/framework/installer/ceph_manifests_v1.3.go b/tests/framework/installer/ceph_manifests_v1.3.go new file mode 100644 index 000000000..e8c06dcde --- /dev/null +++ b/tests/framework/installer/ceph_manifests_v1.3.go @@ -0,0 +1,52 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "fmt" + "testing" + + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/require" +) + +const ( + version1_3 = "master" // v1.3 is not yet released, so test upgrade to master until then +) + +// CephManifestsV1_3 wraps rook yaml definitions for Rook-Ceph v1.3 manifests +type CephManifestsV1_3 struct { + K8sh *utils.K8sHelper + Namespace string + SystemNamespace string + OperatorContainer string + T func() *testing.T +} + +// RookImage returns the rook image under test for v1.2 +func (m *CephManifestsV1_3) RookImage() string { + return fmt.Sprintf("rook/ceph:%s", version1_3) +} + +// UpgradeToV1_3 performs the steps necessary to upgrade a Rook v1.1 cluster to v1.2. It does not +// verify the upgrade but merely starts the upgrade process. +func (m *CephManifestsV1_3) UpgradeToV1_3() { + // no special manifest operations (apply, create, delete) yet + + require.NoError(m.T(), + m.K8sh.SetDeploymentVersion(m.SystemNamespace, m.OperatorContainer, m.OperatorContainer, version1_3)) +} diff --git a/tests/framework/installer/installer.go b/tests/framework/installer/installer.go index d302eed6e..62200a7bc 100644 --- a/tests/framework/installer/installer.go +++ b/tests/framework/installer/installer.go @@ -28,10 +28,9 @@ import ( ) const ( - // Version tag for the latest manifests + // VersionMaster tag for the latest manifests VersionMaster = "master" - // Version tag for Rook v1.0 - Version1_0 = "v1.0.6" + // test suite names CassandraTestSuite = "cassandra" CephTestSuite = "ceph" diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index de2eddaed..7a4bde188 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -32,6 +32,11 @@ import ( "github.com/stretchr/testify/suite" ) +const ( + rbdPodName = "test-pod-upgrade" + operatorContainer = "rook-ceph-operator" +) + // ************************************************ // *** Major scenarios tested by the UpgradeSuite *** // Setup @@ -92,15 +97,16 @@ func (s *UpgradeSuite) TearDownSuite() { func (s *UpgradeSuite) TestUpgradeToMaster() { systemNamespace := installer.SystemNamespace(s.namespace) - // Create block, object, and file storage on 0.8 before the upgrade + // + // Create block, object, and file storage on 1.0 before the upgrade + // poolName := "upgradepool" storageClassName := "block-upgrade" blockName := "block-claim-upgrade" - podName := "test-pod-upgrade" logger.Infof("Initializing block before the upgrade") - setupBlockLite(s.helper, s.k8sh, s.Suite, s.namespace, poolName, storageClassName, blockName, podName, s.op.installer.CephVersion) - createPodWithBlock(s.helper, s.k8sh, s.Suite, s.namespace, blockName, podName) - defer blockTestDataCleanUp(s.helper, s.k8sh, s.Suite, s.namespace, poolName, storageClassName, blockName, podName) + setupBlockLite(s.helper, s.k8sh, s.Suite, s.namespace, poolName, storageClassName, blockName, rbdPodName, s.op.installer.CephVersion) + createPodWithBlock(s.helper, s.k8sh, s.Suite, s.namespace, blockName, rbdPodName) + defer blockTestDataCleanUp(s.helper, s.k8sh, s.Suite, s.namespace, poolName, storageClassName, blockName, rbdPodName) logger.Infof("Initializing file before the upgrade") filesystemName := "upgrade-test-fs" @@ -116,18 +122,18 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.namespace, objectStoreName, 1) // verify that we're actually running the right pre-upgrade image - operatorContainer := "rook-ceph-operator" - version, err := k8sutil.GetDeploymentImage(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer) - assert.NoError(s.T(), err) - assert.Equal(s.T(), "rook/ceph:"+installer.Version1_0, version) + s.VerifyOperatorImage("rook/ceph:" + installer.Version1_0) message := "my simple message" preFilename := "pre-upgrade-file" - assert.NoError(s.T(), s.k8sh.WriteToPod("", podName, preFilename, message)) - assert.NoError(s.T(), s.k8sh.ReadFromPod("", podName, preFilename, message)) + assert.NoError(s.T(), s.k8sh.WriteToPod("", rbdPodName, preFilename, message)) + assert.NoError(s.T(), s.k8sh.ReadFromPod("", rbdPodName, preFilename, message)) assert.NoError(s.T(), s.k8sh.WriteToPod(s.namespace, filePodName, preFilename, message)) assert.NoError(s.T(), s.k8sh.ReadFromPod(s.namespace, filePodName, preFilename, message)) + // we will keep appending to this to continue verifying old files through the upgrades + oldFilesToRead := []string{preFilename} + // Gather logs before Ceph upgrade to help with debugging if installer.Env.Logs == "all" { s.k8sh.PrintPodDescribe(s.namespace) @@ -147,14 +153,11 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { numOSDs := len(osdDeps) // there should be this many upgraded OSDs require.True(s.T(), numOSDs > 0) d := osdDeps[0] - oldRookVersion := d.Labels["rook-version"] // upgraded OSDs should not have this version label oldCephVersion := d.Labels["ceph-version"] // upgraded OSDs should not have this version label - // Update to the next version of cluster roles before the operator is restarted - err = s.updateClusterRoles() - require.NoError(s.T(), err) - - // Upgrade Ceph version + // + // Upgrade Ceph version from Mimic to Nautilus before upgrading to Rook v1.1 + // s.k8sh.Kubectl("-n", s.namespace, "patch", "CephCluster", s.namespace, "--type=merge", "-p", fmt.Sprintf(`{"spec": {"cephVersion": {"image": "%s"}}}`, installer.NautilusVersion.Image)) @@ -192,144 +195,140 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { n = strings.Replace(s.T().Name(), "/", "_", -1) + "_after_ceph_upgrade" s.op.installer.GatherAllRookLogs(n, systemNamespace, s.namespace) - // Upgrade Rook to master - require.NoError(s.T(), s.k8sh.SetDeploymentVersion(systemNamespace, operatorContainer, operatorContainer, installer.VersionMaster)) + // + // Upgrade Rook from v1.0 to v1.1 + // + m1 := installer.CephManifestsV1_1{ + K8sh: s.k8sh, + Namespace: s.namespace, + SystemNamespace: systemNamespace, + OperatorContainer: operatorContainer, + T: s.T, + } + m1.UpgradeToV1_1() + + s.VerifyOperatorImage(m1.RookImage()) + s.VerifyRookUpgrade(numMons, numOSDs) + logger.Infof("Done with automatic upgrade from v1.0 to v1.1") + newFile := "post-upgrade-1_0-to-1_1-file" + s.VerifyFilesAfterUpgrade(filesystemName, newFile, message, oldFilesToRead) + oldFilesToRead = append(oldFilesToRead, newFile) + logger.Infof("Verified upgrade from v1.0 to v1.1") + + // + // Upgrade Rook from v1.1 to v1.2 + // + m2 := installer.CephManifestsV1_2{ + K8sh: s.k8sh, + Namespace: s.namespace, + SystemNamespace: systemNamespace, + OperatorContainer: operatorContainer, + T: s.T, + } + m2.UpgradeToV1_2() + + s.VerifyOperatorImage(m2.RookImage()) + s.VerifyRookUpgrade(numMons, numOSDs) + logger.Infof("Done with automatic upgrade from v1.1 to v1.2") + newFile = "post-upgrade-1_1-to-1_2-file" + s.VerifyFilesAfterUpgrade(filesystemName, newFile, message, oldFilesToRead) + oldFilesToRead = append(oldFilesToRead, newFile) + logger.Infof("Verified upgrade from v1.1 to v1.2") + + // + // Upgrade Rook from v1.2 to v1.3 (master) + // + m3 := installer.CephManifestsV1_3{ + K8sh: s.k8sh, + Namespace: s.namespace, + SystemNamespace: systemNamespace, + OperatorContainer: operatorContainer, + T: s.T, + } + m3.UpgradeToV1_3() + + s.VerifyOperatorImage(m3.RookImage()) + s.VerifyRookUpgrade(numMons, numOSDs) + logger.Infof("Done with automatic upgrade from v1.2 to v1.3") + newFile = "post-upgrade-1_2-to-1_3-file" + s.VerifyFilesAfterUpgrade(filesystemName, newFile, message, oldFilesToRead) + oldFilesToRead = append(oldFilesToRead, newFile) + logger.Infof("Verified upgrade from v1.2 to v1.3") +} + +func (s *UpgradeSuite) VerifyOperatorImage(expectedImage string) { + systemNamespace := installer.SystemNamespace(s.namespace) // verify that the operator spec is updated - version, err = k8sutil.GetDeploymentImage(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer) + version, err := k8sutil.GetDeploymentImage(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer) assert.NoError(s.T(), err) - assert.Equal(s.T(), "rook/ceph:"+installer.VersionMaster, version) + assert.Equal(s.T(), expectedImage, version) +} - // wait for the mon pods to be running. we can no longer check a version in the pod spec, so we just wait a bit. - time.Sleep(15 * time.Second) +func (s *UpgradeSuite) VerifyRookUpgrade(numMons, numOSDs int) { + // Get some info about the currently deployed mons to determine later if they are all updated + monDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mon") + require.NoError(s.T(), err) + require.Equal(s.T(), numMons, len(monDepList.Items)) - monsNotOldVersion = fmt.Sprintf("app=rook-ceph-mon,rook-version!=%s", oldRookVersion) + // Get some info about the currently deployed OSDs to determine later if they are all updated + osdDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd") + require.NoError(s.T(), err) + require.NotZero(s.T(), len(osdDepList.Items)) + require.Equal(s.T(), numOSDs, len(osdDepList.Items)) + + d := osdDepList.Items[0] + oldRookVersion := d.Labels["rook-version"] // upgraded OSDs should not have this version label + + monsNotOldVersion := fmt.Sprintf("app=rook-ceph-mon,rook-version!=%s", oldRookVersion) err = s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, numMons) require.NoError(s.T(), err) err = s.k8sh.WaitForLabeledDeploymentsToBeReady(monsNotOldVersion, s.namespace) require.NoError(s.T(), err) // wait for the osd pods to be updated - osdsNotOldVersion = fmt.Sprintf("app=rook-ceph-osd,rook-version!=%s", oldRookVersion) + osdsNotOldVersion := fmt.Sprintf("app=rook-ceph-osd,rook-version!=%s", oldRookVersion) err = s.k8sh.WaitForDeploymentCount(osdsNotOldVersion, s.namespace, numOSDs) require.NoError(s.T(), err) err = s.k8sh.WaitForLabeledDeploymentsToBeReady(osdsNotOldVersion, s.namespace) require.NoError(s.T(), err) - mdsesNotOldVersion = fmt.Sprintf("app=rook-ceph-mds,rook-version!=%s", oldRookVersion) + mdsesNotOldVersion := fmt.Sprintf("app=rook-ceph-mds,rook-version!=%s", oldRookVersion) err = s.k8sh.WaitForDeploymentCount(mdsesNotOldVersion, s.namespace, 4 /* always expect 4 mdses */) require.NoError(s.T(), err) err = s.k8sh.WaitForLabeledDeploymentsToBeReady(mdsesNotOldVersion, s.namespace) require.NoError(s.T(), err) - logger.Infof("Done with automatic upgrade to master") + // rgwsNotOldVersion + // There is an unlikely corner case with RGWs where the upgrade will fail on upgrades regarding + // a pool added for erasure coding. This hasn't been observed by users, so we ignore this + // currently. // Give a few seconds for the daemons to settle down after the upgrade time.Sleep(5 * time.Second) +} + +func (s *UpgradeSuite) VerifyFilesAfterUpgrade(fsName, newFileToWrite, messageForAllFiles string, oldFilesToRead []string) { + retryCount := 5 // wait for filesystem to be active - err = waitForFilesystemActive(s.k8sh, s.namespace, filesystemName) + err := waitForFilesystemActive(s.k8sh, s.namespace, fsName) require.NoError(s.T(), err) - // Test writing and reading from the pod with cephfs mounted that was created before the upgrade. - postFilename := "post-upgrade-file" - assert.NoError(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, preFilename, message, 5)) - assert.NoError(s.T(), s.k8sh.WriteToPodRetry(s.namespace, filePodName, postFilename, message, 5)) - assert.NoError(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, postFilename, message, 5)) + for _, file := range oldFilesToRead { + // test reading preexisting files in the pod with cephfs mounted + assert.NoError(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, file, messageForAllFiles, retryCount)) - // Test writing and reading from the pod with rbd mounted that was created before the upgrade. - // There is some unreliability right after the upgrade when there is only one osd, so we will retry if needed - assert.NoError(s.T(), s.k8sh.ReadFromPodRetry("", podName, preFilename, message, 5)) - assert.NoError(s.T(), s.k8sh.WriteToPodRetry("", podName, postFilename, message, 5)) - assert.NoError(s.T(), s.k8sh.ReadFromPodRetry("", podName, postFilename, message, 5)) -} + // test reading preexisting files in the pod with rbd mounted + // There is some unreliability right after the upgrade when there is only one osd, so we will retry if needed + assert.NoError(s.T(), s.k8sh.ReadFromPodRetry("", rbdPodName, file, messageForAllFiles, retryCount)) + } -// Update the clusterroles that have been modified in master from the previous release -func (s *UpgradeSuite) updateClusterRoles() error { - logger.Infof("Placeholder: create the new resources that have been added since 1.0") - namespace := s.namespace - newResources := ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - verbs: - - get - - list - - watch - - create - - update - - delete ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-cmd-reporter -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-cmd-reporter-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-osd -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list ---- -# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-osd -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-osd -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: ` + namespace + ` -` - logger.Infof("creating the new resources that have been added since 1.0") - return s.k8sh.ResourceOperation("apply", newResources) + // test writing and reading a new file in the pod with cephfs mounted + assert.NoError(s.T(), s.k8sh.WriteToPodRetry(s.namespace, filePodName, newFileToWrite, messageForAllFiles, retryCount)) + assert.NoError(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, newFileToWrite, messageForAllFiles, retryCount)) + + // test writing and reading a new file in the pod with rbd mounted + assert.NoError(s.T(), s.k8sh.WriteToPodRetry("", rbdPodName, newFileToWrite, messageForAllFiles, retryCount)) + assert.NoError(s.T(), s.k8sh.ReadFromPodRetry("", rbdPodName, newFileToWrite, messageForAllFiles, retryCount)) }