diff --git a/tests/framework/installer/ceph_manifests_v1.0.go b/tests/framework/installer/ceph_manifests_v1.0.go index d38605cf9..30b15a457 100644 --- a/tests/framework/installer/ceph_manifests_v1.0.go +++ b/tests/framework/installer/ceph_manifests_v1.0.go @@ -17,16 +17,27 @@ limitations under the License. package installer import ( + "fmt" "strconv" "github.com/google/uuid" ) -// CephManifestsV1_0 wraps rook yaml definitions +const ( + // Version tag for Rook v1.0 + Version1_0 = "v1.0.6" +) + +// CephManifestsV1_0 wraps rook yaml definitions for Rook-Ceph v1.0 manifests type CephManifestsV1_0 struct { imageTag string } +// RookImage returns the rook image under test for v1.0 +func (m *CephManifestsV1_0) RookImage() string { + return fmt.Sprintf("rook/ceph:%s", Version1_0) +} + func (m *CephManifestsV1_0) GetRookCRDs() string { return ` apiVersion: apiextensions.k8s.io/v1beta1 diff --git a/tests/framework/installer/ceph_manifests_v1.1.go b/tests/framework/installer/ceph_manifests_v1.1.go new file mode 100644 index 000000000..3bda1fd0e --- /dev/null +++ b/tests/framework/installer/ceph_manifests_v1.1.go @@ -0,0 +1,490 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "fmt" + "testing" + + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/require" +) + +const ( + version1_1 = "v1.1.8" +) + +// CephManifestsV1_1 wraps rook yaml definitions for Rook-Ceph v1.1 manifests +type CephManifestsV1_1 struct { + K8sh *utils.K8sHelper + Namespace string + SystemNamespace string + OperatorContainer string + T func() *testing.T +} + +// UpgradeToV1_1 performs the steps necessary to upgrade a Rook v1.0 cluster to v1.1. It does not +// verify the upgrade but merely starts the upgrade process. +func (m *CephManifestsV1_1) UpgradeToV1_1() { + require.NoError(m.T(), m.K8sh.ResourceOperation("create", m.createManifest())) + require.NoError(m.T(), m.K8sh.ResourceOperation("apply", m.applyManifest())) + + require.NoError(m.T(), + m.K8sh.SetDeploymentVersion(m.SystemNamespace, m.OperatorContainer, m.OperatorContainer, version1_1)) +} + +// RookImage returns the rook image under test for v1.1 +func (m *CephManifestsV1_1) RookImage() string { + return fmt.Sprintf("rook/ceph:%s", version1_1) +} + +func (m *CephManifestsV1_1) createManifest() string { + return ` +# Service account, role, and role binding to allow cmd-reporter job to run in Ceph cluster namespace +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-cmd-reporter + namespace: ` + m.Namespace + ` +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cmd-reporter + namespace: ` + m.Namespace + ` +rules: +- apiGroups: + - "" + resources: + - pods + - configmaps + verbs: + - get + - list + - watch + - create + - update + - delete +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-cmd-reporter + namespace: ` + m.Namespace + ` +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-cmd-reporter +subjects: +- kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: ` + m.Namespace + ` + + +# CRDs, role, and binding for Object Buckets +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: objectbuckets.objectbucket.io +spec: + group: objectbucket.io + versions: + - name: v1alpha1 + served: true + storage: true + names: + kind: ObjectBucket + listKind: ObjectBucketList + plural: objectbuckets + singular: objectbucket + shortNames: + - ob + - obs + scope: Cluster + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: objectbucketclaims.objectbucket.io +spec: + versions: + - name: v1alpha1 + served: true + storage: true + group: objectbucket.io + names: + kind: ObjectBucketClaim + listKind: ObjectBucketClaimList + plural: objectbucketclaims + singular: objectbucketclaim + shortNames: + - obc + - obcs + scope: Namespaced + subresources: + status: {} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-object-bucket + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true" +rules: +- apiGroups: + - "" + verbs: + - "*" + resources: + - secrets + - configmaps +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - "objectbucket.io" + verbs: + - "*" + resources: + - "*" +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-object-bucket +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-object-bucket +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: ` + m.SystemNamespace + ` +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: ` + m.SystemNamespace + ` + name: cephfs-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-provisioner-role-cfg + namespace: ` + m.SystemNamespace + ` +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-provisioner-sa + namespace: ` + m.SystemNamespace + ` +roleRef: + kind: Role + name: cephfs-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: ` + m.SystemNamespace + ` + name: rbd-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-role-cfg + namespace: ` + m.SystemNamespace + ` +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-provisioner-sa + namespace: ` + m.SystemNamespace + ` +roleRef: + kind: Role + name: rbd-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io +` +} + +func (m *CephManifestsV1_1) applyManifest() string { + return ` +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-global-rules + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" +rules: +- apiGroups: + - "" + resources: + # Pod access is needed for fencing + - pods + # Node access is needed for determining nodes where mons should run + - nodes + - nodes/proxy + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + # PVs and PVCs are managed by the Rook provisioner + - persistentvolumes + - persistentvolumeclaims + - endpoints + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - ceph.rook.io + resources: + - "*" + verbs: + - "*" +- apiGroups: + - rook.io + resources: + - "*" + verbs: + - "*" +- apiGroups: + - policy + - apps + resources: + #this is for the clusterdisruption controller + - poddisruptionbudgets + #this is for both clusterdisruption and nodedrain controllers + - deployments + verbs: + - "*" +- apiGroups: + - healthchecking.openshift.io + resources: + - machinedisruptionbudgets + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - machine.openshift.io + resources: + - machines + verbs: + - get + - list + - watch + - create + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: rook-ceph-system + namespace: ` + m.SystemNamespace + ` + labels: + operator: rook + storage-backend: ceph +rules: + - apiGroups: + - "" + resources: + - pods + - configmaps + - services + verbs: + - get + - list + - watch + - patch + - create + - update + - delete + - apiGroups: + - apps + resources: + - daemonsets + - statefulsets + - deployments + verbs: + - get + - list + - watch + - create + - update + - delete +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-external-provisioner-runner-rules + labels: + rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true" +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin-rules + labels: + rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true" +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-external-provisioner-runner-rules + labels: + rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true" +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete", "update"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete", "get", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] +` +} diff --git a/tests/framework/installer/ceph_manifests_v1.2.go b/tests/framework/installer/ceph_manifests_v1.2.go new file mode 100644 index 000000000..27547bd80 --- /dev/null +++ b/tests/framework/installer/ceph_manifests_v1.2.go @@ -0,0 +1,213 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "fmt" + "testing" + + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/require" +) + +const ( + version1_2 = "v1.2.1" +) + +// CephManifestsV1_2 wraps rook yaml definitions for Rook-Ceph v1.2 manifests +type CephManifestsV1_2 struct { + K8sh *utils.K8sHelper + Namespace string + SystemNamespace string + OperatorContainer string + T func() *testing.T +} + +// UpgradeToV1_2 performs the steps necessary to upgrade a Rook v1.1 cluster to v1.2. It does not +// verify the upgrade but merely starts the upgrade process. +func (m *CephManifestsV1_2) UpgradeToV1_2() { + require.NoError(m.T(), m.K8sh.ResourceOperation("apply", m.applyManifest())) + + require.NoError(m.T(), + m.K8sh.SetDeploymentVersion(m.SystemNamespace, m.OperatorContainer, m.OperatorContainer, version1_2)) +} + +// RookImage returns the rook image under test for v1.2 +func (m *CephManifestsV1_2) RookImage() string { + return fmt.Sprintf("rook/ceph:%s", version1_2) +} + +func (m *CephManifestsV1_2) applyManifest() string { + return ` +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: rook-ceph-global-rules + labels: + operator: rook + storage-backend: ceph + rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true" +rules: +- apiGroups: + - "" + resources: + # Pod access is needed for fencing + - pods + # Node access is needed for determining nodes where mons should run + - nodes + - nodes/proxy + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + # PVs and PVCs are managed by the Rook provisioner + - persistentvolumes + - persistentvolumeclaims + - endpoints + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - ceph.rook.io + resources: + - "*" + verbs: + - "*" +- apiGroups: + - rook.io + resources: + - "*" + verbs: + - "*" +- apiGroups: + - policy + - apps + resources: + # This is for the clusterdisruption controller + - poddisruptionbudgets + # This is for both clusterdisruption and nodedrain controllers + - deployments + - replicasets + verbs: + - "*" +- apiGroups: + - healthchecking.openshift.io + resources: + - machinedisruptionbudgets + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - machine.openshift.io + resources: + - machines + verbs: + - get + - list + - watch + - create + - update + - delete +- apiGroups: + - storage.k8s.io + resources: + - csidrivers + verbs: + - create +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: cephclients.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephClient + listKind: CephClientList + plural: cephclients + singular: cephclient + scope: Namespaced + version: v1 + validation: + openAPIV3Schema: + properties: + spec: + properties: + caps: + type: object +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-osd + namespace: ` + m.Namespace + ` +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list +--- +# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: rook-ceph-osd +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-osd +subjects: +- kind: ServiceAccount + name: rook-ceph-osd + namespace: ` + m.Namespace + ` +` +} diff --git a/tests/framework/installer/ceph_manifests_v1.3.go b/tests/framework/installer/ceph_manifests_v1.3.go new file mode 100644 index 000000000..e8c06dcde --- /dev/null +++ b/tests/framework/installer/ceph_manifests_v1.3.go @@ -0,0 +1,52 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package installer + +import ( + "fmt" + "testing" + + "github.com/rook/rook/tests/framework/utils" + "github.com/stretchr/testify/require" +) + +const ( + version1_3 = "master" // v1.3 is not yet released, so test upgrade to master until then +) + +// CephManifestsV1_3 wraps rook yaml definitions for Rook-Ceph v1.3 manifests +type CephManifestsV1_3 struct { + K8sh *utils.K8sHelper + Namespace string + SystemNamespace string + OperatorContainer string + T func() *testing.T +} + +// RookImage returns the rook image under test for v1.2 +func (m *CephManifestsV1_3) RookImage() string { + return fmt.Sprintf("rook/ceph:%s", version1_3) +} + +// UpgradeToV1_3 performs the steps necessary to upgrade a Rook v1.1 cluster to v1.2. It does not +// verify the upgrade but merely starts the upgrade process. +func (m *CephManifestsV1_3) UpgradeToV1_3() { + // no special manifest operations (apply, create, delete) yet + + require.NoError(m.T(), + m.K8sh.SetDeploymentVersion(m.SystemNamespace, m.OperatorContainer, m.OperatorContainer, version1_3)) +} diff --git a/tests/framework/installer/installer.go b/tests/framework/installer/installer.go index d302eed6e..62200a7bc 100644 --- a/tests/framework/installer/installer.go +++ b/tests/framework/installer/installer.go @@ -28,10 +28,9 @@ import ( ) const ( - // Version tag for the latest manifests + // VersionMaster tag for the latest manifests VersionMaster = "master" - // Version tag for Rook v1.0 - Version1_0 = "v1.0.6" + // test suite names CassandraTestSuite = "cassandra" CephTestSuite = "ceph" diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index de2eddaed..7a4bde188 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -32,6 +32,11 @@ import ( "github.com/stretchr/testify/suite" ) +const ( + rbdPodName = "test-pod-upgrade" + operatorContainer = "rook-ceph-operator" +) + // ************************************************ // *** Major scenarios tested by the UpgradeSuite *** // Setup @@ -92,15 +97,16 @@ func (s *UpgradeSuite) TearDownSuite() { func (s *UpgradeSuite) TestUpgradeToMaster() { systemNamespace := installer.SystemNamespace(s.namespace) - // Create block, object, and file storage on 0.8 before the upgrade + // + // Create block, object, and file storage on 1.0 before the upgrade + // poolName := "upgradepool" storageClassName := "block-upgrade" blockName := "block-claim-upgrade" - podName := "test-pod-upgrade" logger.Infof("Initializing block before the upgrade") - setupBlockLite(s.helper, s.k8sh, s.Suite, s.namespace, poolName, storageClassName, blockName, podName, s.op.installer.CephVersion) - createPodWithBlock(s.helper, s.k8sh, s.Suite, s.namespace, blockName, podName) - defer blockTestDataCleanUp(s.helper, s.k8sh, s.Suite, s.namespace, poolName, storageClassName, blockName, podName) + setupBlockLite(s.helper, s.k8sh, s.Suite, s.namespace, poolName, storageClassName, blockName, rbdPodName, s.op.installer.CephVersion) + createPodWithBlock(s.helper, s.k8sh, s.Suite, s.namespace, blockName, rbdPodName) + defer blockTestDataCleanUp(s.helper, s.k8sh, s.Suite, s.namespace, poolName, storageClassName, blockName, rbdPodName) logger.Infof("Initializing file before the upgrade") filesystemName := "upgrade-test-fs" @@ -116,18 +122,18 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.namespace, objectStoreName, 1) // verify that we're actually running the right pre-upgrade image - operatorContainer := "rook-ceph-operator" - version, err := k8sutil.GetDeploymentImage(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer) - assert.NoError(s.T(), err) - assert.Equal(s.T(), "rook/ceph:"+installer.Version1_0, version) + s.VerifyOperatorImage("rook/ceph:" + installer.Version1_0) message := "my simple message" preFilename := "pre-upgrade-file" - assert.NoError(s.T(), s.k8sh.WriteToPod("", podName, preFilename, message)) - assert.NoError(s.T(), s.k8sh.ReadFromPod("", podName, preFilename, message)) + assert.NoError(s.T(), s.k8sh.WriteToPod("", rbdPodName, preFilename, message)) + assert.NoError(s.T(), s.k8sh.ReadFromPod("", rbdPodName, preFilename, message)) assert.NoError(s.T(), s.k8sh.WriteToPod(s.namespace, filePodName, preFilename, message)) assert.NoError(s.T(), s.k8sh.ReadFromPod(s.namespace, filePodName, preFilename, message)) + // we will keep appending to this to continue verifying old files through the upgrades + oldFilesToRead := []string{preFilename} + // Gather logs before Ceph upgrade to help with debugging if installer.Env.Logs == "all" { s.k8sh.PrintPodDescribe(s.namespace) @@ -147,14 +153,11 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { numOSDs := len(osdDeps) // there should be this many upgraded OSDs require.True(s.T(), numOSDs > 0) d := osdDeps[0] - oldRookVersion := d.Labels["rook-version"] // upgraded OSDs should not have this version label oldCephVersion := d.Labels["ceph-version"] // upgraded OSDs should not have this version label - // Update to the next version of cluster roles before the operator is restarted - err = s.updateClusterRoles() - require.NoError(s.T(), err) - - // Upgrade Ceph version + // + // Upgrade Ceph version from Mimic to Nautilus before upgrading to Rook v1.1 + // s.k8sh.Kubectl("-n", s.namespace, "patch", "CephCluster", s.namespace, "--type=merge", "-p", fmt.Sprintf(`{"spec": {"cephVersion": {"image": "%s"}}}`, installer.NautilusVersion.Image)) @@ -192,144 +195,140 @@ func (s *UpgradeSuite) TestUpgradeToMaster() { n = strings.Replace(s.T().Name(), "/", "_", -1) + "_after_ceph_upgrade" s.op.installer.GatherAllRookLogs(n, systemNamespace, s.namespace) - // Upgrade Rook to master - require.NoError(s.T(), s.k8sh.SetDeploymentVersion(systemNamespace, operatorContainer, operatorContainer, installer.VersionMaster)) + // + // Upgrade Rook from v1.0 to v1.1 + // + m1 := installer.CephManifestsV1_1{ + K8sh: s.k8sh, + Namespace: s.namespace, + SystemNamespace: systemNamespace, + OperatorContainer: operatorContainer, + T: s.T, + } + m1.UpgradeToV1_1() + + s.VerifyOperatorImage(m1.RookImage()) + s.VerifyRookUpgrade(numMons, numOSDs) + logger.Infof("Done with automatic upgrade from v1.0 to v1.1") + newFile := "post-upgrade-1_0-to-1_1-file" + s.VerifyFilesAfterUpgrade(filesystemName, newFile, message, oldFilesToRead) + oldFilesToRead = append(oldFilesToRead, newFile) + logger.Infof("Verified upgrade from v1.0 to v1.1") + + // + // Upgrade Rook from v1.1 to v1.2 + // + m2 := installer.CephManifestsV1_2{ + K8sh: s.k8sh, + Namespace: s.namespace, + SystemNamespace: systemNamespace, + OperatorContainer: operatorContainer, + T: s.T, + } + m2.UpgradeToV1_2() + + s.VerifyOperatorImage(m2.RookImage()) + s.VerifyRookUpgrade(numMons, numOSDs) + logger.Infof("Done with automatic upgrade from v1.1 to v1.2") + newFile = "post-upgrade-1_1-to-1_2-file" + s.VerifyFilesAfterUpgrade(filesystemName, newFile, message, oldFilesToRead) + oldFilesToRead = append(oldFilesToRead, newFile) + logger.Infof("Verified upgrade from v1.1 to v1.2") + + // + // Upgrade Rook from v1.2 to v1.3 (master) + // + m3 := installer.CephManifestsV1_3{ + K8sh: s.k8sh, + Namespace: s.namespace, + SystemNamespace: systemNamespace, + OperatorContainer: operatorContainer, + T: s.T, + } + m3.UpgradeToV1_3() + + s.VerifyOperatorImage(m3.RookImage()) + s.VerifyRookUpgrade(numMons, numOSDs) + logger.Infof("Done with automatic upgrade from v1.2 to v1.3") + newFile = "post-upgrade-1_2-to-1_3-file" + s.VerifyFilesAfterUpgrade(filesystemName, newFile, message, oldFilesToRead) + oldFilesToRead = append(oldFilesToRead, newFile) + logger.Infof("Verified upgrade from v1.2 to v1.3") +} + +func (s *UpgradeSuite) VerifyOperatorImage(expectedImage string) { + systemNamespace := installer.SystemNamespace(s.namespace) // verify that the operator spec is updated - version, err = k8sutil.GetDeploymentImage(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer) + version, err := k8sutil.GetDeploymentImage(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer) assert.NoError(s.T(), err) - assert.Equal(s.T(), "rook/ceph:"+installer.VersionMaster, version) + assert.Equal(s.T(), expectedImage, version) +} - // wait for the mon pods to be running. we can no longer check a version in the pod spec, so we just wait a bit. - time.Sleep(15 * time.Second) +func (s *UpgradeSuite) VerifyRookUpgrade(numMons, numOSDs int) { + // Get some info about the currently deployed mons to determine later if they are all updated + monDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mon") + require.NoError(s.T(), err) + require.Equal(s.T(), numMons, len(monDepList.Items)) - monsNotOldVersion = fmt.Sprintf("app=rook-ceph-mon,rook-version!=%s", oldRookVersion) + // Get some info about the currently deployed OSDs to determine later if they are all updated + osdDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd") + require.NoError(s.T(), err) + require.NotZero(s.T(), len(osdDepList.Items)) + require.Equal(s.T(), numOSDs, len(osdDepList.Items)) + + d := osdDepList.Items[0] + oldRookVersion := d.Labels["rook-version"] // upgraded OSDs should not have this version label + + monsNotOldVersion := fmt.Sprintf("app=rook-ceph-mon,rook-version!=%s", oldRookVersion) err = s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, numMons) require.NoError(s.T(), err) err = s.k8sh.WaitForLabeledDeploymentsToBeReady(monsNotOldVersion, s.namespace) require.NoError(s.T(), err) // wait for the osd pods to be updated - osdsNotOldVersion = fmt.Sprintf("app=rook-ceph-osd,rook-version!=%s", oldRookVersion) + osdsNotOldVersion := fmt.Sprintf("app=rook-ceph-osd,rook-version!=%s", oldRookVersion) err = s.k8sh.WaitForDeploymentCount(osdsNotOldVersion, s.namespace, numOSDs) require.NoError(s.T(), err) err = s.k8sh.WaitForLabeledDeploymentsToBeReady(osdsNotOldVersion, s.namespace) require.NoError(s.T(), err) - mdsesNotOldVersion = fmt.Sprintf("app=rook-ceph-mds,rook-version!=%s", oldRookVersion) + mdsesNotOldVersion := fmt.Sprintf("app=rook-ceph-mds,rook-version!=%s", oldRookVersion) err = s.k8sh.WaitForDeploymentCount(mdsesNotOldVersion, s.namespace, 4 /* always expect 4 mdses */) require.NoError(s.T(), err) err = s.k8sh.WaitForLabeledDeploymentsToBeReady(mdsesNotOldVersion, s.namespace) require.NoError(s.T(), err) - logger.Infof("Done with automatic upgrade to master") + // rgwsNotOldVersion + // There is an unlikely corner case with RGWs where the upgrade will fail on upgrades regarding + // a pool added for erasure coding. This hasn't been observed by users, so we ignore this + // currently. // Give a few seconds for the daemons to settle down after the upgrade time.Sleep(5 * time.Second) +} + +func (s *UpgradeSuite) VerifyFilesAfterUpgrade(fsName, newFileToWrite, messageForAllFiles string, oldFilesToRead []string) { + retryCount := 5 // wait for filesystem to be active - err = waitForFilesystemActive(s.k8sh, s.namespace, filesystemName) + err := waitForFilesystemActive(s.k8sh, s.namespace, fsName) require.NoError(s.T(), err) - // Test writing and reading from the pod with cephfs mounted that was created before the upgrade. - postFilename := "post-upgrade-file" - assert.NoError(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, preFilename, message, 5)) - assert.NoError(s.T(), s.k8sh.WriteToPodRetry(s.namespace, filePodName, postFilename, message, 5)) - assert.NoError(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, postFilename, message, 5)) + for _, file := range oldFilesToRead { + // test reading preexisting files in the pod with cephfs mounted + assert.NoError(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, file, messageForAllFiles, retryCount)) - // Test writing and reading from the pod with rbd mounted that was created before the upgrade. - // There is some unreliability right after the upgrade when there is only one osd, so we will retry if needed - assert.NoError(s.T(), s.k8sh.ReadFromPodRetry("", podName, preFilename, message, 5)) - assert.NoError(s.T(), s.k8sh.WriteToPodRetry("", podName, postFilename, message, 5)) - assert.NoError(s.T(), s.k8sh.ReadFromPodRetry("", podName, postFilename, message, 5)) -} + // test reading preexisting files in the pod with rbd mounted + // There is some unreliability right after the upgrade when there is only one osd, so we will retry if needed + assert.NoError(s.T(), s.k8sh.ReadFromPodRetry("", rbdPodName, file, messageForAllFiles, retryCount)) + } -// Update the clusterroles that have been modified in master from the previous release -func (s *UpgradeSuite) updateClusterRoles() error { - logger.Infof("Placeholder: create the new resources that have been added since 1.0") - namespace := s.namespace - newResources := ` ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -rules: -- apiGroups: - - "" - resources: - - pods - - configmaps - verbs: - - get - - list - - watch - - create - - update - - delete ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: rook-ceph-cmd-reporter -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-cmd-reporter-psp - namespace: ` + namespace + ` -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: -- kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: ` + namespace + ` ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-osd -rules: -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list ---- -# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: rook-ceph-osd -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: rook-ceph-osd -subjects: -- kind: ServiceAccount - name: rook-ceph-osd - namespace: ` + namespace + ` -` - logger.Infof("creating the new resources that have been added since 1.0") - return s.k8sh.ResourceOperation("apply", newResources) + // test writing and reading a new file in the pod with cephfs mounted + assert.NoError(s.T(), s.k8sh.WriteToPodRetry(s.namespace, filePodName, newFileToWrite, messageForAllFiles, retryCount)) + assert.NoError(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, newFileToWrite, messageForAllFiles, retryCount)) + + // test writing and reading a new file in the pod with rbd mounted + assert.NoError(s.T(), s.k8sh.WriteToPodRetry("", rbdPodName, newFileToWrite, messageForAllFiles, retryCount)) + assert.NoError(s.T(), s.k8sh.ReadFromPodRetry("", rbdPodName, newFileToWrite, messageForAllFiles, retryCount)) }