ceph: upgrade test versions v1.0->v1.1->v1.2->v1.3

In the Ceph upgrade integration test, upgrade from Rook v1.0 with Ceph
Mimic v13 to install legacy disk-based OSDs. Then upgrade Ceph to
Nautilus, then Rook to v1.1, then v1.2, then v1.3 (master currently) to
make sure that Rook is able to run legacy OSDs created without
ceph-volume.

Signed-off-by: Blaine Gardner <blaine.gardner@suse.com>
This commit is contained in:
Blaine Gardner committed 2020-01-13 08:56:35 -07:00
1 parent c40763c5c6
commit 2e242ca886
6 files changed
+895 -131

No files matched your search

@@ -17,16 +17,27 @@ limitations under the License.
package installer
import (
"fmt"
"strconv"
"github.com/google/uuid"
)
// CephManifestsV1_0 wraps rook yaml definitions
const (
// Version tag for Rook v1.0
Version1_0 = "v1.0.6"
)
// CephManifestsV1_0 wraps rook yaml definitions for Rook-Ceph v1.0 manifests
type CephManifestsV1_0 struct {
imageTag string
}
// RookImage returns the rook image under test for v1.0
func (m *CephManifestsV1_0) RookImage() string {
return fmt.Sprintf("rook/ceph:%s", Version1_0)
}
func (m *CephManifestsV1_0) GetRookCRDs() string {
return `
apiVersion: apiextensions.k8s.io/v1beta1
@@ -0,0 +1,490 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package installer
import (
"fmt"
"testing"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/require"
)
const (
version1_1 = "v1.1.8"
)
// CephManifestsV1_1 wraps rook yaml definitions for Rook-Ceph v1.1 manifests
type CephManifestsV1_1 struct {
K8sh *utils.K8sHelper
Namespace string
SystemNamespace string
OperatorContainer string
T func() *testing.T
}
// UpgradeToV1_1 performs the steps necessary to upgrade a Rook v1.0 cluster to v1.1. It does not
// verify the upgrade but merely starts the upgrade process.
func (m *CephManifestsV1_1) UpgradeToV1_1() {
require.NoError(m.T(), m.K8sh.ResourceOperation("create", m.createManifest()))
require.NoError(m.T(), m.K8sh.ResourceOperation("apply", m.applyManifest()))
require.NoError(m.T(),
m.K8sh.SetDeploymentVersion(m.SystemNamespace, m.OperatorContainer, m.OperatorContainer, version1_1))
}
// RookImage returns the rook image under test for v1.1
func (m *CephManifestsV1_1) RookImage() string {
return fmt.Sprintf("rook/ceph:%s", version1_1)
}
func (m *CephManifestsV1_1) createManifest() string {
return `
# Service account, role, and role binding to allow cmd-reporter job to run in Ceph cluster namespace
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + m.Namespace + `
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + m.Namespace + `
rules:
- apiGroups:
- ""
resources:
- pods
- configmaps
verbs:
- get
- list
- watch
- create
- update
- delete
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + m.Namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-cmd-reporter
subjects:
- kind: ServiceAccount
name: rook-ceph-cmd-reporter
namespace: ` + m.Namespace + `
# CRDs, role, and binding for Object Buckets
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: objectbuckets.objectbucket.io
spec:
group: objectbucket.io
versions:
- name: v1alpha1
served: true
storage: true
names:
kind: ObjectBucket
listKind: ObjectBucketList
plural: objectbuckets
singular: objectbucket
shortNames:
- ob
- obs
scope: Cluster
subresources:
status: {}
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: objectbucketclaims.objectbucket.io
spec:
versions:
- name: v1alpha1
served: true
storage: true
group: objectbucket.io
names:
kind: ObjectBucketClaim
listKind: ObjectBucketClaimList
plural: objectbucketclaims
singular: objectbucketclaim
shortNames:
- obc
- obcs
scope: Namespaced
subresources:
status: {}
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-object-bucket
labels:
operator: rook
storage-backend: ceph
rbac.ceph.rook.io/aggregate-to-rook-ceph-mgr-cluster: "true"
rules:
- apiGroups:
- ""
verbs:
- "*"
resources:
- secrets
- configmaps
- apiGroups:
- storage.k8s.io
resources:
- storageclasses
verbs:
- get
- list
- watch
- apiGroups:
- "objectbucket.io"
verbs:
- "*"
resources:
- "*"
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-object-bucket
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-object-bucket
subjects:
- kind: ServiceAccount
name: rook-ceph-system
namespace: ` + m.SystemNamespace + `
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
namespace: ` + m.SystemNamespace + `
name: cephfs-external-provisioner-cfg
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "create", "delete"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-provisioner-role-cfg
namespace: ` + m.SystemNamespace + `
subjects:
- kind: ServiceAccount
name: rook-csi-cephfs-provisioner-sa
namespace: ` + m.SystemNamespace + `
roleRef:
kind: Role
name: cephfs-external-provisioner-cfg
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
namespace: ` + m.SystemNamespace + `
name: rbd-external-provisioner-cfg
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-provisioner-role-cfg
namespace: ` + m.SystemNamespace + `
subjects:
- kind: ServiceAccount
name: rook-csi-rbd-provisioner-sa
namespace: ` + m.SystemNamespace + `
roleRef:
kind: Role
name: rbd-external-provisioner-cfg
apiGroup: rbac.authorization.k8s.io
`
}
func (m *CephManifestsV1_1) applyManifest() string {
return `
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-global-rules
labels:
operator: rook
storage-backend: ceph
rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true"
rules:
- apiGroups:
- ""
resources:
# Pod access is needed for fencing
- pods
# Node access is needed for determining nodes where mons should run
- nodes
- nodes/proxy
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
# PVs and PVCs are managed by the Rook provisioner
- persistentvolumes
- persistentvolumeclaims
- endpoints
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
- apiGroups:
- storage.k8s.io
resources:
- storageclasses
verbs:
- get
- list
- watch
- apiGroups:
- batch
resources:
- jobs
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- ceph.rook.io
resources:
- "*"
verbs:
- "*"
- apiGroups:
- rook.io
resources:
- "*"
verbs:
- "*"
- apiGroups:
- policy
- apps
resources:
#this is for the clusterdisruption controller
- poddisruptionbudgets
#this is for both clusterdisruption and nodedrain controllers
- deployments
verbs:
- "*"
- apiGroups:
- healthchecking.openshift.io
resources:
- machinedisruptionbudgets
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- machine.openshift.io
resources:
- machines
verbs:
- get
- list
- watch
- create
- update
- delete
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: rook-ceph-system
namespace: ` + m.SystemNamespace + `
labels:
operator: rook
storage-backend: ceph
rules:
- apiGroups:
- ""
resources:
- pods
- configmaps
- services
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
- apiGroups:
- apps
resources:
- daemonsets
- statefulsets
- deployments
verbs:
- get
- list
- watch
- create
- update
- delete
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-external-provisioner-runner-rules
labels:
rbac.ceph.rook.io/aggregate-to-cephfs-external-provisioner-runner: "true"
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete", "update"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-nodeplugin-rules
labels:
rbac.ceph.rook.io/aggregate-to-rbd-csi-nodeplugin: "true"
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "update"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list"]
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-external-provisioner-runner-rules
labels:
rbac.ceph.rook.io/aggregate-to-rbd-external-provisioner-runner: "true"
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete", "update"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["create", "list", "watch", "delete", "get", "update"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots/status"]
verbs: ["update"]
`
}
@@ -0,0 +1,213 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package installer
import (
"fmt"
"testing"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/require"
)
const (
version1_2 = "v1.2.1"
)
// CephManifestsV1_2 wraps rook yaml definitions for Rook-Ceph v1.2 manifests
type CephManifestsV1_2 struct {
K8sh *utils.K8sHelper
Namespace string
SystemNamespace string
OperatorContainer string
T func() *testing.T
}
// UpgradeToV1_2 performs the steps necessary to upgrade a Rook v1.1 cluster to v1.2. It does not
// verify the upgrade but merely starts the upgrade process.
func (m *CephManifestsV1_2) UpgradeToV1_2() {
require.NoError(m.T(), m.K8sh.ResourceOperation("apply", m.applyManifest()))
require.NoError(m.T(),
m.K8sh.SetDeploymentVersion(m.SystemNamespace, m.OperatorContainer, m.OperatorContainer, version1_2))
}
// RookImage returns the rook image under test for v1.2
func (m *CephManifestsV1_2) RookImage() string {
return fmt.Sprintf("rook/ceph:%s", version1_2)
}
func (m *CephManifestsV1_2) applyManifest() string {
return `
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: rook-ceph-global-rules
labels:
operator: rook
storage-backend: ceph
rbac.ceph.rook.io/aggregate-to-rook-ceph-global: "true"
rules:
- apiGroups:
- ""
resources:
# Pod access is needed for fencing
- pods
# Node access is needed for determining nodes where mons should run
- nodes
- nodes/proxy
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
# PVs and PVCs are managed by the Rook provisioner
- persistentvolumes
- persistentvolumeclaims
- endpoints
verbs:
- get
- list
- watch
- patch
- create
- update
- delete
- apiGroups:
- storage.k8s.io
resources:
- storageclasses
verbs:
- get
- list
- watch
- apiGroups:
- batch
resources:
- jobs
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- ceph.rook.io
resources:
- "*"
verbs:
- "*"
- apiGroups:
- rook.io
resources:
- "*"
verbs:
- "*"
- apiGroups:
- policy
- apps
resources:
# This is for the clusterdisruption controller
- poddisruptionbudgets
# This is for both clusterdisruption and nodedrain controllers
- deployments
- replicasets
verbs:
- "*"
- apiGroups:
- healthchecking.openshift.io
resources:
- machinedisruptionbudgets
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- machine.openshift.io
resources:
- machines
verbs:
- get
- list
- watch
- create
- update
- delete
- apiGroups:
- storage.k8s.io
resources:
- csidrivers
verbs:
- create
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: cephclients.ceph.rook.io
spec:
group: ceph.rook.io
names:
kind: CephClient
listKind: CephClientList
plural: cephclients
singular: cephclient
scope: Namespaced
version: v1
validation:
openAPIV3Schema:
properties:
spec:
properties:
caps:
type: object
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-osd
namespace: ` + m.Namespace + `
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- list
---
# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-osd
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-osd
subjects:
- kind: ServiceAccount
name: rook-ceph-osd
namespace: ` + m.Namespace + `
`
}
@@ -0,0 +1,52 @@
/*
Copyright 2016 The Rook Authors. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package installer
import (
"fmt"
"testing"
"github.com/rook/rook/tests/framework/utils"
"github.com/stretchr/testify/require"
)
const (
version1_3 = "master" // v1.3 is not yet released, so test upgrade to master until then
)
// CephManifestsV1_3 wraps rook yaml definitions for Rook-Ceph v1.3 manifests
type CephManifestsV1_3 struct {
K8sh *utils.K8sHelper
Namespace string
SystemNamespace string
OperatorContainer string
T func() *testing.T
}
// RookImage returns the rook image under test for v1.2
func (m *CephManifestsV1_3) RookImage() string {
return fmt.Sprintf("rook/ceph:%s", version1_3)
}
// UpgradeToV1_3 performs the steps necessary to upgrade a Rook v1.1 cluster to v1.2. It does not
// verify the upgrade but merely starts the upgrade process.
func (m *CephManifestsV1_3) UpgradeToV1_3() {
// no special manifest operations (apply, create, delete) yet
require.NoError(m.T(),
m.K8sh.SetDeploymentVersion(m.SystemNamespace, m.OperatorContainer, m.OperatorContainer, version1_3))
}
+2 -3
View File
@@ -28,10 +28,9 @@ import (
)
const (
// Version tag for the latest manifests
// VersionMaster tag for the latest manifests
VersionMaster = "master"
// Version tag for Rook v1.0
Version1_0 = "v1.0.6"
// test suite names
CassandraTestSuite = "cassandra"
CephTestSuite = "ceph"
+126 -127
View File
@@ -32,6 +32,11 @@ import (
"github.com/stretchr/testify/suite"
)
const (
rbdPodName = "test-pod-upgrade"
operatorContainer = "rook-ceph-operator"
)
// ************************************************
// *** Major scenarios tested by the UpgradeSuite ***
// Setup
@@ -92,15 +97,16 @@ func (s *UpgradeSuite) TearDownSuite() {
func (s *UpgradeSuite) TestUpgradeToMaster() {
systemNamespace := installer.SystemNamespace(s.namespace)
// Create block, object, and file storage on 0.8 before the upgrade
//
// Create block, object, and file storage on 1.0 before the upgrade
//
poolName := "upgradepool"
storageClassName := "block-upgrade"
blockName := "block-claim-upgrade"
podName := "test-pod-upgrade"
logger.Infof("Initializing block before the upgrade")
setupBlockLite(s.helper, s.k8sh, s.Suite, s.namespace, poolName, storageClassName, blockName, podName, s.op.installer.CephVersion)
createPodWithBlock(s.helper, s.k8sh, s.Suite, s.namespace, blockName, podName)
defer blockTestDataCleanUp(s.helper, s.k8sh, s.Suite, s.namespace, poolName, storageClassName, blockName, podName)
setupBlockLite(s.helper, s.k8sh, s.Suite, s.namespace, poolName, storageClassName, blockName, rbdPodName, s.op.installer.CephVersion)
createPodWithBlock(s.helper, s.k8sh, s.Suite, s.namespace, blockName, rbdPodName)
defer blockTestDataCleanUp(s.helper, s.k8sh, s.Suite, s.namespace, poolName, storageClassName, blockName, rbdPodName)
logger.Infof("Initializing file before the upgrade")
filesystemName := "upgrade-test-fs"
@@ -116,18 +122,18 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.namespace, objectStoreName, 1)
// verify that we're actually running the right pre-upgrade image
operatorContainer := "rook-ceph-operator"
version, err := k8sutil.GetDeploymentImage(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer)
assert.NoError(s.T(), err)
assert.Equal(s.T(), "rook/ceph:"+installer.Version1_0, version)
s.VerifyOperatorImage("rook/ceph:" + installer.Version1_0)
message := "my simple message"
preFilename := "pre-upgrade-file"
assert.NoError(s.T(), s.k8sh.WriteToPod("", podName, preFilename, message))
assert.NoError(s.T(), s.k8sh.ReadFromPod("", podName, preFilename, message))
assert.NoError(s.T(), s.k8sh.WriteToPod("", rbdPodName, preFilename, message))
assert.NoError(s.T(), s.k8sh.ReadFromPod("", rbdPodName, preFilename, message))
assert.NoError(s.T(), s.k8sh.WriteToPod(s.namespace, filePodName, preFilename, message))
assert.NoError(s.T(), s.k8sh.ReadFromPod(s.namespace, filePodName, preFilename, message))
// we will keep appending to this to continue verifying old files through the upgrades
oldFilesToRead := []string{preFilename}
// Gather logs before Ceph upgrade to help with debugging
if installer.Env.Logs == "all" {
s.k8sh.PrintPodDescribe(s.namespace)
@@ -147,14 +153,11 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
numOSDs := len(osdDeps) // there should be this many upgraded OSDs
require.True(s.T(), numOSDs > 0)
d := osdDeps[0]
oldRookVersion := d.Labels["rook-version"] // upgraded OSDs should not have this version label
oldCephVersion := d.Labels["ceph-version"] // upgraded OSDs should not have this version label
// Update to the next version of cluster roles before the operator is restarted
err = s.updateClusterRoles()
require.NoError(s.T(), err)
// Upgrade Ceph version
//
// Upgrade Ceph version from Mimic to Nautilus before upgrading to Rook v1.1
//
s.k8sh.Kubectl("-n", s.namespace, "patch", "CephCluster", s.namespace, "--type=merge",
"-p", fmt.Sprintf(`{"spec": {"cephVersion": {"image": "%s"}}}`, installer.NautilusVersion.Image))
@@ -192,144 +195,140 @@ func (s *UpgradeSuite) TestUpgradeToMaster() {
n = strings.Replace(s.T().Name(), "/", "_", -1) + "_after_ceph_upgrade"
s.op.installer.GatherAllRookLogs(n, systemNamespace, s.namespace)
// Upgrade Rook to master
require.NoError(s.T(), s.k8sh.SetDeploymentVersion(systemNamespace, operatorContainer, operatorContainer, installer.VersionMaster))
//
// Upgrade Rook from v1.0 to v1.1
//
m1 := installer.CephManifestsV1_1{
K8sh: s.k8sh,
Namespace: s.namespace,
SystemNamespace: systemNamespace,
OperatorContainer: operatorContainer,
T: s.T,
}
m1.UpgradeToV1_1()
s.VerifyOperatorImage(m1.RookImage())
s.VerifyRookUpgrade(numMons, numOSDs)
logger.Infof("Done with automatic upgrade from v1.0 to v1.1")
newFile := "post-upgrade-1_0-to-1_1-file"
s.VerifyFilesAfterUpgrade(filesystemName, newFile, message, oldFilesToRead)
oldFilesToRead = append(oldFilesToRead, newFile)
logger.Infof("Verified upgrade from v1.0 to v1.1")
//
// Upgrade Rook from v1.1 to v1.2
//
m2 := installer.CephManifestsV1_2{
K8sh: s.k8sh,
Namespace: s.namespace,
SystemNamespace: systemNamespace,
OperatorContainer: operatorContainer,
T: s.T,
}
m2.UpgradeToV1_2()
s.VerifyOperatorImage(m2.RookImage())
s.VerifyRookUpgrade(numMons, numOSDs)
logger.Infof("Done with automatic upgrade from v1.1 to v1.2")
newFile = "post-upgrade-1_1-to-1_2-file"
s.VerifyFilesAfterUpgrade(filesystemName, newFile, message, oldFilesToRead)
oldFilesToRead = append(oldFilesToRead, newFile)
logger.Infof("Verified upgrade from v1.1 to v1.2")
//
// Upgrade Rook from v1.2 to v1.3 (master)
//
m3 := installer.CephManifestsV1_3{
K8sh: s.k8sh,
Namespace: s.namespace,
SystemNamespace: systemNamespace,
OperatorContainer: operatorContainer,
T: s.T,
}
m3.UpgradeToV1_3()
s.VerifyOperatorImage(m3.RookImage())
s.VerifyRookUpgrade(numMons, numOSDs)
logger.Infof("Done with automatic upgrade from v1.2 to v1.3")
newFile = "post-upgrade-1_2-to-1_3-file"
s.VerifyFilesAfterUpgrade(filesystemName, newFile, message, oldFilesToRead)
oldFilesToRead = append(oldFilesToRead, newFile)
logger.Infof("Verified upgrade from v1.2 to v1.3")
}
func (s *UpgradeSuite) VerifyOperatorImage(expectedImage string) {
systemNamespace := installer.SystemNamespace(s.namespace)
// verify that the operator spec is updated
version, err = k8sutil.GetDeploymentImage(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer)
version, err := k8sutil.GetDeploymentImage(s.k8sh.Clientset, systemNamespace, operatorContainer, operatorContainer)
assert.NoError(s.T(), err)
assert.Equal(s.T(), "rook/ceph:"+installer.VersionMaster, version)
assert.Equal(s.T(), expectedImage, version)
}
// wait for the mon pods to be running. we can no longer check a version in the pod spec, so we just wait a bit.
time.Sleep(15 * time.Second)
func (s *UpgradeSuite) VerifyRookUpgrade(numMons, numOSDs int) {
// Get some info about the currently deployed mons to determine later if they are all updated
monDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-mon")
require.NoError(s.T(), err)
require.Equal(s.T(), numMons, len(monDepList.Items))
monsNotOldVersion = fmt.Sprintf("app=rook-ceph-mon,rook-version!=%s", oldRookVersion)
// Get some info about the currently deployed OSDs to determine later if they are all updated
osdDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd")
require.NoError(s.T(), err)
require.NotZero(s.T(), len(osdDepList.Items))
require.Equal(s.T(), numOSDs, len(osdDepList.Items))
d := osdDepList.Items[0]
oldRookVersion := d.Labels["rook-version"] // upgraded OSDs should not have this version label
monsNotOldVersion := fmt.Sprintf("app=rook-ceph-mon,rook-version!=%s", oldRookVersion)
err = s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, numMons)
require.NoError(s.T(), err)
err = s.k8sh.WaitForLabeledDeploymentsToBeReady(monsNotOldVersion, s.namespace)
require.NoError(s.T(), err)
// wait for the osd pods to be updated
osdsNotOldVersion = fmt.Sprintf("app=rook-ceph-osd,rook-version!=%s", oldRookVersion)
osdsNotOldVersion := fmt.Sprintf("app=rook-ceph-osd,rook-version!=%s", oldRookVersion)
err = s.k8sh.WaitForDeploymentCount(osdsNotOldVersion, s.namespace, numOSDs)
require.NoError(s.T(), err)
err = s.k8sh.WaitForLabeledDeploymentsToBeReady(osdsNotOldVersion, s.namespace)
require.NoError(s.T(), err)
mdsesNotOldVersion = fmt.Sprintf("app=rook-ceph-mds,rook-version!=%s", oldRookVersion)
mdsesNotOldVersion := fmt.Sprintf("app=rook-ceph-mds,rook-version!=%s", oldRookVersion)
err = s.k8sh.WaitForDeploymentCount(mdsesNotOldVersion, s.namespace, 4 /* always expect 4 mdses */)
require.NoError(s.T(), err)
err = s.k8sh.WaitForLabeledDeploymentsToBeReady(mdsesNotOldVersion, s.namespace)
require.NoError(s.T(), err)
logger.Infof("Done with automatic upgrade to master")
// rgwsNotOldVersion
// There is an unlikely corner case with RGWs where the upgrade will fail on upgrades regarding
// a pool added for erasure coding. This hasn't been observed by users, so we ignore this
// currently.
// Give a few seconds for the daemons to settle down after the upgrade
time.Sleep(5 * time.Second)
}
func (s *UpgradeSuite) VerifyFilesAfterUpgrade(fsName, newFileToWrite, messageForAllFiles string, oldFilesToRead []string) {
retryCount := 5
// wait for filesystem to be active
err = waitForFilesystemActive(s.k8sh, s.namespace, filesystemName)
err := waitForFilesystemActive(s.k8sh, s.namespace, fsName)
require.NoError(s.T(), err)
// Test writing and reading from the pod with cephfs mounted that was created before the upgrade.
postFilename := "post-upgrade-file"
assert.NoError(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, preFilename, message, 5))
assert.NoError(s.T(), s.k8sh.WriteToPodRetry(s.namespace, filePodName, postFilename, message, 5))
assert.NoError(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, postFilename, message, 5))
for _, file := range oldFilesToRead {
// test reading preexisting files in the pod with cephfs mounted
assert.NoError(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, file, messageForAllFiles, retryCount))
// Test writing and reading from the pod with rbd mounted that was created before the upgrade.
// There is some unreliability right after the upgrade when there is only one osd, so we will retry if needed
assert.NoError(s.T(), s.k8sh.ReadFromPodRetry("", podName, preFilename, message, 5))
assert.NoError(s.T(), s.k8sh.WriteToPodRetry("", podName, postFilename, message, 5))
assert.NoError(s.T(), s.k8sh.ReadFromPodRetry("", podName, postFilename, message, 5))
}
// test reading preexisting files in the pod with rbd mounted
// There is some unreliability right after the upgrade when there is only one osd, so we will retry if needed
assert.NoError(s.T(), s.k8sh.ReadFromPodRetry("", rbdPodName, file, messageForAllFiles, retryCount))
}
// Update the clusterroles that have been modified in master from the previous release
func (s *UpgradeSuite) updateClusterRoles() error {
logger.Infof("Placeholder: create the new resources that have been added since 1.0")
namespace := s.namespace
newResources := `
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
rules:
- apiGroups:
- ""
resources:
- pods
- configmaps
verbs:
- get
- list
- watch
- create
- update
- delete
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: rook-ceph-cmd-reporter
subjects:
- kind: ServiceAccount
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: rook-ceph-cmd-reporter-psp
namespace: ` + namespace + `
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: psp:rook
subjects:
- kind: ServiceAccount
name: rook-ceph-cmd-reporter
namespace: ` + namespace + `
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-osd
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- list
---
# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: rook-ceph-osd
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: rook-ceph-osd
subjects:
- kind: ServiceAccount
name: rook-ceph-osd
namespace: ` + namespace + `
`
logger.Infof("creating the new resources that have been added since 1.0")
return s.k8sh.ResourceOperation("apply", newResources)
// test writing and reading a new file in the pod with cephfs mounted
assert.NoError(s.T(), s.k8sh.WriteToPodRetry(s.namespace, filePodName, newFileToWrite, messageForAllFiles, retryCount))
assert.NoError(s.T(), s.k8sh.ReadFromPodRetry(s.namespace, filePodName, newFileToWrite, messageForAllFiles, retryCount))
// test writing and reading a new file in the pod with rbd mounted
assert.NoError(s.T(), s.k8sh.WriteToPodRetry("", rbdPodName, newFileToWrite, messageForAllFiles, retryCount))
assert.NoError(s.T(), s.k8sh.ReadFromPodRetry("", rbdPodName, newFileToWrite, messageForAllFiles, retryCount))
}