ceph: refactor upgrade integration test checks for daemon versions

The upgrade test had duplicated code for checking that daemons
were upgraded after the Rook upgrade vs the Ceph upgrade.
This is now refactored for consistent checks after upgrade.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
Travis Nielsen committed 2020-02-04 12:17:27 -07:00
1 parent babd65d131
commit 8659cdbb3d
1 file changed
+17 -38
+17 -38
View File
@@ -193,38 +193,12 @@ func (s *UpgradeSuite) upgradeCephVersion(newCephImage string, numOSDs int) {
oldCephVersion := osdDepList.Items[0].Labels["ceph-version"] // upgraded OSDs should not have this version label
//
// Upgrade Ceph version from Mimic to Nautilus before upgrading to Rook v1.1.
// Set the skipUpgradeChecks flag since we're not fully waiting for Ceph health during the tests.
// Upgrade Ceph version, for example from Mimic to Nautilus.
//
s.k8sh.Kubectl("-n", s.namespace, "patch", "CephCluster", s.namespace, "--type=merge",
"-p", fmt.Sprintf(`{"spec": {"cephVersion": {"image": "%s"}}}`, newCephImage))
// we need to make sure Ceph is fully updated (including RGWs and MDSes) before proceeding to
// upgrade rook; we do not support upgrading Ceph simultaneously with Rook upgrade
monsNotOldVersion := fmt.Sprintf("app=rook-ceph-mon,ceph-version!=%s", oldCephVersion)
err = s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, s.op.mons)
require.NoError(s.T(), err)
err = s.k8sh.WaitForLabeledDeploymentsToBeReady(monsNotOldVersion, s.namespace)
require.NoError(s.T(), err)
osdsNotOldVersion := fmt.Sprintf("app=rook-ceph-osd,ceph-version!=%s", oldCephVersion)
// It can take a LONG time to update the mgr modules, so wait an extra long time here
err = s.k8sh.WaitForDeploymentCountWithRetries(osdsNotOldVersion, s.namespace, numOSDs, utils.RetryLoop*2)
require.NoError(s.T(), err)
err = s.k8sh.WaitForLabeledDeploymentsToBeReady(osdsNotOldVersion, s.namespace)
require.NoError(s.T(), err)
mdsesNotOldVersion := fmt.Sprintf("app=rook-ceph-mds,ceph-version!=%s", oldCephVersion)
err = s.k8sh.WaitForDeploymentCount(mdsesNotOldVersion, s.namespace, 4 /* always expect 4 mdses */)
require.NoError(s.T(), err)
err = s.k8sh.WaitForLabeledDeploymentsToBeReady(mdsesNotOldVersion, s.namespace)
require.NoError(s.T(), err)
rgwsNotOldVersion := fmt.Sprintf("app=rook-ceph-rgw,ceph-version!=%s", oldCephVersion)
err = s.k8sh.WaitForDeploymentCount(rgwsNotOldVersion, s.namespace, 1 /* always expect 1 rgw */)
require.NoError(s.T(), err)
err = s.k8sh.WaitForLabeledDeploymentsToBeReady(rgwsNotOldVersion, s.namespace)
require.NoError(s.T(), err)
s.waitForUpgradedDaemons(oldCephVersion, "ceph-version", numOSDs)
}
func (s *UpgradeSuite) verifyOperatorImage(expectedImage string) {
@@ -256,38 +230,43 @@ func (s *UpgradeSuite) verifyRookUpgrade(numOSDs int) {
d := osdDepList.Items[0]
oldRookVersion := d.Labels["rook-version"] // upgraded OSDs should not have this version label
s.waitForUpgradedDaemons(oldRookVersion, "rook-version", numOSDs)
}
func (s *UpgradeSuite) waitForUpgradedDaemons(previousVersion, versionLabel string, numOSDs int) {
// wait for the mon(s) to be updated
monsNotOldVersion := fmt.Sprintf("app=rook-ceph-mon,rook-version!=%s", oldRookVersion)
err = s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, s.op.mons)
monsNotOldVersion := fmt.Sprintf("app=rook-ceph-mon,%s!=%s", versionLabel, previousVersion)
err := s.k8sh.WaitForDeploymentCount(monsNotOldVersion, s.namespace, s.op.mons)
require.NoError(s.T(), err, "mon(s) didn't update")
err = s.k8sh.WaitForLabeledDeploymentsToBeReady(monsNotOldVersion, s.namespace)
require.NoError(s.T(), err)
// wait for the mgr to be updated
mgrNotOldVersion := fmt.Sprintf("app=rook-ceph-mgr,rook-version!=%s", oldRookVersion)
mgrNotOldVersion := fmt.Sprintf("app=rook-ceph-mgr,%s!=%s", versionLabel, previousVersion)
err = s.k8sh.WaitForDeploymentCount(mgrNotOldVersion, s.namespace, 1)
require.NoError(s.T(), err, "mgr didn't update")
err = s.k8sh.WaitForLabeledDeploymentsToBeReady(mgrNotOldVersion, s.namespace)
require.NoError(s.T(), err)
// wait for the osd pods to be updated
osdsNotOldVersion := fmt.Sprintf("app=rook-ceph-osd,rook-version!=%s", oldRookVersion)
osdsNotOldVersion := fmt.Sprintf("app=rook-ceph-osd,%s!=%s", versionLabel, previousVersion)
err = s.k8sh.WaitForDeploymentCount(osdsNotOldVersion, s.namespace, numOSDs)
require.NoError(s.T(), err, "osd(s) didn't update")
err = s.k8sh.WaitForLabeledDeploymentsToBeReady(osdsNotOldVersion, s.namespace)
require.NoError(s.T(), err)
// wait for the mds pods to be updated
mdsesNotOldVersion := fmt.Sprintf("app=rook-ceph-mds,rook-version!=%s", oldRookVersion)
err = s.k8sh.WaitForDeploymentCount(mdsesNotOldVersion, s.namespace, 4 /* always expect 4 mdses */)
mdsesNotOldVersion := fmt.Sprintf("app=rook-ceph-mds,%s!=%s", versionLabel, previousVersion)
err = s.k8sh.WaitForDeploymentCount(mdsesNotOldVersion, s.namespace, 2 /* always expect 2 mdses */)
require.NoError(s.T(), err)
err = s.k8sh.WaitForLabeledDeploymentsToBeReady(mdsesNotOldVersion, s.namespace)
require.NoError(s.T(), err)
// rgwsNotOldVersion
// There is an unlikely corner case with RGWs where the upgrade will fail on upgrades regarding
// a pool added for erasure coding. This hasn't been observed by users, so we ignore this
// currently.
rgwsNotOldVersion := fmt.Sprintf("app=rook-ceph-rgw,%s!=%s", versionLabel, previousVersion)
err = s.k8sh.WaitForDeploymentCount(rgwsNotOldVersion, s.namespace, 1 /* always expect 1 rgw */)
require.NoError(s.T(), err)
err = s.k8sh.WaitForLabeledDeploymentsToBeReady(rgwsNotOldVersion, s.namespace)
require.NoError(s.T(), err)
// Give a few seconds for the daemons to settle down after the upgrade
time.Sleep(5 * time.Second)