diff --git a/pkg/daemon/ceph/client/filesystem.go b/pkg/daemon/ceph/client/filesystem.go index 958240409..52a06164f 100644 --- a/pkg/daemon/ceph/client/filesystem.go +++ b/pkg/daemon/ceph/client/filesystem.go @@ -25,7 +25,6 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" - cephver "github.com/rook/rook/pkg/operator/ceph/version" "k8s.io/apimachinery/pkg/util/wait" ) @@ -178,72 +177,16 @@ func IsMultiFSEnabled() bool { } // SetNumMDSRanks sets the number of mds ranks (max_mds) for a Ceph filesystem. -func SetNumMDSRanks(context *clusterd.Context, cephVersion cephver.CephVersion, clusterName, fsName string, activeMDSCount int32) error { - // Noted sections 1 and 2 are necessary for reducing max_mds. - // See more: [1] http://docs.ceph.com/docs/nautilus/cephfs/upgrading/ - // [2] https://tracker.ceph.com/issues/23172 - - // * Noted section 1 - See note at top of function - fsAtStart, errAtStart := GetFilesystem(context, clusterName, fsName) - // collect information now, but don't check error yet - // * End of Noted section 1 +func SetNumMDSRanks(context *clusterd.Context, clusterName, fsName string, activeMDSCount int32) error { // Always tell Ceph to set the new max_mds value args := []string{"fs", "set", fsName, "max_mds", strconv.Itoa(int(activeMDSCount))} if _, err := NewCephCommand(context, clusterName, args).Run(); err != nil { return errors.Wrapf(err, "failed to set filesystem %s num mds ranks (max_mds) to %d", fsName, activeMDSCount) } - - // ** Noted section 2 - See note at top of function - // Now check the error to see if we can even determine whether we should reduce or not - if errAtStart != nil { - return errors.Wrapf(errAtStart, `failed to get filesystem %s info needed to ensure mds rank can be changed correctly, -if num active mdses (max_mds) was lowered, USER should deactivate extra active mdses manually`, - fsName) - } - if int(activeMDSCount) > fsAtStart.MDSMap.MaxMDS { - return nil // No need to deactivate mdses if we are raising max_mds - } - logger.Debugf("deactivating some running mdses for filesystem %s", fsName) - // Deactivate all mdses except desired number (N); arbitrarily choose first N to live - fs, err := GetFilesystem(context, clusterName, fsName) - if err != nil { - logger.Warningf( - fmt.Sprintf("Failed to get filesystem %s info needed to deactivate running mdses. ", fsName) + - "using slightly stale info, this could (rarely) result in momentary loss of filesystem availability" + - fmt.Sprintf(": %v", err), - ) - fs = fsAtStart // <-- Do the best we can to disable mdses that were active when we started - // Effects of stale info should be non-destructive, & unlikely the info is actually bad - } - // Deactivate any mdses with a higher rank than the desired max rank - // Ceph only allows mdses to be deactivated in reverse order starting with the highest rank - for gid := int(len(fs.MDSMap.In)) - 1; gid >= int(activeMDSCount); gid-- { - if err := deactivateMdsWithRetry(context, gid, clusterName, fsName); err != nil { - logger.Warningf("in luminous this is non-ideal but not necessarily critical: %v", err) - } - } - // ** End of noted section 2 - return nil } -func deactivateMdsWithRetry(context *clusterd.Context, mdsGid int, namespace, fsName string) error { - retries := 10 - retrySleep := 5 * time.Second - var err error - for i := 1; i <= retries; i++ { - args := []string{"mds", "deactivate", fmt.Sprintf("%s:%d", fsName, mdsGid)} - if _, err = NewCephCommand(context, namespace, args).Run(); err == nil { - logger.Infof("successfully disabled mds with rank %d on attempt %d", mdsGid, i) - return nil - } - time.Sleep(retrySleep) - } - // report most recent error with additional err info - return errors.Wrapf(err, "failed to deactivate mds w/ gid %d for filesystem %q", mdsGid, fsName) -} - // WaitForActiveRanks waits for the filesystem's number of active ranks to equal the desired count. // It times out with an error if the number of active ranks does not become desired in time. // Param 'moreIsOkay' will allow success condition if num of ranks is more than active count given. diff --git a/pkg/operator/ceph/file/filesystem.go b/pkg/operator/ceph/file/filesystem.go index c8c1932ef..e3928321a 100644 --- a/pkg/operator/ceph/file/filesystem.go +++ b/pkg/operator/ceph/file/filesystem.go @@ -75,7 +75,7 @@ func createFilesystem( // set the number of active mds instances if fs.Spec.MetadataServer.ActiveCount > 1 { - if err = client.SetNumMDSRanks(context, clusterInfo.CephVersion, fs.Namespace, fs.Name, fs.Spec.MetadataServer.ActiveCount); err != nil { + if err = client.SetNumMDSRanks(context, fs.Namespace, fs.Name, fs.Spec.MetadataServer.ActiveCount); err != nil { logger.Warningf("failed setting active mds count to %d. %v", fs.Spec.MetadataServer.ActiveCount, err) } } @@ -195,7 +195,7 @@ func (f *Filesystem) doFilesystemCreate(context *clusterd.Context, cephVersion c logger.Infof("filesystem %s already exists", f.Name) // Even if the fs already exists, the num active mdses may have changed - if err := client.SetNumMDSRanks(context, cephVersion, f.Namespace, f.Name, spec.MetadataServer.ActiveCount); err != nil { + if err := client.SetNumMDSRanks(context, f.Namespace, f.Name, spec.MetadataServer.ActiveCount); err != nil { logger.Errorf( fmt.Sprintf("failed to set num mds ranks (max_mds) to %d for filesystem %s, still continuing. ", spec.MetadataServer.ActiveCount, f.Name) + "this error is not critical, but mdses may not be as failure tolerant as desired. " + diff --git a/pkg/operator/ceph/file/mds/mds.go b/pkg/operator/ceph/file/mds/mds.go index 478b8d635..0a5436098 100644 --- a/pkg/operator/ceph/file/mds/mds.go +++ b/pkg/operator/ceph/file/mds/mds.go @@ -283,21 +283,15 @@ func prepareForDaemonUpgrade( clusterName, fsName string, timeout time.Duration, ) error { + // upgrade guide according to nautilus https://docs.ceph.com/docs/nautilus/cephfs/upgrading/#upgrading-the-mds-cluster logger.Infof("preparing filesystem %s for daemon upgrade", fsName) - // * Beginning of noted section 1 - // This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3. - // See more: https://ceph.com/releases/v13-2-0-mimic-released/ - // http://docs.ceph.com/docs/mimic/cephfs/upgrading/ - // As of Oct. 2018, this is only necessary for Luminous and Mimic. - if err := client.SetNumMDSRanks(context, cephVersion, clusterName, fsName, 1); err != nil { - return errors.Wrapf(err, "Could not Prepare filesystem %s for daemon upgrade", fsName) + if err := client.SetNumMDSRanks(context, clusterName, fsName, 1); err != nil { + return errors.Wrapf(err, "Could not prepare filesystem %q for daemon upgrade", fsName) } if err := client.WaitForActiveRanks(context, clusterName, fsName, 1, false, timeout); err != nil { return err } - // * End of Noted section 1 - - logger.Infof("Filesystem %s successfully prepared for mds daemon upgrade", fsName) + logger.Infof("Filesystem %q successfully prepared for MDS daemon upgrade", fsName) return nil } @@ -310,14 +304,11 @@ func finishedWithDaemonUpgrade( activeMDSCount int32, ) error { logger.Debugf("restoring filesystem %s from daemon upgrade", fsName) - logger.Debugf("bringing num active mds daemons for fs %s back to %d", fsName, activeMDSCount) - // * Beginning of noted section 1 - // This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3. - // See more: https://ceph.com/releases/v13-2-0-mimic-released/ - // http://docs.ceph.com/docs/mimic/cephfs/upgrading/ - // TODO: Unknown (Oct. 2018) if any parts can be removed once Rook no longer supports Mimic. - if err := client.SetNumMDSRanks(context, cephVersion, clusterName, fsName, activeMDSCount); err != nil { + logger.Debugf("bringing num active MDS daemons for fs %s back to %d", fsName, activeMDSCount) + // TODO: Unknown (Apr 2020) if this can be removed once Rook no longer supports Nautilus. + // upgrade guide according to nautilus https://docs.ceph.com/docs/nautilus/cephfs/upgrading/#upgrading-the-mds-cluster + if err := client.SetNumMDSRanks(context, clusterName, fsName, activeMDSCount); err != nil { return errors.Wrapf(err, "Failed to restore filesystem %s following daemon upgrade", fsName) - } // * End of noted section 1 + } return nil }