feedc0de/my-rook-config
1
0 Fork 0
forked from rook/rook
Code Pull Requests Activity

Ceph: Newer mechanism to scale MDS for Nautilus+

Browse Source 
ceph mds deactivate is not valid command for Nautilus, changing the mechnaism of upgrading so that it can be done in the way Nautilus does it.

Closes: https://github.com/rook/rook/issues/5278
Signed-off-by: RAJAT SINGH <rajasing@redhat.com>
This commit is contained in:
RAJAT SINGH committed 2020-05-06 13:33:26 +05:30
1 parent e342f7ea30
commit 9feb0287c5
3 files changed
+12 -78
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pkg/daemon/ceph/client/filesystem.go
+1 -58
View File
@@ -25,7 +25,6 @@ import (
"github.com/pkg/errors"
"github.com/rook/rook/pkg/clusterd"
cephver "github.com/rook/rook/pkg/operator/ceph/version"
"k8s.io/apimachinery/pkg/util/wait"
)
@@ -178,72 +177,16 @@ func IsMultiFSEnabled() bool {
}
// SetNumMDSRanks sets the number of mds ranks (max_mds) for a Ceph filesystem.
func SetNumMDSRanks(context *clusterd.Context, cephVersion cephver.CephVersion, clusterName, fsName string, activeMDSCount int32) error {
// Noted sections 1 and 2 are necessary for reducing max_mds.
// See more: [1] http://docs.ceph.com/docs/nautilus/cephfs/upgrading/
// [2] https://tracker.ceph.com/issues/23172
// * Noted section 1 - See note at top of function
fsAtStart, errAtStart := GetFilesystem(context, clusterName, fsName)
// collect information now, but don't check error yet
// * End of Noted section 1
func SetNumMDSRanks(context *clusterd.Context, clusterName, fsName string, activeMDSCount int32) error {
// Always tell Ceph to set the new max_mds value
args := []string{"fs", "set", fsName, "max_mds", strconv.Itoa(int(activeMDSCount))}
if _, err := NewCephCommand(context, clusterName, args).Run(); err != nil {
return errors.Wrapf(err, "failed to set filesystem %s num mds ranks (max_mds) to %d", fsName, activeMDSCount)
}
// ** Noted section 2 - See note at top of function
// Now check the error to see if we can even determine whether we should reduce or not
if errAtStart != nil {
return errors.Wrapf(errAtStart, `failed to get filesystem %s info needed to ensure mds rank can be changed correctly,
if num active mdses (max_mds) was lowered, USER should deactivate extra active mdses manually`,
fsName)
}
if int(activeMDSCount) > fsAtStart.MDSMap.MaxMDS {
return nil // No need to deactivate mdses if we are raising max_mds
}
logger.Debugf("deactivating some running mdses for filesystem %s", fsName)
// Deactivate all mdses except desired number (N); arbitrarily choose first N to live
fs, err := GetFilesystem(context, clusterName, fsName)
if err != nil {
logger.Warningf(
fmt.Sprintf("Failed to get filesystem %s info needed to deactivate running mdses. ", fsName) +
"using slightly stale info, this could (rarely) result in momentary loss of filesystem availability" +
fmt.Sprintf(": %v", err),
)
fs = fsAtStart // <-- Do the best we can to disable mdses that were active when we started
// Effects of stale info should be non-destructive, & unlikely the info is actually bad
}
// Deactivate any mdses with a higher rank than the desired max rank
// Ceph only allows mdses to be deactivated in reverse order starting with the highest rank
for gid := int(len(fs.MDSMap.In)) - 1; gid >= int(activeMDSCount); gid-- {
if err := deactivateMdsWithRetry(context, gid, clusterName, fsName); err != nil {
logger.Warningf("in luminous this is non-ideal but not necessarily critical: %v", err)
}
}
// ** End of noted section 2
return nil
}
func deactivateMdsWithRetry(context *clusterd.Context, mdsGid int, namespace, fsName string) error {
retries := 10
retrySleep := 5 * time.Second
var err error
for i := 1; i <= retries; i++ {
args := []string{"mds", "deactivate", fmt.Sprintf("%s:%d", fsName, mdsGid)}
if _, err = NewCephCommand(context, namespace, args).Run(); err == nil {
logger.Infof("successfully disabled mds with rank %d on attempt %d", mdsGid, i)
return nil
}
time.Sleep(retrySleep)
}
// report most recent error with additional err info
return errors.Wrapf(err, "failed to deactivate mds w/ gid %d for filesystem %q", mdsGid, fsName)
}
// WaitForActiveRanks waits for the filesystem's number of active ranks to equal the desired count.
// It times out with an error if the number of active ranks does not become desired in time.
// Param 'moreIsOkay' will allow success condition if num of ranks is more than active count given.
pkg/operator/ceph/file/filesystem.go
+2 -2
View File
@@ -75,7 +75,7 @@ func createFilesystem(
// set the number of active mds instances
if fs.Spec.MetadataServer.ActiveCount > 1 {
if err = client.SetNumMDSRanks(context, clusterInfo.CephVersion, fs.Namespace, fs.Name, fs.Spec.MetadataServer.ActiveCount); err != nil {
if err = client.SetNumMDSRanks(context, fs.Namespace, fs.Name, fs.Spec.MetadataServer.ActiveCount); err != nil {
logger.Warningf("failed setting active mds count to %d. %v", fs.Spec.MetadataServer.ActiveCount, err)
}
}
@@ -195,7 +195,7 @@ func (f *Filesystem) doFilesystemCreate(context *clusterd.Context, cephVersion c
logger.Infof("filesystem %s already exists", f.Name)
// Even if the fs already exists, the num active mdses may have changed
if err := client.SetNumMDSRanks(context, cephVersion, f.Namespace, f.Name, spec.MetadataServer.ActiveCount); err != nil {
if err := client.SetNumMDSRanks(context, f.Namespace, f.Name, spec.MetadataServer.ActiveCount); err != nil {
logger.Errorf(
fmt.Sprintf("failed to set num mds ranks (max_mds) to %d for filesystem %s, still continuing. ", spec.MetadataServer.ActiveCount, f.Name) +
"this error is not critical, but mdses may not be as failure tolerant as desired. " +
pkg/operator/ceph/file/mds/mds.go
+9 -18
View File
@@ -283,21 +283,15 @@ func prepareForDaemonUpgrade(
clusterName, fsName string,
timeout time.Duration,
) error {
// upgrade guide according to nautilus https://docs.ceph.com/docs/nautilus/cephfs/upgrading/#upgrading-the-mds-cluster
logger.Infof("preparing filesystem %s for daemon upgrade", fsName)
// * Beginning of noted section 1
// This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3.
// See more: https://ceph.com/releases/v13-2-0-mimic-released/
// http://docs.ceph.com/docs/mimic/cephfs/upgrading/
// As of Oct. 2018, this is only necessary for Luminous and Mimic.
if err := client.SetNumMDSRanks(context, cephVersion, clusterName, fsName, 1); err != nil {
return errors.Wrapf(err, "Could not Prepare filesystem %s for daemon upgrade", fsName)
if err := client.SetNumMDSRanks(context, clusterName, fsName, 1); err != nil {
return errors.Wrapf(err, "Could not prepare filesystem %q for daemon upgrade", fsName)
}
if err := client.WaitForActiveRanks(context, clusterName, fsName, 1, false, timeout); err != nil {
return err
}
// * End of Noted section 1
logger.Infof("Filesystem %s successfully prepared for mds daemon upgrade", fsName)
logger.Infof("Filesystem %q successfully prepared for MDS daemon upgrade", fsName)
return nil
}
@@ -310,14 +304,11 @@ func finishedWithDaemonUpgrade(
activeMDSCount int32,
) error {
logger.Debugf("restoring filesystem %s from daemon upgrade", fsName)
logger.Debugf("bringing num active mds daemons for fs %s back to %d", fsName, activeMDSCount)
// * Beginning of noted section 1
// This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3.
// See more: https://ceph.com/releases/v13-2-0-mimic-released/
// http://docs.ceph.com/docs/mimic/cephfs/upgrading/
// TODO: Unknown (Oct. 2018) if any parts can be removed once Rook no longer supports Mimic.
if err := client.SetNumMDSRanks(context, cephVersion, clusterName, fsName, activeMDSCount); err != nil {
logger.Debugf("bringing num active MDS daemons for fs %s back to %d", fsName, activeMDSCount)
// TODO: Unknown (Apr 2020) if this can be removed once Rook no longer supports Nautilus.
// upgrade guide according to nautilus https://docs.ceph.com/docs/nautilus/cephfs/upgrading/#upgrading-the-mds-cluster
if err := client.SetNumMDSRanks(context, clusterName, fsName, activeMDSCount); err != nil {
return errors.Wrapf(err, "Failed to restore filesystem %s following daemon upgrade", fsName)
} // * End of noted section 1
}
return nil
}