diff --git a/pkg/operator/ceph/file/filesystem.go b/pkg/operator/ceph/file/filesystem.go index d9aa76bf7..1fd444461 100644 --- a/pkg/operator/ceph/file/filesystem.go +++ b/pkg/operator/ceph/file/filesystem.go @@ -18,13 +18,13 @@ package file import ( "fmt" - "time" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" "github.com/rook/rook/pkg/daemon/ceph/model" + "github.com/rook/rook/pkg/operator/ceph/file/mds" "github.com/rook/rook/pkg/operator/ceph/pool" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -81,8 +81,8 @@ func createFilesystem( } logger.Infof("start running mdses for filesystem %s", fs.Name) - c := newCluster(clusterInfo, context, rookVersion, cephVersion, hostNetwork, fs, filesystem, ownerRefs) - if err := c.start(); err != nil { + c := mds.NewCluster(clusterInfo, context, rookVersion, cephVersion, hostNetwork, fs, filesystem, ownerRefs) + if err := c.Start(); err != nil { return err } @@ -104,7 +104,7 @@ func deleteFilesystem(context *clusterd.Context, fs cephv1.CephFilesystem) error } } - return deleteMdsCluster(context, fs.Namespace, fs.Name) + return mds.DeleteCluster(context, fs.Namespace, fs.Name) } func validateFilesystem(context *clusterd.Context, f cephv1.CephFilesystem) error { @@ -230,50 +230,3 @@ func downFilesystem(context *clusterd.Context, clusterName, filesystemName strin logger.Infof("Removed filesystem %s", filesystemName) return nil } - -// prepareForDaemonUpgrade performs all actions necessary to ensure the filesystem is prepared -// to have its daemon(s) updated. This helps ensure there is no aberrant behavior during upgrades. -// If the mds is not prepared within the timeout window, an error will be reported. -// Ceph docs: http://docs.ceph.com/docs/master/cephfs/upgrading/ -func prepareForDaemonUpgrade( - context *clusterd.Context, - clusterName, fsName string, - timeout time.Duration, -) error { - logger.Infof("preparing filesystem %s for daemon upgrade", fsName) - // * Beginning of noted section 1 - // This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3. - // See more: https://ceph.com/releases/v13-2-0-mimic-released/ - // http://docs.ceph.com/docs/mimic/cephfs/upgrading/ - // As of Oct. 2018, this is only necessary for Luminous and Mimic. - if err := client.SetNumMDSRanks(context, clusterName, fsName, 1); err != nil { - return fmt.Errorf("Could not Prepare filesystem %s for daemon upgrade: %+v", fsName, err) - } - if err := client.WaitForActiveRanks(context, clusterName, fsName, 1, false, timeout); err != nil { - return err - } - // * End of Noted section 1 - - logger.Infof("Filesystem %s successfully prepared for mds daemon upgrade", fsName) - return nil -} - -// finishedWithDaemonUpgrade performs all actions necessary to bring the filesystem back to its -// ideal state following an upgrade of its daemon(s). -func finishedWithDaemonUpgrade( - context *clusterd.Context, - clusterName, fsName string, - activeMDSCount int32, -) error { - logger.Debugf("restoring filesystem %s from daemon upgrade", fsName) - logger.Debugf("bringing num active mds daemons for fs %s back to %d", fsName, activeMDSCount) - // * Beginning of noted section 1 - // This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3. - // See more: https://ceph.com/releases/v13-2-0-mimic-released/ - // http://docs.ceph.com/docs/mimic/cephfs/upgrading/ - // TODO: Unknown (Oct. 2018) if any parts can be removed once Rook no longer supports Mimic. - if err := client.SetNumMDSRanks(context, clusterName, fsName, activeMDSCount); err != nil { - return fmt.Errorf("Failed to restore filesystem %s following daemon upgrade: %+v", fsName, err) - } // * End of noted section 1 - return nil -} diff --git a/pkg/operator/ceph/file/filesystem_test.go b/pkg/operator/ceph/file/filesystem_test.go index 89ca1cb0b..076f2b38c 100644 --- a/pkg/operator/ceph/file/filesystem_test.go +++ b/pkg/operator/ceph/file/filesystem_test.go @@ -28,6 +28,7 @@ import ( "github.com/rook/rook/pkg/clusterd" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" cephtest "github.com/rook/rook/pkg/daemon/ceph/test" + "github.com/rook/rook/pkg/operator/ceph/file/mds" testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test" testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" @@ -69,7 +70,7 @@ func TestValidateSpec(t *testing.T) { func TestCreateFilesystem(t *testing.T) { var deploymentsUpdated *[]*apps.Deployment - updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub() + mds.UpdateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub() configDir, _ := ioutil.TempDir("", "") // Output to check multiple filesystem creation diff --git a/pkg/operator/ceph/file/config.go b/pkg/operator/ceph/file/mds/config.go similarity index 96% rename from pkg/operator/ceph/file/config.go rename to pkg/operator/ceph/file/mds/config.go index aa7263de7..f18fdb9f1 100644 --- a/pkg/operator/ceph/file/config.go +++ b/pkg/operator/ceph/file/mds/config.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package file +package mds import ( "fmt" @@ -35,7 +35,7 @@ caps mds = "allow" ` ) -func (c *cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error { +func (c *Cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error { user := fmt.Sprintf("mds.%s", m.DaemonID) access := []string{"osd", "allow *", "mds", "allow", "mon", "allow profile mds"} ownerRef := &metav1.OwnerReference{ diff --git a/pkg/operator/ceph/file/mds.go b/pkg/operator/ceph/file/mds/mds.go similarity index 69% rename from pkg/operator/ceph/file/mds.go rename to pkg/operator/ceph/file/mds/mds.go index 647d64f94..c718be343 100644 --- a/pkg/operator/ceph/file/mds.go +++ b/pkg/operator/ceph/file/mds/mds.go @@ -14,24 +14,27 @@ See the License for the specific language governing permissions and limitations under the License. */ -package file +// Package mds provides methods for managing a Ceph mds cluster. +package mds import ( "fmt" "strconv" "time" + "github.com/coreos/pkg/capnslog" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" cephconfig "github.com/rook/rook/pkg/daemon/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/k8sutil" - apps "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mds") + const ( // AppName is the name of Rook's Ceph mds (File) sub-app AppName = "rook-ceph-mds" @@ -42,7 +45,8 @@ const ( fsWaitForActiveTimeout = 3 * time.Minute ) -type cluster struct { +// Cluster represents a Ceph mds cluster. +type Cluster struct { clusterInfo *cephconfig.ClusterInfo context *clusterd.Context rookVersion string @@ -59,7 +63,8 @@ type mdsConfig struct { DataPathMap *config.DataPathMap // location to store data in container } -func newCluster( +// NewCluster creates a Ceph mds cluster representation. +func NewCluster( clusterInfo *cephconfig.ClusterInfo, context *clusterd.Context, rookVersion string, @@ -68,8 +73,8 @@ func newCluster( fs cephv1.CephFilesystem, fsdetails *client.CephFilesystemDetails, ownerRefs []metav1.OwnerReference, -) *cluster { - return &cluster{ +) *Cluster { + return &Cluster{ clusterInfo: clusterInfo, context: context, rookVersion: rookVersion, @@ -81,9 +86,11 @@ func newCluster( } } -var updateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait +// UpdateDeploymentAndWait can be overridden for unit tests. Do not alter this for runtime operation. +var UpdateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait -func (c *cluster) start() error { +// Start starts or updates a Ceph mds cluster in Kubernetes. +func (c *Cluster) Start() error { // If attempt was made to prepare daemons for upgrade, make sure that an attempt is made to // bring fs state back to desired when this method returns with any error or success. var fsPreparedForUpgrade = false @@ -126,7 +133,7 @@ func (c *cluster) start() error { // TODO: need to prepare for upgrade here each time. Also, before a given deployment is // terminated, I think we should somehow make sure that it isn't running the single // active daemon. If it is, then we should have another daemon take over as active. @Jan? - if createdDeployment, err = updateDeploymentAndWait(c.context, d, c.fs.Namespace); err != nil { + if createdDeployment, err = UpdateDeploymentAndWait(c.context, d, c.fs.Namespace); err != nil { return fmt.Errorf("failed to update mds deployment %s. %+v", mdsConfig.ResourceName, err) } } @@ -138,6 +145,14 @@ func (c *cluster) start() error { } } + if err := c.scaleDownDeployments(replicas, desiredDeployments); err != nil { + return fmt.Errorf("failed to scale down mds deployments. %+v", err) + } + + return nil +} + +func (c *Cluster) scaleDownDeployments(replicas int32, desiredDeployments map[string]bool) error { // Remove extraneous mds deployments if they exist deps, err := getMdsDeployments(c.context, c.fs.Namespace, c.fs.Name) if err != nil { @@ -186,7 +201,8 @@ func (c *cluster) start() error { return nil } -func deleteMdsCluster(context *clusterd.Context, namespace, fsName string) error { +// DeleteCluster deletes a Ceph mds cluster from Kubernetes. +func DeleteCluster(context *clusterd.Context, namespace, fsName string) error { // Try to delete all mds deployments and secret keys serving the filesystem, and aggregate // failures together to report all at once at the end. deps, err := getMdsDeployments(context, namespace, fsName) @@ -207,36 +223,49 @@ func deleteMdsCluster(context *clusterd.Context, namespace, fsName string) error return nil } -func getMdsDeployments(context *clusterd.Context, namespace, fsName string) (*apps.DeploymentList, error) { - fsLabelSelector := fmt.Sprintf("rook_file_system=%s", fsName) - deps, err := k8sutil.GetDeployments(context.Clientset, namespace, fsLabelSelector) - if err != nil { - return nil, fmt.Errorf("could not get deployments for filesystem %s (matching label selector '%s'): %+v", fsName, fsLabelSelector, err) +// prepareForDaemonUpgrade performs all actions necessary to ensure the filesystem is prepared +// to have its daemon(s) updated. This helps ensure there is no aberrant behavior during upgrades. +// If the mds is not prepared within the timeout window, an error will be reported. +// Ceph docs: http://docs.ceph.com/docs/master/cephfs/upgrading/ +func prepareForDaemonUpgrade( + context *clusterd.Context, + clusterName, fsName string, + timeout time.Duration, +) error { + logger.Infof("preparing filesystem %s for daemon upgrade", fsName) + // * Beginning of noted section 1 + // This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3. + // See more: https://ceph.com/releases/v13-2-0-mimic-released/ + // http://docs.ceph.com/docs/mimic/cephfs/upgrading/ + // As of Oct. 2018, this is only necessary for Luminous and Mimic. + if err := client.SetNumMDSRanks(context, clusterName, fsName, 1); err != nil { + return fmt.Errorf("Could not Prepare filesystem %s for daemon upgrade: %+v", fsName, err) } - return deps, nil -} + if err := client.WaitForActiveRanks(context, clusterName, fsName, 1, false, timeout); err != nil { + return err + } + // * End of Noted section 1 -func deleteMdsDeployment(context *clusterd.Context, namespace string, deployment *apps.Deployment) error { - errCount := 0 + logger.Infof("Filesystem %s successfully prepared for mds daemon upgrade", fsName) + return nil +} - // Delete the mds deployment - logger.Infof("deleting mds deployment %s", deployment.Name) - var gracePeriod int64 - propagation := metav1.DeletePropagationForeground - options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} - if err := context.Clientset.Apps().Deployments(namespace).Delete(deployment.GetName(), options); err != nil { - errCount++ - logger.Errorf("failed to delete mds deployment %s: %+v", deployment.GetName(), err) - } - - // Delete the mds secret - err := context.Clientset.CoreV1().Secrets(namespace).Delete(deployment.GetName(), &metav1.DeleteOptions{}) - if err != nil && !errors.IsNotFound(err) { - errCount++ - logger.Errorf("failed to delete mds secret %s: %+v", deployment.GetName(), err) - } - if errCount > 0 { - return fmt.Errorf("%d error(s) during deletion of mds deployment %s, see logs above", errCount, deployment.GetName()) - } +// finishedWithDaemonUpgrade performs all actions necessary to bring the filesystem back to its +// ideal state following an upgrade of its daemon(s). +func finishedWithDaemonUpgrade( + context *clusterd.Context, + clusterName, fsName string, + activeMDSCount int32, +) error { + logger.Debugf("restoring filesystem %s from daemon upgrade", fsName) + logger.Debugf("bringing num active mds daemons for fs %s back to %d", fsName, activeMDSCount) + // * Beginning of noted section 1 + // This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3. + // See more: https://ceph.com/releases/v13-2-0-mimic-released/ + // http://docs.ceph.com/docs/mimic/cephfs/upgrading/ + // TODO: Unknown (Oct. 2018) if any parts can be removed once Rook no longer supports Mimic. + if err := client.SetNumMDSRanks(context, clusterName, fsName, activeMDSCount); err != nil { + return fmt.Errorf("Failed to restore filesystem %s following daemon upgrade: %+v", fsName, err) + } // * End of noted section 1 return nil } diff --git a/pkg/operator/ceph/file/spec.go b/pkg/operator/ceph/file/mds/spec.go similarity index 68% rename from pkg/operator/ceph/file/spec.go rename to pkg/operator/ceph/file/mds/spec.go index 112521605..99fe32d28 100644 --- a/pkg/operator/ceph/file/spec.go +++ b/pkg/operator/ceph/file/mds/spec.go @@ -14,11 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package file +package mds import ( + "fmt" "strconv" + "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/ceph/config" opspec "github.com/rook/rook/pkg/operator/ceph/spec" "github.com/rook/rook/pkg/operator/k8sutil" @@ -31,7 +33,7 @@ const ( mdsDaemonCommand = "ceph-mds" ) -func (c *cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment { +func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment { podSpec := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: mdsConfig.ResourceName, @@ -75,7 +77,7 @@ func (c *cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment { return d } -func (c *cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { +func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { return v1.Container{ Name: "mds", Command: []string{ @@ -96,8 +98,31 @@ func (c *cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container { } } -func (c *cluster) podLabels(mdsConfig *mdsConfig) map[string]string { +func (c *Cluster) podLabels(mdsConfig *mdsConfig) map[string]string { labels := opspec.PodLabels(AppName, c.fs.Namespace, "mds", mdsConfig.DaemonID) labels["rook_file_system"] = c.fs.Name return labels } + +func getMdsDeployments(context *clusterd.Context, namespace, fsName string) (*apps.DeploymentList, error) { + fsLabelSelector := fmt.Sprintf("rook_file_system=%s", fsName) + deps, err := k8sutil.GetDeployments(context.Clientset, namespace, fsLabelSelector) + if err != nil { + return nil, fmt.Errorf("could not get deployments for filesystem %s (matching label selector '%s'): %+v", fsName, fsLabelSelector, err) + } + return deps, nil +} + +func deleteMdsDeployment(context *clusterd.Context, namespace string, deployment *apps.Deployment) error { + errCount := 0 + // Delete the mds deployment + logger.Infof("deleting mds deployment %s", deployment.Name) + var gracePeriod int64 + propagation := metav1.DeletePropagationForeground + options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation} + if err := context.Clientset.Apps().Deployments(namespace).Delete(deployment.GetName(), options); err != nil { + errCount++ + logger.Errorf("failed to delete mds deployment %s: %+v", deployment.GetName(), err) + } + return nil +} diff --git a/pkg/operator/ceph/file/spec_test.go b/pkg/operator/ceph/file/mds/spec_test.go similarity index 98% rename from pkg/operator/ceph/file/spec_test.go rename to pkg/operator/ceph/file/mds/spec_test.go index 069657664..d2bdc62a5 100644 --- a/pkg/operator/ceph/file/spec_test.go +++ b/pkg/operator/ceph/file/mds/spec_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package file +package mds import ( "testing" @@ -52,7 +52,7 @@ func testDeploymentObject(hostNetwork bool) *apps.Deployment { }}}} clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"} - c := newCluster( + c := NewCluster( clusterInfo, &clusterd.Context{Clientset: testop.New(1)}, "rook/rook:myversion",