ceph file: tidy up file/mds operator code

Separate `file` operator from `mds` tasks logically for easier code
navigation.

Signed-off-by: Blaine Gardner <blaine.gardner@suse.com>

pkg/operator/ceph/file/filesystem.go

@@ -18,13 +18,13 @@ package file
 
 import (
 	"fmt"
-	"time"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/daemon/ceph/client"
 	cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
 	"github.com/rook/rook/pkg/daemon/ceph/model"
+	"github.com/rook/rook/pkg/operator/ceph/file/mds"
 	"github.com/rook/rook/pkg/operator/ceph/pool"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -81,8 +81,8 @@ func createFilesystem(
 	}
 
 	logger.Infof("start running mdses for filesystem %s", fs.Name)
-	c := newCluster(clusterInfo, context, rookVersion, cephVersion, hostNetwork, fs, filesystem, ownerRefs)
-	if err := c.start(); err != nil {
+	c := mds.NewCluster(clusterInfo, context, rookVersion, cephVersion, hostNetwork, fs, filesystem, ownerRefs)
+	if err := c.Start(); err != nil {
 		return err
 	}
 
@@ -104,7 +104,7 @@ func deleteFilesystem(context *clusterd.Context, fs cephv1.CephFilesystem) error
 		}
 	}
 
-	return deleteMdsCluster(context, fs.Namespace, fs.Name)
+	return mds.DeleteCluster(context, fs.Namespace, fs.Name)
 }
 
 func validateFilesystem(context *clusterd.Context, f cephv1.CephFilesystem) error {
@@ -230,50 +230,3 @@ func downFilesystem(context *clusterd.Context, clusterName, filesystemName strin
 	logger.Infof("Removed filesystem %s", filesystemName)
 	return nil
 }
-
-// prepareForDaemonUpgrade performs all actions necessary to ensure the filesystem is prepared
-// to have its daemon(s) updated. This helps ensure there is no aberrant behavior during upgrades.
-// If the mds is not prepared within the timeout window, an error will be reported.
-// Ceph docs: http://docs.ceph.com/docs/master/cephfs/upgrading/
-func prepareForDaemonUpgrade(
-	context *clusterd.Context,
-	clusterName, fsName string,
-	timeout time.Duration,
-) error {
-	logger.Infof("preparing filesystem %s for daemon upgrade", fsName)
-	// * Beginning of noted section 1
-	// This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3.
-	//   See more:  https://ceph.com/releases/v13-2-0-mimic-released/
-	//              http://docs.ceph.com/docs/mimic/cephfs/upgrading/
-	// As of Oct. 2018, this is only necessary for Luminous and Mimic.
-	if err := client.SetNumMDSRanks(context, clusterName, fsName, 1); err != nil {
-		return fmt.Errorf("Could not Prepare filesystem %s for daemon upgrade: %+v", fsName, err)
-	}
-	if err := client.WaitForActiveRanks(context, clusterName, fsName, 1, false, timeout); err != nil {
-		return err
-	}
-	// * End of Noted section 1
-
-	logger.Infof("Filesystem %s successfully prepared for mds daemon upgrade", fsName)
-	return nil
-}
-
-// finishedWithDaemonUpgrade performs all actions necessary to bring the filesystem back to its
-// ideal state following an upgrade of its daemon(s).
-func finishedWithDaemonUpgrade(
-	context *clusterd.Context,
-	clusterName, fsName string,
-	activeMDSCount int32,
-) error {
-	logger.Debugf("restoring filesystem %s from daemon upgrade", fsName)
-	logger.Debugf("bringing num active mds daemons for fs %s back to %d", fsName, activeMDSCount)
-	// * Beginning of noted section 1
-	// This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3.
-	//   See more:  https://ceph.com/releases/v13-2-0-mimic-released/
-	//              http://docs.ceph.com/docs/mimic/cephfs/upgrading/
-	// TODO: Unknown (Oct. 2018) if any parts can be removed once Rook no longer supports Mimic.
-	if err := client.SetNumMDSRanks(context, clusterName, fsName, activeMDSCount); err != nil {
-		return fmt.Errorf("Failed to restore filesystem %s following daemon upgrade: %+v", fsName, err)
-	} // * End of noted section 1
-	return nil
-}

pkg/operator/ceph/file/filesystem_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/rook/rook/pkg/clusterd"
 	cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
 	cephtest "github.com/rook/rook/pkg/daemon/ceph/test"
+	"github.com/rook/rook/pkg/operator/ceph/file/mds"
 	testopk8s "github.com/rook/rook/pkg/operator/k8sutil/test"
 	testop "github.com/rook/rook/pkg/operator/test"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
@@ -69,7 +70,7 @@ func TestValidateSpec(t *testing.T) {
 
 func TestCreateFilesystem(t *testing.T) {
 	var deploymentsUpdated *[]*apps.Deployment
-	updateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub()
+	mds.UpdateDeploymentAndWait, deploymentsUpdated = testopk8s.UpdateDeploymentAndWaitStub()
 
 	configDir, _ := ioutil.TempDir("", "")
 	// Output to check multiple filesystem creation

pkg/operator/ceph/file/mds/config.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package file
+package mds
 
 import (
 	"fmt"
@@ -35,7 +35,7 @@ caps mds = "allow"
 `
 )
 
-func (c *cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error {
+func (c *Cluster) generateKeyring(m *mdsConfig, deploymentUID types.UID) error {
 	user := fmt.Sprintf("mds.%s", m.DaemonID)
 	access := []string{"osd", "allow *", "mds", "allow", "mon", "allow profile mds"}
 	ownerRef := &metav1.OwnerReference{

pkg/operator/ceph/file/mds/mds.go

@@ -14,24 +14,27 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package file
+// Package mds provides methods for managing a Ceph mds cluster.
+package mds
 
 import (
 	"fmt"
 	"strconv"
 	"time"
 
+	"github.com/coreos/pkg/capnslog"
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/daemon/ceph/client"
 	cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
 	"github.com/rook/rook/pkg/operator/ceph/config"
 	"github.com/rook/rook/pkg/operator/k8sutil"
-	apps "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-mds")
+
 const (
 	// AppName is the name of Rook's Ceph mds (File) sub-app
 	AppName = "rook-ceph-mds"
@@ -42,7 +45,8 @@ const (
 	fsWaitForActiveTimeout = 3 * time.Minute
 )
 
-type cluster struct {
+// Cluster represents a Ceph mds cluster.
+type Cluster struct {
 	clusterInfo *cephconfig.ClusterInfo
 	context     *clusterd.Context
 	rookVersion string
@@ -59,7 +63,8 @@ type mdsConfig struct {
 	DataPathMap  *config.DataPathMap // location to store data in container
 }
 
-func newCluster(
+// NewCluster creates a Ceph mds cluster representation.
+func NewCluster(
 	clusterInfo *cephconfig.ClusterInfo,
 	context *clusterd.Context,
 	rookVersion string,
@@ -68,8 +73,8 @@ func newCluster(
 	fs cephv1.CephFilesystem,
 	fsdetails *client.CephFilesystemDetails,
 	ownerRefs []metav1.OwnerReference,
-) *cluster {
-	return &cluster{
+) *Cluster {
+	return &Cluster{
 		clusterInfo: clusterInfo,
 		context:     context,
 		rookVersion: rookVersion,
@@ -81,9 +86,11 @@ func newCluster(
 	}
 }
 
-var updateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait
+// UpdateDeploymentAndWait can be overridden for unit tests. Do not alter this for runtime operation.
+var UpdateDeploymentAndWait = k8sutil.UpdateDeploymentAndWait
 
-func (c *cluster) start() error {
+// Start starts or updates a Ceph mds cluster in Kubernetes.
+func (c *Cluster) Start() error {
 	// If attempt was made to prepare daemons for upgrade, make sure that an attempt is made to
 	// bring fs state back to desired when this method returns with any error or success.
 	var fsPreparedForUpgrade = false
@@ -126,7 +133,7 @@ func (c *cluster) start() error {
 			// TODO: need to prepare for upgrade here each time. Also, before a given deployment is
 			// terminated, I think we should somehow make sure that it isn't running the single
 			// active daemon. If it is, then we should have another daemon take over as active. @Jan?
-			if createdDeployment, err = updateDeploymentAndWait(c.context, d, c.fs.Namespace); err != nil {
+			if createdDeployment, err = UpdateDeploymentAndWait(c.context, d, c.fs.Namespace); err != nil {
 				return fmt.Errorf("failed to update mds deployment %s. %+v", mdsConfig.ResourceName, err)
 			}
 		}
@@ -138,6 +145,14 @@ func (c *cluster) start() error {
 		}
 	}
 
+	if err := c.scaleDownDeployments(replicas, desiredDeployments); err != nil {
+		return fmt.Errorf("failed to scale down mds deployments. %+v", err)
+	}
+
+	return nil
+}
+
+func (c *Cluster) scaleDownDeployments(replicas int32, desiredDeployments map[string]bool) error {
 	// Remove extraneous mds deployments if they exist
 	deps, err := getMdsDeployments(c.context, c.fs.Namespace, c.fs.Name)
 	if err != nil {
@@ -186,7 +201,8 @@ func (c *cluster) start() error {
 	return nil
 }
 
-func deleteMdsCluster(context *clusterd.Context, namespace, fsName string) error {
+// DeleteCluster deletes a Ceph mds cluster from Kubernetes.
+func DeleteCluster(context *clusterd.Context, namespace, fsName string) error {
 	// Try to delete all mds deployments and secret keys serving the filesystem, and aggregate
 	// failures together to report all at once at the end.
 	deps, err := getMdsDeployments(context, namespace, fsName)
@@ -207,36 +223,49 @@ func deleteMdsCluster(context *clusterd.Context, namespace, fsName string) error
 	return nil
 }
 
-func getMdsDeployments(context *clusterd.Context, namespace, fsName string) (*apps.DeploymentList, error) {
-	fsLabelSelector := fmt.Sprintf("rook_file_system=%s", fsName)
-	deps, err := k8sutil.GetDeployments(context.Clientset, namespace, fsLabelSelector)
-	if err != nil {
-		return nil, fmt.Errorf("could not get deployments for filesystem %s (matching label selector '%s'): %+v", fsName, fsLabelSelector, err)
+// prepareForDaemonUpgrade performs all actions necessary to ensure the filesystem is prepared
+// to have its daemon(s) updated. This helps ensure there is no aberrant behavior during upgrades.
+// If the mds is not prepared within the timeout window, an error will be reported.
+// Ceph docs: http://docs.ceph.com/docs/master/cephfs/upgrading/
+func prepareForDaemonUpgrade(
+	context *clusterd.Context,
+	clusterName, fsName string,
+	timeout time.Duration,
+) error {
+	logger.Infof("preparing filesystem %s for daemon upgrade", fsName)
+	// * Beginning of noted section 1
+	// This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3.
+	//   See more:  https://ceph.com/releases/v13-2-0-mimic-released/
+	//              http://docs.ceph.com/docs/mimic/cephfs/upgrading/
+	// As of Oct. 2018, this is only necessary for Luminous and Mimic.
+	if err := client.SetNumMDSRanks(context, clusterName, fsName, 1); err != nil {
+		return fmt.Errorf("Could not Prepare filesystem %s for daemon upgrade: %+v", fsName, err)
 	}
-	return deps, nil
-}
+	if err := client.WaitForActiveRanks(context, clusterName, fsName, 1, false, timeout); err != nil {
+		return err
+	}
+	// * End of Noted section 1
 
-func deleteMdsDeployment(context *clusterd.Context, namespace string, deployment *apps.Deployment) error {
-	errCount := 0
+	logger.Infof("Filesystem %s successfully prepared for mds daemon upgrade", fsName)
+	return nil
+}
 
-	// Delete the mds deployment
-	logger.Infof("deleting mds deployment %s", deployment.Name)
-	var gracePeriod int64
-	propagation := metav1.DeletePropagationForeground
-	options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
-	if err := context.Clientset.Apps().Deployments(namespace).Delete(deployment.GetName(), options); err != nil {
-		errCount++
-		logger.Errorf("failed to delete mds deployment %s: %+v", deployment.GetName(), err)
-	}
-
-	// Delete the mds secret
-	err := context.Clientset.CoreV1().Secrets(namespace).Delete(deployment.GetName(), &metav1.DeleteOptions{})
-	if err != nil && !errors.IsNotFound(err) {
-		errCount++
-		logger.Errorf("failed to delete mds secret %s: %+v", deployment.GetName(), err)
-	}
-	if errCount > 0 {
-		return fmt.Errorf("%d error(s) during deletion of mds deployment %s, see logs above", errCount, deployment.GetName())
-	}
+// finishedWithDaemonUpgrade performs all actions necessary to bring the filesystem back to its
+// ideal state following an upgrade of its daemon(s).
+func finishedWithDaemonUpgrade(
+	context *clusterd.Context,
+	clusterName, fsName string,
+	activeMDSCount int32,
+) error {
+	logger.Debugf("restoring filesystem %s from daemon upgrade", fsName)
+	logger.Debugf("bringing num active mds daemons for fs %s back to %d", fsName, activeMDSCount)
+	// * Beginning of noted section 1
+	// This section is necessary for upgrading to Mimic and to/past Luminous 12.2.3.
+	//   See more:  https://ceph.com/releases/v13-2-0-mimic-released/
+	//              http://docs.ceph.com/docs/mimic/cephfs/upgrading/
+	// TODO: Unknown (Oct. 2018) if any parts can be removed once Rook no longer supports Mimic.
+	if err := client.SetNumMDSRanks(context, clusterName, fsName, activeMDSCount); err != nil {
+		return fmt.Errorf("Failed to restore filesystem %s following daemon upgrade: %+v", fsName, err)
+	} // * End of noted section 1
 	return nil
 }

pkg/operator/ceph/file/mds/spec.go

@@ -14,11 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package file
+package mds
 
 import (
+	"fmt"
 	"strconv"
 
+	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/operator/ceph/config"
 	opspec "github.com/rook/rook/pkg/operator/ceph/spec"
 	"github.com/rook/rook/pkg/operator/k8sutil"
@@ -31,7 +33,7 @@ const (
 	mdsDaemonCommand = "ceph-mds"
 )
 
-func (c *cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment {
+func (c *Cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment {
 	podSpec := v1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        mdsConfig.ResourceName,
@@ -75,7 +77,7 @@ func (c *cluster) makeDeployment(mdsConfig *mdsConfig) *apps.Deployment {
 	return d
 }
 
-func (c *cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container {
+func (c *Cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container {
 	return v1.Container{
 		Name: "mds",
 		Command: []string{
@@ -96,8 +98,31 @@ func (c *cluster) makeMdsDaemonContainer(mdsConfig *mdsConfig) v1.Container {
 	}
 }
 
-func (c *cluster) podLabels(mdsConfig *mdsConfig) map[string]string {
+func (c *Cluster) podLabels(mdsConfig *mdsConfig) map[string]string {
 	labels := opspec.PodLabels(AppName, c.fs.Namespace, "mds", mdsConfig.DaemonID)
 	labels["rook_file_system"] = c.fs.Name
 	return labels
 }
+
+func getMdsDeployments(context *clusterd.Context, namespace, fsName string) (*apps.DeploymentList, error) {
+	fsLabelSelector := fmt.Sprintf("rook_file_system=%s", fsName)
+	deps, err := k8sutil.GetDeployments(context.Clientset, namespace, fsLabelSelector)
+	if err != nil {
+		return nil, fmt.Errorf("could not get deployments for filesystem %s (matching label selector '%s'): %+v", fsName, fsLabelSelector, err)
+	}
+	return deps, nil
+}
+
+func deleteMdsDeployment(context *clusterd.Context, namespace string, deployment *apps.Deployment) error {
+	errCount := 0
+	// Delete the mds deployment
+	logger.Infof("deleting mds deployment %s", deployment.Name)
+	var gracePeriod int64
+	propagation := metav1.DeletePropagationForeground
+	options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
+	if err := context.Clientset.Apps().Deployments(namespace).Delete(deployment.GetName(), options); err != nil {
+		errCount++
+		logger.Errorf("failed to delete mds deployment %s: %+v", deployment.GetName(), err)
+	}
+	return nil
+}

pkg/operator/ceph/file/mds/spec_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package file
+package mds
 
 import (
 	"testing"
@@ -52,7 +52,7 @@ func testDeploymentObject(hostNetwork bool) *apps.Deployment {
 				}}}}
 	clusterInfo := &cephconfig.ClusterInfo{FSID: "myfsid"}
 
-	c := newCluster(
+	c := NewCluster(
 		clusterInfo,
 		&clusterd.Context{Clientset: testop.New(1)},
 		"rook/rook:myversion",