Merge pull request #1199 from kokhang/cephfs-crd

Handle cephfs mount for kernel older than 4.7

Documentation/common-problems.md

@@ -157,6 +157,17 @@ If `uname -a` shows that you have a kernel version older than `3.15`, you'll nee
 * Disable some Ceph features by starting the [rook toolbox](./toolbox.md) and running `ceph osd crush tunables bobtail`
 * Upgrade your kernel to `3.15` or later.
 
+### Filesystem Mounting
+
+In the `rook-agent` pod logs, you may see a snippet similar to the following:
+```console
+2017-11-07 00:04:37.808870 I | rook-flexdriver: WARNING: The node kernel version is 4.4.0-87-generic, which do not support multiple ceph filesystems. The kernel version has to be at least 4.7. If you have multiple ceph filesystems, the result could be inconsistent
+```
+
+This will happen in kernels with versions older than 4.7, where the option `mds_namespace` is not supported. This option is used to specify a filesystem namespace.
+
+In this case, if there is only one filesystem in the Rook cluster, there should be no issues and the mount should succeed. If you have more than one filesystem, inconsistent results may arise and the filesystem mounted may not be the one you specified.
+
 If the issue is still not resolved from the steps above, please come chat with us on the **#general** channel of our [Rook Slack](https://rook-slackin.herokuapp.com/).
 We want to help you get your storage working and learn from those lessons to prevent users in the future from seeing the same issue.
 

Documentation/k8s-filesystem.md

@@ -117,6 +117,8 @@ spec:
 
 You now have a docker registry which is HA with persistent storage.
 
+**NOTE:** If the Rook cluster has more than one filesystem and the application pod is scheduled to a node with kernel version older than 4.7, inconsistent results may arise since kernels older than 4.7 do not support specifying filesystem namespaces.
+
 ## Test the storage
 
 Once you have pushed an image to the registry (see the [instructions](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/registry) to expose and use the kube-registry), verify that kube-registry is using the filesystem that was configured above by mounting the shared file system in the toolbox pod.

cmd/rookflex/cmd/mount.go

@@ -28,9 +28,14 @@ import (
 	"github.com/rook/rook/pkg/model"
 	"github.com/spf13/cobra"
 	k8smount "k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/util/version"
 	"k8s.io/kubernetes/pkg/volume/util"
 )
 
+const (
+	mds_namespace_kernel_support = "4.7"
+)
+
 var (
 	mountCmd = &cobra.Command{
 		Use:   "mount",
@@ -214,8 +219,32 @@ func mountCephFS(client *rpc.Client, opts *flexvolume.AttachOptions) error {
 		}
 	}
 
+	options := []string{fmt.Sprintf("name=%s", clientAccessInfo.UserName), fmt.Sprintf("secret=%s", clientAccessInfo.SecretKey)}
+
+	// Get kernel version
+	var kernelVersion string
+	err = client.Call("FlexvolumeController.GetKernelVersion", struct{}{} /* no inputs */, &kernelVersion)
+	if err != nil {
+		log(client, fmt.Sprintf("WARNING: The node kernel version cannot be detected. The kernel version has to be at least %s in order to specify a filesystem namespace."+
+			" If you have multiple ceph filesystems, the result could be inconsistent", mds_namespace_kernel_support), false)
+	} else {
+		kernelVersionParsed, err := version.ParseGeneric(kernelVersion)
+		if err != nil {
+			log(client, fmt.Sprintf("WARNING: The node kernel version %s cannot be parsed. The kernel version has to be at least %s in order to specify a filesystem namespace."+
+				" If you have multiple ceph filesystems, the result could be inconsistent", kernelVersion, mds_namespace_kernel_support), false)
+		} else {
+			if kernelVersionParsed.AtLeast(version.MustParseGeneric(mds_namespace_kernel_support)) {
+				options = append(options, fmt.Sprintf("mds_namespace=%s", opts.FsName))
+			} else {
+				log(client,
+					fmt.Sprintf("WARNING: The node kernel version is %s, which do not support multiple ceph filesystems. "+
+						"The kernel version has to be at least %s. If you have multiple ceph filesystems, the result could be inconsistent",
+						kernelVersion, mds_namespace_kernel_support), false)
+			}
+		}
+	}
+
 	devicePath := fmt.Sprintf("%s:%s", strings.Join(clientAccessInfo.MonAddresses, ","), path)
-	options := []string{fmt.Sprintf("name=%s", clientAccessInfo.UserName), fmt.Sprintf("secret=%s", clientAccessInfo.SecretKey), fmt.Sprintf("mds_namespace=%s", opts.FsName)}
 
 	log(client, fmt.Sprintf("mounting ceph filesystem %s on %s to %s", opts.FsName, devicePath, opts.MountDir), false)
 	mounter := getMounter()

pkg/agent/flexvolume/controller.go

@@ -335,6 +335,17 @@ func (c *FlexvolumeController) GetClientAccessInfo(clusterName string, clientAcc
 	return nil
 }
 
+// GetKernelVersion returns the kernel version of the current node.
+func (c *FlexvolumeController) GetKernelVersion(_ *struct{} /* no inputs */, kernelVersion *string) error {
+	nodeName := os.Getenv(k8sutil.NodeNameEnvVar)
+	node, err := c.context.Clientset.Core().Nodes().Get(nodeName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get kernel version from node information for node %s: %+v", nodeName, err)
+	}
+	*kernelVersion = node.Status.NodeInfo.KernelVersion
+	return nil
+}
+
 // getKubeletRootDir queries the kubelet configuration to find the kubelet root dir. Defaults to /var/lib/kubelet
 func (c *FlexvolumeController) getKubeletRootDir() string {
 	nodeConfigURI, err := k8sutil.NodeConfigURI()

pkg/operator/agent/agent.go

@@ -48,7 +48,7 @@ var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-agent")
 var clusterAccessRules = []v1beta1.PolicyRule{
 	{
 		APIGroups: []string{""},
-		Resources: []string{"pods", "secrets", "configmaps", "persistentvolumes", "nodes/proxy"},
+		Resources: []string{"pods", "secrets", "configmaps", "persistentvolumes", "nodes", "nodes/proxy"},
 		Verbs:     []string{"get", "list"},
 	},
 	{

pkg/operator/mds/mds.go

@@ -42,7 +42,6 @@ const (
 	dataPoolSuffix     = "-data"
 	metadataPoolSuffix = "-metadata"
 	keyringName        = "keyring"
-	adminSecretName    = "rook-admin"
 )
 
 func CreateFileSystem(context *clusterd.Context, clusterName string, f *model.FilesystemRequest, version string, hostNetwork bool) error {