From 724a78cc6b01f68cc79c94084d99bebefe2d447f Mon Sep 17 00:00:00 2001 From: Steve Leon Date: Mon, 9 Oct 2017 14:36:21 -0700 Subject: [PATCH] Moving FindRBDMappedFile to ceph/util/ Fixes https://github.com/rook/rook/issues/1056 --- cmd/rookctl/block/list.go | 7 ++- cmd/rookctl/block/map.go | 5 +- cmd/rookctl/block/unmap.go | 5 +- pkg/agent/flexvolume/manager/ceph/manager.go | 12 ++-- pkg/ceph/util/util.go | 62 ++++++++++++++++++++ pkg/ceph/util/util_test.go | 43 ++++++++++++++ pkg/util/sys/device.go | 42 ++----------- pkg/util/sys/device_test.go | 18 ------ 8 files changed, 126 insertions(+), 68 deletions(-) create mode 100644 pkg/ceph/util/util.go create mode 100644 pkg/ceph/util/util_test.go diff --git a/cmd/rookctl/block/list.go b/cmd/rookctl/block/list.go index 1915a647e..0cb851c17 100644 --- a/cmd/rookctl/block/list.go +++ b/cmd/rookctl/block/list.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/rook/rook/cmd/rookctl/rook" + "github.com/rook/rook/pkg/ceph/util" "github.com/rook/rook/pkg/rook/client" "github.com/rook/rook/pkg/util/display" "github.com/rook/rook/pkg/util/exec" @@ -50,7 +51,7 @@ func listBlocksEntry(cmd *cobra.Command, args []string) error { c := rook.NewRookNetworkRestClient() e := &exec.CommandExecutor{} - out, err := listBlocks(sys.RBDSysBusPathDefault, c, e) + out, err := listBlocks(util.RBDSysBusPathDefault, c, e) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -75,8 +76,8 @@ func listBlocks(rbdSysBusPath string, c client.RookRestClient, executor exec.Exe image := &(images[i]) // look up local device and mount point, ignoring errors - imageFile, _ := sys.FindRBDMappedFile(image.Name, image.PoolName, rbdSysBusPath) - devPath := sys.RBDDevicePathPrefix + imageFile + imageFile, _ := util.FindRBDMappedFile(image.Name, image.PoolName, rbdSysBusPath) + devPath := util.RBDDevicePathPrefix + imageFile dev := strings.TrimPrefix(devPath, devicePathPrefix) var mountPoint string if dev != "" { diff --git a/cmd/rookctl/block/map.go b/cmd/rookctl/block/map.go index 845922509..f98e2c16e 100644 --- a/cmd/rookctl/block/map.go +++ b/cmd/rookctl/block/map.go @@ -24,6 +24,7 @@ import ( rc "github.com/rook/rook/cmd/rookctl/client" "github.com/rook/rook/cmd/rookctl/rook" + "github.com/rook/rook/pkg/ceph/util" "github.com/rook/rook/pkg/model" "github.com/rook/rook/pkg/rook/client" "github.com/rook/rook/pkg/util/exec" @@ -72,7 +73,7 @@ func mapBlockEntry(cmd *cobra.Command, args []string) error { } c := rook.NewRookNetworkRestClient() e := &exec.CommandExecutor{} - out, err := mapBlock(mapImageName, mapImagePoolName, mapImagePath, sys.RBDSysBusPathDefault, sys.RBDDevicePathPrefix, mapFormatRequested, c, e) + out, err := mapBlock(mapImageName, mapImagePoolName, mapImagePath, util.RBDSysBusPathDefault, util.RBDDevicePathPrefix, mapFormatRequested, c, e) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -204,7 +205,7 @@ func openRBDFile(hasSingleMajor bool, singleMajorPath, path string) (*os.File, e func waitForDevicePath(imageName, poolName, rbdSysBusPath, rbdDevicePathPrefix string, maxRetries, sleepSecs int) (string, error) { retryCount := 0 for { - mappedFile, err := sys.FindRBDMappedFile(imageName, poolName, rbdSysBusPath) + mappedFile, err := util.FindRBDMappedFile(imageName, poolName, rbdSysBusPath) if err != nil { return "", fmt.Errorf("failed to find mapped image: %+v", err) } diff --git a/cmd/rookctl/block/unmap.go b/cmd/rookctl/block/unmap.go index 2743e549d..71ed00468 100644 --- a/cmd/rookctl/block/unmap.go +++ b/cmd/rookctl/block/unmap.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/rook/rook/cmd/rookctl/rook" + "github.com/rook/rook/pkg/ceph/util" "github.com/rook/rook/pkg/util/exec" "github.com/rook/rook/pkg/util/flags" "github.com/rook/rook/pkg/util/sys" @@ -53,7 +54,7 @@ func unmapBlockEntry(cmd *cobra.Command, args []string) error { } e := &exec.CommandExecutor{} - out, err := unmapBlock(unmapDeviceName, unmapPath, sys.RBDSysBusPathDefault, e) + out, err := unmapBlock(unmapDeviceName, unmapPath, util.RBDSysBusPathDefault, e) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -93,7 +94,7 @@ func unmapBlock(device, path, rbdSysBusPath string, executor exec.Executor) (str return "", fmt.Errorf("failed to unmount rbd device %s: %+v", device, err) } - rbdNum := strings.TrimPrefix(device, sys.RBDDevicePathPrefix) + rbdNum := strings.TrimPrefix(device, util.RBDDevicePathPrefix) logger.Infof("removing rbd device %s (%s)", rbdNum, device) // determine if the rbd kernel module supports single_major and open the diff --git a/pkg/agent/flexvolume/manager/ceph/manager.go b/pkg/agent/flexvolume/manager/ceph/manager.go index 4e4b76fdd..c9bafd681 100644 --- a/pkg/agent/flexvolume/manager/ceph/manager.go +++ b/pkg/agent/flexvolume/manager/ceph/manager.go @@ -25,14 +25,14 @@ import ( "strings" "time" - cephmon "github.com/rook/rook/pkg/ceph/mon" - "github.com/rook/rook/pkg/model" - "github.com/rook/rook/pkg/util/sys" - "github.com/coreos/pkg/capnslog" cephclient "github.com/rook/rook/pkg/ceph/client" + cephmon "github.com/rook/rook/pkg/ceph/mon" + "github.com/rook/rook/pkg/ceph/util" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/model" "github.com/rook/rook/pkg/operator/mon" + "github.com/rook/rook/pkg/util/sys" ) const ( @@ -194,13 +194,13 @@ func getClusterInfo(context *clusterd.Context, clusterName string) (string, stri // FindDevicePath polls and wait for the mapped ceph image device to show up func (f *devicePathFinder) FindDevicePath(image, pool, clusterName string) (string, error) { - mappedFile, err := sys.FindRBDMappedFile(image, pool, sys.RBDSysBusPathDefault) + mappedFile, err := util.FindRBDMappedFile(image, pool, util.RBDSysBusPathDefault) if err != nil { return "", fmt.Errorf("failed to find mapped image: %+v", err) } if mappedFile != "" { - devicePath := sys.RBDDevicePathPrefix + mappedFile + devicePath := util.RBDDevicePathPrefix + mappedFile if _, err := os.Lstat(devicePath); err != nil { return "", fmt.Errorf("sysfs information for image '%s' in pool '%s' found but the rbd device path %s does not exist", image, pool, devicePath) } diff --git a/pkg/ceph/util/util.go b/pkg/ceph/util/util.go new file mode 100644 index 000000000..264b9c717 --- /dev/null +++ b/pkg/ceph/util/util.go @@ -0,0 +1,62 @@ +/* +Copyright 2017 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Some of the code below came from https://github.com/coreos/etcd-operator +which also has the apache 2.0 license. +*/ + +package util + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +const ( + RBDSysBusPathDefault = "/sys/bus/rbd" + RBDDevicesDir = "devices" + RBDDevicePathPrefix = "/dev/rbd" +) + +// FindRBDMappedFile search for the mapped RBD volume and returns its device path +func FindRBDMappedFile(imageName, poolName, sysBusDir string) (string, error) { + + sysBusDeviceDir := filepath.Join(sysBusDir, RBDDevicesDir) + // if sysPath does not exist, no attachments has happened + if _, err := os.Stat(sysBusDeviceDir); os.IsNotExist(err) { + return "", nil + } + + files, err := ioutil.ReadDir(sysBusDeviceDir) + if err != nil { + return "", fmt.Errorf("failed to read rbd device dir: %+v", err) + } + + for _, idFile := range files { + nameContent, err := ioutil.ReadFile(filepath.Join(sysBusDeviceDir, idFile.Name(), "name")) + if err == nil && imageName == strings.TrimSpace(string(nameContent)) { + // the image for the current rbd device matches, now try to match pool + poolContent, err := ioutil.ReadFile(filepath.Join(sysBusDeviceDir, idFile.Name(), "pool")) + if err == nil && poolName == strings.TrimSpace(string(poolContent)) { + // match current device matches both image name and pool name, return the device + return idFile.Name(), nil + } + } + } + return "", nil +} diff --git a/pkg/ceph/util/util_test.go b/pkg/ceph/util/util_test.go new file mode 100644 index 000000000..47531089c --- /dev/null +++ b/pkg/ceph/util/util_test.go @@ -0,0 +1,43 @@ +/* +Copyright 2017 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Some of the code below came from https://github.com/coreos/etcd-operator +which also has the apache 2.0 license. +*/ +package util + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFindDevicePath(t *testing.T) { + // set up a mock RBD sys bus file system + mockRBDSysBusPath, err := ioutil.TempDir("", "TestFindDevicePath") + if err != nil { + t.Fatalf("failed to create temp rbd sys bus dir: %+v", err) + } + defer os.RemoveAll(mockRBDSysBusPath) + dev0Path := filepath.Join(mockRBDSysBusPath, "devices", "3") + os.MkdirAll(dev0Path, 0777) + ioutil.WriteFile(filepath.Join(dev0Path, "name"), []byte("myimage1"), 0777) + ioutil.WriteFile(filepath.Join(dev0Path, "pool"), []byte("mypool1"), 0777) + mappedImageFile, _ := FindRBDMappedFile("myimage1", "mypool1", mockRBDSysBusPath) + assert.Equal(t, "3", mappedImageFile) +} diff --git a/pkg/util/sys/device.go b/pkg/util/sys/device.go index 288d83fb7..6ad5b67e6 100644 --- a/pkg/util/sys/device.go +++ b/pkg/util/sys/device.go @@ -17,7 +17,6 @@ package sys import ( "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -29,14 +28,11 @@ import ( ) const ( - DiskType = "disk" - SSDType = "ssd" - PartType = "part" - sgdisk = "sgdisk" - mountCmd = "mount" - RBDSysBusPathDefault = "/sys/bus/rbd" - RBDDevicesDir = "devices" - RBDDevicePathPrefix = "/dev/rbd" + DiskType = "disk" + SSDType = "ssd" + PartType = "part" + sgdisk = "sgdisk" + mountCmd = "mount" ) type Partition struct { @@ -326,31 +322,3 @@ func parseKeyValuePairString(propsRaw string) map[string]string { return propMap } - -// FindRBDMappedFile search for the mapped RBD volume and returns its device path -func FindRBDMappedFile(imageName, poolName, sysBusDir string) (string, error) { - - sysBusDeviceDir := filepath.Join(sysBusDir, RBDDevicesDir) - // if sysPath does not exist, no attachments has happened - if _, err := os.Stat(sysBusDeviceDir); os.IsNotExist(err) { - return "", nil - } - - files, err := ioutil.ReadDir(sysBusDeviceDir) - if err != nil { - return "", fmt.Errorf("failed to read rbd device dir: %+v", err) - } - - for _, idFile := range files { - nameContent, err := ioutil.ReadFile(filepath.Join(sysBusDeviceDir, idFile.Name(), "name")) - if err == nil && imageName == strings.TrimSpace(string(nameContent)) { - // the image for the current rbd device matches, now try to match pool - poolContent, err := ioutil.ReadFile(filepath.Join(sysBusDeviceDir, idFile.Name(), "pool")) - if err == nil && poolName == strings.TrimSpace(string(poolContent)) { - // match current device matches both image name and pool name, return the device - return idFile.Name(), nil - } - } - } - return "", nil -} diff --git a/pkg/util/sys/device_test.go b/pkg/util/sys/device_test.go index 39d96f785..75ff98779 100644 --- a/pkg/util/sys/device_test.go +++ b/pkg/util/sys/device_test.go @@ -17,9 +17,6 @@ package sys import ( "fmt" - "io/ioutil" - "os" - "path/filepath" "strings" "testing" @@ -197,18 +194,3 @@ NAME="sda6" SIZE="134217728" TYPE="part" PKNAME="sda"`, nil assert.Nil(t, err) assert.Equal(t, 0, len(partitions)) } - -func TestFindDevicePath(t *testing.T) { - // set up a mock RBD sys bus file system - mockRBDSysBusPath, err := ioutil.TempDir("", "TestFindDevicePath") - if err != nil { - t.Fatalf("failed to create temp rbd sys bus dir: %+v", err) - } - defer os.RemoveAll(mockRBDSysBusPath) - dev0Path := filepath.Join(mockRBDSysBusPath, "devices", "3") - os.MkdirAll(dev0Path, 0777) - ioutil.WriteFile(filepath.Join(dev0Path, "name"), []byte("myimage1"), 0777) - ioutil.WriteFile(filepath.Join(dev0Path, "pool"), []byte("mypool1"), 0777) - mappedImageFile, _ := FindRBDMappedFile("myimage1", "mypool1", mockRBDSysBusPath) - assert.Equal(t, "3", mappedImageFile) -}