From 6c11ff52d47a30448861fa25dcc76076c3c619f4 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Thu, 5 Apr 2018 18:39:20 +0000 Subject: [PATCH] add device discovery daemon to operator: run "rook discover" on storage nodes and discover devices on each node. The discovered disks are saved in a per node configmap, local-device-nodename. Device information consits of name and persistent names, uuid, partition, filesystem, rotational, readonly, size, etc. Signed-off-by: Huamin Chen --- cmd/rook/discover.go | 54 ++++ cmd/rook/main.go | 1 + pkg/clusterd/context.go | 3 +- pkg/clusterd/disk.go | 64 +++-- pkg/clusterd/disk_test.go | 22 +- pkg/daemon/ceph/osd/agent_test.go | 23 +- pkg/daemon/ceph/osd/daemon.go | 2 +- pkg/daemon/ceph/osd/daemon_test.go | 40 ++- pkg/daemon/ceph/osd/device.go | 36 +-- pkg/daemon/ceph/osd/device_test.go | 62 ++-- pkg/daemon/discover/discover.go | 164 +++++++++++ pkg/daemon/discover/discover_test.go | 100 +++++++ pkg/operator/cluster/ceph/mon/util.go | 2 +- pkg/operator/cluster/ceph/osd/osd.go | 14 +- pkg/operator/discover/discover.go | 264 ++++++++++++++++++ pkg/operator/discover/discover_test.go | 127 +++++++++ pkg/operator/k8sutil/rbac.go | 16 +- pkg/operator/k8sutil/rbac_test.go | 9 +- pkg/operator/operator.go | 6 + pkg/util/sys/device.go | 173 +++++++++--- pkg/util/sys/device_test.go | 117 +++++--- pkg/util/sys/parse.go | 4 +- pkg/util/sys/parse_test.go | 2 +- tests/framework/utils/helm_helper.go | 2 +- tests/integration/base_deploy_test.go | 2 + .../integration/multi_cluster_deploy_test.go | 3 + 26 files changed, 1105 insertions(+), 207 deletions(-) create mode 100644 cmd/rook/discover.go create mode 100644 pkg/daemon/discover/discover.go create mode 100644 pkg/daemon/discover/discover_test.go create mode 100644 pkg/operator/discover/discover.go create mode 100644 pkg/operator/discover/discover_test.go diff --git a/cmd/rook/discover.go b/cmd/rook/discover.go new file mode 100644 index 000000000..8f84c897f --- /dev/null +++ b/cmd/rook/discover.go @@ -0,0 +1,54 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "fmt" + "github.com/rook/rook/pkg/daemon/discover" + "github.com/spf13/cobra" +) + +var discoverCmd = &cobra.Command{ + Use: "discover", + Short: "Discover devices", + Hidden: true, +} + +func init() { + discoverCmd.RunE = startDiscover +} + +func startDiscover(cmd *cobra.Command, args []string) error { + setLogLevel() + + logStartupInfo(discoverCmd.Flags()) + + clientset, _, rookClientset, err := getClientset() + if err != nil { + terminateFatal(fmt.Errorf("failed to init k8s client. %+v\n", err)) + } + + context := createContext() + context.Clientset = clientset + context.RookClientset = rookClientset + + err = discover.Run(context) + if err != nil { + terminateFatal(err) + } + + return nil +} diff --git a/cmd/rook/main.go b/cmd/rook/main.go index bca84df3c..7c7c89f61 100644 --- a/cmd/rook/main.go +++ b/cmd/rook/main.go @@ -100,6 +100,7 @@ func addCommands() { rootCmd.AddCommand(mdsCmd) rootCmd.AddCommand(agentCmd) rootCmd.AddCommand(operatorCmd) + rootCmd.AddCommand(discoverCmd) } func setLogLevel() { diff --git a/pkg/clusterd/context.go b/pkg/clusterd/context.go index ab5200a5a..ddff82b0d 100644 --- a/pkg/clusterd/context.go +++ b/pkg/clusterd/context.go @@ -19,6 +19,7 @@ import ( "github.com/coreos/pkg/capnslog" rookclient "github.com/rook/rook/pkg/client/clientset/versioned" "github.com/rook/rook/pkg/util/exec" + "github.com/rook/rook/pkg/util/sys" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/client-go/kubernetes" ) @@ -50,5 +51,5 @@ type Context struct { NetworkInfo NetworkInfo // The local devices detected on the node - Devices []*LocalDisk + Devices []*sys.LocalDisk } diff --git a/pkg/clusterd/disk.go b/pkg/clusterd/disk.go index 86f39d0e3..0cdc0fc20 100644 --- a/pkg/clusterd/disk.go +++ b/pkg/clusterd/disk.go @@ -30,22 +30,7 @@ var ( isRBD = regexp.MustCompile("^rbd[0-9]+p?[0-9]{0,}$") ) -type LocalDisk struct { - Name string `json:"name"` - ID string `json:"id"` - UUID string `json:"uuid"` - Size uint64 `json:"size"` - Rotational bool `json:"rotational"` - Readonly bool `json:"readonly"` - FileSystem string `json:"fileSystem"` - MountPoint string `json:"mountPoint"` - Type string `json:"type"` - Parent string `json:"parent"` - HasChildren bool `json:"hasChildren"` - Empty bool `json:"empty"` -} - -func GetAvailableDevices(devices []*LocalDisk) []string { +func GetAvailableDevices(devices []*sys.LocalDisk) []string { var available []string for _, device := range devices { @@ -60,8 +45,8 @@ func GetAvailableDevices(devices []*LocalDisk) []string { } // check whether a device is completely empty -func getDeviceEmpty(device *LocalDisk) bool { - return device.Parent == "" && (device.Type == sys.DiskType || device.Type == sys.SSDType || device.Type == sys.CryptType) && device.FileSystem == "" +func getDeviceEmpty(device *sys.LocalDisk) bool { + return device.Parent == "" && (device.Type == sys.DiskType || device.Type == sys.SSDType || device.Type == sys.CryptType) && device.Filesystem == "" } func ignoreDevice(d string) bool { @@ -69,9 +54,9 @@ func ignoreDevice(d string) bool { } // Discover all the details of devices available on the local node -func DiscoverDevices(executor exec.Executor) ([]*LocalDisk, error) { +func DiscoverDevices(executor exec.Executor) ([]*sys.LocalDisk, error) { - var disks []*LocalDisk + var disks []*sys.LocalDisk devices, err := sys.ListDevices(executor) if err != nil { return nil, err @@ -99,19 +84,20 @@ func DiscoverDevices(executor exec.Executor) ([]*LocalDisk, error) { // get the UUID for disks var diskUUID string if diskType != sys.PartType { - diskUUID, err = sys.GetDiskUUID(d, executor) + diskUUID, err = sys.GetFSUUID(d, executor) if err != nil { logger.Warningf("skipping device %s with an unknown uuid. %+v", d, err) continue } } - fs, err := sys.GetDeviceFilesystems(d, executor) + udevInfo, err := sys.GetUdevInfo(d, executor) if err != nil { - return nil, err + logger.Warningf("failed to get udev info for device %s: %+v", d, err) + continue } - disk := &LocalDisk{Name: d, UUID: diskUUID, FileSystem: fs} + disk := &sys.LocalDisk{Name: d, UUID: diskUUID} if val, ok := diskProps["TYPE"]; ok { disk.Type = val @@ -137,6 +123,36 @@ func DiscoverDevices(executor exec.Executor) ([]*LocalDisk, error) { disk.Empty = getDeviceEmpty(disk) + // parse udev info output + if val, ok := udevInfo["DEVLINKS"]; ok { + disk.DevLinks = val + } + if val, ok := udevInfo["ID_FS_UUID"]; ok { + disk.UUID = val + } + if val, ok := udevInfo["ID_FS_TYPE"]; ok { + disk.Filesystem = val + } + if val, ok := udevInfo["ID_SERIAL"]; ok { + disk.Serial = val + } + + if val, ok := udevInfo["ID_VENDOR"]; ok { + disk.Vendor = val + } + + if val, ok := udevInfo["ID_MODEL"]; ok { + disk.Model = val + } + + if val, ok := udevInfo["ID_WWN_WITH_EXTENSION"]; ok { + disk.WWNVendorExtension = val + } + + if val, ok := udevInfo["ID_WWN"]; ok { + disk.WWN = val + } + disks = append(disks, disk) } diff --git a/pkg/clusterd/disk_test.go b/pkg/clusterd/disk_test.go index cf7b517ac..7b745d026 100644 --- a/pkg/clusterd/disk_test.go +++ b/pkg/clusterd/disk_test.go @@ -26,34 +26,34 @@ import ( func TestAvailableDisks(t *testing.T) { // no disks discovered for a node is an error - disks := GetAvailableDevices([]*LocalDisk{}) + disks := GetAvailableDevices([]*sys.LocalDisk{}) assert.Equal(t, 0, len(disks)) // no available disks because of the formatting - d1 := &LocalDisk{Name: "sda", UUID: "myuuid1", Size: 123, Rotational: true, Readonly: false, FileSystem: "btrfs", MountPoint: "/mnt/abc", Type: sys.DiskType, HasChildren: true} - disks = GetAvailableDevices([]*LocalDisk{d1}) + d1 := &sys.LocalDisk{Name: "sda", UUID: "myuuid1", Size: 123, Rotational: true, Readonly: false, Filesystem: "btrfs", Type: sys.DiskType, HasChildren: true} + disks = GetAvailableDevices([]*sys.LocalDisk{d1}) assert.Equal(t, 0, len(disks)) // multiple available disks - d2 := &LocalDisk{Name: "sdb", UUID: "myuuid2", Size: 123, Rotational: true, Readonly: false, Type: sys.DiskType, HasChildren: true} - d3 := &LocalDisk{Name: "sdc", UUID: "myuuid3", Size: 123, Rotational: true, Readonly: false, Type: sys.DiskType, HasChildren: true} - disks = GetAvailableDevices([]*LocalDisk{d1, d2, d3}) + d2 := &sys.LocalDisk{Name: "sdb", UUID: "myuuid2", Size: 123, Rotational: true, Readonly: false, Type: sys.DiskType, HasChildren: true} + d3 := &sys.LocalDisk{Name: "sdc", UUID: "myuuid3", Size: 123, Rotational: true, Readonly: false, Type: sys.DiskType, HasChildren: true} + disks = GetAvailableDevices([]*sys.LocalDisk{d1, d2, d3}) assert.Equal(t, 2, len(disks)) assert.Equal(t, "sdb", disks[0]) assert.Equal(t, "sdc", disks[1]) // partitions don't result in more available devices - d4 := &LocalDisk{Name: "sdb1", UUID: "myuuid4", Size: 123, Rotational: true, Readonly: false, Type: sys.PartType, HasChildren: true} - d5 := &LocalDisk{Name: "sdb2", UUID: "myuuid5", Size: 123, Rotational: true, Readonly: false, Type: sys.PartType, HasChildren: true} - disks = GetAvailableDevices([]*LocalDisk{d1, d2, d3, d4, d5}) + d4 := &sys.LocalDisk{Name: "sdb1", UUID: "myuuid4", Size: 123, Rotational: true, Readonly: false, Type: sys.PartType, HasChildren: true} + d5 := &sys.LocalDisk{Name: "sdb2", UUID: "myuuid5", Size: 123, Rotational: true, Readonly: false, Type: sys.PartType, HasChildren: true} + disks = GetAvailableDevices([]*sys.LocalDisk{d1, d2, d3, d4, d5}) assert.Equal(t, 2, len(disks)) assert.Equal(t, "sdb", disks[0]) assert.Equal(t, "sdc", disks[1]) // Crypt disk type results in available disk - d6 := &LocalDisk{Name: "sdd", UUID: "myuuid2", Size: 123, Rotational: true, Readonly: false, Type: sys.CryptType, HasChildren: true} - disks = GetAvailableDevices([]*LocalDisk{d6}) + d6 := &sys.LocalDisk{Name: "sdd", UUID: "myuuid2", Size: 123, Rotational: true, Readonly: false, Type: sys.CryptType, HasChildren: true} + disks = GetAvailableDevices([]*sys.LocalDisk{d6}) assert.Equal(t, 1, len(disks)) } diff --git a/pkg/daemon/ceph/osd/agent_test.go b/pkg/daemon/ceph/osd/agent_test.go index de8282e0b..e002284fa 100644 --- a/pkg/daemon/ceph/osd/agent_test.go +++ b/pkg/daemon/ceph/osd/agent_test.go @@ -34,6 +34,7 @@ import ( "github.com/rook/rook/pkg/operator/k8sutil" testop "github.com/rook/rook/pkg/operator/test" exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/rook/rook/pkg/util/sys" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -218,7 +219,7 @@ func testOSDAgentWithDevicesHelper(t *testing.T, storeConfig rookalpha.StoreConf _, _, sdxUUID := mockPartitionSchemeEntry(t, 23, "sdx", &storeConfig, agent.kv, agent.nodeName) // note only sdx already has a UUID (it's been through partitioning) - context.Devices = []*clusterd.LocalDisk{ + context.Devices = []*sys.LocalDisk{ {Name: "sdx", Size: 1234567890, UUID: sdxUUID}, {Name: "sdy", Size: 1234567890}, } @@ -285,7 +286,7 @@ func TestOSDAgentNoDevices(t *testing.T) { // set up expected ProcManager commands context := &clusterd.Context{ - Devices: []*clusterd.LocalDisk{}, + Devices: []*sys.LocalDisk{}, Executor: executor, ConfigDir: configDir, } @@ -372,12 +373,12 @@ func createTestKeyring(t *testing.T, configRoot string, args []string) { func TestGetPartitionPerfScheme(t *testing.T) { configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) - context := &clusterd.Context{Devices: []*clusterd.LocalDisk{}, ConfigDir: configDir} + context := &clusterd.Context{Devices: []*sys.LocalDisk{}, ConfigDir: configDir} test.CreateConfigDir(configDir) // 3 disks: 2 for data and 1 for the metadata of both disks (2 WALs and 2 DBs) a := &OsdAgent{devices: "sda,sdb", metadataDevice: "sdc", kv: mockKVStore(), nodeName: "a"} - context.Devices = []*clusterd.LocalDisk{ + context.Devices = []*sys.LocalDisk{ {Name: "sda", Size: 107374182400}, // 100 GB {Name: "sdb", Size: 107374182400}, // 100 GB {Name: "sdc", Size: 44158681088}, // 1 MB (starting offset) + 2 * (576 MB + 20 GB) = 41.125 GB @@ -412,7 +413,7 @@ func TestGetPartitionPerfScheme(t *testing.T) { if command == "blkid" { return "", nil } - if command == "df" { + if command == "udevadm" { return "", nil } return "", fmt.Errorf("unexpected command %s %v", command, args) @@ -476,14 +477,14 @@ func TestGetPartitionSchemeDiskInUse(t *testing.T) { if command == "blkid" { return "", nil } - if command == "df" { + if command == "udevadm" { return "", nil } return "", fmt.Errorf("unexpected command %s %v", command, args) }, } context := &clusterd.Context{ - Devices: []*clusterd.LocalDisk{}, + Devices: []*sys.LocalDisk{}, ConfigDir: configDir, Executor: executor, } @@ -491,7 +492,7 @@ func TestGetPartitionSchemeDiskInUse(t *testing.T) { a := &OsdAgent{devices: "sda", kv: mockKVStore()} _, _, sdaUUID := mockPartitionSchemeEntry(t, 1, "sda", nil, a.kv, a.nodeName) - context.Devices = []*clusterd.LocalDisk{ + context.Devices = []*sys.LocalDisk{ {Name: "sda", Size: 107374182400, UUID: sdaUUID}, // 100 GB } @@ -540,14 +541,14 @@ func TestGetPartitionSchemeDiskNameChanged(t *testing.T) { if command == "blkid" { return "", nil } - if command == "df" { + if command == "udevadm" { return "", nil } return "", fmt.Errorf("unexpected command %s %v", command, args) }, } context := &clusterd.Context{ - Devices: []*clusterd.LocalDisk{}, + Devices: []*sys.LocalDisk{}, ConfigDir: configDir, Executor: executor, } @@ -558,7 +559,7 @@ func TestGetPartitionSchemeDiskNameChanged(t *testing.T) { // setup an existing partition schme with metadata on nvme01 and data on sda _, metadataUUID, sdaUUID := mockDistributedPartitionScheme(t, 1, "nvme01", "sda", a.kv, a.nodeName) - context.Devices = []*clusterd.LocalDisk{ + context.Devices = []*sys.LocalDisk{ {Name: "nvme01-changed", Size: 107374182400, UUID: metadataUUID}, {Name: "sda-changed", Size: 107374182400, UUID: sdaUUID}, } diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index 32c3724f6..ce0c77208 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -184,7 +184,7 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices string, metad if device.Type == sys.PartType { continue } - ownPartitions, fs, err := checkIfDeviceAvailable(context.Executor, device.Name) + ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, device.Name) if err != nil { return nil, fmt.Errorf("failed to get device %s info. %+v", device.Name, err) } diff --git a/pkg/daemon/ceph/osd/daemon_test.go b/pkg/daemon/ceph/osd/daemon_test.go index ed7580bc4..3105e61ca 100644 --- a/pkg/daemon/ceph/osd/daemon_test.go +++ b/pkg/daemon/ceph/osd/daemon_test.go @@ -26,9 +26,43 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/cluster/ceph/osd/config" exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/rook/rook/pkg/util/sys" "github.com/stretchr/testify/assert" ) +const udevFSOutput = ` +DEVNAME=/dev/sdk +DEVPATH=/devices/platform/host6/session2/target6:0:0/6:0:0:0/block/sdk +DEVTYPE=disk +ID_BUS=scsi +ID_FS_TYPE=ext2 +ID_FS_USAGE=filesystem +ID_FS_UUID=f2d38cba-37da-411d-b7ba-9a6696c58174 +ID_FS_UUID_ENC=f2d38cba-37da-411d-b7ba-9a6696c58174 +ID_FS_VERSION=1.0 +ID_MODEL=disk01 +ID_MODEL_ENC=disk01\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20 +ID_PATH=ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-0 +ID_PATH_TAG=ip-127_0_0_1_3260-iscsi-iqn_2016-06_world_srv_storage_target01-lun-0 +ID_REVISION=4.0 +ID_SCSI=1 +ID_SCSI_SERIAL=d27e5d89-8829-468b-90ce-4ef8c02f07fe +ID_SERIAL=36001405d27e5d898829468b90ce4ef8c +ID_SERIAL_SHORT=6001405d27e5d898829468b90ce4ef8c +ID_TARGET_PORT=0 +ID_TYPE=disk +ID_VENDOR=LIO-ORG +ID_VENDOR_ENC=LIO-ORG\x20 +ID_WWN=0x6001405d27e5d898 +ID_WWN_VENDOR_EXTENSION=0x829468b90ce4ef8c +ID_WWN_WITH_EXTENSION=0x6001405d27e5d898829468b90ce4ef8c +MAJOR=8 +MINOR=160 +SUBSYSTEM=block +TAGS=:systemd: +USEC_INITIALIZED=15981915740802 +` + func TestRunDaemon(t *testing.T) { configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) @@ -159,10 +193,10 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil // partition sdb1 has a label MY-PART return "MY-PART", nil } - } else if command == "df" { + } else if command == "udevadm" { if strings.Index(name, "sdc") != -1 { // /dev/sdc has a file system - return "/dev/sdc ext4", nil + return udevFSOutput, nil } return "", nil } @@ -171,7 +205,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil } context := &clusterd.Context{Executor: executor} - context.Devices = []*clusterd.LocalDisk{ + context.Devices = []*sys.LocalDisk{ {Name: "sda"}, {Name: "sdb"}, {Name: "sdc"}, diff --git a/pkg/daemon/ceph/osd/device.go b/pkg/daemon/ceph/osd/device.go index 1b7ccee54..497fbab1b 100644 --- a/pkg/daemon/ceph/osd/device.go +++ b/pkg/daemon/ceph/osd/device.go @@ -90,7 +90,7 @@ func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool } // check if partitions belong to rook - ownPartitions, devFS, err := checkIfDeviceAvailable(context.Executor, dataDetails.Device) + ownPartitions, devFS, err := sys.CheckIfDeviceAvailable(context.Executor, dataDetails.Device) if err != nil { return fmt.Errorf("failed to format device. %+v", err) } @@ -125,38 +125,6 @@ func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool return nil } -func checkIfDeviceAvailable(executor exec.Executor, name string) (bool, string, error) { - ownPartitions := true - partitions, _, err := sys.GetDevicePartitions(name, executor) - if err != nil { - return false, "", fmt.Errorf("failed to get %s partitions. %+v", name, err) - } - if !rookOwnsPartitions(partitions) { - ownPartitions = false - } - - // check if there is a file system on the device - devFS, err := sys.GetDeviceFilesystems(name, executor) - if err != nil { - return false, "", fmt.Errorf("failed to get device %s filesystem: %+v", name, err) - } - - return ownPartitions, devFS, nil -} - -func rookOwnsPartitions(partitions []*sys.Partition) bool { - - // if there are partitions, they must all have the rook osd label - for _, p := range partitions { - if !strings.HasPrefix(p.Label, "ROOK-OSD") { - return false - } - } - - // if there are no partitions, or the partitions are all from rook OSDs, then rook owns the device - return true -} - // partitions a given device exclusively for metadata usage func partitionMetadata(context *clusterd.Context, info *config.MetadataDeviceInfo, kv *k8sutil.ConfigMapKVStore, storeName string) error { if len(info.Partitions) == 0 { @@ -181,7 +149,7 @@ func partitionMetadata(context *clusterd.Context, info *config.MetadataDeviceInf } // check one last time to make sure it's OK for us to format this metadata device - ownPartitions, fs, err := checkIfDeviceAvailable(context.Executor, info.Device) + ownPartitions, fs, err := sys.CheckIfDeviceAvailable(context.Executor, info.Device) if err != nil { return fmt.Errorf("failed to get metadata device %s info: %+v", info.Device, err) } else if fs != "" || !ownPartitions { diff --git a/pkg/daemon/ceph/osd/device_test.go b/pkg/daemon/ceph/osd/device_test.go index 7d893f4e1..7e6565500 100644 --- a/pkg/daemon/ceph/osd/device_test.go +++ b/pkg/daemon/ceph/osd/device_test.go @@ -93,26 +93,7 @@ func TestOverwriteRookOwnedPartitions(t *testing.T) { defer os.RemoveAll(configDir) // set up mock execute so we can verify the partitioning happens on sda - execCount := 0 executor := &exectest.MockExecutor{} - executor.MockExecuteCommand = func(debug bool, name string, command string, args ...string) error { - logger.Infof("RUN %d for '%s'. %s %+v", execCount, name, command, args) - assert.Equal(t, "sgdisk", command) - switch execCount { - case 0: - assert.Equal(t, []string{"--zap-all", "/dev/sda"}, args) - case 1: - assert.Equal(t, []string{"--clear", "--mbrtogpt", "/dev/sda"}, args) - case 2: - assert.Equal(t, 11, len(args)) - assert.Equal(t, "--change-name=1:ROOK-OSD1-WAL", args[1]) - assert.Equal(t, "--change-name=2:ROOK-OSD1-DB", args[4]) - assert.Equal(t, "--change-name=3:ROOK-OSD1-BLOCK", args[7]) - assert.Equal(t, "/dev/sda", args[10]) - } - execCount++ - return nil - } // set up a mock function to return "rook owned" partitions on the device and it does not have a filesystem outputExecCount := 0 @@ -120,23 +101,29 @@ func TestOverwriteRookOwnedPartitions(t *testing.T) { logger.Infof("OUTPUT %d for %s. %s %+v", outputExecCount, name, command, args) var output string switch outputExecCount { - case 0, 4: // we'll call this twice, once explicitly below to verify rook owns the partitions and a 2nd time within formatDevice + case 0, 7: // we'll call this twice, once explicitly below to verify rook owns the partitions and a 2nd time within formatDevice assert.Equal(t, command, "lsblk") output = `NAME="sda" SIZE="65" TYPE="disk" PKNAME="" NAME="sda1" SIZE="30" TYPE="part" PKNAME="sda" NAME="sda2" SIZE="10" TYPE="part" PKNAME="sda" NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda"` - case 1, 5: - assert.Equal(t, "blkid /dev/sda1", name) - output = "ROOK-OSD0-WAL" - case 2, 6: - assert.Equal(t, "blkid /dev/sda2", name) - output = "ROOK-OSD0-DB" - case 3, 7: - assert.Equal(t, "blkid /dev/sda3", name) - output = "ROOK-OSD0-BLOCK" - case 8: - assert.Equal(t, command, "df") + case 1, 8: + assert.Equal(t, "udevadm /dev/sda1", name) + output = "ID_PART_ENTRY_NAME=ROOK-OSD0-WAL" + case 2, 9: + assert.Equal(t, "get filesystem type for sda1", name) + output = "" + case 3, 10: + assert.Equal(t, "udevadm /dev/sda2", name) + output = "ID_PART_ENTRY_NAME=ROOK-OSD0-DB" + case 4, 11: + assert.Equal(t, "get filesystem type for sda2", name) + output = "" + case 5, 12: + assert.Equal(t, "udevadm /dev/sda3", name) + output = "ID_PART_ENTRY_NAME=ROOK-OSD0-BLOCK" + case 6, 13: + assert.Equal(t, "get filesystem type for sda3", name) output = "" } outputExecCount++ @@ -152,7 +139,7 @@ NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda"` config.PopulateCollocatedPerfSchemeEntry(entry, "sda", storeConfig) context := &clusterd.Context{Executor: executor, ConfigDir: configDir} - context.Devices = []*clusterd.LocalDisk{ + context.Devices = []*sys.LocalDisk{ {Name: "sda", Size: 65}, } config := &osdConfig{configRoot: configDir, rootPath: filepath.Join(configDir, "osd1"), id: entry.ID, @@ -161,14 +148,13 @@ NAME="sda3" SIZE="20" TYPE="part" PKNAME="sda"` // ensure that our mocking makes it look like rook owns the partitions on sda partitions, _, err := sys.GetDevicePartitions("sda", context.Executor) assert.Nil(t, err) - assert.True(t, rookOwnsPartitions(partitions)) + assert.True(t, sys.RookOwnsPartitions(partitions)) // try to format the device. even though the device has existing partitions, they are owned by rook, so it is safe // to format and the format/partitioning will happen. err = formatDevice(context, config, false, storeConfig) assert.Nil(t, err) - assert.Equal(t, 3, execCount) - assert.Equal(t, 9, outputExecCount) + assert.Equal(t, 15, outputExecCount) } func TestPartitionBluestoreMetadata(t *testing.T) { @@ -238,10 +224,10 @@ func TestPartitionBluestoreMetadataSafe(t *testing.T) { nodeID := "node123" executor := &exectest.MockExecutor{} executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { - if command == "df" { + if command == "udevadm" { // mock that the metadata device already has a filesytem, this should abort the partition effort if strings.Index(name, "nvme01") != -1 { - return "/dev/nvme01 ext4", nil + return udevFSOutput, nil } } @@ -320,7 +306,7 @@ func testPartitionOSDHelper(t *testing.T, storeConfig rookalpha.StoreConfig) { // setup a context with 1 disk: sda context := &clusterd.Context{Executor: executor, ConfigDir: configDir} - context.Devices = []*clusterd.LocalDisk{ + context.Devices = []*sys.LocalDisk{ {Name: "sda", Size: 100}, } diff --git a/pkg/daemon/discover/discover.go b/pkg/daemon/discover/discover.go new file mode 100644 index 000000000..b5ca5da9e --- /dev/null +++ b/pkg/daemon/discover/discover.go @@ -0,0 +1,164 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package discover to discover unused devices. +package discover + +import ( + "encoding/json" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/coreos/pkg/capnslog" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/sys" + + "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var ( + logger = capnslog.NewPackageLogger("github.com/rook/rook", "rook-discover") + AppName = "rook-discover" + NodeAttr = "rook.io/node" + LocalDiskCMData = "devices" + LocalDiskCMName = "local-device-" + probeInterval = 30 * time.Second + nodeName, namespace, lastDevice, cmName string + cm *v1.ConfigMap +) + +func Run(context *clusterd.Context) error { + if context == nil { + return fmt.Errorf("nil context") + } + nodeName = os.Getenv(k8sutil.NodeNameEnvVar) + namespace = os.Getenv(k8sutil.PodNamespaceEnvVar) + cmName = LocalDiskCMName + nodeName + tick := time.NewTicker(probeInterval).C + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, syscall.SIGTERM) + for { + select { + case <-sigc: + logger.Infof("shutdown signal received, exiting...") + return nil + case <-tick: + err := updateDeviceCM(context) + if err != nil { + logger.Infof("failed to update device configmap: %v", err) + } + } + } +} +func updateDeviceCM(context *clusterd.Context) error { + logger.Infof("updating device configmap") + devices, err := probeDevices(context) + if err != nil { + logger.Infof("failed to probe devices: %v", err) + return err + } + deviceJson, err := json.Marshal(devices) + if err != nil { + logger.Infof("failed to marshal: %v", err) + return err + } + deviceStr := string(deviceJson) + if cm == nil { + cm, err = context.Clientset.CoreV1().ConfigMaps(namespace).Get(cmName, metav1.GetOptions{}) + } + if err == nil { + lastDevice = cm.Data[LocalDiskCMData] + logger.Debugf("last devices %s", lastDevice) + } else { + if !errors.IsNotFound(err) { + logger.Infof("failed to get configmap: %v", err) + return err + } + + data := make(map[string]string, 1) + data[LocalDiskCMData] = deviceStr + // the map doesn't exist yet, create it now + cm = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: namespace, + Labels: map[string]string{ + k8sutil.AppAttr: AppName, + NodeAttr: nodeName, + }, + }, + Data: data, + } + cm, err = context.Clientset.CoreV1().ConfigMaps(namespace).Create(cm) + if err != nil { + logger.Infof("failed to create configmap: %v", err) + return fmt.Errorf("failed to create local device map %s: %+v", cmName, err) + } + lastDevice = deviceStr + } + if deviceStr != lastDevice { + data := make(map[string]string, 1) + data[LocalDiskCMData] = deviceStr + cm.Data = data + cm, err = context.Clientset.CoreV1().ConfigMaps(namespace).Update(cm) + if err != nil { + logger.Infof("failed to update configmap %s: %v", cmName, err) + return err + } + } + return nil +} + +func probeDevices(context *clusterd.Context) ([]sys.LocalDisk, error) { + devices := make([]sys.LocalDisk, 0) + localDevices, err := clusterd.DiscoverDevices(context.Executor) + if err != nil { + return devices, fmt.Errorf("failed initial hardware discovery. %+v", err) + } + for _, device := range localDevices { + if device == nil { + continue + } + if device.Type == sys.PartType { + continue + } + + partitions, _, err := sys.GetDevicePartitions(device.Name, context.Executor) + if err != nil { + logger.Infof("failed to check device partitions %s: %v", device.Name, err) + continue + } + + // check if there is a file system on the device + fs, err := sys.GetDeviceFilesystems(device.Name, context.Executor) + if err != nil { + logger.Infof("failed to check device filesystem %s: %v", device.Name, err) + continue + } + device.Partitions = partitions + device.Filesystem = fs + devices = append(devices, *device) + } + + logger.Infof("available devices: %+v", devices) + return devices, nil +} diff --git a/pkg/daemon/discover/discover_test.go b/pkg/daemon/discover/discover_test.go new file mode 100644 index 000000000..e36e8ed39 --- /dev/null +++ b/pkg/daemon/discover/discover_test.go @@ -0,0 +1,100 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package discover to discover unused devices. +package discover + +import ( + "testing" + + "github.com/rook/rook/pkg/clusterd" + exectest "github.com/rook/rook/pkg/util/exec/test" + + "github.com/stretchr/testify/assert" +) + +const udevOutput = `DEVLINKS=/dev/disk/by-id/scsi-36001405d27e5d898829468b90ce4ef8c /dev/disk/by-id/wwn-0x6001405d27e5d898829468b90ce4ef8c /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-0 /dev/disk/by-uuid/f2d38cba-37da-411d-b7ba-9a6696c58174 +DEVNAME=/dev/sdk +DEVPATH=/devices/platform/host6/session2/target6:0:0/6:0:0:0/block/sdk +DEVTYPE=disk +ID_BUS=scsi +ID_FS_TYPE=ext2 +ID_FS_USAGE=filesystem +ID_FS_UUID=f2d38cba-37da-411d-b7ba-9a6696c58174 +ID_FS_UUID_ENC=f2d38cba-37da-411d-b7ba-9a6696c58174 +ID_FS_VERSION=1.0 +ID_MODEL=disk01 +ID_MODEL_ENC=disk01\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20 +ID_PATH=ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-0 +ID_PATH_TAG=ip-127_0_0_1_3260-iscsi-iqn_2016-06_world_srv_storage_target01-lun-0 +ID_REVISION=4.0 +ID_SCSI=1 +ID_SCSI_SERIAL=d27e5d89-8829-468b-90ce-4ef8c02f07fe +ID_SERIAL=36001405d27e5d898829468b90ce4ef8c +ID_SERIAL_SHORT=6001405d27e5d898829468b90ce4ef8c +ID_TARGET_PORT=0 +ID_TYPE=disk +ID_VENDOR=LIO-ORG +ID_VENDOR_ENC=LIO-ORG\x20 +ID_WWN=0x6001405d27e5d898 +ID_WWN_VENDOR_EXTENSION=0x829468b90ce4ef8c +ID_WWN_WITH_EXTENSION=0x6001405d27e5d898829468b90ce4ef8c +MAJOR=8 +MINOR=160 +SUBSYSTEM=block +TAGS=:systemd: +USEC_INITIALIZED=15981915740802 +` + +func TestProbeDevices(t *testing.T) { + // set up mock execute so we can verify the partitioning happens on sda + executor := &exectest.MockExecutor{} + executor.MockExecuteCommandWithOutput = func(debug bool, name string, command string, args ...string) (string, error) { + logger.Infof("RUN Command for '%s'. %s arg %+v", name, command, args) + output := "" + switch name { + case "lsblk all": + output = "testa" + case "lsblk /dev/testa": + output = `SIZE="249510756352" ROTA="1" RO="0" TYPE="disk" PKNAME=""` + case "get filesystem type for testa": + output = udevOutput + case "get parent for device testa": + output = ` testa +testa testa1 +testa testa2 +testa2 centos_host13-root +testa2 centos_host13-swap +testa2 centos_host13-home +` + case "get disk testa fs uuid": + output = udevOutput + + case "get disk testa fs serial": + output = udevOutput + } + + return output, nil + } + + context := &clusterd.Context{Executor: executor} + + devices, err := probeDevices(context) + assert.Nil(t, err) + assert.Equal(t, 1, len(devices)) + assert.Equal(t, "ext2", devices[0].Filesystem) + +} diff --git a/pkg/operator/cluster/ceph/mon/util.go b/pkg/operator/cluster/ceph/mon/util.go index 888899119..2d379978a 100644 --- a/pkg/operator/cluster/ceph/mon/util.go +++ b/pkg/operator/cluster/ceph/mon/util.go @@ -311,7 +311,7 @@ func genSecret(executor exec.Executor, configDir, name string, args []string) (s } func extractKey(contents string) (string, error) { - secret := sys.Awk(sys.Grep(string(contents), "key"), 3) + secret := sys.Awk(sys.Grep(string(contents), "key"), 3, " ") if secret == "" { return "", fmt.Errorf("failed to parse secret") } diff --git a/pkg/operator/cluster/ceph/osd/osd.go b/pkg/operator/cluster/ceph/osd/osd.go index d36e6d926..1d4947cd0 100644 --- a/pkg/operator/cluster/ceph/osd/osd.go +++ b/pkg/operator/cluster/ceph/osd/osd.go @@ -28,6 +28,7 @@ import ( "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/cluster/ceph/osd/config" + "github.com/rook/rook/pkg/operator/discover" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/util/display" "k8s.io/api/core/v1" @@ -105,7 +106,7 @@ func (c *Cluster) Start() error { logger.Infof("start running osds in namespace %s", c.Namespace) // create the artifacts for the osd to work with RBAC enabled - err := k8sutil.MakeRole(c.context.Clientset, c.Namespace, appName, clusterAccessRules, c.ownerRef) + err := k8sutil.MakeRole(c.context.Clientset, c.Namespace, appName, clusterAccessRules, &c.ownerRef) if err != nil { logger.Warningf("failed to init RBAC for OSDs. %+v", err) } @@ -168,9 +169,16 @@ func (c *Cluster) Start() error { errorMessages = append(errorMessages, fmt.Sprintf("failed to set orchestration starting status for node %s: %+v", n.Name, err)) continue } - + devicesToUse := n.Devices + availDev, deviceErr := discover.GetAvailableDevices(c.context, n.Name, c.Namespace, n.Devices, n.Selection.DeviceFilter, n.Selection.GetUseAllDevices()) + if deviceErr != nil { + logger.Warningf("failed to get devices for node %s cluster %s: %v", n.Name, c.Namespace, deviceErr) + } else { + devicesToUse = availDev + logger.Infof("avail devices for node %s: %+v", n.Name, availDev) + } // create the replicaSet that will run the OSDs for this node - rs := c.makeReplicaSet(n.Name, n.Devices, n.Selection, resources, n.Config) + rs := c.makeReplicaSet(n.Name, devicesToUse, n.Selection, resources, n.Config) _, err := c.context.Clientset.Extensions().ReplicaSets(c.Namespace).Create(rs) if err != nil { if !errors.IsAlreadyExists(err) { diff --git a/pkg/operator/discover/discover.go b/pkg/operator/discover/discover.go new file mode 100644 index 000000000..8b40bc29e --- /dev/null +++ b/pkg/operator/discover/discover.go @@ -0,0 +1,264 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package discover to discover devices on storage nodes. +package discover + +import ( + "encoding/json" + "fmt" + "os" + "regexp" + + "github.com/coreos/pkg/capnslog" + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha1" + "github.com/rook/rook/pkg/clusterd" + discoverDaemon "github.com/rook/rook/pkg/daemon/discover" + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/sys" + + "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + "k8s.io/api/rbac/v1beta1" + kserrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + discoverDaemonsetName = "rook-discover" + discoverDaemonsetTolerationEnv = "DISCOVER_TOLERATION" + discoverDaemonsetTolerationKeyEnv = "DISCOVER_TOLERATION_KEY" +) + +var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-discover") + +var accessRules = []v1beta1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"configmaps"}, + Verbs: []string{"get", "list", "update", "create", "delete"}, + }, +} + +// Discover reference to be deployed +type Discover struct { + clientset kubernetes.Interface +} + +// New creates an instance of Discover +func New(clientset kubernetes.Interface) *Discover { + return &Discover{ + clientset: clientset, + } +} + +// Start the discover +func (d *Discover) Start(namespace, discoverImage string) error { + + err := k8sutil.MakeRole(d.clientset, namespace, discoverDaemonsetName, accessRules, nil) + if err != nil { + return fmt.Errorf("failed to init RBAC for rook-discover. %+v", err) + } + + err = d.createDiscoverDaemonSet(namespace, discoverImage) + if err != nil { + return fmt.Errorf("Error starting discover daemonset: %v", err) + } + return nil +} + +func (d *Discover) createDiscoverDaemonSet(namespace, discoverImage string) error { + privileged := false + ds := &extensions.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: discoverDaemonsetName, + }, + Spec: extensions.DaemonSetSpec{ + UpdateStrategy: extensions.DaemonSetUpdateStrategy{ + Type: extensions.RollingUpdateDaemonSetStrategyType, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": discoverDaemonsetName, + }, + }, + Spec: v1.PodSpec{ + ServiceAccountName: discoverDaemonsetName, + Containers: []v1.Container{ + { + Name: discoverDaemonsetName, + Image: discoverImage, + Args: []string{"discover"}, + SecurityContext: &v1.SecurityContext{ + Privileged: &privileged, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "dev", + MountPath: "/dev", + ReadOnly: true, + }, + { + Name: "sys", + MountPath: "/sys", + ReadOnly: true, + }, + { + Name: "udev", + MountPath: "/run/udev", + ReadOnly: true, + }, + }, + Env: []v1.EnvVar{ + k8sutil.NamespaceEnvVar(), + k8sutil.NodeEnvVar(), + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "dev", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/dev", + }, + }, + }, + { + Name: "sys", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/sys", + }, + }, + }, + { + Name: "udev", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/run/udev", + }, + }, + }, + }, + HostNetwork: false, + }, + }, + }, + } + + // Add toleration if any + tolerationValue := os.Getenv(discoverDaemonsetTolerationEnv) + if tolerationValue != "" { + ds.Spec.Template.Spec.Tolerations = []v1.Toleration{ + { + Effect: v1.TaintEffect(tolerationValue), + Operator: v1.TolerationOpExists, + Key: os.Getenv(discoverDaemonsetTolerationKeyEnv), + }, + } + } + + _, err := d.clientset.Extensions().DaemonSets(namespace).Create(ds) + if err != nil { + if !kserrors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create rook-discover daemon set. %+v", err) + } + logger.Infof("rook-discover daemonset already exists") + } else { + logger.Infof("rook-discover daemonset started") + } + return nil + +} + +func ListDevices(context *clusterd.Context, namespace, nodeName string) (map[string][]sys.LocalDisk, error) { + var devices map[string][]sys.LocalDisk + listOpts := metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", k8sutil.AppAttr, discoverDaemon.AppName)} + cms, err := context.Clientset.CoreV1().ConfigMaps(namespace).List(listOpts) + if err != nil { + return devices, fmt.Errorf("failed to list device configmaps: %+v", err) + } + devices = make(map[string][]sys.LocalDisk, len(cms.Items)) + for _, cm := range cms.Items { + node := cm.ObjectMeta.Labels[discoverDaemon.NodeAttr] + if len(nodeName) > 0 && node != nodeName { + continue + } + deviceJson := cm.Data[discoverDaemon.LocalDiskCMData] + logger.Debugf("node %s, device %s", node, deviceJson) + + if len(node) == 0 || len(deviceJson) == 0 { + continue + } + var d []sys.LocalDisk + err = json.Unmarshal([]byte(deviceJson), &d) + if err != nil { + logger.Warningf("failed to unmarshal %s", deviceJson) + continue + } + devices[node] = d + } + logger.Debugf("devices %+v", devices) + return devices, nil +} + +func GetAvailableDevices(context *clusterd.Context, nodeName, clusterName string, devices []rookalpha.Device, filter string, useAllDevices bool) ([]rookalpha.Device, error) { + results := []rookalpha.Device{} + if len(devices) == 0 && len(filter) == 0 && !useAllDevices { + return results, nil + } + namespace := os.Getenv(k8sutil.PodNamespaceEnvVar) + allDevices, err := ListDevices(context, namespace, nodeName) + if err != nil { + return results, err + } + nodeDevices, ok := allDevices[nodeName] + if !ok { + return results, fmt.Errorf("node %s has no devices", nodeName) + } + if len(devices) > 0 { + for i := range devices { + for j := range nodeDevices { + if devices[i].Name == nodeDevices[j].Name { + results = append(results, devices[i]) + } + } + } + } else if len(filter) >= 0 { + for i := range nodeDevices { + //TODO support filter based on other keys + matched, err := regexp.Match(filter, []byte(nodeDevices[i].Name)) + if err == nil && matched { + d := rookalpha.Device{ + Name: nodeDevices[i].Name, + } + results = append(results, d) + } + } + } else if useAllDevices { + for i := range nodeDevices { + d := rookalpha.Device{ + Name: nodeDevices[i].Name, + } + results = append(results, d) + } + } + + return results, nil +} diff --git a/pkg/operator/discover/discover_test.go b/pkg/operator/discover/discover_test.go new file mode 100644 index 000000000..22a335ba7 --- /dev/null +++ b/pkg/operator/discover/discover_test.go @@ -0,0 +1,127 @@ +/* +Copyright 2018 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package discover to discover devices on storage nodes. +package discover + +import ( + "os" + "testing" + + rookalpha "github.com/rook/rook/pkg/apis/rook.io/v1alpha1" + "github.com/rook/rook/pkg/clusterd" + discoverDaemon "github.com/rook/rook/pkg/daemon/discover" + "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/operator/test" + + "github.com/stretchr/testify/assert" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestStartDiscoveryDaemonset(t *testing.T) { + clientset := test.New(3) + + os.Setenv(k8sutil.PodNamespaceEnvVar, "rook-system") + defer os.Unsetenv(k8sutil.PodNamespaceEnvVar) + + os.Setenv(k8sutil.PodNameEnvVar, "rook-operator") + defer os.Unsetenv(k8sutil.PodNameEnvVar) + + namespace := "ns" + a := New(clientset) + + // start a basic cluster + err := a.Start(namespace, "rook/rook:myversion") + assert.Nil(t, err) + + // check clusters rbac roles + _, err = clientset.CoreV1().ServiceAccounts(namespace).Get("rook-discover", metav1.GetOptions{}) + assert.Nil(t, err) + + _, err = clientset.RbacV1beta1().Roles(namespace).Get("rook-discover", metav1.GetOptions{}) + assert.Nil(t, err) + + // check daemonset parameters + agentDS, err := clientset.Extensions().DaemonSets(namespace).Get("rook-discover", metav1.GetOptions{}) + assert.Nil(t, err) + assert.Equal(t, namespace, agentDS.Namespace) + assert.Equal(t, "rook-discover", agentDS.Name) + assert.False(t, *agentDS.Spec.Template.Spec.Containers[0].SecurityContext.Privileged) + volumes := agentDS.Spec.Template.Spec.Volumes + assert.Equal(t, 3, len(volumes)) + volumeMounts := agentDS.Spec.Template.Spec.Containers[0].VolumeMounts + assert.Equal(t, 3, len(volumeMounts)) + envs := agentDS.Spec.Template.Spec.Containers[0].Env + assert.Equal(t, 2, len(envs)) + image := agentDS.Spec.Template.Spec.Containers[0].Image + assert.Equal(t, "rook/rook:myversion", image) + assert.Nil(t, agentDS.Spec.Template.Spec.Tolerations) +} + +func TestGetAvailableDevices(t *testing.T) { + clientset := test.New(3) + + ns := "rook-system" + nodeName := "node123" + os.Setenv(k8sutil.PodNamespaceEnvVar, ns) + defer os.Unsetenv(k8sutil.PodNamespaceEnvVar) + + os.Setenv(k8sutil.PodNameEnvVar, "rook-operator") + defer os.Unsetenv(k8sutil.PodNameEnvVar) + + data := make(map[string]string, 1) + data[discoverDaemon.LocalDiskCMData] = `[{"name":"sdd","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/scsi-36001405f826bd553d8c4dbf9f41c18be /dev/disk/by-id/wwn-0x6001405f826bd553d8c4dbf9f41c18be /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-1","size":10737418240,"uuid":"","serial":"36001405f826bd553d8c4dbf9f41c18be","type":"disk","rotational":true,"readOnly":false,"ownPartition":true,"filesystem":"","vendor":"LIO-ORG","model":"disk02","wwn":"0x6001405f826bd553","wwnVendorExtension":"0x6001405f826bd553d8c4dbf9f41c18be","empty":true},{"name":"sdb","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/scsi-3600140577f462d9908b409d94114e042 /dev/disk/by-id/wwn-0x600140577f462d9908b409d94114e042 /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-3","size":5368709120,"uuid":"","serial":"3600140577f462d9908b409d94114e042","type":"disk","rotational":true,"readOnly":false,"ownPartition":false,"filesystem":"","vendor":"LIO-ORG","model":"disk04","wwn":"0x600140577f462d99","wwnVendorExtension":"0x600140577f462d9908b409d94114e042","empty":true},{"name":"sdc","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/scsi-3600140568c0bd28d4ee43769387c9f02 /dev/disk/by-id/wwn-0x600140568c0bd28d4ee43769387c9f02 /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-2","size":5368709120,"uuid":"","serial":"3600140568c0bd28d4ee43769387c9f02","type":"disk","rotational":true,"readOnly":false,"ownPartition":true,"filesystem":"","vendor":"LIO-ORG","model":"disk03","wwn":"0x600140568c0bd28d","wwnVendorExtension":"0x600140568c0bd28d4ee43769387c9f02","empty":true},{"name":"sda","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/scsi-36001405fc00c75fb4c243aa9d61987bd /dev/disk/by-id/wwn-0x6001405fc00c75fb4c243aa9d61987bd /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-0","size":10737418240,"uuid":"","serial":"36001405fc00c75fb4c243aa9d61987bd","type":"disk","rotational":true,"readOnly":false,"ownPartition":false,"filesystem":"","vendor":"LIO-ORG","model":"disk01","wwn":"0x6001405fc00c75fb","wwnVendorExtension":"0x6001405fc00c75fb4c243aa9d61987bd","empty":true},{"name":"nvme0n1","parent":"","hasChildren":false,"devLinks":"/dev/disk/by-id/nvme-eui.002538c5710091a7","size":512110190592,"uuid":"","serial":"","type":"disk","rotational":false,"readOnly":false,"ownPartition":false,"filesystem":"","vendor":"","model":"","wwn":"","wwnVendorExtension":"","empty":true}]` + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "local-device-" + nodeName, + Namespace: ns, + Labels: map[string]string{ + k8sutil.AppAttr: discoverDaemon.AppName, + discoverDaemon.NodeAttr: nodeName, + }, + }, + Data: data, + } + _, err := clientset.CoreV1().ConfigMaps(ns).Create(cm) + assert.Nil(t, err) + context := &clusterd.Context{ + Clientset: clientset, + } + d := []rookalpha.Device{ + { + Name: "sdc", + }, + { + Name: "foo", + }, + } + + nodeDevices, err := ListDevices(context, ns, "" /* all nodes */) + assert.Nil(t, err) + assert.Equal(t, 1, len(nodeDevices)) + + devices, err := GetAvailableDevices(context, nodeName, ns, d, "^sd.", false) + assert.Nil(t, err) + assert.Equal(t, 1, len(devices)) + devices, err = GetAvailableDevices(context, nodeName, ns, nil, "^sd.", false) + assert.Nil(t, err) + assert.Equal(t, 4, len(devices)) + devices, err = GetAvailableDevices(context, nodeName, ns, nil, "", true) + assert.Nil(t, err) + assert.Equal(t, 5, len(devices)) +} diff --git a/pkg/operator/k8sutil/rbac.go b/pkg/operator/k8sutil/rbac.go index f2acd88b0..9439df6f1 100644 --- a/pkg/operator/k8sutil/rbac.go +++ b/pkg/operator/k8sutil/rbac.go @@ -32,9 +32,9 @@ const ( enableRBACEnv = "RBAC_ENABLED" ) -func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1beta1.PolicyRule, ownerRef metav1.OwnerReference) error { +func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1beta1.PolicyRule, ownerRef *metav1.OwnerReference) error { - err := makeServiceAccount(clientset, namespace, name, &ownerRef) + err := makeServiceAccount(clientset, namespace, name, ownerRef) if err != nil { return err } @@ -48,11 +48,13 @@ func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1 // the create will fail with an error that we're changing the permissions. role := &v1beta1.Role{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - OwnerReferences: []metav1.OwnerReference{ownerRef}, + Name: name, }, Rules: rules, } + if ownerRef != nil { + role.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*ownerRef} + } _, err = clientset.RbacV1beta1().Roles(namespace).Get(role.Name, metav1.GetOptions{}) if errors.IsNotFound(err) { logger.Infof("creating role %s in namespace %s", name, namespace) @@ -67,8 +69,7 @@ func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1 binding := &v1beta1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - OwnerReferences: []metav1.OwnerReference{ownerRef}, + Name: name, }, RoleRef: v1beta1.RoleRef{ Name: name, @@ -79,6 +80,9 @@ func MakeRole(clientset kubernetes.Interface, namespace, name string, rules []v1 {Kind: "ServiceAccount", Name: name, Namespace: namespace}, }, } + if ownerRef != nil { + binding.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*ownerRef} + } _, err = clientset.RbacV1beta1().RoleBindings(namespace).Create(binding) if err != nil && !errors.IsAlreadyExists(err) { return fmt.Errorf("failed to create %s role binding in namespace %s. %+v", name, namespace, err) diff --git a/pkg/operator/k8sutil/rbac_test.go b/pkg/operator/k8sutil/rbac_test.go index 5f3e10684..169908118 100644 --- a/pkg/operator/k8sutil/rbac_test.go +++ b/pkg/operator/k8sutil/rbac_test.go @@ -56,7 +56,8 @@ func TestMakeRole(t *testing.T) { }, } - err := MakeRole(clientset, namespace, name, rules, metav1.OwnerReference{}) + ownerRef := metav1.OwnerReference{} + err := MakeRole(clientset, namespace, name, rules, &ownerRef) role, err := clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{}) assert.Nil(t, err) @@ -82,7 +83,7 @@ func TestMakeRole(t *testing.T) { }, } - err = MakeRole(clientset, namespace, name, newRules, metav1.OwnerReference{}) + err = MakeRole(clientset, namespace, name, newRules, &ownerRef) assert.Nil(t, err) role, err = clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{}) assert.Nil(t, err) @@ -175,8 +176,8 @@ func TestMakeRoleRBACDisabled(t *testing.T) { Verbs: []string{"get", "list"}, }, } - - err := MakeRole(clientset, namespace, name, rules, metav1.OwnerReference{}) + ownerRef := metav1.OwnerReference{} + err := MakeRole(clientset, namespace, name, rules, &ownerRef) _, err = clientset.RbacV1beta1().Roles(namespace).Get(name, metav1.GetOptions{}) assert.NotNil(t, err) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 6718f740f..5155a8132 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -29,6 +29,7 @@ import ( "github.com/rook/rook/pkg/daemon/agent/flexvolume/attachment" "github.com/rook/rook/pkg/operator/agent" "github.com/rook/rook/pkg/operator/cluster" + "github.com/rook/rook/pkg/operator/discover" "github.com/rook/rook/pkg/operator/file" "github.com/rook/rook/pkg/operator/k8sutil" "github.com/rook/rook/pkg/operator/object" @@ -86,6 +87,11 @@ func (o *Operator) Run() error { return fmt.Errorf("Error starting agent daemonset: %v", err) } + rookDiscover := discover.New(o.context.Clientset) + if err := rookDiscover.Start(namespace, o.rookImage); err != nil { + return fmt.Errorf("Error starting device discovery daemonset: %v", err) + } + signalChan := make(chan os.Signal, 1) stopChan := make(chan struct{}) signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) diff --git a/pkg/util/sys/device.go b/pkg/util/sys/device.go index 561d3f294..27182570b 100644 --- a/pkg/util/sys/device.go +++ b/pkg/util/sys/device.go @@ -19,9 +19,8 @@ import ( "fmt" "os" "path/filepath" - "strings" - "strconv" + "strings" "github.com/google/uuid" "github.com/rook/rook/pkg/util/exec" @@ -37,9 +36,48 @@ const ( ) type Partition struct { - Name string - Size uint64 - Label string + Name string + Size uint64 + Label string + Filesystem string +} + +// LocalDevice contains information about an unformatted block device +type LocalDisk struct { + // Name is the device name + Name string `json:"name"` + // Parent is the device parent's name + Parent string `json:"parent"` + // HasChildren is whether the device has a children device + HasChildren bool `json:"hasChildren"` + // DevLinks is the persistent device path on the host + DevLinks string `json:"devLinks"` + // Size is the device capacity in byte + Size uint64 `json:"size"` + // UUID is used by /dev/disk/by-uuid + UUID string `json:"uuid"` + // Serial is the disk serial used by /dev/disk/by-id + Serial string `json:"serial"` + // Type is disk type + Type string `json:"type"` + // Rotational is the boolean whether the device is rotational: true for hdd, false for ssd and nvme + Rotational bool `json:"rotational"` + // ReadOnly is the boolean whether the device is readonly + Readonly bool `json:"readOnly"` + // Partitions is a partition slice + Partitions []Partition + // Filesystem is the filesystem currently on the device + Filesystem string `json:"filesystem"` + // Vendor is the device vendor + Vendor string `json:"vendor"` + // Model is the device model + Model string `json:"model"` + // WWN is the world wide name of the device + WWN string `json:"wwn"` + // WWNVendorExtension is the WWN_VENDOR_EXTENSION from udev info + WWNVendorExtension string `json:"wwnVendorExtension"` + // Empty checks whether the device is completely empty + Empty bool `json:"empty"` } func ListDevices(executor exec.Executor) ([]string, error) { @@ -52,7 +90,7 @@ func ListDevices(executor exec.Executor) ([]string, error) { return strings.Split(devices, "\n"), nil } -func GetDevicePartitions(device string, executor exec.Executor) (partitions []*Partition, unusedSpace uint64, err error) { +func GetDevicePartitions(device string, executor exec.Executor) (partitions []Partition, unusedSpace uint64, err error) { cmd := fmt.Sprintf("lsblk /dev/%s", device) output, err := executor.ExecuteCommandWithOutput(false, cmd, "lsblk", fmt.Sprintf("/dev/%s", device), "--bytes", "--pairs", "--output", "NAME,SIZE,TYPE,PKNAME") @@ -74,7 +112,7 @@ func GetDevicePartitions(device string, executor exec.Executor) (partitions []*P } } else if props["PKNAME"] == device && props["TYPE"] == PartType { // found a partition - p := &Partition{Name: name} + p := Partition{Name: name} p.Size, err = strconv.ParseUint(props["SIZE"], 10, 64) if err != nil { return nil, 0, fmt.Errorf("failed to get partition %s size. %+v", name, err) @@ -87,6 +125,12 @@ func GetDevicePartitions(device string, executor exec.Executor) (partitions []*P } p.Label = label + fs, err := GetDeviceFilesystems(name, executor) + if err != nil { + return nil, 0, err + } + p.Filesystem = fs + partitions = append(partitions, p) } } @@ -120,15 +164,25 @@ func GetDevicePropertiesFromPath(devicePath string, executor exec.Executor) (map return parseKeyValuePairString(output), nil } +func GetUdevInfo(device string, executor exec.Executor) (map[string]string, error) { + cmd := fmt.Sprintf("udevadm info %s", device) + output, err := executor.ExecuteCommandWithOutput(false, cmd, "udevadm", "info", "--query=property", fmt.Sprintf("/dev/%s", device)) + if err != nil { + return nil, err + } + + return parseUdevInfo(output), nil +} + // get the file systems availab func GetDeviceFilesystems(device string, executor exec.Executor) (string, error) { cmd := fmt.Sprintf("get filesystem type for %s", device) - output, err := executor.ExecuteCommandWithOutput(false, cmd, "df", "--output=source,fstype") + output, err := executor.ExecuteCommandWithOutput(false, cmd, "udevadm", "info", "--query=property", fmt.Sprintf("/dev/%s", device)) if err != nil { return "", fmt.Errorf("command %s failed: %+v", cmd, err) } - return parseDFOutput(device, output), nil + return parseFS(output), nil } func RemovePartitions(device string, executor exec.Executor) error { @@ -177,13 +231,14 @@ func GetPartitionLabel(deviceName string, executor exec.Executor) (string, error // look up the partition's label with blkid because lsblk relies on udev which is // not available in containers devicePath := fmt.Sprintf("/dev/%s", deviceName) - cmd := fmt.Sprintf("blkid %s", devicePath) - output, err := executor.ExecuteCommandWithOutput(false, cmd, "blkid", devicePath, "-s", "PARTLABEL", "-o", "value") + cmd := fmt.Sprintf("udevadm %s", devicePath) + output, err := executor.ExecuteCommandWithOutput(false, cmd, + "udevadm", "info", "--query=property", devicePath) if err != nil { return "", fmt.Errorf("failed to get partition label for device %s: %+v", deviceName, err) } - return output, nil + return parsePartLabel(output), nil } // look up the mount point of the given device. empty string returned if device is not mounted. @@ -195,7 +250,7 @@ func GetDeviceMountPoint(deviceName string, executor exec.Executor) (string, err } searchFor := fmt.Sprintf("^/dev/%s on", deviceName) - mountPoint := Awk(Grep(output, searchFor), 3) + mountPoint := Awk(Grep(output, searchFor), 3, " ") return mountPoint, nil } @@ -208,7 +263,7 @@ func GetDeviceFromMountPoint(mountPoint string, executor exec.Executor) (string, } searchFor := fmt.Sprintf("on %s ", mountPoint) - device := Awk(Grep(output, searchFor), 1) + device := Awk(Grep(output, searchFor), 1, " ") return device, nil } @@ -254,31 +309,46 @@ func UnmountDevice(devicePath string, executor exec.Executor) error { return nil } -func DoesDeviceHaveChildren(device string, executor exec.Executor) (bool, error) { - cmd := fmt.Sprintf("check children for device %s", device) - output, err := executor.ExecuteCommandWithOutput(false, cmd, "lsblk --all -n -l --output PKNAME") +func GetFSUUID(device string, executor exec.Executor) (string, error) { + cmd := fmt.Sprintf("get disk %s fs uuid", device) + output, err := executor.ExecuteCommandWithOutput(false, cmd, + "udevadm", "info", "--query=property", fmt.Sprintf("/dev/%s", device)) if err != nil { - return false, fmt.Errorf("command %s failed: %+v", cmd, err) + return "", err } - - searchFor := fmt.Sprintf("^%s$", device) - children := Grep(output, searchFor) - - return children != "", nil + return parseFSUUID(output), nil } -// finds the file system(s) for the device in the output of 'df' -func parseDFOutput(device, output string) string { - var fs []string - lines := strings.Split(output, "\n") - for _, line := range lines { - if strings.HasPrefix(line, fmt.Sprintf("/dev/%s", device)) { - words := strings.Split(line, " ") - fs = append(fs, words[len(words)-1]) +func CheckIfDeviceAvailable(executor exec.Executor, name string) (bool, string, error) { + ownPartitions := true + partitions, _, err := GetDevicePartitions(name, executor) + if err != nil { + return false, "", fmt.Errorf("failed to get %s partitions. %+v", name, err) + } + if !RookOwnsPartitions(partitions) { + ownPartitions = false + } + + // check if there is a file system on the device + devFS, err := GetDeviceFilesystems(name, executor) + if err != nil { + return false, "", fmt.Errorf("failed to get device %s filesystem: %+v", name, err) + } + + return ownPartitions, devFS, nil +} + +func RookOwnsPartitions(partitions []Partition) bool { + + // if there are partitions, they must all have the rook osd label + for _, p := range partitions { + if !strings.HasPrefix(p.Label, "ROOK-OSD") { + return false } } - return strings.Join(fs, ",") + // if there are no partitions, or the partitions are all from rook OSDs, then rook owns the device + return true } // finds the disk uuid in the output of sgdisk @@ -323,3 +393,42 @@ func parseKeyValuePairString(propsRaw string) map[string]string { return propMap } + +// find fs from udevadm info +func parseFS(output string) string { + m := parseUdevInfo(output) + if v, ok := m["ID_FS_TYPE"]; ok { + return v + } + return "" +} + +// find fs from udevadm info +func parseFSUUID(output string) string { + m := parseUdevInfo(output) + if v, ok := m["ID_FS_UUID"]; ok { + return v + } + return "" +} + +// find partition label from udevadm info +func parsePartLabel(output string) string { + m := parseUdevInfo(output) + if v, ok := m["ID_PART_ENTRY_NAME"]; ok { + return v + } + return "" +} + +func parseUdevInfo(output string) map[string]string { + lines := strings.Split(output, "\n") + result := make(map[string]string, len(lines)) + for _, v := range lines { + pairs := strings.Split(v, "=") + if len(pairs) > 1 { + result[pairs[0]] = pairs[1] + } + } + return result +} diff --git a/pkg/util/sys/device_test.go b/pkg/util/sys/device_test.go index 75ff98779..8454a31b0 100644 --- a/pkg/util/sys/device_test.go +++ b/pkg/util/sys/device_test.go @@ -24,6 +24,68 @@ import ( "github.com/stretchr/testify/assert" ) +const ( + udevOutput = `DEVLINKS=/dev/disk/by-id/scsi-36001405d27e5d898829468b90ce4ef8c /dev/disk/by-id/wwn-0x6001405d27e5d898829468b90ce4ef8c /dev/disk/by-path/ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-0 /dev/disk/by-uuid/f2d38cba-37da-411d-b7ba-9a6696c58174 +DEVNAME=/dev/sdk +DEVPATH=/devices/platform/host6/session2/target6:0:0/6:0:0:0/block/sdk +DEVTYPE=disk +ID_BUS=scsi +ID_FS_TYPE=ext2 +ID_FS_USAGE=filesystem +ID_FS_UUID=f2d38cba-37da-411d-b7ba-9a6696c58174 +ID_FS_UUID_ENC=f2d38cba-37da-411d-b7ba-9a6696c58174 +ID_FS_VERSION=1.0 +ID_MODEL=disk01 +ID_MODEL_ENC=disk01\x20\x20\x20\x20\x20\x20\x20\x20\x20\x20 +ID_PATH=ip-127.0.0.1:3260-iscsi-iqn.2016-06.world.srv:storage.target01-lun-0 +ID_PATH_TAG=ip-127_0_0_1_3260-iscsi-iqn_2016-06_world_srv_storage_target01-lun-0 +ID_REVISION=4.0 +ID_SCSI=1 +ID_SCSI_SERIAL=d27e5d89-8829-468b-90ce-4ef8c02f07fe +ID_SERIAL=36001405d27e5d898829468b90ce4ef8c +ID_SERIAL_SHORT=6001405d27e5d898829468b90ce4ef8c +ID_TARGET_PORT=0 +ID_TYPE=disk +ID_VENDOR=LIO-ORG +ID_VENDOR_ENC=LIO-ORG\x20 +ID_WWN=0x6001405d27e5d898 +ID_WWN_VENDOR_EXTENSION=0x829468b90ce4ef8c +ID_WWN_WITH_EXTENSION=0x6001405d27e5d898829468b90ce4ef8c +MAJOR=8 +MINOR=160 +SUBSYSTEM=block +TAGS=:systemd: +USEC_INITIALIZED=15981915740802 +` + udevPartOutput = `ID_PART_ENTRY_DISK=8:32 +ID_PART_ENTRY_NAME=%s +ID_PART_ENTRY_NUMBER=3 +ID_PART_ENTRY_OFFSET=3278848 +ID_PART_ENTRY_SCHEME=gpt +ID_PART_ENTRY_SIZE=7206879 +ID_PART_ENTRY_TYPE=0fc63daf-8483-4772-8e79-3d69d8477de4 +ID_PART_ENTRY_UUID=2089640e-bdeb-4fb4-aaec-88e165780b88 +ID_PART_TABLE_TYPE=gpt +ID_PART_TABLE_UUID=46242f96-6cf7-4e5d-b4bd-9d046e6ad920 +ID_REVISION=4.0 +ID_SCSI=1 +ID_SCSI_SERIAL=68c0bd28-d4ee-4376-9387-c9f02c53b3f2 +ID_SERIAL=3600140568c0bd28d4ee43769387c9f02 +ID_SERIAL_SHORT=600140568c0bd28d4ee43769387c9f02 +ID_TARGET_PORT=0 +ID_TYPE=disk +ID_VENDOR=LIO-ORG +ID_VENDOR_ENC=LIO-ORG\x20 +ID_WWN=0x600140568c0bd28d +ID_WWN_VENDOR_EXTENSION=0x4ee43769387c9f02 +ID_WWN_WITH_EXTENSION=0x600140568c0bd28d4ee43769387c9f02 +MAJOR=8 +MINOR=35 +PARTN=3 +SUBSYSTEM=block +` +) + func TestFindUUID(t *testing.T) { output := `Disk /dev/sdb: 10485760 sectors, 5.0 GiB Logical sector size: 512 bytes @@ -39,24 +101,10 @@ Total free space is 20971453 sectors (10.0 GiB) } func TestParseFileSystem(t *testing.T) { - output := `Filesystem Type -devtmpfs devtmpfs -/dev/sda9 ext4 -/dev/sda3 ext4 -/dev/sda1 vfat -tmpfs tmpfs -tmpfs tmpfs -/dev/sda6 ext4 -sdc tmpfs` + output := udevOutput - result := parseDFOutput("sda", output) - assert.Equal(t, "ext4,ext4,vfat,ext4", result) - - result = parseDFOutput("sdb", output) - assert.Equal(t, "", result) - - result = parseDFOutput("sdc", output) - assert.Equal(t, "", result) + result := parseFS(output) + assert.Equal(t, "ext2", result) } func TestGetDeviceFromMountPoint(t *testing.T) { @@ -127,6 +175,7 @@ func TestGetPartitions(t *testing.T) { executor := &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(debug bool, actionName string, command string, arg ...string) (string, error) { run++ + logger.Infof("run %d action %s", run, actionName) switch { case run == 1: return `NAME="sdc" SIZE="100000" TYPE="disk" PKNAME=""`, nil @@ -136,12 +185,15 @@ NAME="sdb2" SIZE="10" TYPE="part" PKNAME="sdb" NAME="sdb3" SIZE="20" TYPE="part" PKNAME="sdb" NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil case run == 3: - return "ROOK-OSD0-DB", nil - case run == 4: - return "ROOK-OSD0-BLOCK", nil + return fmt.Sprintf(udevPartOutput, "ROOK-OSD0-DB"), nil + case run == 4 || run == 6 || run == 8: + // get filesystem + return "", nil case run == 5: - return "ROOK-OSD0-WAL", nil - case run == 6: + return fmt.Sprintf(udevPartOutput, "ROOK-OSD0-BLOCK"), nil + case run == 7: + return fmt.Sprintf(udevPartOutput, "ROOK-OSD0-WAL"), nil + case run == 9: return `NAME="sda" SIZE="19818086400" TYPE="disk" PKNAME="" NAME="sda4" SIZE="1073741824" TYPE="part" PKNAME="sda" NAME="sda2" SIZE="2097152" TYPE="part" PKNAME="sda" @@ -151,22 +203,14 @@ NAME="sda3" SIZE="1073741824" TYPE="part" PKNAME="sda" NAME="usr" SIZE="1065345024" TYPE="crypt" PKNAME="sda3" NAME="sda1" SIZE="134217728" TYPE="part" PKNAME="sda" NAME="sda6" SIZE="134217728" TYPE="part" PKNAME="sda"`, nil - case run == 7: - return "USR-B", nil - case run == 8: - return "BIOS-BOOT", nil case run == 9: - return "ROOT", nil + return fmt.Sprintf(udevPartOutput, "ROOT"), nil case run == 10: - return "OEM-CONFIG", nil + return fmt.Sprintf(udevPartOutput, "OEM-CONFIG"), nil case run == 11: - return "USR-A", nil + return fmt.Sprintf(udevPartOutput, "USR-A"), nil case run == 12: - return "EFI-SYSTEM", nil - case run == 13: - return "OEM", nil - case run == 14: - return "", nil + return fmt.Sprintf(udevPartOutput, "EFI-SYSTEM"), nil } return "", nil }, @@ -194,3 +238,8 @@ NAME="sda6" SIZE="134217728" TYPE="part" PKNAME="sda"`, nil assert.Nil(t, err) assert.Equal(t, 0, len(partitions)) } + +func TestParseUdevInfo(t *testing.T) { + m := parseUdevInfo(udevOutput) + assert.Equal(t, m["ID_FS_TYPE"], "ext2") +} diff --git a/pkg/util/sys/parse.go b/pkg/util/sys/parse.go index 95c465acf..69ccabbf5 100644 --- a/pkg/util/sys/parse.go +++ b/pkg/util/sys/parse.go @@ -37,13 +37,13 @@ func Grep(input, searchFor string) string { // awk finds the space-delimited token at the given position. // 0 returns the whole line, while 1 is the first token. -func Awk(input string, position int) string { +func Awk(input string, position int, delimit string) string { logger.Debugf("awk %d from %s", position, input) if position == 0 { return input } - words := strings.Split(strings.TrimSpace(input), " ") + words := strings.Split(strings.TrimSpace(input), delimit) position-- if position < 0 || position >= len(words) { // out of range diff --git a/pkg/util/sys/parse_test.go b/pkg/util/sys/parse_test.go index 30be0455f..023583ef7 100644 --- a/pkg/util/sys/parse_test.go +++ b/pkg/util/sys/parse_test.go @@ -56,6 +56,6 @@ func TestAwk(t *testing.T) { } func testAwk(t *testing.T, index int, input, expected string) { - out := Awk(input, index) + out := Awk(input, index, " ") assert.Equal(t, expected, out) } diff --git a/tests/framework/utils/helm_helper.go b/tests/framework/utils/helm_helper.go index 21e0f71cd..ed2eac075 100644 --- a/tests/framework/utils/helm_helper.go +++ b/tests/framework/utils/helm_helper.go @@ -62,7 +62,7 @@ func (h *HelmHelper) GetLocalRookHelmChartVersion(chartName string) (string, err } cd := strings.Replace(sys.Grep(result, chartName), "\t", " ", 2) - return sys.Awk(cd, 2), nil + return sys.Awk(cd, 2, " "), nil } //InstallLocalRookHelmChart installs a give helm chart diff --git a/tests/integration/base_deploy_test.go b/tests/integration/base_deploy_test.go index 75afd5c99..08913f37f 100644 --- a/tests/integration/base_deploy_test.go +++ b/tests/integration/base_deploy_test.go @@ -45,6 +45,8 @@ func checkIfRookClusterIsInstalled(s suite.Suite, k8sh *utils.K8sHelper, opNames "Make sure there is 1 rook-operator present in Running state") assert.True(s.T(), k8sh.CheckPodCountAndState("rook-agent", opNamespace, 1, "Running"), "Make sure there is 1 rook-agent present in Running state") + assert.True(s.T(), k8sh.CheckPodCountAndState("rook-discover", opNamespace, 1, "Running"), + "Make sure there is 1 rook-discover present in Running state") assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mgr", clusterNamespace, 1, "Running"), "Make sure there is 1 rook-ceph-mgr present in Running state") assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-osd", clusterNamespace, 1, "Running"), diff --git a/tests/integration/multi_cluster_deploy_test.go b/tests/integration/multi_cluster_deploy_test.go index 17fc79842..007626b00 100644 --- a/tests/integration/multi_cluster_deploy_test.go +++ b/tests/integration/multi_cluster_deploy_test.go @@ -160,6 +160,9 @@ func (o MCTestOperations) SetUp() { require.True(o.T(), o.kh.IsPodInExpectedState("rook-agent", installer.SystemNamespace(o.namespace1), "Running"), "Make sure rook-agent is in running state") + require.True(o.T(), o.kh.IsPodInExpectedState("rook-discover", installer.SystemNamespace(o.namespace1), "Running"), + "Make sure rook-discover is in running state") + time.Sleep(10 * time.Second) // start the two clusters in parallel