diff --git a/pkg/api/device.go b/pkg/api/device.go deleted file mode 100644 index ff41325a6..000000000 --- a/pkg/api/device.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2016 The Rook Authors. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package api - -import ( - "encoding/json" - "net/http" - - "github.com/rook/rook/pkg/cephmgr/osd" -) - -// Adds the device and configures an OSD on the device. -// POST -// /device -func (h *Handler) AddDevice(w http.ResponseWriter, r *http.Request) { - device, ok := handleLoadDeviceFromBody(w, r) - if !ok { - return - } - - err := osd.AddDesiredDevice(h.context.EtcdClient, device.Name, device.NodeID) - if err != nil { - logger.Errorf("failed to add device. %v", err) - w.WriteHeader(http.StatusInternalServerError) - return - } -} - -// Stops the OSD and removes a device from participating in the cluster. -// POST -// /device/remove -func (h *Handler) RemoveDevice(w http.ResponseWriter, r *http.Request) { - device, ok := handleLoadDeviceFromBody(w, r) - if !ok { - return - } - - err := osd.RemoveDesiredDevice(h.context.EtcdClient, device.Name, device.NodeID) - if err != nil { - logger.Errorf("failed to remove device. %v", err) - w.WriteHeader(http.StatusInternalServerError) - return - } -} - -func handleLoadDeviceFromBody(w http.ResponseWriter, r *http.Request) (*osd.Device, bool) { - body, ok := handleReadBody(w, r, "load device") - if !ok { - return nil, false - } - - var device osd.Device - if err := json.Unmarshal(body, &device); err != nil { - logger.Errorf("failed to unmarshal request body '%s': %+v", string(body), err) - w.WriteHeader(http.StatusBadRequest) - return nil, false - } - - if device.Name == "" || device.NodeID == "" { - logger.Errorf("missing name or nodeId: %+v", device) - w.WriteHeader(http.StatusBadRequest) - return nil, false - } - - return &device, true -} diff --git a/pkg/api/handlers_test.go b/pkg/api/handlers_test.go index b899609b2..9da2b32df 100644 --- a/pkg/api/handlers_test.go +++ b/pkg/api/handlers_test.go @@ -16,6 +16,7 @@ limitations under the License. package api import ( + "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -40,50 +41,8 @@ const ( SuccessGetPoolECPool1Response = `{"pool":"ecPool1","pool_id":1,"size":3}{"pool":"ecPool1","pool_id":1,"min_size":3}{"pool":"ecPool1","pool_id":1,"crash_replay_interval":0}{"pool":"ecPool1","pool_id":1,"pg_num":100}{"pool":"ecPool1","pool_id":1,"pgp_num":100}{"pool":"ecPool1","pool_id":1,"crush_ruleset":1}{"pool":"ecPool1","pool_id":1,"hashpspool":"true"}{"pool":"ecPool1","pool_id":1,"nodelete":"false"}{"pool":"ecPool1","pool_id":1,"nopgchange":"false"}{"pool":"ecPool1","pool_id":1,"nosizechange":"false"}{"pool":"ecPool1","pool_id":1,"write_fadvise_dontneed":"false"}{"pool":"ecPool1","pool_id":1,"noscrub":"false"}{"pool":"ecPool1","pool_id":1,"nodeep-scrub":"false"}{"pool":"ecPool1","pool_id":1,"use_gmt_hitset":true}{"pool":"ecPool1","pool_id":1,"auid":0}{"pool":"ecPool1","pool_id":1,"erasure_code_profile":"ecPool1_ecprofile"}{"pool":"ecPool1","pool_id":1,"min_write_recency_for_promote":0}{"pool":"ecPool1","pool_id":1,"fast_read":0}{"pool":"ecPool1","pool_id":1}{"pool":"ecPool1","pool_id":1}{"pool":"ecPool1","pool_id":1}{"pool":"ecPool1","pool_id":1}{"pool":"ecPool1","pool_id":1}{"pool":"ecPool1","pool_id":1}` ) -func TestAddRemoveDeviceHandler(t *testing.T) { - etcdClient := util.NewMockEtcdClient() - context := &clusterd.Context{EtcdClient: etcdClient} - etcdClient.SetValue("/rook/nodes/config/123/disks/foo/uuid", "12345") - - req, err := http.NewRequest("POST", "http://10.0.0.100/device", strings.NewReader(`{"name":"foo"}`)) - assert.Nil(t, err) - - w := httptest.NewRecorder() - h := NewHandler(context, nil, nil) - - // missing the node id - h.AddDevice(w, req) - assert.Equal(t, http.StatusBadRequest, w.Code) - - // successful request - req, err = http.NewRequest("POST", "http://10.0.0.100/device", strings.NewReader(`{"name":"foo","nodeId":"123"}`)) - assert.Nil(t, err) - h = NewHandler(context, nil, nil) - w = httptest.NewRecorder() - h.AddDevice(w, req) - assert.Equal(t, http.StatusOK, w.Code) - - devices := etcdClient.GetChildDirs("/rook/services/ceph/osd/desired/123/device") - assert.Equal(t, 1, devices.Count()) - assert.True(t, devices.Contains("foo")) - - // remove the device - req, err = http.NewRequest("POST", "http://10.0.0.100/device/remove", strings.NewReader(`{"name":"foo","nodeId":"123"}`)) - assert.Nil(t, err) - h = NewHandler(context, nil, nil) - w = httptest.NewRecorder() - h.RemoveDevice(w, req) - assert.Equal(t, http.StatusOK, w.Code) - - // check the desired state - devices = etcdClient.GetChildDirs("/rook/services/ceph/osd/desired/123/device") - assert.Equal(t, 0, devices.Count()) -} - -func TestRemoveDeviceHandler(t *testing.T) { -} - func TestGetNodesHandler(t *testing.T) { + nodeID := "node1" etcdClient := util.NewMockEtcdClient() context := &clusterd.Context{EtcdClient: etcdClient} @@ -102,16 +61,18 @@ func TestGetNodesHandler(t *testing.T) { assert.Equal(t, http.StatusOK, w.Code) assert.Equal(t, `[]`, w.Body.String()) + // add the disks to etcd + disks := []inventory.Disk{ + inventory.Disk{Type: inventory.DiskType, Size: 50, Rotational: true, Available: true}, + inventory.Disk{Type: inventory.DiskType, Size: 100, Rotational: false, Available: true}, + } + output, _ := json.Marshal(disks) + etcdClient.SetValue(path.Join(inventory.NodesConfigKey, nodeID, "disks"), string(output)) + // set up a discovered node in etcd - inventory.SetIPAddress(etcdClient, "node1", "1.2.3.4", "10.0.0.11") - inventory.SetLocation(etcdClient, "node1", "root=default,dc=datacenter1") - nodeConfigKey := path.Join(inventory.NodesConfigKey, "node1") - etcdClient.CreateDir(nodeConfigKey) - inventory.TestSetDiskInfo(etcdClient, nodeConfigKey, "sda", "123d4869-29ee-4bfd-bf21-dfd597bd222e", - 100, true, false, "btrfs", "/mnt/abc", inventory.Disk, "", false) - inventory.TestSetDiskInfo(etcdClient, nodeConfigKey, "sdb", "321d4869-29ee-4bfd-bf21-dfd597bdffff", - 50, false, false, "ext4", "/mnt/def", inventory.Disk, "", false) - appliedOSDKey := "/rook/services/ceph/osd/applied/node1" + inventory.SetIPAddress(etcdClient, nodeID, "1.2.3.4", "10.0.0.11") + inventory.SetLocation(etcdClient, nodeID, "root=default,dc=datacenter1") + appliedOSDKey := path.Join("/rook/services/ceph/osd/applied", nodeID) etcdClient.SetValue(path.Join(appliedOSDKey, "12", "disk-uuid"), "123d4869-29ee-4bfd-bf21-dfd597bd222e") etcdClient.SetValue(path.Join(appliedOSDKey, "13", "disk-uuid"), "321d4869-29ee-4bfd-bf21-dfd597bdffff") diff --git a/pkg/api/node.go b/pkg/api/node.go index 9d959c2c6..4f43ee250 100644 --- a/pkg/api/node.go +++ b/pkg/api/node.go @@ -19,7 +19,6 @@ import ( "net/http" "github.com/rook/rook/pkg/cephmgr/mon" - "github.com/rook/rook/pkg/cephmgr/osd" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/clusterd/inventory" "github.com/rook/rook/pkg/model" @@ -46,22 +45,11 @@ func (h *Handler) GetNodes(w http.ResponseWriter, r *http.Request) { nodes := make([]model.Node, len(clusterInventory.Nodes)) i := 0 for nodeID, n := range clusterInventory.Nodes { - // look up all the disks that the current node has applied OSDs on - appliedIDs, err := osd.GetAppliedOSDs(nodeID, h.context.EtcdClient) - if err != nil { - logger.Errorf("failed to get applied OSDs for node %s: %+v", nodeID, err) - w.WriteHeader(http.StatusInternalServerError) - return - } - storage := uint64(0) for _, d := range n.Disks { - for _, uuid := range appliedIDs { - if d.UUID == uuid { - // current disk is in applied OSD set, add its storage to the running total - storage += d.Size - } - } + // Add up the space of all devices. + // We should have a separate metric for osd devices, but keep it simple for now. + storage += d.Size } // determine the node's state/health diff --git a/pkg/api/routes.go b/pkg/api/routes.go index f88a4fd97..5f1d3fe4a 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -107,18 +107,6 @@ func (h *Handler) GetRoutes() []Route { "/filesystem/remove", h.RemoveFileSystem, }, - { - "AddDevice", - "POST", - "/device", - h.AddDevice, - }, - { - "RemoveDevice", - "POST", - "/device/remove", - h.RemoveDevice, - }, { "SetLogLevel", "POST", diff --git a/pkg/cephmgr/osd/agent.go b/pkg/cephmgr/osd/agent.go index 99dc87a9a..7de8eafc7 100644 --- a/pkg/cephmgr/osd/agent.go +++ b/pkg/cephmgr/osd/agent.go @@ -48,14 +48,15 @@ const ( ) type osdAgent struct { - cluster *mon.ClusterInfo - forceFormat bool - location string - factory client.ConnectionFactory - osdProc map[int]*proc.MonitoredProc - devices string - configCounter int32 - osdsCompleted chan struct{} + cluster *mon.ClusterInfo + forceFormat bool + location string + factory client.ConnectionFactory + osdProc map[int]*proc.MonitoredProc + desiredDevices []string + devices string + configCounter int32 + osdsCompleted chan struct{} } func NewAgent(factory client.ConnectionFactory, devices string, forceFormat bool, location string) *osdAgent { @@ -72,19 +73,13 @@ func (a *osdAgent) Initialize(context *clusterd.Context) error { if len(a.devices) > 0 { // add the devices to desired state - devices := strings.Split(a.devices, ",") - for _, device := range devices { - logger.Infof("Adding device %s to desired state", device) - err := AddDesiredDevice(context.EtcdClient, device, context.NodeID) - if err != nil { - return fmt.Errorf("failed to add desired device %s. %v", device, err) - } - } + a.desiredDevices = strings.Split(a.devices, ",") + logger.Infof("desired devices for osds: %+v", a.desiredDevices) } // if no devices or directories were specified, use the current directory for an osd - if len(a.devices) == 0 { - logger.Infof("Adding local path to local directory %s", context.ConfigDir) + if len(a.desiredDevices) == 0 { + logger.Infof("Adding local path %s to desired state", context.ConfigDir) err := AddDesiredDir(context.EtcdClient, context.ConfigDir, context.NodeID) if err != nil { return fmt.Errorf("failed to add current dir %s. %v", context.ConfigDir, err) @@ -180,7 +175,7 @@ func (a *osdAgent) decrementConfigCounter() { } func (a *osdAgent) stopUndesiredDevices(context *clusterd.Context, connection client.Connection) error { - desiredDevices, err := loadDesiredDevices(context.EtcdClient, context.NodeID) + desiredDevices, err := a.loadDesiredDevices(context) if err != nil { return fmt.Errorf("failed to load desired devices. %v", err) } @@ -277,7 +272,7 @@ func getAppliedKey(nodeID string) string { // create and initalize OSDs for all the devices specified in the given config func (a *osdAgent) createDesiredOSDs(adminConn client.Connection, context *clusterd.Context) error { - devices, err := loadDesiredDevices(context.EtcdClient, context.NodeID) + devices, err := a.loadDesiredDevices(context) if err != nil { return fmt.Errorf("failed to load desired devices. %v", err) } @@ -355,6 +350,9 @@ func (a *osdAgent) configureDevices(context *clusterd.Context, devices map[strin go func() { defer connection.Shutdown() + // set the signal that the osd config is completed + defer close(a.osdsCompleted) + a.incrementConfigCounter() defer a.decrementConfigCounter() @@ -371,9 +369,6 @@ func (a *osdAgent) configureDevices(context *clusterd.Context, devices map[strin } logger.Infof("%d/%d bluestore osds succeeded on this node", succeeded, len(devices)) - - // set the signal that the osd config is completed - close(a.osdsCompleted) }() return nil @@ -387,13 +382,12 @@ func (a *osdAgent) createOrStartOSD(context *clusterd.Context, connection client return err } - name := config.deviceName + // set the desired state of the dir with the osd id. If a device, we will delay setting this until we have the device uuid if dir { - name = configRoot - } - err = setOSDOnDevice(context.EtcdClient, context.NodeID, name, config.id, dir) - if err != nil { - return err + err = setOSDOnDevice(context.EtcdClient, context.NodeID, configRoot, config.id, dir) + if err != nil { + return fmt.Errorf("failed to associate osd id %d with the data dir", config.id) + } } } @@ -537,10 +531,29 @@ func getPseudoDir(path string) string { return strings.Replace(path, "/", "_", -1) } -func loadDesiredDevices(etcdClient etcd.KeysAPI, nodeID string) (map[string]int, error) { +func (a *osdAgent) loadDesiredDevices(context *clusterd.Context) (map[string]int, error) { + // get the device UUID to device name mapping + uuidToName := map[string]string{} + for _, disk := range context.Inventory.Local.Disks { + if disk.UUID != "" { + uuidToName[disk.UUID] = disk.Name + } + } + logger.Debugf("uuid to name map: %+v", uuidToName) + + // ensure all the desired devices are in the list devices := map[string]int{} - key := path.Join(fmt.Sprintf(deviceDesiredKey, nodeID)) - devKeys, err := etcdClient.Get(ctx.Background(), key, &etcd.GetOptions{Recursive: true}) + for _, name := range a.desiredDevices { + if _, ok := devices[name]; !ok { + // add the device to the desired list + devices[name] = unassignedOSDID + } + } + logger.Debugf("desired osd id mapping when new: %+v", devices) + + // parse the desired devices from etcd, which are based on the disk uuid. + key := path.Join(fmt.Sprintf(deviceDesiredKey, context.NodeID)) + devKeys, err := context.EtcdClient.Get(ctx.Background(), key, &etcd.GetOptions{Recursive: true}) if err != nil { if util.IsEtcdKeyNotFound(err) { return devices, nil @@ -548,23 +561,30 @@ func loadDesiredDevices(etcdClient etcd.KeysAPI, nodeID string) (map[string]int, return nil, err } - // parse the dirs from etcd for _, dev := range devKeys.Node.Nodes { - device := util.GetLeafKeyPath(dev.Key) + uuid := util.GetLeafKeyPath(dev.Key) osdID := unassignedOSDID for _, setting := range dev.Nodes { if strings.HasSuffix(setting.Key, "/osd-id") { id, err := strconv.Atoi(setting.Value) if err == nil { + logger.Debugf("found osd id %d for disk uuid %s", osdID, uuid) osdID = id } } } - devices[device] = osdID + // translate the disk uuid to the device name + if name, ok := uuidToName[uuid]; ok { + devices[name] = osdID + } else { + logger.Warningf("did not find name for disk uuid %s", uuid) + } } + logger.Debugf("final osd id mapping: %+v", devices) + return devices, nil } @@ -585,11 +605,11 @@ func setOSDOnDevice(etcdClient etcd.KeysAPI, nodeID, name string, id int, dir bo } // add a device to the desired state -func AddDesiredDevice(etcdClient etcd.KeysAPI, device, nodeID string) error { - key := path.Join(fmt.Sprintf(deviceDesiredKey, nodeID), device) +func AddDesiredDevice(etcdClient etcd.KeysAPI, nodeID, uuid string, osdID int) error { + key := path.Join(fmt.Sprintf(deviceDesiredKey, nodeID), uuid) err := util.CreateEtcdDir(etcdClient, key) if err != nil { - return fmt.Errorf("failed to add device %s on node %s to desired. %v", device, nodeID, err) + return fmt.Errorf("failed to add device %s on node %s to desired. %v", uuid, nodeID, err) } return nil @@ -641,12 +661,12 @@ func AddDesiredDir(etcdClient etcd.KeysAPI, dir, nodeID string) error { } // remove a device from the desired state -func RemoveDesiredDevice(etcdClient etcd.KeysAPI, device, nodeID string) error { +func RemoveDesiredDevice(etcdClient etcd.KeysAPI, nodeID, uuid string) error { - key := path.Join(fmt.Sprintf(deviceDesiredKey, nodeID), device) + key := path.Join(fmt.Sprintf(deviceDesiredKey, nodeID), uuid) _, err := etcdClient.Delete(ctx.Background(), key, &etcd.DeleteOptions{Dir: true}) if err != nil { - return fmt.Errorf("failed to remove device %s on node %s from desired. %v", device, nodeID, err) + return fmt.Errorf("failed to remove device uuid %s on node %s from desired. %v", uuid, nodeID, err) } return nil diff --git a/pkg/cephmgr/osd/agent_test.go b/pkg/cephmgr/osd/agent_test.go index 4d5e80645..f9766bd44 100644 --- a/pkg/cephmgr/osd/agent_test.go +++ b/pkg/cephmgr/osd/agent_test.go @@ -93,18 +93,16 @@ func TestOSDAgentWithDevices(t *testing.T) { NodeID: nodeID, ConfigDir: "/tmp", ProcMan: proc.New(executor), + Inventory: createInventory(), + } + context.Inventory.Local.Disks = []inventory.LocalDisk{ + inventory.LocalDisk{Name: "sdx", Size: 1234567890, UUID: "12345"}, + inventory.LocalDisk{Name: "sdy", Size: 2234567890, UUID: "54321"}, } - - // prep the etcd keys that would have been discovered by inventory - disksKey := path.Join(inventory.GetNodeConfigKey(context.NodeID), "disks") - etcdClient.SetValue(path.Join(disksKey, "sdx", "uuid"), "12345") - etcdClient.SetValue(path.Join(disksKey, "sdy", "uuid"), "54321") - etcdClient.SetValue(path.Join(disksKey, "sdx", "size"), "1234567890") - etcdClient.SetValue(path.Join(disksKey, "sdy", "size"), "2234567890") // Set one device as already having an assigned osd id. The other device will go through id selection, // which is mocked in the createTestAgent method to return an id of 3. - etcdClient.SetValue(fmt.Sprintf("/rook/services/ceph/osd/desired/%s/device/sdx/osd-id", nodeID), "23") + etcdClient.SetValue(fmt.Sprintf("/rook/services/ceph/osd/desired/%s/device/12345/osd-id", nodeID), "23") // prep the OSD agent and related orcehstration data prepAgentOrchestrationData(t, agent, etcdClient, context, clusterName) @@ -168,6 +166,7 @@ func TestOSDAgentNoDevices(t *testing.T) { NodeID: nodeID, ProcMan: proc.New(executor), ConfigDir: "/tmp", + Inventory: createInventory(), } // prep the OSD agent and related orcehstration data @@ -206,12 +205,6 @@ func TestAppliedDevices(t *testing.T) { assert.Equal(t, 0, len(osds)) // two applied osds - nodeConfigKey := path.Join(inventory.NodesConfigKey, nodeID) - etcdClient.CreateDir(nodeConfigKey) - inventory.TestSetDiskInfo(etcdClient, nodeConfigKey, "sda", "ff6d4869-29ee-4bfd-bf21-dfd597bd222e", - 100, true, false, "btrfs", "/mnt/xyz", inventory.Disk, "", false) - inventory.TestSetDiskInfo(etcdClient, nodeConfigKey, "sdb", "ff6d4869-29ee-4bfd-bf21-dfd597bd222e", - 50, false, false, "ext4", "/mnt/zyx", inventory.Disk, "", false) appliedOSDKey := "/rook/services/ceph/osd/applied/abc" etcdClient.SetValue(path.Join(appliedOSDKey, "1", "disk-uuid"), "1234") etcdClient.SetValue(path.Join(appliedOSDKey, "2", "disk-uuid"), "2345") @@ -228,8 +221,9 @@ func TestRemoveDevice(t *testing.T) { etcdClient, agent, conn := createTestAgent(t, nodeID, "") executor := &exectest.MockExecutor{} - context := &clusterd.Context{EtcdClient: etcdClient, NodeID: nodeID, Executor: executor, ProcMan: proc.New(executor)} - etcdClient.SetValue("/rook/services/ceph/osd/desired/a/device/sda/osd-id", "23") + context := &clusterd.Context{EtcdClient: etcdClient, NodeID: nodeID, Executor: executor, ProcMan: proc.New(executor), Inventory: createInventory()} + context.Inventory.Local.Disks = []inventory.LocalDisk{inventory.LocalDisk{Name: "sda", Size: 1234567890, UUID: "5435435333"}} + etcdClient.SetValue("/rook/services/ceph/osd/desired/a/device/5435435333/osd-id", "23") // create two applied osds, one of which is desired appliedRoot := "/rook/services/ceph/osd/applied/" + nodeID @@ -298,27 +292,67 @@ func createTestKeyring(t *testing.T, args []string) { } func TestDesiredDeviceState(t *testing.T) { + nodeID := "a" etcdClient := util.NewMockEtcdClient() - etcdClient.CreateDir("/rook/nodes/config/a/disks/foo") // add a device - err := AddDesiredDevice(etcdClient, "foo", "a") + err := AddDesiredDevice(etcdClient, nodeID, "myuuid", 23) assert.Nil(t, err) devices := etcdClient.GetChildDirs("/rook/services/ceph/osd/desired/a/device") assert.Equal(t, 1, devices.Count()) - assert.True(t, devices.Contains("foo")) + assert.True(t, devices.Contains("myuuid")) // remove the device - err = RemoveDesiredDevice(etcdClient, "foo", "a") + err = RemoveDesiredDevice(etcdClient, nodeID, "myuuid") assert.Nil(t, err) devices = etcdClient.GetChildDirs("/rook/services/ceph/osd/desired/a/device") assert.Equal(t, 0, devices.Count()) // removing a non-existent device is a no-op - err = RemoveDesiredDevice(etcdClient, "foo", "a") + err = RemoveDesiredDevice(etcdClient, nodeID, "foo") assert.Nil(t, err) } +func TestLoadDesiredDevices(t *testing.T) { + etcdClient := util.NewMockEtcdClient() + a := &osdAgent{desiredDevices: []string{}} + + // no devices are desired + context := &clusterd.Context{EtcdClient: etcdClient, Inventory: createInventory(), NodeID: "a"} + desired, err := a.loadDesiredDevices(context) + assert.Nil(t, err) + assert.Equal(t, 0, len(desired)) + + // two devices are desired and it is a new config + context.Inventory.Local.Disks = []inventory.LocalDisk{ + inventory.LocalDisk{Name: "sda", Size: 1234567890, UUID: "12345"}, + inventory.LocalDisk{Name: "sdb", Size: 2234567890, UUID: "54321"}, + } + a.desiredDevices = []string{"sda", "sdb"} + desired, err = a.loadDesiredDevices(context) + assert.Nil(t, err) + assert.Equal(t, 2, len(desired)) + assert.Equal(t, -1, desired["sda"]) + assert.Equal(t, -1, desired["sdb"]) + + // two devices are desired and they have previously been configured + etcdClient.SetValue("/rook/services/ceph/osd/desired/a/device/12345/osd-id", "23") + etcdClient.SetValue("/rook/services/ceph/osd/desired/a/device/54321/osd-id", "24") + desired, err = a.loadDesiredDevices(context) + assert.Nil(t, err) + assert.Equal(t, 2, len(desired)) + assert.Equal(t, 23, desired["sda"]) + assert.Equal(t, 24, desired["sdb"]) + + // no devices are desired and they have previously been configured + a.desiredDevices = []string{} + desired, err = a.loadDesiredDevices(context) + assert.Nil(t, err) + assert.Equal(t, 2, len(desired)) + assert.Equal(t, 23, desired["sda"]) + assert.Equal(t, 24, desired["sdb"]) +} + func TestDesiredDirsState(t *testing.T) { etcdClient := util.NewMockEtcdClient() @@ -336,3 +370,7 @@ func TestDesiredDirsState(t *testing.T) { assert.Equal(t, 1, len(loadedDirs)) assert.Equal(t, unassignedOSDID, loadedDirs["/my/dir"]) } + +func createInventory() *inventory.Config { + return &inventory.Config{Local: &inventory.Hardware{Disks: []inventory.LocalDisk{}}} +} diff --git a/pkg/cephmgr/osd/device.go b/pkg/cephmgr/osd/device.go index 9c0c142d8..2bfa0f390 100644 --- a/pkg/cephmgr/osd/device.go +++ b/pkg/cephmgr/osd/device.go @@ -99,9 +99,9 @@ func formatDevice(context *clusterd.Context, config *osdConfig, forceFormat bool // Partitions a device for use by a bluestore osd. // If there are any partitions or formatting already on the device, it will be wiped. func partitionBluestoreDevice(context *clusterd.Context, config *osdConfig) error { - size, err := inventory.GetDeviceSize(config.deviceName, context.NodeID, context.EtcdClient) + size, err := getDiskSize(context, config.deviceName) if err != nil { - return fmt.Errorf("failed to get device %s size. %+v", config.deviceName, err) + return fmt.Errorf("failed to get device size for osd %d. %+v", config.id, err) } cmd := fmt.Sprintf("zap %s", config.deviceName) @@ -110,7 +110,7 @@ func partitionBluestoreDevice(context *clusterd.Context, config *osdConfig) erro return fmt.Errorf("failed to zap partitions on /dev/%s: %+v", config.deviceName, err) } - scheme, err := partition.GetSimpleScheme(int(size / 1024 / 1024)) + scheme, err := partition.GetSimpleScheme(config.id, int(size/1024/1024)) if err != nil { return fmt.Errorf("failed to get simple scheme. %+v", err) } @@ -129,20 +129,31 @@ func partitionBluestoreDevice(context *clusterd.Context, config *osdConfig) erro return fmt.Errorf("failed to partition /dev/%s. %+v", config.deviceName, err) } - err = inventory.SetDeviceUUID(context.NodeID, config.deviceName, scheme.DiskUUID, context.EtcdClient) - if err != nil { - return fmt.Errorf("failed to set uuid %s. %+v", scheme.DiskUUID, err) - } - // save the scheme err = scheme.Save(config.rootPath) if err != nil { return fmt.Errorf("failed to save partition scheme. %+v", err) } + // save the desired state of the osd for this device + err = setOSDOnDevice(context.EtcdClient, context.NodeID, scheme.DiskUUID, config.id, false) + if err != nil { + return fmt.Errorf("failed to associate osd id %d with device %s (%s)", config.id, config.deviceName, scheme.DiskUUID) + } + return nil } +func getDiskSize(context *clusterd.Context, name string) (uint64, error) { + for _, device := range context.Inventory.Local.Disks { + if device.Name == name { + return device.Size, nil + } + } + + return 0, fmt.Errorf("device %s not found", name) +} + func registerOSD(bootstrapConn client.Connection, config *osdConfig) error { var err error config.uuid, err = uuid.NewRandom() diff --git a/pkg/cephmgr/osd/partition/scheme.go b/pkg/cephmgr/osd/partition/scheme.go index 94d9b80a0..a8d5efc21 100644 --- a/pkg/cephmgr/osd/partition/scheme.go +++ b/pkg/cephmgr/osd/partition/scheme.go @@ -39,13 +39,14 @@ const ( type Scheme struct { Version int `json:"version"` SizeMB int `json:"sizeMb"` + ID int `json:"id"` DiskUUID string `json:"diskUuid"` PartitionUUIDs map[string]string `json:"partitionUuids"` } // This is a simple scheme to create the wal and db each of size 10% of the disk, // with the remainder of the disk being allocated for the raw data. -func GetSimpleScheme(sizeMB int) (*Scheme, error) { +func GetSimpleScheme(id, sizeMB int) (*Scheme, error) { diskUUID, err := uuid.NewRandom() if err != nil { return nil, fmt.Errorf("failed to get disk uuid. %+v", err) @@ -72,6 +73,7 @@ func GetSimpleScheme(sizeMB int) (*Scheme, error) { return &Scheme{ Version: SimpleVersion, SizeMB: sizeMB, + ID: id, DiskUUID: diskUUID.String(), PartitionUUIDs: uuids, }, nil @@ -87,11 +89,11 @@ func (s *Scheme) GetArgs(name string) []string { tenPercent := s.SizeMB / 10 // append args for individual partitions - args := s.getPartitionArgs(WalPartitionName, walPartition, offset, tenPercent, fmt.Sprintf("osd-wal-%s", s.DiskUUID)) + args := s.getPartitionArgs(WalPartitionName, walPartition, offset, tenPercent, fmt.Sprintf("ROOK-OSD%d-WAL", s.ID)) offset += tenPercent - args = append(args, s.getPartitionArgs(DatabasePartitionName, databasePartition, offset, tenPercent, fmt.Sprintf("osd-db-%s", s.DiskUUID))...) + args = append(args, s.getPartitionArgs(DatabasePartitionName, databasePartition, offset, tenPercent, fmt.Sprintf("ROOK-OSD%d-DB", s.ID))...) offset += tenPercent - args = append(args, s.getPartitionArgs(BlockPartitionName, blockPartition, offset, UseRemainingSpace, fmt.Sprintf("osd-block-%s", s.DiskUUID))...) + args = append(args, s.getPartitionArgs(BlockPartitionName, blockPartition, offset, UseRemainingSpace, fmt.Sprintf("ROOK-OSD%d-BLOCK", s.ID))...) // append args for the whole device args = append(args, fmt.Sprintf("--disk-guid=%s", s.DiskUUID)) diff --git a/pkg/cephmgr/osd/partition/scheme_test.go b/pkg/cephmgr/osd/partition/scheme_test.go index 1b1a0643f..b14eb6c91 100644 --- a/pkg/cephmgr/osd/partition/scheme_test.go +++ b/pkg/cephmgr/osd/partition/scheme_test.go @@ -24,11 +24,12 @@ import ( ) func TestSimpleScheme(t *testing.T) { - scheme, err := GetSimpleScheme(123) + scheme, err := GetSimpleScheme(1, 123) assert.Nil(t, err) assert.Equal(t, 123, scheme.SizeMB) assert.Equal(t, 36, len(scheme.DiskUUID)) assert.Equal(t, 3, len(scheme.PartitionUUIDs)) + assert.Equal(t, 1, scheme.ID) args := scheme.GetArgs("foo") assert.Equal(t, 11, len(args)) diff --git a/pkg/clusterd/clusterMember.go b/pkg/clusterd/clusterMember.go index c1f112dbf..15f82d147 100644 --- a/pkg/clusterd/clusterMember.go +++ b/pkg/clusterd/clusterMember.go @@ -90,12 +90,6 @@ func (r *ClusterMember) initialize() error { return fmt.Errorf("failed to elect leader. %+v", err) } - // initialize the hardware inventory - err = r.discoverHardware() - if err != nil { - return fmt.Errorf("failed to detect initial hardware. %+v", err) - } - // in a goroutine, begin the monitor cluster loop for changes in membership, leadership, etc. go func() { r.refreshLeader() @@ -271,5 +265,11 @@ func (r *ClusterMember) discoverHardware() error { } // Discover current state of the disks and other node properties - return inventory.DiscoverHardware(r.context.NodeID, r.context.EtcdClient, r.context.Executor) + var err error + r.context.Inventory.Local, err = inventory.DiscoverHardware(r.context.EtcdClient, r.context.Executor, r.context.NodeID) + if err != nil { + return fmt.Errorf("failed to perform discovery. %+v", err) + } + + return nil } diff --git a/pkg/clusterd/clusterMember_test.go b/pkg/clusterd/clusterMember_test.go index 8012344c3..042fd3671 100644 --- a/pkg/clusterd/clusterMember_test.go +++ b/pkg/clusterd/clusterMember_test.go @@ -589,7 +589,7 @@ func createDefaultDependencies() (*util.MockEtcdClient, *Context, *mockLeaseMana mockLeaseManager := &mockLeaseManager{} etcdClient := util.NewMockEtcdClient() machineId := "8e8f532fe96dcae6b1ce335822e5b03c" - context := &Context{EtcdClient: etcdClient, NodeID: machineId, Executor: &exectest.MockExecutor{}} + context := &Context{EtcdClient: etcdClient, NodeID: machineId, Executor: &exectest.MockExecutor{}, Inventory: &inventory.Config{}} return etcdClient, context, mockLeaseManager, &MockLeader{} } diff --git a/pkg/clusterd/inventory/disk.go b/pkg/clusterd/inventory/disk.go index d811b49a3..5b09c222c 100644 --- a/pkg/clusterd/inventory/disk.go +++ b/pkg/clusterd/inventory/disk.go @@ -16,186 +16,87 @@ limitations under the License. package inventory import ( - "errors" + "encoding/json" "fmt" "path" "strconv" - "strings" etcd "github.com/coreos/etcd/client" - "github.com/rook/rook/pkg/util" + ctx "golang.org/x/net/context" + "github.com/rook/rook/pkg/util/exec" "github.com/rook/rook/pkg/util/sys" - ctx "golang.org/x/net/context" ) -const ( - diskUUIDKey = "uuid" - diskSizeKey = "size" - diskRotationalKey = "rotational" - diskReadonlyKey = "readonly" - diskFileSystemKey = "filesystem" - diskTypeKey = "type" - diskParentKey = "parent" - diskHasChildrenKey = "children" - diskMountPointKey = "mountpoint" -) +func GetAvailableDevices(devices []LocalDisk) []string { -func DiskTypeToStr(diskType DiskType) string { - switch diskType { - case Disk: - return "disk" - case Part: - return "part" - default: - return "unknown" - } -} - -func StrToDiskType(diskType string) (DiskType, error) { - diskType = strings.ToLower(diskType) - switch diskType { - case "disk": - return Disk, nil - case "part": - return Part, nil - default: - return -1, errors.New(fmt.Sprintf("unknown disk type: %s", diskType)) - } -} - -func GetDeviceSize(name, nodeID string, etcdClient etcd.KeysAPI) (uint64, error) { - key := path.Join(GetNodeConfigKey(nodeID), disksKey, name, diskSizeKey) - resp, err := etcdClient.Get(ctx.Background(), key, nil) - if err != nil { - return 0, err - } - - size, err := strconv.ParseUint(resp.Node.Value, 10, 64) - if err != nil { - return 0, err - } - - return size, nil -} - -func GetDeviceFromUUID(uuid, nodeID string, etcdClient etcd.KeysAPI) (string, error) { - key := path.Join(GetNodeConfigKey(nodeID), disksKey) - devices, err := util.GetDirChildKeys(etcdClient, key) - if err != nil { - return "", err - } - - for d := range devices.Iter() { - resp, err := etcdClient.Get(ctx.Background(), path.Join(key, d, diskUUIDKey), nil) - if err != nil || resp == nil || resp.Node == nil { - continue - } - - if resp.Node.Value == uuid { - return d, nil + var available []string + for _, device := range devices { + logger.Debugf("Evaluating device %+v", device) + if getDeviceAvailable(device) { + logger.Debugf("Available device: %s", device.Name) + available = append(available, device.Name) } } - return "", fmt.Errorf("device for uuid %s not found", uuid) + return available } -func SetDeviceUUID(nodeID, device, uuid string, etcdClient etcd.KeysAPI) error { - key := path.Join(GetNodeConfigKey(nodeID), disksKey, device, diskUUIDKey) - _, err := etcdClient.Set(ctx.Background(), key, uuid, nil) - return err -} - -func GetDeviceUUID(device, nodeID string, etcdClient etcd.KeysAPI) (string, error) { - key := path.Join(GetNodeConfigKey(nodeID), disksKey, device, diskUUIDKey) - resp, err := etcdClient.Get(ctx.Background(), key, nil) +func storeDevices(etcdClient etcd.KeysAPI, nodeID string, devices []LocalDisk) error { + // store the basic device info in etcd + disks := toClusterDisks(devices) + output, err := json.Marshal(disks) if err != nil { - return "", err + return fmt.Errorf("failed to marshal disks. %+v", err) } - return resp.Node.Value, nil + key := path.Join(NodesConfigKey, nodeID, disksKey) + _, err = etcdClient.Set(ctx.Background(), key, string(output), nil) + if err != nil { + return fmt.Errorf("failed to store disks in etcd. %+v", err) + } + + return nil } -func getDiskInfo(diskInfo *etcd.Node) (*DiskConfig, error) { - disk := &DiskConfig{} - disk.Name = util.GetLeafKeyPath(diskInfo.Key) +func loadDisksConfig(nodeConfig *NodeConfig, rawDisks string) error { + var disks []Disk + if err := json.Unmarshal([]byte(rawDisks), &disks); err != nil { + return fmt.Errorf("failed to deserialize disks. %+v", err) + } + nodeConfig.Disks = disks + return nil +} - // iterate over all properties of the disk - for _, diskProperty := range diskInfo.Nodes { - diskPropertyName := util.GetLeafKeyPath(diskProperty.Key) - switch diskPropertyName { - case diskUUIDKey: - disk.UUID = diskProperty.Value - case diskSizeKey: - size, err := strconv.ParseUint(diskProperty.Value, 10, 64) - if err != nil { - return nil, err - } else { - disk.Size = size - } - case diskRotationalKey: - rotational, err := strconv.ParseInt(diskProperty.Value, 10, 64) - if err != nil { - return nil, err - } else { - disk.Rotational = itob(rotational) - } - case diskReadonlyKey: - readonly, err := strconv.ParseInt(diskProperty.Value, 10, 64) - if err != nil { - return nil, err - } else { - disk.Readonly = itob(readonly) - } - case diskFileSystemKey: - disk.FileSystem = diskProperty.Value - case diskMountPointKey: - disk.MountPoint = diskProperty.Value - case diskTypeKey: - diskType, err := StrToDiskType(diskProperty.Value) - if err != nil { - return nil, err - } else { - disk.Type = diskType - } - case diskParentKey: - disk.Parent = diskProperty.Value - case diskHasChildrenKey: - hasChildren, err := strconv.ParseInt(diskProperty.Value, 10, 64) - if err != nil { - return nil, err - } else { - disk.HasChildren = itob(hasChildren) - } - default: - logger.Warningf("unknown disk property key %s, skipping...", diskPropertyName) +// Extract the basic disk info that will be used in cluster-wide orchestration decisions. +func toClusterDisks(devices []LocalDisk) []Disk { + var disks []Disk + for _, device := range devices { + if device.Type == DiskType || device.Type == SSDType { + disk := Disk{ + Type: device.Type, + Size: device.Size, + Rotational: device.Rotational, + Available: getDeviceAvailable(device)} + disks = append(disks, disk) } } - return disk, nil + return disks } -func btoi(b bool) int { - if b { - return 1 - } else { - return 0 - } +// check whether a device is available for use by bluestore +func getDeviceAvailable(device LocalDisk) bool { + return device.Parent == "" && device.Type == DiskType && device.FileSystem == "" } -func itob(i int64) bool { - if i == 0 { - return false - } else { - return true - } -} - -func discoverDisks(nodeConfigKey string, etcdClient etcd.KeysAPI, executor exec.Executor) error { +// Discover all the details of devices available on the local node +func discoverDevices(executor exec.Executor) ([]LocalDisk, error) { + var disks []LocalDisk devices, err := sys.ListDevices(executor) if err != nil { - return err + return nil, err } for _, d := range devices { @@ -207,7 +108,7 @@ func discoverDisks(nodeConfigKey string, etcdClient etcd.KeysAPI, executor exec. } diskType, ok := diskProps["TYPE"] - if !ok || (diskType != "ssd" && diskType != "disk" && diskType != "part") { + if !ok || (diskType != SSDType && diskType != DiskType && diskType != PartType) { // unsupported disk type, just continue continue } @@ -224,71 +125,35 @@ func discoverDisks(nodeConfigKey string, etcdClient etcd.KeysAPI, executor exec. fs, err := sys.GetDeviceFilesystems(d, executor) if err != nil { - return err + return nil, err } - dkey := path.Join(nodeConfigKey, disksKey, d) + disk := LocalDisk{Name: d, UUID: diskUUID, FileSystem: fs} - if _, err := etcdClient.Set(ctx.Background(), path.Join(dkey, diskUUIDKey), diskUUID, nil); err != nil { - return err + if val, ok := diskProps["TYPE"]; ok { + disk.Type = val } - if err := setSimpleDiskProperty("SIZE", diskSizeKey, dkey, diskProps, etcdClient); err != nil { - return err + if val, ok := diskProps["SIZE"]; ok { + if size, err := strconv.ParseUint(val, 10, 64); err == nil { + disk.Size = size + } } - if err := setSimpleDiskProperty("ROTA", diskRotationalKey, dkey, diskProps, etcdClient); err != nil { - return err + if val, ok := diskProps["ROTA"]; ok { + if rotates, err := strconv.ParseBool(val); err == nil { + disk.Rotational = rotates + } } - if err := setSimpleDiskProperty("RO", diskReadonlyKey, dkey, diskProps, etcdClient); err != nil { - return err + if val, ok := diskProps["RO"]; ok { + if ro, err := strconv.ParseBool(val); err == nil { + disk.Readonly = ro + } } - if err := setSimpleDiskProperty("PKNAME", diskParentKey, dkey, diskProps, etcdClient); err != nil { - return err - } - if _, err := etcdClient.Set(ctx.Background(), path.Join(dkey, diskFileSystemKey), fs, nil); err != nil { - return err + if val, ok := diskProps["PKNAME"]; ok { + disk.Parent = val } + + disks = append(disks, disk) } - return nil -} - -func setSimpleDiskProperty(propName, keyName, diskKey string, diskPropMap map[string]string, etcdClient etcd.KeysAPI) error { - val, ok := diskPropMap[propName] - if !ok { - return fmt.Errorf("disk property %s not found in map: %+v", propName, diskPropMap) - } - if _, err := etcdClient.Set(ctx.Background(), path.Join(diskKey, keyName), val, nil); err != nil { - return err - } - - return nil -} - -// test usage only -func TestSetDiskInfo(etcdClient *util.MockEtcdClient, hardwareKey string, name string, uuid string, size uint64, rotational bool, readonly bool, - filesystem string, mountPoint string, diskType DiskType, parent string, hasChildren bool) DiskConfig { - - diskKey := path.Join(hardwareKey, disksKey, name) - etcdClient.Set(ctx.Background(), path.Join(diskKey, diskUUIDKey), uuid, nil) - etcdClient.Set(ctx.Background(), path.Join(diskKey, diskSizeKey), strconv.FormatUint(size, 10), nil) - etcdClient.Set(ctx.Background(), path.Join(diskKey, diskRotationalKey), strconv.Itoa(btoi(rotational)), nil) - etcdClient.Set(ctx.Background(), path.Join(diskKey, diskReadonlyKey), strconv.Itoa(btoi(readonly)), nil) - etcdClient.Set(ctx.Background(), path.Join(diskKey, diskFileSystemKey), filesystem, nil) - etcdClient.Set(ctx.Background(), path.Join(diskKey, diskMountPointKey), mountPoint, nil) - etcdClient.Set(ctx.Background(), path.Join(diskKey, diskTypeKey), DiskTypeToStr(diskType), nil) - etcdClient.Set(ctx.Background(), path.Join(diskKey, diskParentKey), parent, nil) - etcdClient.Set(ctx.Background(), path.Join(diskKey, diskHasChildrenKey), strconv.Itoa(btoi(hasChildren)), nil) - - return DiskConfig{ - Name: name, - UUID: uuid, - Size: size, - Rotational: rotational, - Readonly: readonly, - FileSystem: filesystem, - MountPoint: mountPoint, - Type: diskType, - Parent: parent, - HasChildren: hasChildren, - } + return disks, nil } diff --git a/pkg/clusterd/inventory/disk_test.go b/pkg/clusterd/inventory/disk_test.go new file mode 100644 index 000000000..4b2f40609 --- /dev/null +++ b/pkg/clusterd/inventory/disk_test.go @@ -0,0 +1,98 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package inventory + +import ( + "encoding/json" + "path" + "testing" + + "github.com/rook/rook/pkg/util" + exectest "github.com/rook/rook/pkg/util/exec/test" + "github.com/stretchr/testify/assert" +) + +func TestSerializeClusterDisks(t *testing.T) { + nodeID := "df1c87e8266843f2ab822c0d72f584d3" + etcdClient := &util.MockEtcdClient{} + d1 := LocalDisk{Name: "sda", UUID: "u1", Size: 23, Rotational: true, Readonly: false, + FileSystem: "btrfs", MountPoint: "/mnt/abc", Type: DiskType, HasChildren: true} + d2 := LocalDisk{Name: "sdb", UUID: "u2", Size: 24, Rotational: true, Readonly: false, + Type: DiskType, HasChildren: true} + + err := storeDevices(etcdClient, nodeID, []LocalDisk{d1, d2}) + assert.Nil(t, err) + + key := path.Join(NodesConfigKey, nodeID, disksKey) + rawDisk := etcdClient.GetValue(key) + var disks []Disk + err = json.Unmarshal([]byte(rawDisk), &disks) + assert.Nil(t, err) + assert.Equal(t, 2, len(disks)) + assert.False(t, disks[0].Available) + assert.True(t, disks[0].Rotational) + assert.Equal(t, d1.Size, disks[0].Size) + assert.Equal(t, d1.Type, disks[0].Type) + + assert.True(t, disks[1].Available) + assert.True(t, disks[1].Rotational) + assert.Equal(t, d2.Size, disks[1].Size) + assert.Equal(t, d2.Type, disks[1].Type) +} + +func TestAvailableDisks(t *testing.T) { + + // no disks discovered for a node is an error + disks := GetAvailableDevices([]LocalDisk{}) + assert.Equal(t, 0, len(disks)) + + // no available disks because of the formatting + d1 := LocalDisk{Name: "sda", UUID: "myuuid1", Size: 123, Rotational: true, Readonly: false, FileSystem: "btrfs", MountPoint: "/mnt/abc", Type: DiskType, HasChildren: true} + disks = GetAvailableDevices([]LocalDisk{d1}) + assert.Equal(t, 0, len(disks)) + + // multiple available disks + d2 := LocalDisk{Name: "sdb", UUID: "myuuid2", Size: 123, Rotational: true, Readonly: false, Type: DiskType, HasChildren: true} + d3 := LocalDisk{Name: "sdc", UUID: "myuuid3", Size: 123, Rotational: true, Readonly: false, Type: DiskType, HasChildren: true} + disks = GetAvailableDevices([]LocalDisk{d1, d2, d3}) + assert.Equal(t, 2, len(disks)) + assert.Equal(t, "sdb", disks[0]) + assert.Equal(t, "sdc", disks[1]) + + // partitions don't result in more available devices + d4 := LocalDisk{Name: "sdb1", UUID: "myuuid4", Size: 123, Rotational: true, Readonly: false, Type: PartType, HasChildren: true} + d5 := LocalDisk{Name: "sdb2", UUID: "myuuid5", Size: 123, Rotational: true, Readonly: false, Type: PartType, HasChildren: true} + disks = GetAvailableDevices([]LocalDisk{d1, d2, d3, d4, d5}) + assert.Equal(t, 2, len(disks)) + assert.Equal(t, "sdb", disks[0]) + assert.Equal(t, "sdc", disks[1]) +} + +func TestDiscoverDevices(t *testing.T) { + executor := &exectest.MockExecutor{ + MockExecuteCommand: func(actionName string, command string, arg ...string) error { + logger.Infof("mock execute. %s. %s", actionName, command) + return nil + }, + MockExecuteCommandWithOutput: func(actionName string, command string, arg ...string) (string, error) { + logger.Infof("mock execute with output. %s. %s", actionName, command) + return "", nil + }, + } + devices, err := discoverDevices(executor) + assert.Nil(t, err) + assert.Equal(t, 0, len(devices)) +} diff --git a/pkg/clusterd/inventory/hardware.go b/pkg/clusterd/inventory/hardware.go index adab7635b..f0e9c8904 100644 --- a/pkg/clusterd/inventory/hardware.go +++ b/pkg/clusterd/inventory/hardware.go @@ -17,53 +17,56 @@ package inventory import "time" -type DiskType int - const ( - MemoryTotalSizeKey = "total" - NetworkIPv4AddressKey = "ipv4" - NetworkIPv6AddressKey = "ipv6" - NetworkSpeedKey = "speed" - ProcPhysicalIDKey = "physical-id" - ProcSiblingsKey = "siblings" - ProcCoreIDKey = "core-id" - ProcNumCoresKey = "cores" - ProcSpeedKey = "speed" - ProcBitsKey = "arch" - Disk DiskType = iota - Part + DiskType = "disk" + SSDType = "ssd" + PartType = "part" ) type Config struct { Nodes map[string]*NodeConfig `json:"nodes"` + Local *Hardware `json:"local"` } +// Basic config info for a node in the cluster type NodeConfig struct { - Disks []DiskConfig `json:"disks"` + Disks []Disk `json:"disks"` Processors []ProcessorConfig `json:"processors"` - Memory MemoryConfig `json:"memory"` NetworkAdapters []NetworkConfig `json:"networkAdapters"` + Memory uint64 `json:"memory"` PublicIP string `json:"publicIp"` PrivateIP string `json:"privateIp"` HeartbeatAge time.Duration `json:"heartbeatAge"` Location string `json:"location"` } -type DiskConfig struct { - Name string `json:"name"` - UUID string `json:"uuid"` - Size uint64 `json:"size"` - Rotational bool `json:"rotational"` - Readonly bool `json:"readonly"` - FileSystem string `json:"fileSystem"` - MountPoint string `json:"mountPoint"` - Type DiskType `json:"type"` - Parent string `json:"parent"` - HasChildren bool `json:"hasChildren"` +type Disk struct { + Available bool `json:"available"` + Type string `json:"type"` + Size uint64 `json:"size"` + Rotational bool `json:"rotational"` } -type MemoryConfig struct { - TotalSize uint64 `json:"totalSize"` +// Local hardware info +type Hardware struct { + Disks []LocalDisk `json:"disks"` + Processors []ProcessorConfig `json:"processors"` + NetworkAdapters []NetworkConfig `json:"networkAdapters"` + Memory uint64 `json:"memory"` +} + +type LocalDisk struct { + Name string `json:"name"` + ID string `json:"id"` + UUID string `json:"uuid"` + Size uint64 `json:"size"` + Rotational bool `json:"rotational"` + Readonly bool `json:"readonly"` + FileSystem string `json:"fileSystem"` + MountPoint string `json:"mountPoint"` + Type string `json:"type"` + Parent string `json:"parent"` + HasChildren bool `json:"hasChildren"` } type ProcessorConfig struct { diff --git a/pkg/clusterd/inventory/inventory.go b/pkg/clusterd/inventory/inventory.go index 93ec82ee2..e91bc28c9 100644 --- a/pkg/clusterd/inventory/inventory.go +++ b/pkg/clusterd/inventory/inventory.go @@ -23,6 +23,7 @@ import ( ctx "golang.org/x/net/context" "github.com/rook/rook/pkg/util" + "github.com/rook/rook/pkg/util/exec" ) const ( @@ -31,6 +32,28 @@ const ( TriggerHardwareDetectionKey = "trigger-hardware-detection" ) +// Discover all the hardware properties for this node. +// Store the important properties in etcd and more detailed info for the local node in the context. +func DiscoverHardware(etcdClient etcd.KeysAPI, executor exec.Executor, nodeID string) (*Hardware, error) { + devices, err := discoverDevices(executor) + if err != nil { + return nil, fmt.Errorf("failed to discover devices. %+v", err) + } + + if err = storeDevices(etcdClient, nodeID, devices); err != nil { + return nil, fmt.Errorf("failed to store disks in etcd. %+v", err) + } + + mem := getSystemMemory() + if err = storeMemory(etcdClient, nodeID, mem); err != nil { + return nil, fmt.Errorf("failed to store system memory in etcd. %+v", err) + } + + // TODO: Discover network adapters, processors, ... + + return &Hardware{Disks: devices, Memory: mem}, nil +} + func LoadDiscoveredNodes(etcdClient etcd.KeysAPI) (*Config, error) { // Get the discovered state of the infrastructure diff --git a/pkg/clusterd/inventory/memory.go b/pkg/clusterd/inventory/memory.go new file mode 100644 index 000000000..b9ffb1bad --- /dev/null +++ b/pkg/clusterd/inventory/memory.go @@ -0,0 +1,51 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package inventory + +// #include +import "C" + +import ( + "fmt" + "path" + "strconv" + + etcd "github.com/coreos/etcd/client" + ctx "golang.org/x/net/context" +) + +func getSystemMemory() uint64 { + return uint64(C.sysconf(C._SC_PHYS_PAGES) * C.sysconf(C._SC_PAGE_SIZE)) +} + +func storeMemory(etcdClient etcd.KeysAPI, nodeID string, size uint64) error { + key := path.Join(NodesConfigKey, nodeID, memoryKey) + _, err := etcdClient.Set(ctx.Background(), key, strconv.FormatUint(size, 10), nil) + if err != nil { + return fmt.Errorf("failed to store memory in etcd. %+v", err) + } + + return nil +} + +func loadMemoryConfig(nodeConfig *NodeConfig, rawMemory string) error { + size, err := strconv.ParseUint(rawMemory, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse memory. %+v", err) + } + nodeConfig.Memory = size + return nil +} diff --git a/pkg/clusterd/inventory/node.go b/pkg/clusterd/inventory/node.go index d24ccb965..0e3c1dfe4 100644 --- a/pkg/clusterd/inventory/node.go +++ b/pkg/clusterd/inventory/node.go @@ -19,13 +19,13 @@ import ( "errors" "fmt" "path" - "strconv" "time" ctx "golang.org/x/net/context" "github.com/rook/rook/pkg/util" - "github.com/rook/rook/pkg/util/exec" + + "encoding/json" etcd "github.com/coreos/etcd/client" ) @@ -34,8 +34,8 @@ const ( HeartbeatKey = "heartbeat" disksKey = "disks" processorsKey = "cpu" - networkKey = "net" - memoryKey = "mem" + networkKey = "network" + memoryKey = "memory" locationKey = "location" heartbeatTtlSeconds = 60 * 60 publicIpAddressKey = "publicIp" @@ -46,17 +46,6 @@ var ( HeartbeatTtlDuration = time.Duration(heartbeatTtlSeconds) * time.Second ) -func DiscoverHardware(nodeID string, etcdClient etcd.KeysAPI, executor exec.Executor) error { - nodeConfigKey := GetNodeConfigKey(nodeID) - if err := discoverDisks(nodeConfigKey, etcdClient, executor); err != nil { - return err - } - - // TODO: discover more hardware properties - - return nil -} - // gets the key under which all node hardware/config will be stored func GetNodeConfigKey(nodeID string) string { return path.Join(NodesConfigKey, nodeID) @@ -131,7 +120,6 @@ func loadNodeHealth(etcdClient etcd.KeysAPI, nodes map[string]*NodeConfig) error } func loadSingleNodeHealth(node *NodeConfig, health *etcd.Node) error { - for _, prop := range health.Nodes { switch util.GetLeafKeyPath(prop.Key) { case HeartbeatKey: @@ -151,7 +139,6 @@ func SetIPAddress(etcdClient etcd.KeysAPI, nodeId, publicIP, privateIP string) e if err != nil { return err } - return setConfigProperty(etcdClient, nodeId, privateIpAddressKey, privateIP) } @@ -176,25 +163,25 @@ func loadHardwareConfig(nodeId string, nodeConfig *NodeConfig, nodeInfo *etcd.No var err error switch key { case disksKey: - err = loadDisksConfig(nodeConfig, nodeConfigRoot) + err = loadDisksConfig(nodeConfig, nodeConfigRoot.Value) case processorsKey: - err = loadProcessorsConfig(nodeConfig, nodeConfigRoot) - - case memoryKey: - err = loadMemoryConfig(nodeConfig, nodeConfigRoot) + err = loadProcessorsConfig(nodeConfig, nodeConfigRoot.Value) case networkKey: - err = loadNetworkConfig(nodeConfig, nodeConfigRoot) + err = loadNetworkConfig(nodeConfig, nodeConfigRoot.Value) + + case memoryKey: + err = loadMemoryConfig(nodeConfig, nodeConfigRoot.Value) case publicIpAddressKey: - err = loadSimpleConfigStringProperty(&(nodeConfig.PublicIP), nodeConfigRoot, "Public IP") + nodeConfig.PublicIP = nodeConfigRoot.Value case privateIpAddressKey: - err = loadSimpleConfigStringProperty(&(nodeConfig.PrivateIP), nodeConfigRoot, "Private IP") + nodeConfig.PrivateIP = nodeConfigRoot.Value case locationKey: - err = loadSimpleConfigStringProperty(&(nodeConfig.Location), nodeConfigRoot, "Location") + nodeConfig.Location = nodeConfigRoot.Value default: logger.Warningf("unexpected hardware component: %s, skipping...", nodeConfigRoot) @@ -209,161 +196,50 @@ func loadHardwareConfig(nodeId string, nodeConfig *NodeConfig, nodeInfo *etcd.No return nil } -func loadDisksConfig(nodeConfig *NodeConfig, disksRootNode *etcd.Node) error { - numDisks := 0 - if disksRootNode.Nodes != nil { - numDisks = len(disksRootNode.Nodes) +func loadProcessorsConfig(nodeConfig *NodeConfig, rawProcs string) error { + var processors []ProcessorConfig + if err := json.Unmarshal([]byte(rawProcs), &processors); err != nil { + return fmt.Errorf("failed to deserialize processors. %+v", err) + } + nodeConfig.Processors = processors + return nil +} + +func loadNetworkConfig(nodeConfig *NodeConfig, rawAdapters string) error { + var adapters []NetworkConfig + if err := json.Unmarshal([]byte(rawAdapters), &adapters); err != nil { + return fmt.Errorf("failed to deserialize network adapters. %+v", err) + } + nodeConfig.NetworkAdapters = adapters + return nil +} + +func storeNetworkConfig(etcdClient etcd.KeysAPI, nodeID string, config []NetworkConfig) error { + output, err := json.Marshal(config) + if err != nil { + return fmt.Errorf("failed to marshal network config. %+v", err) } - nodeConfig.Disks = make([]DiskConfig, numDisks) - - // iterate over all disks from etcd - for i, diskInfo := range disksRootNode.Nodes { - disk, err := getDiskInfo(diskInfo) - if err != nil { - logger.Errorf("Failed to get disk. err=%v", err) - return err - } - - nodeConfig.Disks[i] = *disk - + key := path.Join(NodesConfigKey, nodeID, networkKey) + _, err = etcdClient.Set(ctx.Background(), key, string(output), nil) + if err != nil { + return fmt.Errorf("failed to store network config in etcd. %+v", err) } return nil } -func loadProcessorsConfig(nodeConfig *NodeConfig, procsRootNode *etcd.Node) error { - numProcs := 0 - if procsRootNode.Nodes != nil { - numProcs = len(procsRootNode.Nodes) +func storeProcessorConfig(etcdClient etcd.KeysAPI, nodeID string, config []ProcessorConfig) error { + output, err := json.Marshal(config) + if err != nil { + return fmt.Errorf("failed to marshal processor config. %+v", err) } - nodeConfig.Processors = make([]ProcessorConfig, numProcs) - - // iterate over all processors from etcd - for i, procInfo := range procsRootNode.Nodes { - proc := ProcessorConfig{} - if procID, err := strconv.ParseUint(util.GetLeafKeyPath(procInfo.Key), 10, 32); err != nil { - return err - } else { - proc.ID = uint(procID) - } - - // iterate over all properties of the processor - for _, procProperty := range procInfo.Nodes { - procPropertyName := util.GetLeafKeyPath(procProperty.Key) - switch procPropertyName { - case ProcPhysicalIDKey: - if phsyicalId, err := strconv.ParseUint(procProperty.Value, 10, 32); err != nil { - return err - } else { - proc.PhysicalID = uint(phsyicalId) - } - case ProcSiblingsKey: - if siblings, err := strconv.ParseUint(procProperty.Value, 10, 32); err != nil { - return err - } else { - proc.Siblings = uint(siblings) - } - case ProcCoreIDKey: - if coreId, err := strconv.ParseUint(procProperty.Value, 10, 32); err != nil { - return err - } else { - proc.CoreID = uint(coreId) - } - case ProcNumCoresKey: - if numCores, err := strconv.ParseUint(procProperty.Value, 10, 32); err != nil { - return err - } else { - proc.NumCores = uint(numCores) - } - case ProcSpeedKey: - if speed, err := strconv.ParseFloat(procProperty.Value, 64); err != nil { - return err - } else { - proc.Speed = speed - } - case ProcBitsKey: - if numBits, err := strconv.ParseUint(procProperty.Value, 10, 32); err != nil { - return err - } else { - proc.Bits = uint(numBits) - } - default: - logger.Warningf("unknown processor property key %s, skipping", procPropertyName) - } - } - - nodeConfig.Processors[i] = proc + key := path.Join(NodesConfigKey, nodeID, processorsKey) + _, err = etcdClient.Set(ctx.Background(), key, string(output), nil) + if err != nil { + return fmt.Errorf("failed to store processor config in etcd. %+v", err) } return nil } - -func loadMemoryConfig(nodeConfig *NodeConfig, memoryRootNode *etcd.Node) error { - mem := MemoryConfig{} - for _, memProperty := range memoryRootNode.Nodes { - memPropertyName := util.GetLeafKeyPath(memProperty.Key) - switch memPropertyName { - case MemoryTotalSizeKey: - if size, err := strconv.ParseUint(memProperty.Value, 10, 64); err != nil { - return err - } else { - mem.TotalSize = size - } - default: - logger.Warningf("unknown memory property key %s, skipping", memPropertyName) - } - } - - nodeConfig.Memory = mem - return nil -} - -func loadNetworkConfig(nodeConfig *NodeConfig, networkRootNode *etcd.Node) error { - numNics := 0 - if networkRootNode.Nodes != nil { - numNics = len(networkRootNode.Nodes) - } - - nodeConfig.NetworkAdapters = make([]NetworkConfig, numNics) - - // iterate over all network adapters from etcd - for i, netInfo := range networkRootNode.Nodes { - net := NetworkConfig{} - net.Name = util.GetLeafKeyPath(netInfo.Key) - - // iterate over all properties of the network adapter - for _, netProperty := range netInfo.Nodes { - netPropertyName := util.GetLeafKeyPath(netProperty.Key) - switch netPropertyName { - case NetworkIPv4AddressKey: - net.IPv4Address = netProperty.Value - case NetworkIPv6AddressKey: - net.IPv6Address = netProperty.Value - case NetworkSpeedKey: - if netProperty.Value == "" { - net.Speed = 0 - } else if speed, err := strconv.ParseUint(netProperty.Value, 10, 64); err != nil { - return err - } else { - net.Speed = speed - } - default: - logger.Warningf("unknown network adapter property key %s, skipping", netPropertyName) - } - } - - nodeConfig.NetworkAdapters[i] = net - } - - return nil -} - -func loadSimpleConfigStringProperty(cfgField *string, cfgNode *etcd.Node, propName string) error { - if cfgNode.Dir { - return fmt.Errorf("%s node '%s' is a directory, but it's expected to be a key", propName, cfgNode.Key) - } - *cfgField = cfgNode.Value - return nil -} diff --git a/pkg/clusterd/inventory/node_test.go b/pkg/clusterd/inventory/node_test.go index 160665b74..9b91e42b7 100644 --- a/pkg/clusterd/inventory/node_test.go +++ b/pkg/clusterd/inventory/node_test.go @@ -17,13 +17,9 @@ package inventory import ( "path" - "strconv" "testing" "time" - etcd "github.com/coreos/etcd/client" - ctx "golang.org/x/net/context" - "github.com/rook/rook/pkg/util" "github.com/stretchr/testify/assert" ) @@ -61,45 +57,40 @@ func TestLoadDiscoveredNodes(t *testing.T) { } func TestLoadHardwareConfig(t *testing.T) { - machineId := "df1c87e8266843f2ab822c0d72f584d3" + nodeID := "12345" etcdClient := util.NewMockEtcdClient() - // set up the hardware date in etcd - hardwareKey := path.Join(NodesConfigKey, machineId) - etcdClient.CreateDir(hardwareKey) - // setup disk info in etcd - disksConfig := make([]DiskConfig, 2) - disksConfig[0] = TestSetDiskInfo(etcdClient, hardwareKey, "sda", "12344869-29ee-4bfd-bf21-dfd597bd222e", - 10737418240, true, false, "btrfs", "/mnt/abc", Disk, "", true) - disksConfig[1] = TestSetDiskInfo(etcdClient, hardwareKey, "sda2", "23454869-29ee-4bfd-bf21-dfd597bd222e", - 2097152, false, true, "", "", Part, "sda", false) + d1 := LocalDisk{Name: "sda", UUID: "uuid1", Size: 10737418240, Rotational: true, Readonly: false, Type: DiskType, HasChildren: true} + d2 := LocalDisk{Name: "sda2", UUID: "uuid2", Size: 2097152, Rotational: false, Readonly: true, Type: PartType, HasChildren: false} + err := storeDevices(etcdClient, nodeID, []LocalDisk{d1, d2}) + assert.Nil(t, err) // setup processor info in etcd - procsKey := path.Join(hardwareKey, processorsKey) - etcdClient.Set(ctx.Background(), procsKey, "", &etcd.SetOptions{Dir: true}) - procsConfig := make([]ProcessorConfig, 3) - procsConfig[0] = setProcInfo(etcdClient, procsKey, 0, 0, 1, 0, 1, 1234.56, 64) - procsConfig[1] = setProcInfo(etcdClient, procsKey, 1, 1, 2, 0, 2, 8000.00, 32) - procsConfig[2] = setProcInfo(etcdClient, procsKey, 2, 1, 2, 1, 2, 4000.01, 32) + p1 := ProcessorConfig{ID: 0, PhysicalID: 3, Siblings: 1, CoreID: 6, NumCores: 1, Speed: 1234.56, Bits: 64} + p2 := ProcessorConfig{ID: 1, PhysicalID: 4, Siblings: 2, CoreID: 7, NumCores: 2, Speed: 8000.00, Bits: 32} + p3 := ProcessorConfig{ID: 2, PhysicalID: 5, Siblings: 0, CoreID: 8, NumCores: 4, Speed: 4000.01, Bits: 32} + err = storeProcessorConfig(etcdClient, nodeID, []ProcessorConfig{p1, p2, p3}) + assert.Nil(t, err) // setup memory info in etcd - memKey := path.Join(hardwareKey, memoryKey) - etcdClient.Set(ctx.Background(), memKey, "", &etcd.SetOptions{Dir: true}) - memConfig := setMemoryInfo(etcdClient, memKey, 4149252096) + mem := getSystemMemory() + err = storeMemory(etcdClient, nodeID, mem) + assert.Nil(t, err) // set up network info in etcd - netKey := path.Join(hardwareKey, networkKey) - etcdClient.Set(ctx.Background(), netKey, "", &etcd.SetOptions{Dir: true}) - netsConfig := make([]NetworkConfig, 2) - netsConfig[0] = setNetInfo(etcdClient, netKey, "eth0", "172.17.42.1/16", "fe80::42:4aff:fefe:13d7/64", 0) - netsConfig[1] = setNetInfo(etcdClient, netKey, "veth2b6453a", "", "fe80::7c0f:acff:feff:478d/64", 10000) + n1 := NetworkConfig{Name: "eth0", IPv4Address: "172.17.42.1/16", IPv6Address: "fe80::42:4aff:fefe:13d7/64", Speed: 0} + n2 := NetworkConfig{Name: "veth2b6453a", IPv6Address: "fe80::7c0f:acff:feff:478d/64", Speed: 10000} + err = storeNetworkConfig(etcdClient, nodeID, []NetworkConfig{n1, n2}) + assert.Nil(t, err) // set IP address in etcd - SetIPAddress(etcdClient, machineId, "10.0.0.43", "10.2.3.4") + publicIP := "10.0.0.43" + privateIP := "10.2.3.4" + SetIPAddress(etcdClient, nodeID, publicIP, privateIP) // set location in etcd - SetLocation(etcdClient, machineId, "root=default,dc=datacenter1") + SetLocation(etcdClient, nodeID, "root=default,dc=datacenter1") // load the discovered node config nodeConfig, err := loadNodeConfig(etcdClient) @@ -107,132 +98,29 @@ func TestLoadHardwareConfig(t *testing.T) { assert.NotNil(t, nodeConfig, "loaded node config should not be nil") assert.Equal(t, 1, len(nodeConfig)) - // verify all the hardware configuration retrieved from etcd - verifyDiskConfig(t, nodeConfig[machineId], disksConfig) - verifyProcConfig(t, nodeConfig[machineId], procsConfig) - verifyMemoryConfig(t, nodeConfig[machineId], memConfig) - verifyNetworkConfig(t, nodeConfig[machineId], netsConfig) - assert.Equal(t, "10.0.0.43", nodeConfig[machineId].PublicIP) - assert.Equal(t, "root=default,dc=datacenter1", nodeConfig[machineId].Location) -} + cfg := nodeConfig[nodeID] -func setProcInfo(etcdClient *util.MockEtcdClient, procsKey string, procId uint, physicalId uint, siblings uint, - coreID uint, numCores uint, speed float64, bits uint) ProcessorConfig { + // verify the single disk (the partition is not saved) + assert.Equal(t, 1, len(cfg.Disks)) + assert.True(t, cfg.Disks[0].Available) + assert.Equal(t, d1.Rotational, cfg.Disks[0].Rotational) + assert.Equal(t, d1.Size, cfg.Disks[0].Size) + assert.Equal(t, d1.Type, cfg.Disks[0].Type) - procKey := path.Join(procsKey, strconv.FormatUint(uint64(procId), 10)) - etcdClient.Set(ctx.Background(), procKey, "", &etcd.SetOptions{Dir: true}) + // verify the processors + assert.Equal(t, 3, len(cfg.Processors)) - etcdClient.Set(ctx.Background(), path.Join(procKey, ProcPhysicalIDKey), strconv.FormatUint(uint64(physicalId), 10), nil) - etcdClient.Set(ctx.Background(), path.Join(procKey, ProcSiblingsKey), strconv.FormatUint(uint64(siblings), 10), nil) - etcdClient.Set(ctx.Background(), path.Join(procKey, ProcCoreIDKey), strconv.FormatUint(uint64(coreID), 10), nil) - etcdClient.Set(ctx.Background(), path.Join(procKey, ProcNumCoresKey), strconv.FormatUint(uint64(numCores), 10), nil) - etcdClient.Set(ctx.Background(), path.Join(procKey, ProcSpeedKey), strconv.FormatFloat(speed, 'f', 2, 64), nil) - etcdClient.Set(ctx.Background(), path.Join(procKey, ProcBitsKey), strconv.FormatUint(uint64(bits), 10), nil) + // verify the network adapters + assert.Equal(t, 2, len(cfg.NetworkAdapters)) - return ProcessorConfig{ - ID: procId, - PhysicalID: physicalId, - Siblings: siblings, - CoreID: coreID, - NumCores: numCores, - Speed: speed, - Bits: bits, - } -} + // verify the simple values + assert.Equal(t, mem, cfg.Memory) + assert.Equal(t, "10.0.0.43", cfg.PublicIP) + assert.Equal(t, "root=default,dc=datacenter1", cfg.Location) -func setMemoryInfo(etcdClient *util.MockEtcdClient, memKey string, totalMem uint64) MemoryConfig { - etcdClient.Set(ctx.Background(), path.Join(memKey, MemoryTotalSizeKey), strconv.FormatUint(totalMem, 10), nil) - return MemoryConfig{TotalSize: totalMem} -} + // verify that some values are in etcd + key := path.Join(NodesConfigKey, nodeID) + assert.Equal(t, publicIP, etcdClient.GetValue(path.Join(key, publicIpAddressKey))) + assert.Equal(t, privateIP, etcdClient.GetValue(path.Join(key, privateIpAddressKey))) -func setNetInfo(etcdClient *util.MockEtcdClient, netsKey string, name string, ipv4Addr string, ipv6Addr string, speed uint64) NetworkConfig { - netKey := path.Join(netsKey, name) - etcdClient.Set(ctx.Background(), netKey, "", &etcd.SetOptions{Dir: true}) - - etcdClient.Set(ctx.Background(), path.Join(netKey, NetworkIPv4AddressKey), ipv4Addr, nil) - etcdClient.Set(ctx.Background(), path.Join(netKey, NetworkIPv6AddressKey), ipv6Addr, nil) - speedStr := "" - if speed > 0 { - speedStr = strconv.FormatUint(speed, 10) - } - etcdClient.Set(ctx.Background(), path.Join(netKey, NetworkSpeedKey), speedStr, nil) - - return NetworkConfig{ - Name: name, - IPv4Address: ipv4Addr, - IPv6Address: ipv6Addr, - Speed: speed, - } -} - -func verifyDiskConfig(t *testing.T, nodeConfig *NodeConfig, expectedDisksConfig []DiskConfig) { - assert.Equal(t, len(expectedDisksConfig), len(nodeConfig.Disks)) - - for _, expectedDisk := range expectedDisksConfig { - var matchingActual DiskConfig - for _, actualDisk := range nodeConfig.Disks { - if actualDisk.UUID == expectedDisk.UUID { - matchingActual = actualDisk - break - } - } - - assert.NotNil(t, matchingActual, "missing actual disk %s", expectedDisk.UUID) - assert.Equal(t, expectedDisk, matchingActual) - } -} - -func verifyProcConfig(t *testing.T, nodeConfig *NodeConfig, expectedProcsConfig []ProcessorConfig) { - assert.Equal(t, len(expectedProcsConfig), len(nodeConfig.Processors)) - - for _, expectedProc := range expectedProcsConfig { - var matchingActual ProcessorConfig - for _, actualProc := range nodeConfig.Processors { - if actualProc.ID == expectedProc.ID { - matchingActual = actualProc - break - } - } - - assert.NotNil(t, matchingActual, "missing actual proc %d", expectedProc.ID) - assert.Equal(t, expectedProc, matchingActual) - } -} - -func verifyMemoryConfig(t *testing.T, nodeConfig *NodeConfig, expectedMemConfig MemoryConfig) { - assert.NotNil(t, nodeConfig.Memory) - assert.Equal(t, expectedMemConfig, nodeConfig.Memory) -} - -func verifyNetworkConfig(t *testing.T, nodeConfig *NodeConfig, expectedNetsConfig []NetworkConfig) { - assert.Equal(t, len(expectedNetsConfig), len(nodeConfig.NetworkAdapters)) - - for _, expectedNet := range expectedNetsConfig { - var matchingActual NetworkConfig - for _, actualNet := range nodeConfig.NetworkAdapters { - if actualNet.Name == expectedNet.Name { - matchingActual = actualNet - break - } - } - - assert.NotNil(t, matchingActual, "missing actual network adapter %s", expectedNet.Name) - assert.Equal(t, expectedNet, matchingActual) - } -} - -func TestGetSimpleDiskPropertiesFromSerial(t *testing.T) { - nodeID := "df1c87e8266843f2ab822c0d72f584d3" - etcdClient := &util.MockEtcdClient{} - hardwareKey := path.Join(NodesConfigKey, nodeID) - etcdClient.Set(ctx.Background(), hardwareKey, "", &etcd.SetOptions{Dir: true}) - TestSetDiskInfo(etcdClient, hardwareKey, "sda", "abcd4869-29ee-4bfd-bf21-dfd597bd222e", - 10737418240, true, false, "btrfs", "/mnt/abc", Disk, "", true) - - diskNode, _ := etcdClient.Get(ctx.Background(), path.Join(hardwareKey, "disks", "sda"), nil) - disk, err := getDiskInfo(diskNode.Node) - assert.Nil(t, err) - - assert.Equal(t, "sda", disk.Name) - assert.Equal(t, "abcd4869-29ee-4bfd-bf21-dfd597bd222e", disk.UUID) } diff --git a/pkg/clusterd/join.go b/pkg/clusterd/join.go index 461c30526..7f0ef1bbb 100644 --- a/pkg/clusterd/join.go +++ b/pkg/clusterd/join.go @@ -77,7 +77,15 @@ func StartJoinCluster(services []*ClusterService, configDir, nodeID, discoveryUR ProcMan: proc.New(executor), ConfigDir: configDir, LogLevel: logLevel, + Inventory: &inventory.Config{}, } + + // initialize the device inventory + context.Inventory.Local, err = inventory.DiscoverHardware(context.EtcdClient, context.Executor, context.NodeID) + if err != nil { + return nil, fmt.Errorf("failed initial hardware discovery. %+v", err) + } + clusterLeader := newServicesLeader(context) clusterMember := newClusterMember(context, leaseManager, clusterLeader) clusterLeader.parent = clusterMember