diff --git a/pkg/api/handlers.go b/pkg/api/handlers.go index d1e313e8d..7c57a8661 100644 --- a/pkg/api/handlers.go +++ b/pkg/api/handlers.go @@ -2,6 +2,7 @@ package api import ( "encoding/json" + "fmt" "io" "io/ioutil" "log" @@ -112,6 +113,65 @@ type overallMonStatus struct { Desired []*cephmgr.CephMonitorConfig `json:"desired"` } +// Adds the device and configures an OSD on the device. +// POST +// /device +func (h *Handler) AddDevice(w http.ResponseWriter, r *http.Request) { + device, err := loadDeviceFromBody(w, r) + if err != nil { + log.Printf("failed to add device. %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + + err = cephmgr.AddDesiredDevice(h.EtcdClient, device) + if err != nil { + log.Printf("failed to add device. %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } +} + +// Stops the OSD and removes a device from participating in the cluster. +// POST +// /device/remove +func (h *Handler) RemoveDevice(w http.ResponseWriter, r *http.Request) { + device, err := loadDeviceFromBody(w, r) + if err != nil { + log.Printf("failed to remove device. %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + + err = cephmgr.RemoveDesiredDevice(h.EtcdClient, device) + if err != nil { + log.Printf("failed to remove device. %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } +} + +func loadDeviceFromBody(w http.ResponseWriter, r *http.Request) (*cephmgr.Device, error) { + // read/unmarshal the new pool to create from the request body + body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1024)) + if err == nil { + r.Body.Close() + } else { + return nil, fmt.Errorf("failed to read request body: %+v", err) + } + + var device cephmgr.Device + if err := json.Unmarshal(body, &device); err != nil { + return nil, fmt.Errorf("failed to unmarshal request body '%s': %+v", string(body), err) + } + + if device.Name == "" || device.NodeID == "" { + return nil, fmt.Errorf("missing name or nodeId") + } + + return &device, nil +} + // Gets the monitors that have been created in this cluster. // GET // /mon diff --git a/pkg/api/handlers_test.go b/pkg/api/handlers_test.go index 9d671c61b..5b34eab79 100644 --- a/pkg/api/handlers_test.go +++ b/pkg/api/handlers_test.go @@ -21,6 +21,46 @@ import ( "github.com/stretchr/testify/assert" ) +func TestAddRemoveDeviceHandler(t *testing.T) { + req, err := http.NewRequest("POST", "http://10.0.0.100/device", strings.NewReader(`{"name":"foo"}`)) + assert.Nil(t, err) + + w := httptest.NewRecorder() + etcdClient := util.NewMockEtcdClient() + h := NewHandler(etcdClient, nil, nil) + + // missing the node id + h.AddDevice(w, req) + assert.Equal(t, http.StatusBadRequest, w.Code) + + // successful request + req, err = http.NewRequest("POST", "http://10.0.0.100/device", strings.NewReader(`{"name":"foo","nodeId":"123"}`)) + assert.Nil(t, err) + h = NewHandler(etcdClient, nil, nil) + w = httptest.NewRecorder() + h.AddDevice(w, req) + assert.Equal(t, http.StatusOK, w.Code) + + devices := etcdClient.GetChildDirs("/castle/services/ceph/osd/desired/123/device") + assert.Equal(t, 1, devices.Count()) + assert.True(t, devices.Contains("foo")) + + // remove the device + req, err = http.NewRequest("POST", "http://10.0.0.100/device/remove", strings.NewReader(`{"name":"foo","nodeId":"123"}`)) + assert.Nil(t, err) + h = NewHandler(etcdClient, nil, nil) + w = httptest.NewRecorder() + h.RemoveDevice(w, req) + assert.Equal(t, http.StatusOK, w.Code) + + // check the desired state + devices = etcdClient.GetChildDirs("/castle/services/ceph/osd/desired/123/device") + assert.Equal(t, 0, devices.Count()) +} + +func TestRemoveDeviceHandler(t *testing.T) { +} + func TestGetNodesHandler(t *testing.T) { req, err := http.NewRequest("GET", "http://10.0.0.100/node", nil) if err != nil { diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 406b552bf..e796abe38 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -26,5 +26,17 @@ func (h *Handler) GetRoutes() []Route { "/mon", h.GetMonitors, }, + { + "AddDevice", + "POST", + "/device", + h.AddDevice, + }, + { + "RemoveDevice", + "POST", + "/device/remove", + h.RemoveDevice, + }, } } diff --git a/pkg/cephmgr/osd.go b/pkg/cephmgr/osd.go index 083b10252..ac7818b62 100644 --- a/pkg/cephmgr/osd.go +++ b/pkg/cephmgr/osd.go @@ -14,16 +14,21 @@ import ( "syscall" "time" + etcd "github.com/coreos/etcd/client" + ctx "golang.org/x/net/context" + "github.com/google/uuid" "github.com/quantum/castle/pkg/cephmgr/client" "github.com/quantum/castle/pkg/clusterd" "github.com/quantum/castle/pkg/clusterd/inventory" "github.com/quantum/castle/pkg/proc" + "github.com/quantum/castle/pkg/util" ) const ( DevicesValue = "devices" ForceFormatValue = "forceFormat" + deviceDesiredKey = "/castle/services/ceph/osd/desired/%s/device/%s" bootstrapOSDKeyringTemplate = ` [client.bootstrap-osd] key = %s @@ -31,9 +36,36 @@ const ( ` ) +type Device struct { + Name string `json:"name"` + NodeID string `json:"nodeId"` +} + // request the current user once and stash it in this global variable var currentUser *user.User +// add a device to the desired state +func AddDesiredDevice(etcdClient etcd.KeysAPI, device *Device) error { + key := fmt.Sprintf(deviceDesiredKey, device.NodeID, device.Name) + err := util.CreateEtcdDir(etcdClient, key) + if err != nil { + return fmt.Errorf("failed to add device %s on node %s to desired. %v", device.Name, device.NodeID, err) + } + + return nil +} + +// remove a device from the desired state +func RemoveDesiredDevice(etcdClient etcd.KeysAPI, device *Device) error { + key := fmt.Sprintf(deviceDesiredKey, device.NodeID, device.Name) + _, err := etcdClient.Delete(ctx.Background(), key, &etcd.DeleteOptions{Dir: true}) + if err != nil { + return fmt.Errorf("failed to remove device %s on node %s from desired. %v", device.Name, device.NodeID, err) + } + + return nil +} + // get the bootstrap OSD root dir func getBootstrapOSDDir() string { return "/tmp/bootstrap-osd" diff --git a/pkg/cephmgr/osd_test.go b/pkg/cephmgr/osd_test.go index 89ea348c4..a5dbe8f3f 100644 --- a/pkg/cephmgr/osd_test.go +++ b/pkg/cephmgr/osd_test.go @@ -15,6 +15,28 @@ import ( "github.com/stretchr/testify/assert" ) +func TestDesiredDeviceState(t *testing.T) { + etcdClient := util.NewMockEtcdClient() + + // add a device + device := &Device{Name: "foo", NodeID: "a"} + err := AddDesiredDevice(etcdClient, device) + assert.Nil(t, err) + devices := etcdClient.GetChildDirs("/castle/services/ceph/osd/desired/a/device") + assert.Equal(t, 1, devices.Count()) + assert.True(t, devices.Contains("foo")) + + // remove the device + err = RemoveDesiredDevice(etcdClient, device) + assert.Nil(t, err) + devices = etcdClient.GetChildDirs("/castle/services/ceph/osd/desired/a/device") + assert.Equal(t, 0, devices.Count()) + + // removing a non-existent device is a no-op + err = RemoveDesiredDevice(etcdClient, device) + assert.Nil(t, err) +} + func TestOSDBootstrap(t *testing.T) { clusterName := "mycluster" targetPath := getBootstrapOSDKeyringPath(clusterName)