diff --git a/cmd/rook/main.go b/cmd/rook/main.go index df80243fa..47b90bad6 100644 --- a/cmd/rook/main.go +++ b/cmd/rook/main.go @@ -22,6 +22,7 @@ import ( "github.com/rook/rook/cmd/rook/block" "github.com/rook/rook/cmd/rook/filesystem" "github.com/rook/rook/cmd/rook/node" + "github.com/rook/rook/cmd/rook/object" "github.com/rook/rook/cmd/rook/pool" "github.com/rook/rook/cmd/rook/rook" "github.com/rook/rook/cmd/rook/status" @@ -41,6 +42,7 @@ func addCommands() { rook.RootCmd.AddCommand(pool.Cmd) rook.RootCmd.AddCommand(block.Cmd) rook.RootCmd.AddCommand(filesystem.Cmd) + rook.RootCmd.AddCommand(object.Cmd) rook.RootCmd.AddCommand(status.Cmd) rook.RootCmd.AddCommand(version.Cmd) } diff --git a/cmd/rook/node/list.go b/cmd/rook/node/list.go index 4208e5eb6..2d4c1cf28 100644 --- a/cmd/rook/node/list.go +++ b/cmd/rook/node/list.go @@ -67,7 +67,7 @@ func listNodes(c client.RookRestClient) (string, error) { w := rook.NewTableWriter(&buffer) // write header columns - fmt.Fprintln(w, "PUBLIC\tPRIVATE\tSTATE\tCLUSTER\tSIZE\tLOCATION\tUPDATED\t") + fmt.Fprintln(w, "PUBLIC\tPRIVATE\tSTATE\tCLUSTER\tSIZE\tLOCATION\tUPDATED") // print a row for each node for _, n := range nodes { diff --git a/cmd/rook/node/list_test.go b/cmd/rook/node/list_test.go index 61cb19bf6..76dd2008f 100644 --- a/cmd/rook/node/list_test.go +++ b/cmd/rook/node/list_test.go @@ -48,7 +48,7 @@ func TestListNodes(t *testing.T) { out, err := listNodes(c) assert.Nil(t, err) - expectedOut := "PUBLIC PRIVATE STATE CLUSTER SIZE LOCATION UPDATED \n" + + expectedOut := "PUBLIC PRIVATE STATE CLUSTER SIZE LOCATION UPDATED\n" + "187.1.2.3 10.0.0.100 OK cluster1 100 B root=default,dc=datacenter5 1s ago \n" assert.Equal(t, expectedOut, out) } diff --git a/cmd/rook/object/connection.go b/cmd/rook/object/connection.go new file mode 100644 index 000000000..2ec85848b --- /dev/null +++ b/cmd/rook/object/connection.go @@ -0,0 +1,83 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package object + +import ( + "bytes" + "fmt" + "net/http" + "os" + + "github.com/rook/rook/cmd/rook/rook" + "github.com/rook/rook/pkg/rook/client" + "github.com/rook/rook/pkg/util/flags" + "github.com/spf13/cobra" +) + +var connectionCmd = &cobra.Command{ + Use: "connection", + Short: "Gets connection information that will allow a client to access object storage in the cluster", + Aliases: []string{"conn"}, +} + +func init() { + connectionCmd.RunE = getConnectionInfoEntry +} + +func getConnectionInfoEntry(cmd *cobra.Command, args []string) error { + rook.SetupLogging() + + if err := flags.VerifyRequiredFlags(cmd, []string{}); err != nil { + return err + } + + c := client.NewRookNetworkRestClient(client.GetRestURL(rook.APIServerEndpoint), http.DefaultClient) + out, err := getConnectionInfo(c) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + fmt.Print(out) + return nil +} + +func getConnectionInfo(c client.RookRestClient) (string, error) { + connInfo, err := c.GetObjectStoreConnectionInfo() + + if err != nil { + if client.IsHttpNotFound(err) { + return "object store connection info is not ready, if \"object create\" has already been run, please be patient\n", nil + } + + return "", fmt.Errorf("failed to get object store connection info: %+v", err) + } + + var buffer bytes.Buffer + w := rook.NewTableWriter(&buffer) + + // write header columns + fmt.Fprintln(w, "NAME\tVALUE") + + // write object store connection info + fmt.Fprintf(w, "%s\t%s\t\n", "S3_HOST", connInfo.Host) + fmt.Fprintf(w, "%s\t%s\t\n", "S3_ENDPOINT", connInfo.IPEndpoint) + fmt.Fprintf(w, "%s\t%s\t\n", "S3_ACCESS_KEY", connInfo.AccessKey) + fmt.Fprintf(w, "%s\t%s\t\n", "S3_SECRET_KEY", connInfo.SecretKey) + + w.Flush() + return buffer.String(), nil +} diff --git a/cmd/rook/object/connection_test.go b/cmd/rook/object/connection_test.go new file mode 100644 index 000000000..d9af63812 --- /dev/null +++ b/cmd/rook/object/connection_test.go @@ -0,0 +1,77 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package object + +import ( + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/rook/rook/pkg/model" + "github.com/rook/rook/pkg/rook/client" + "github.com/rook/rook/pkg/rook/test" +) + +func TestGetConnectionInfo(t *testing.T) { + c := &test.MockRookRestClient{ + MockGetObjectStoreConnectionInfo: func() (model.ObjectStoreS3Info, error) { + return model.ObjectStoreS3Info{ + Host: "rook-rgw:12345", + IPEndpoint: "1.2.3.4:12345", + AccessKey: "UST0JAP8CE61FDE0Q4BE", + SecretKey: "tVCuH20xTokjEpVJc7mKjL8PLTfGh4NZ3le3zg9X", + }, nil + }, + } + + expectedOut := "NAME VALUE\n" + + "S3_HOST rook-rgw:12345 \n" + + "S3_ENDPOINT 1.2.3.4:12345 \n" + + "S3_ACCESS_KEY UST0JAP8CE61FDE0Q4BE \n" + + "S3_SECRET_KEY tVCuH20xTokjEpVJc7mKjL8PLTfGh4NZ3le3zg9X \n" + + out, err := getConnectionInfo(c) + assert.Nil(t, err) + assert.Equal(t, expectedOut, out) +} + +func TestGetConnectionInfoNotFound(t *testing.T) { + c := &test.MockRookRestClient{ + MockGetObjectStoreConnectionInfo: func() (model.ObjectStoreS3Info, error) { + return model.ObjectStoreS3Info{}, client.RookRestError{ + Status: http.StatusNotFound, + } + }, + } + + out, err := getConnectionInfo(c) + assert.Nil(t, err) + assert.Equal(t, "object store connection info is not ready, if \"object create\" has already been run, please be patient\n", out) +} + +func TestGetConnectionInfoError(t *testing.T) { + c := &test.MockRookRestClient{ + MockGetObjectStoreConnectionInfo: func() (model.ObjectStoreS3Info, error) { + return model.ObjectStoreS3Info{}, fmt.Errorf("mock get connection info failed") + }, + } + + out, err := getConnectionInfo(c) + assert.NotNil(t, err) + assert.Equal(t, "", out) +} diff --git a/cmd/rook/object/create.go b/cmd/rook/object/create.go new file mode 100644 index 000000000..45ec50d7d --- /dev/null +++ b/cmd/rook/object/create.go @@ -0,0 +1,65 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package object + +import ( + "fmt" + "net/http" + "os" + + "github.com/rook/rook/cmd/rook/rook" + "github.com/rook/rook/pkg/rook/client" + "github.com/rook/rook/pkg/util/flags" + "github.com/spf13/cobra" +) + +var createCmd = &cobra.Command{ + Use: "create", + Short: "Creates a new object storage instance in the cluster", +} + +func init() { + createCmd.RunE = createObjectStoreEntry +} + +func createObjectStoreEntry(cmd *cobra.Command, args []string) error { + rook.SetupLogging() + + if err := flags.VerifyRequiredFlags(cmd, []string{}); err != nil { + return err + } + + c := client.NewRookNetworkRestClient(client.GetRestURL(rook.APIServerEndpoint), http.DefaultClient) + out, err := createObjectStore(c) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + fmt.Println(out) + return nil +} + +func createObjectStore(c client.RookRestClient) (string, error) { + _, err := c.CreateObjectStore() + + // HTTP 202 Accepted is expected + if err != nil && !client.IsHttpAccepted(err) { + return "", fmt.Errorf("failed to create new object store: %+v", err) + } + + return fmt.Sprintf("succeeded starting creation of object store"), nil +} diff --git a/cmd/rook/object/create_test.go b/cmd/rook/object/create_test.go new file mode 100644 index 000000000..1bb69f731 --- /dev/null +++ b/cmd/rook/object/create_test.go @@ -0,0 +1,45 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package object + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/rook/rook/pkg/rook/test" +) + +func TestCreateObjectStore(t *testing.T) { + c := &test.MockRookRestClient{} + + out, err := createObjectStore(c) + assert.Nil(t, err) + assert.Equal(t, "succeeded starting creation of object store", out) +} + +func TestCreateObjectStoreError(t *testing.T) { + c := &test.MockRookRestClient{ + MockCreateObjectStore: func() (string, error) { + return "", fmt.Errorf("mock create object store failed") + }, + } + + out, err := createObjectStore(c) + assert.NotNil(t, err) + assert.Equal(t, "", out) +} diff --git a/cmd/rook/object/object.go b/cmd/rook/object/object.go new file mode 100644 index 000000000..ddab61177 --- /dev/null +++ b/cmd/rook/object/object.go @@ -0,0 +1,29 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package object + +import "github.com/spf13/cobra" + +var Cmd = &cobra.Command{ + Use: "object", + Aliases: []string{"obj"}, + Short: "Performs commands and operations on object stores in the cluster", +} + +func init() { + Cmd.AddCommand(createCmd) + Cmd.AddCommand(connectionCmd) +} diff --git a/pkg/api/client.go b/pkg/api/client.go index cae4635f6..345892312 100644 --- a/pkg/api/client.go +++ b/pkg/api/client.go @@ -27,6 +27,7 @@ import ( // /client func (h *Handler) GetClientAccessInfo(w http.ResponseWriter, r *http.Request) { // TODO: auth is extremely important here because we are returning cephx credentials + // https://github.com/rook/rook/issues/209 adminConn, ok := h.handleConnectToCeph(w) if !ok { diff --git a/pkg/api/object.go b/pkg/api/object.go index 526402fed..572f7919b 100644 --- a/pkg/api/object.go +++ b/pkg/api/object.go @@ -19,11 +19,14 @@ import ( "net/http" "github.com/rook/rook/pkg/cephmgr/rgw" + "github.com/rook/rook/pkg/clusterd/inventory" + "github.com/rook/rook/pkg/model" + "github.com/rook/rook/pkg/util" ) // Creates a new object store in this cluster. // POST -// /object +// /objectstore func (h *Handler) CreateObjectStore(w http.ResponseWriter, r *http.Request) { if err := rgw.EnableObjectStore(h.context); err != nil { @@ -38,7 +41,7 @@ func (h *Handler) CreateObjectStore(w http.ResponseWriter, r *http.Request) { // Removes the object store from this cluster. // POST -// /object/remove +// /objectstore/remove func (h *Handler) RemoveObjectStore(w http.ResponseWriter, r *http.Request) { if err := rgw.RemoveObjectStore(h.context); err != nil { logger.Errorf("failed to remove object store: %+v", err) @@ -49,3 +52,55 @@ func (h *Handler) RemoveObjectStore(w http.ResponseWriter, r *http.Request) { logger.Debugf("started async deletion of the object store") w.WriteHeader(http.StatusAccepted) } + +// Gets connection information to the object store in this cluster. +// GET +// /objectstore/connectioninfo +func (h *Handler) GetObjectStoreConnectionInfo(w http.ResponseWriter, r *http.Request) { + // TODO: auth is extremely important here because we are returning RGW credentials + // https://github.com/rook/rook/issues/209 + + // TODO: support other types of connection info to object store (such as swift) + // https://github.com/rook/rook/issues/255 + + accessKey, secretKey, err := rgw.GetBuiltinUserAccessInfo(h.context.EtcdClient) + if err != nil { + handleConnectionLookupFailure(err, "builtin user access info", w) + return + } + + clusterInventory, err := inventory.LoadDiscoveredNodes(h.context.EtcdClient) + if err != nil { + logger.Errorf("failed to load discovered nodes: %+v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + host, ipEndpoint, found, err := rgw.GetRGWEndpoints(h.context.EtcdClient, clusterInventory) + if err != nil { + handleConnectionLookupFailure(err, "rgw endpoints", w) + return + } else if !found { + logger.Errorf("failed to find rgw endpoints") + w.WriteHeader(http.StatusNotFound) + return + } + + s3Info := model.ObjectStoreS3Info{ + Host: host, + IPEndpoint: ipEndpoint, + AccessKey: accessKey, + SecretKey: secretKey, + } + + FormatJsonResponse(w, s3Info) +} + +func handleConnectionLookupFailure(err error, action string, w http.ResponseWriter) { + logger.Errorf("failed to get %s: %+v", action, err) + if util.IsEtcdKeyNotFound(err) { + w.WriteHeader(http.StatusNotFound) + } else { + w.WriteHeader(http.StatusInternalServerError) + } +} diff --git a/pkg/api/object_test.go b/pkg/api/object_test.go new file mode 100644 index 000000000..a7784d7f4 --- /dev/null +++ b/pkg/api/object_test.go @@ -0,0 +1,120 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package api + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + testceph "github.com/rook/rook/pkg/cephmgr/client/test" + "github.com/rook/rook/pkg/cephmgr/test" + "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/clusterd/inventory" + "github.com/rook/rook/pkg/model" + "github.com/rook/rook/pkg/util" + "github.com/stretchr/testify/assert" +) + +func TestCreateObjectStoreHandler(t *testing.T) { + etcdClient := util.NewMockEtcdClient() + context := &clusterd.Context{EtcdClient: etcdClient} + + req, err := http.NewRequest("POST", "http://10.0.0.100/objectstore", nil) + if err != nil { + logger.Fatal(err) + } + + cephFactory := &testceph.MockConnectionFactory{Fsid: "myfsid", SecretKey: "mykey"} + connFactory := &test.MockConnectionFactory{} + + // call the CreateObjectStore handler, which should return http 202 Accepted and record info + // about the file system request in etcd + w := httptest.NewRecorder() + h := NewHandler(context, connFactory, cephFactory) + h.CreateObjectStore(w, req) + assert.Equal(t, http.StatusAccepted, w.Code) + assert.Equal(t, "1", etcdClient.GetValue("/rook/services/ceph/object/desired/state")) +} + +func TestRemoveObjectStoreHandler(t *testing.T) { + // simulate object store already being installed by setting the desired key in etcd + etcdClient := util.NewMockEtcdClient() + etcdClient.SetValue("/rook/services/ceph/object/desired/state", "1") + + context := &clusterd.Context{EtcdClient: etcdClient} + + req, err := http.NewRequest("POST", "http://10.0.0.100/objectstore/remove", nil) + if err != nil { + logger.Fatal(err) + } + + cephFactory := &testceph.MockConnectionFactory{Fsid: "myfsid", SecretKey: "mykey"} + connFactory := &test.MockConnectionFactory{} + + // call RemoveObjectStore handler and verify the response is 202 Accepted and the desired + // key has been deleted from etcd + w := httptest.NewRecorder() + h := NewHandler(context, connFactory, cephFactory) + h.RemoveObjectStore(w, req) + assert.Equal(t, http.StatusAccepted, w.Code) + assert.Equal(t, 0, etcdClient.GetChildDirs("/rook/services/ceph/object/desired").Count()) +} + +func TestGetObjectStoreConnectionInfoHandler(t *testing.T) { + etcdClient := util.NewMockEtcdClient() + inventory.SetIPAddress(etcdClient, "123", "1.2.3.4", "2.3.4.5") + context := &clusterd.Context{EtcdClient: etcdClient} + + req, err := http.NewRequest("GET", "http://10.0.0.100/objectstore/connectioninfo", nil) + if err != nil { + logger.Fatal(err) + } + + cephFactory := &testceph.MockConnectionFactory{Fsid: "myfsid", SecretKey: "mykey"} + connFactory := &test.MockConnectionFactory{} + + // before RGW has been installed or any user accounts have been created, the handler will return 404 not found + w := httptest.NewRecorder() + h := NewHandler(context, connFactory, cephFactory) + h.GetObjectStoreConnectionInfo(w, req) + assert.Equal(t, http.StatusNotFound, w.Code) + + // simulate RGW being installed and the built in user being created + etcdClient.SetValue("/rook/services/ceph/rgw/applied/node/123", "") + etcdClient.SetValue("/rook/services/ceph/object/applied/admin/id", "UST0JAP8CE61FDE0Q4BE") + etcdClient.SetValue("/rook/services/ceph/object/applied/admin/_secret", "tVCuH20xTokjEpVJc7mKjL8PLTfGh4NZ3le3zg9X") + + w = httptest.NewRecorder() + h = NewHandler(context, connFactory, cephFactory) + h.GetObjectStoreConnectionInfo(w, req) + assert.Equal(t, http.StatusOK, w.Code) + + expectedRespObj := model.ObjectStoreS3Info{ + Host: "rook-rgw:53390", + IPEndpoint: "1.2.3.4:53390", + AccessKey: "UST0JAP8CE61FDE0Q4BE", + SecretKey: "tVCuH20xTokjEpVJc7mKjL8PLTfGh4NZ3le3zg9X", + } + + // unmarshal the http response to get the actual object and compare it to the expected object + var actualResultObj model.ObjectStoreS3Info + bodyBytes, _ := ioutil.ReadAll(w.Body) + json.Unmarshal(bodyBytes, &actualResultObj) + assert.Equal(t, expectedRespObj, actualResultObj) +} diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 215b498c4..f88a4fd97 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -83,6 +83,12 @@ func (h *Handler) GetRoutes() []Route { "/objectstore/remove", h.RemoveObjectStore, }, + { + "GetObjectStoreConnectionInfo", + "GET", + "/objectstore/connectioninfo", + h.GetObjectStoreConnectionInfo, + }, { "GetFileSystems", "GET", diff --git a/pkg/cephmgr/rgw/agent.go b/pkg/cephmgr/rgw/agent.go index 3c95f5c58..d39442ddd 100644 --- a/pkg/cephmgr/rgw/agent.go +++ b/pkg/cephmgr/rgw/agent.go @@ -35,8 +35,8 @@ import ( ) const ( - dnsName = "rook-rgw" - rgwPort = 53390 + DNSName = "rook-rgw" + RGWPort = 53390 rgwAgentName = "rgw" keyringTemplate = `[client.radosgw.gateway] key = %s @@ -118,10 +118,10 @@ func (a *rgwAgent) generateConfigFiles(context *clusterd.Context, cluster *mon.C } settings := map[string]string{ - "host": dnsName, - "rgw port": strconv.Itoa(rgwPort), + "host": DNSName, + "rgw port": strconv.Itoa(RGWPort), "rgw data": dataDir, - "rgw dns name": fmt.Sprintf("%s:%d", dnsName, rgwPort), + "rgw dns name": fmt.Sprintf("%s:%d", DNSName, RGWPort), "rgw log nonexistent bucket": "true", "rgw intent log object name utc": "true", "rgw enable usage log": "true", @@ -182,7 +182,7 @@ func (a *rgwAgent) startRGW(context *clusterd.Context, cluster *mon.ClusterInfo) fmt.Sprintf("--cluster=%s", cluster.Name), fmt.Sprintf("--conf=%s", getRGWConfFilePath(context.ConfigDir, cluster.Name)), fmt.Sprintf("--keyring=%s", getRGWKeyringPath(context.ConfigDir)), - fmt.Sprintf("--rgw-frontends=civetweb port=%d", rgwPort), + fmt.Sprintf("--rgw-frontends=civetweb port=%d", RGWPort), fmt.Sprintf("--rgw-mime-types-file=%s", getMimeTypesPath(context.ConfigDir))) if err != nil { return fmt.Errorf("failed to start rgw: %+v", err) diff --git a/pkg/cephmgr/rgw/leader.go b/pkg/cephmgr/rgw/leader.go index cd93fbfd2..10a0ba2f3 100644 --- a/pkg/cephmgr/rgw/leader.go +++ b/pkg/cephmgr/rgw/leader.go @@ -25,6 +25,7 @@ import ( "github.com/rook/rook/pkg/cephmgr/client" "github.com/rook/rook/pkg/cephmgr/mon" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/clusterd/inventory" "github.com/rook/rook/pkg/util" ) @@ -104,6 +105,25 @@ func RemoveObjectStore(context *clusterd.Context) error { return nil } +func GetRGWEndpoints(etcdClient etcd.KeysAPI, clusterInventory *inventory.Config) (host, ipEndpoint string, found bool, err error) { + appliedNodes, err := util.GetDirChildKeys(etcdClient, getRGWNodesKey(true)) + if err != nil { + return "", "", false, err + } + + for nodeID := range appliedNodes.Iter() { + // just return the details of the first RGW node we can find + nodeDetails, ok := clusterInventory.Nodes[nodeID] + if ok { + host = getRGWEndpoint(DNSName) + ipEndpoint = getRGWEndpoint(nodeDetails.PublicIP) + return host, ipEndpoint, true, nil + } + } + + return "", "", false, nil +} + // Configure the single instance of object storage in the cluster. func getObjectStoreState(context *clusterd.Context, applied bool) (bool, error) { var state string @@ -230,3 +250,7 @@ func (r *Leader) getDesiredRGWNodes(context *clusterd.Context, count int) ([]str return nodes.ToSlice(), nil } + +func getRGWEndpoint(addr string) string { + return fmt.Sprintf("%s:%d", addr, RGWPort) +} diff --git a/pkg/cephmgr/rgw/leader_test.go b/pkg/cephmgr/rgw/leader_test.go index 230a6f3ac..e8536d91a 100644 --- a/pkg/cephmgr/rgw/leader_test.go +++ b/pkg/cephmgr/rgw/leader_test.go @@ -16,10 +16,14 @@ limitations under the License. package rgw import ( + "fmt" "os" "path" "testing" + etcd "github.com/coreos/etcd/client" + ctx "golang.org/x/net/context" + testceph "github.com/rook/rook/pkg/cephmgr/client/test" cephtest "github.com/rook/rook/pkg/cephmgr/test" "github.com/rook/rook/pkg/clusterd/inventory" @@ -80,6 +84,13 @@ func TestRGWConfig(t *testing.T) { assert.Equal(t, "myaccessid", etcdClient.GetValue("/rook/services/ceph/object/applied/admin/id")) assert.Equal(t, "mybigsecretkey", etcdClient.GetValue("/rook/services/ceph/object/applied/admin/_secret")) + // Get the RGW endpoints and verify + host, ipAddr, found, err := GetRGWEndpoints(etcdClient, context.Inventory) + assert.Nil(t, err) + assert.True(t, found) + assert.Equal(t, "rook-rgw:53390", host) + assert.True(t, ipAddr == "1.2.3.4:53390" || ipAddr == "2.3.4.5:53390", fmt.Sprintf("unexpected rgw IP endpoint: %s", ipAddr)) + // Remove the object service err = RemoveObjectStore(context) assert.Nil(t, err) @@ -175,3 +186,28 @@ func TestGetDesiredNodes(t *testing.T) { desired, err = leader.getDesiredRGWNodes(context, 3) assert.NotNil(t, err) } + +func TestGetRGWEndpointsFailure(t *testing.T) { + etcdClient := util.NewMockEtcdClient() + etcdClient.MockGet = func(context ctx.Context, key string, opts *etcd.GetOptions) (*etcd.Response, error) { + return nil, fmt.Errorf("mock etcd GET error") + } + + host, ipAddr, found, err := GetRGWEndpoints(etcdClient, nil) + assert.Equal(t, "", host) + assert.Equal(t, "", ipAddr) + assert.False(t, found) + assert.NotNil(t, err) +} + +func TestGetRGWEndpointsNotFound(t *testing.T) { + etcdClient := util.NewMockEtcdClient() + nodes := map[string]*inventory.NodeConfig{} + clusterInventory := &inventory.Config{Nodes: nodes} + + host, ipAddr, found, err := GetRGWEndpoints(etcdClient, clusterInventory) + assert.Equal(t, "", host) + assert.Equal(t, "", ipAddr) + assert.False(t, found) + assert.Nil(t, err) +} diff --git a/pkg/cephmgr/rgw/user.go b/pkg/cephmgr/rgw/user.go index 287317cfd..f01b31755 100644 --- a/pkg/cephmgr/rgw/user.go +++ b/pkg/cephmgr/rgw/user.go @@ -20,14 +20,18 @@ import ( "fmt" "path" + etcd "github.com/coreos/etcd/client" ctx "golang.org/x/net/context" "github.com/rook/rook/pkg/cephmgr/mon" "github.com/rook/rook/pkg/clusterd" + "github.com/rook/rook/pkg/util" ) const ( builtinUserKey = "admin" + idKey = "id" + secretKey = "_secret" ) type user struct { @@ -36,7 +40,7 @@ type user struct { type keyInfo struct { User string `json:"user"` - AccessID string `json:"access_key"` + AccessKey string `json:"access_key"` SecretKey string `json:"secret_key"` } @@ -62,19 +66,38 @@ func createBuiltinUser(context *clusterd.Context) error { } userkey := u.Keys[0] - if userkey.AccessID == "" || userkey.SecretKey == "" { + if userkey.AccessKey == "" || userkey.SecretKey == "" { return fmt.Errorf("missing user properties in %s", result) } // store the creds in etcd key := path.Join(mon.CephKey, ObjectStoreKey, clusterd.AppliedKey, builtinUserKey) - if _, err := context.EtcdClient.Set(ctx.Background(), path.Join(key, "id"), userkey.AccessID, nil); err != nil { + if _, err := context.EtcdClient.Set(ctx.Background(), path.Join(key, idKey), userkey.AccessKey, nil); err != nil { return fmt.Errorf("failed to store access id. %+v", err) } - if _, err := context.EtcdClient.Set(ctx.Background(), path.Join(key, "_secret"), userkey.SecretKey, nil); err != nil { + if _, err := context.EtcdClient.Set(ctx.Background(), path.Join(key, secretKey), userkey.SecretKey, nil); err != nil { return fmt.Errorf("failed to store access id. %+v", err) } return nil } + +func GetBuiltinUserAccessInfo(etcdClient etcd.KeysAPI) (string, string, error) { + return getUserAccessInfo(builtinUserKey, etcdClient) +} + +func getUserAccessInfo(userName string, etcdClient etcd.KeysAPI) (string, string, error) { + userKey := path.Join(mon.CephKey, ObjectStoreKey, clusterd.AppliedKey, userName) + keys := map[string]string{ + idKey: path.Join(userKey, idKey), + secretKey: path.Join(userKey, secretKey), + } + + vals, err := util.GetEtcdValues(etcdClient, keys) + if err != nil { + return "", "", err + } + + return vals[idKey], vals[secretKey], nil +} diff --git a/pkg/cephmgr/rgw/user_test.go b/pkg/cephmgr/rgw/user_test.go index 8f815a967..8ed94e5ca 100644 --- a/pkg/cephmgr/rgw/user_test.go +++ b/pkg/cephmgr/rgw/user_test.go @@ -114,3 +114,15 @@ func TestBuiltinUser(t *testing.T) { assert.Equal(t, "foo", etcdClient.GetValue("/rook/services/ceph/object/applied/admin/id")) assert.Equal(t, "bar", etcdClient.GetValue("/rook/services/ceph/object/applied/admin/_secret")) } + +func TestGetBuiltinUserAccessInfo(t *testing.T) { + // simulate the built in user having been created + etcdClient := util.NewMockEtcdClient() + etcdClient.SetValue("/rook/services/ceph/object/applied/admin/id", "UST0JAP8CE61FDE0Q4BE") + etcdClient.SetValue("/rook/services/ceph/object/applied/admin/_secret", "tVCuH20xTokjEpVJc7mKjL8PLTfGh4NZ3le3zg9X") + + id, secret, err := GetBuiltinUserAccessInfo(etcdClient) + assert.Nil(t, err) + assert.Equal(t, "UST0JAP8CE61FDE0Q4BE", id) + assert.Equal(t, "tVCuH20xTokjEpVJc7mKjL8PLTfGh4NZ3le3zg9X", secret) +} diff --git a/pkg/model/object.go b/pkg/model/object.go new file mode 100644 index 000000000..8c8ea2371 --- /dev/null +++ b/pkg/model/object.go @@ -0,0 +1,23 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package model + +type ObjectStoreS3Info struct { + Host string `json:"host"` + IPEndpoint string `json:"ipEndpoint"` + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` +} diff --git a/pkg/rook/client/client.go b/pkg/rook/client/client.go index 472bc2c1c..6942d6a5e 100644 --- a/pkg/rook/client/client.go +++ b/pkg/rook/client/client.go @@ -41,6 +41,8 @@ type RookRestClient interface { CreateFilesystem(model.FilesystemRequest) (string, error) DeleteFilesystem(model.FilesystemRequest) (string, error) GetStatusDetails() (model.StatusDetails, error) + CreateObjectStore() (string, error) + GetObjectStoreConnectionInfo() (model.ObjectStoreS3Info, error) } type RookNetworkRestClient struct { @@ -70,12 +72,20 @@ func (e RookRestError) Error() string { } func IsHttpAccepted(err error) bool { + return IsHttpStatusCode(err, http.StatusAccepted) +} + +func IsHttpNotFound(err error) bool { + return IsHttpStatusCode(err, http.StatusNotFound) +} + +func IsHttpStatusCode(err error, statusCode int) bool { if err == nil { return false } rrErr, ok := err.(RookRestError) - return ok && rrErr.Status == http.StatusAccepted + return ok && rrErr.Status == statusCode } func (a *RookNetworkRestClient) URL() string { diff --git a/pkg/rook/client/client_test.go b/pkg/rook/client/client_test.go index abb719348..24f7b34b0 100644 --- a/pkg/rook/client/client_test.go +++ b/pkg/rook/client/client_test.go @@ -27,13 +27,14 @@ import ( ) const ( - SuccessGetNodesContent = `[{"nodeID": "node1","publicIp": "1.2.3.100","privateIp": "10.0.0.100","storage": 100},{"nodeID": "node2","ipAddr": "10.0.0.101","storage": 200}]` - SuccessGetPoolsContent = "[{\"poolName\":\"rbd\",\"poolNum\":0,\"type\":0,\"replicationConfig\":{\"size\":1},\"erasureCodedConfig\":{\"dataChunkCount\":0,\"codingChunkCount\":0,\"algorithm\":\"\"}},{\"poolName\":\"ecPool1\",\"poolNum\":1,\"type\":1,\"replicationConfig\":{\"size\":0},\"erasureCodedConfig\":{\"dataChunkCount\":2,\"codingChunkCount\":1,\"algorithm\":\"jerasure::reed_sol_van\"}}]" - SuccessCreatePoolContent = `pool 'ecPool1' created` - SuccessGetBlockImagesContent = `[{"imageName":"myimage1","poolName":"rbd","size":10485760,"device":"","mountPoint":""},{"imageName":"myimage2","poolName":"rbd2","size":10485761,"device":"","mountPoint":""}]` - SuccessCreateBlockImageContent = `succeeded created image myimage3` - SuccessGetClientAccessInfoContent = `{"monAddresses":["10.37.129.214:6790/0"],"userName":"admin","secretKey":"AQBsCv1X5oD9GhAARHVU9N+kFRWDjyLA1dqzIg=="}` - SuccessGetFilesystemsContent = `[{"name":"myfs1","metadataPool":"myfs1-metadata","dataPools":["myfs1-data"]}]` + SuccessGetNodesContent = `[{"nodeID": "node1","publicIp": "1.2.3.100","privateIp": "10.0.0.100","storage": 100},{"nodeID": "node2","ipAddr": "10.0.0.101","storage": 200}]` + SuccessGetPoolsContent = "[{\"poolName\":\"rbd\",\"poolNum\":0,\"type\":0,\"replicationConfig\":{\"size\":1},\"erasureCodedConfig\":{\"dataChunkCount\":0,\"codingChunkCount\":0,\"algorithm\":\"\"}},{\"poolName\":\"ecPool1\",\"poolNum\":1,\"type\":1,\"replicationConfig\":{\"size\":0},\"erasureCodedConfig\":{\"dataChunkCount\":2,\"codingChunkCount\":1,\"algorithm\":\"jerasure::reed_sol_van\"}}]" + SuccessCreatePoolContent = `pool 'ecPool1' created` + SuccessGetBlockImagesContent = `[{"imageName":"myimage1","poolName":"rbd","size":10485760,"device":"","mountPoint":""},{"imageName":"myimage2","poolName":"rbd2","size":10485761,"device":"","mountPoint":""}]` + SuccessCreateBlockImageContent = `succeeded created image myimage3` + SuccessGetClientAccessInfoContent = `{"monAddresses":["10.37.129.214:6790/0"],"userName":"admin","secretKey":"AQBsCv1X5oD9GhAARHVU9N+kFRWDjyLA1dqzIg=="}` + SuccessGetFilesystemsContent = `[{"name":"myfs1","metadataPool":"myfs1-metadata","dataPools":["myfs1-data"]}]` + SuccessGetObjectStoreConnectionInfoContent = `{"host":"rook-rgw:12345", "accessKey":"UST0JAP8CE61FDE0Q4BE", "secretKey":"tVCuH20xTokjEpVJc7mKjL8PLTfGh4NZ3le3zg9X"}` ) func TestURL(t *testing.T) { @@ -229,6 +230,35 @@ func TestDeleteFilesystem(t *testing.T) { assert.Equal(t, "", resp) } +func TestCreateObjectStore(t *testing.T) { + mockServer := NewMockHttpServer(202, "") + defer mockServer.Close() + mockHttpClient := NewMockHttpClient(mockServer.URL) + client := NewRookNetworkRestClient(mockServer.URL, mockHttpClient) + + resp, err := client.CreateObjectStore() + assert.NotNil(t, err) + assert.True(t, IsHttpAccepted(err)) + assert.Equal(t, "", resp) +} + +func TestGetObjectStoreConnectionInfo(t *testing.T) { + mockServer := NewMockHttpServer(200, SuccessGetObjectStoreConnectionInfoContent) + defer mockServer.Close() + mockHttpClient := NewMockHttpClient(mockServer.URL) + client := NewRookNetworkRestClient(mockServer.URL, mockHttpClient) + + expectedResp := model.ObjectStoreS3Info{ + Host: "rook-rgw:12345", + AccessKey: "UST0JAP8CE61FDE0Q4BE", + SecretKey: "tVCuH20xTokjEpVJc7mKjL8PLTfGh4NZ3le3zg9X", + } + + resp, err := client.GetObjectStoreConnectionInfo() + assert.Nil(t, err) + assert.Equal(t, expectedResp, resp) +} + func TestGetNodesFailure(t *testing.T) { ClientFailureHelper(t, func(client RookRestClient) (interface{}, error) { return client.GetNodes() }) } @@ -288,6 +318,25 @@ func TestDeleteFilesystemFailure(t *testing.T) { ClientFailureHelperWithVerification(t, clientFunc, verifyFunc) } +func TestCreateObjectStoreFailure(t *testing.T) { + clientFunc := func(client RookRestClient) (interface{}, error) { + return client.CreateObjectStore() + } + verifyFunc := getStringVerifyFunc(t) + ClientFailureHelperWithVerification(t, clientFunc, verifyFunc) +} + +func TestGetObjectStoreConnectionInfoFailure(t *testing.T) { + clientFunc := func(client RookRestClient) (interface{}, error) { + return client.GetObjectStoreConnectionInfo() + } + verifyFunc := func(resp interface{}, err error) { + assert.NotNil(t, err) + assert.Equal(t, model.ObjectStoreS3Info{}, resp.(model.ObjectStoreS3Info)) + } + ClientFailureHelperWithVerification(t, clientFunc, verifyFunc) +} + func ClientFailureHelper(t *testing.T, clientFunc func(RookRestClient) (interface{}, error)) { verifyFunc := func(resp interface{}, err error) { assert.NotNil(t, err) diff --git a/pkg/rook/client/object.go b/pkg/rook/client/object.go new file mode 100644 index 000000000..fe8165af2 --- /dev/null +++ b/pkg/rook/client/object.go @@ -0,0 +1,52 @@ +/* +Copyright 2016 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package client + +import ( + "encoding/json" + "path" + + "github.com/rook/rook/pkg/model" +) + +const ( + objectStoreQueryName = "objectstore" + connectionInfoQueryName = "connectioninfo" +) + +func (c *RookNetworkRestClient) CreateObjectStore() (string, error) { + resp, err := c.DoPost(objectStoreQueryName, nil) + if err != nil { + return "", err + } + + return string(resp), nil +} + +func (c *RookNetworkRestClient) GetObjectStoreConnectionInfo() (model.ObjectStoreS3Info, error) { + body, err := c.DoGet(path.Join(objectStoreQueryName, connectionInfoQueryName)) + if err != nil { + return model.ObjectStoreS3Info{}, err + } + + var connInfo model.ObjectStoreS3Info + err = json.Unmarshal(body, &connInfo) + if err != nil { + return model.ObjectStoreS3Info{}, err + } + + return connInfo, nil +} diff --git a/pkg/rook/test/mockclient.go b/pkg/rook/test/mockclient.go index ddf79b6d1..3f89ae4d0 100644 --- a/pkg/rook/test/mockclient.go +++ b/pkg/rook/test/mockclient.go @@ -30,16 +30,18 @@ func (*RookMockRestClientFactory) CreateRookRestClient(url string, httpClient *h // Mock Rook REST Client implementation type MockRookRestClient struct { - MockGetNodes func() ([]model.Node, error) - MockGetPools func() ([]model.Pool, error) - MockCreatePool func(pool model.Pool) (string, error) - MockGetBlockImages func() ([]model.BlockImage, error) - MockCreateBlockImage func(image model.BlockImage) (string, error) - MockGetClientAccessInfo func() (model.ClientAccessInfo, error) - MockGetFilesystems func() ([]model.Filesystem, error) - MockCreateFilesystem func(model.FilesystemRequest) (string, error) - MockDeleteFilesystem func(model.FilesystemRequest) (string, error) - MockGetStatusDetails func() (model.StatusDetails, error) + MockGetNodes func() ([]model.Node, error) + MockGetPools func() ([]model.Pool, error) + MockCreatePool func(pool model.Pool) (string, error) + MockGetBlockImages func() ([]model.BlockImage, error) + MockCreateBlockImage func(image model.BlockImage) (string, error) + MockGetClientAccessInfo func() (model.ClientAccessInfo, error) + MockGetFilesystems func() ([]model.Filesystem, error) + MockCreateFilesystem func(model.FilesystemRequest) (string, error) + MockDeleteFilesystem func(model.FilesystemRequest) (string, error) + MockGetStatusDetails func() (model.StatusDetails, error) + MockCreateObjectStore func() (string, error) + MockGetObjectStoreConnectionInfo func() (model.ObjectStoreS3Info, error) } func (m *MockRookRestClient) GetNodes() ([]model.Node, error) { @@ -122,6 +124,22 @@ func (m *MockRookRestClient) GetStatusDetails() (model.StatusDetails, error) { return model.StatusDetails{}, nil } +func (m *MockRookRestClient) CreateObjectStore() (string, error) { + if m.MockCreateObjectStore != nil { + return m.MockCreateObjectStore() + } + + return "", nil +} + +func (m *MockRookRestClient) GetObjectStoreConnectionInfo() (model.ObjectStoreS3Info, error) { + if m.MockGetObjectStoreConnectionInfo != nil { + return m.MockGetObjectStoreConnectionInfo() + } + + return model.ObjectStoreS3Info{}, nil +} + func (m *MockRookRestClient) URL() string { return "" }