diff --git a/cmd/rook/rgw.go b/cmd/rook/rgw.go index 5cae94ed5..1b2a14a99 100644 --- a/cmd/rook/rgw.go +++ b/cmd/rook/rgw.go @@ -32,12 +32,14 @@ var rgwCmd = &cobra.Command{ } var ( + rgwName string rgwKeyring string rgwHost string rgwPort int ) func init() { + rgwCmd.Flags().StringVar(&rgwName, "rgw-name", "", "name of the object store") rgwCmd.Flags().StringVar(&rgwKeyring, "rgw-keyring", "", "the rgw keyring") rgwCmd.Flags().StringVar(&rgwHost, "rgw-host", "", "dns host name") rgwCmd.Flags().IntVar(&rgwPort, "rgw-port", 0, "rgw port number") @@ -49,7 +51,7 @@ func init() { } func startRGW(cmd *cobra.Command, args []string) error { - required := []string{"mon-endpoints", "cluster-name", "mon-secret", "admin-secret", "rgw-host", "rgw-keyring", "public-ipv4", "private-ipv4"} + required := []string{"mon-endpoints", "cluster-name", "mon-secret", "admin-secret", "rgw-name", "rgw-host", "rgw-keyring", "public-ipv4", "private-ipv4"} if err := flags.VerifyRequiredFlags(rgwCmd, required); err != nil { return err } @@ -65,6 +67,7 @@ func startRGW(cmd *cobra.Command, args []string) error { clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints) config := &rgw.Config{ ClusterInfo: &clusterInfo, + Name: rgwName, Keyring: rgwKeyring, Host: rgwHost, Port: rgwPort, diff --git a/pkg/api/cluster_handler.go b/pkg/api/cluster_handler.go index 9d08b14f2..99f485457 100644 --- a/pkg/api/cluster_handler.go +++ b/pkg/api/cluster_handler.go @@ -29,9 +29,9 @@ import ( type ClusterHandler interface { GetClusterInfo() (*mon.ClusterInfo, error) - EnableObjectStore() error - RemoveObjectStore() error - GetObjectStoreConnectionInfo() (s3info *model.ObjectStoreConnectInfo, found bool, err error) + EnableObjectStore(name string) error + RemoveObjectStore(name string) error + GetObjectStoreConnectionInfo(name string) (s3info *model.ObjectStoreConnectInfo, found bool, err error) StartFileSystem(fs *model.FilesystemRequest) error RemoveFileSystem(fs *model.FilesystemRequest) error GetMonitors() (map[string]*mon.CephMonitorConfig, error) @@ -50,15 +50,15 @@ func (e *etcdHandler) GetClusterInfo() (*mon.ClusterInfo, error) { return mon.LoadClusterInfo(e.context.EtcdClient) } -func (e *etcdHandler) EnableObjectStore() error { +func (e *etcdHandler) EnableObjectStore(name string) error { return rgw.EnableObjectStore(e.context.EtcdClient) } -func (e *etcdHandler) RemoveObjectStore() error { +func (e *etcdHandler) RemoveObjectStore(name string) error { return rgw.RemoveObjectStore(e.context.EtcdClient) } -func (e *etcdHandler) GetObjectStoreConnectionInfo() (*model.ObjectStoreConnectInfo, bool, error) { +func (e *etcdHandler) GetObjectStoreConnectionInfo(name string) (*model.ObjectStoreConnectInfo, bool, error) { clusterInventory, err := inventory.LoadDiscoveredNodes(e.context.EtcdClient) if err != nil { diff --git a/pkg/api/k8s/cluster_handler.go b/pkg/api/k8s/cluster_handler.go index 0a457e8a7..6ff5b50e6 100644 --- a/pkg/api/k8s/cluster_handler.go +++ b/pkg/api/k8s/cluster_handler.go @@ -43,11 +43,11 @@ func (s *clusterHandler) GetClusterInfo() (*mon.ClusterInfo, error) { return s.clusterInfo, nil } -func (s *clusterHandler) EnableObjectStore() error { +func (s *clusterHandler) EnableObjectStore(name string) error { logger.Infof("Starting the Object store") // Passing an empty Placement{} as the api doesn't know about placement // information. This should be resolved with the transition to CRD (TPR). - r := k8srgw.New(s.context, s.namespace, s.versionTag, k8sutil.Placement{}) + r := k8srgw.New(s.context, name, s.namespace, s.versionTag, k8sutil.Placement{}) err := r.Start() if err != nil { return fmt.Errorf("failed to start rgw. %+v", err) @@ -55,20 +55,20 @@ func (s *clusterHandler) EnableObjectStore() error { return nil } -func (s *clusterHandler) RemoveObjectStore() error { +func (s *clusterHandler) RemoveObjectStore(name string) error { logger.Infof("TODO: Remove the object store") return nil } -func (s *clusterHandler) GetObjectStoreConnectionInfo() (*model.ObjectStoreConnectInfo, bool, error) { +func (s *clusterHandler) GetObjectStoreConnectionInfo(name string) (*model.ObjectStoreConnectInfo, bool, error) { logger.Infof("Getting the object store connection info") - service, err := s.context.Clientset.CoreV1().Services(s.namespace).Get("rook-ceph-rgw", metav1.GetOptions{}) + service, err := s.context.Clientset.CoreV1().Services(s.namespace).Get(k8srgw.InstanceName(name), metav1.GetOptions{}) if err != nil { return nil, false, fmt.Errorf("failed to get rgw service. %+v", err) } info := &model.ObjectStoreConnectInfo{ - Host: "rook-ceph-rgw", + Host: k8srgw.InstanceName(name), IPEndpoint: rgw.GetRGWEndpoint(service.Spec.ClusterIP), } logger.Infof("Object store connection: %+v", info) diff --git a/pkg/api/object.go b/pkg/api/object.go index fee3e7091..45a845f6e 100644 --- a/pkg/api/object.go +++ b/pkg/api/object.go @@ -25,12 +25,18 @@ import ( "github.com/rook/rook/pkg/model" ) +const objectStoreName = "default" + +func (h *Handler) objectContext() *rgw.Context { + return rgw.NewContext(h.context, objectStoreName, h.config.ClusterHandler.GetClusterInfo) +} + // CreateObjectStore creates a new object store in this cluster. // POST // /objectstore func (h *Handler) CreateObjectStore(w http.ResponseWriter, r *http.Request) { - if err := h.config.ClusterHandler.EnableObjectStore(); err != nil { + if err := h.config.ClusterHandler.EnableObjectStore(objectStoreName); err != nil { logger.Errorf("failed to create object store: %+v", err) w.WriteHeader(http.StatusInternalServerError) return @@ -44,7 +50,7 @@ func (h *Handler) CreateObjectStore(w http.ResponseWriter, r *http.Request) { // DELETE // /objectstore func (h *Handler) RemoveObjectStore(w http.ResponseWriter, r *http.Request) { - if err := h.config.ClusterHandler.RemoveObjectStore(); err != nil { + if err := h.config.ClusterHandler.RemoveObjectStore(objectStoreName); err != nil { logger.Errorf("failed to remove object store: %+v", err) w.WriteHeader(http.StatusInternalServerError) return @@ -59,7 +65,7 @@ func (h *Handler) RemoveObjectStore(w http.ResponseWriter, r *http.Request) { // /objectstore/connectioninfo func (h *Handler) GetObjectStoreConnectionInfo(w http.ResponseWriter, r *http.Request) { - s3Info, found, err := h.config.ClusterHandler.GetObjectStoreConnectionInfo() + s3Info, found, err := h.config.ClusterHandler.GetObjectStoreConnectionInfo(objectStoreName) if err != nil { logger.Errorf("failed get object store info. %+v", err) if found { @@ -77,7 +83,7 @@ func (h *Handler) GetObjectStoreConnectionInfo(w http.ResponseWriter, r *http.Re // GET // /objectstore/users func (h *Handler) ListUsers(w http.ResponseWriter, r *http.Request) { - userNames, _, err := rgw.ListUsers(h.context, h.config.ClusterHandler.GetClusterInfo) + userNames, _, err := rgw.ListUsers(h.objectContext()) if err != nil { logger.Errorf("Error listing users: %+v", err) w.WriteHeader(http.StatusInternalServerError) @@ -86,7 +92,7 @@ func (h *Handler) ListUsers(w http.ResponseWriter, r *http.Request) { users := []model.ObjectUser{} for _, userName := range userNames { - user, _, err := rgw.GetUser(h.context, userName, h.config.ClusterHandler.GetClusterInfo) + user, _, err := rgw.GetUser(h.objectContext(), userName) if err != nil { logger.Errorf("Error listing users: %+v", err) w.WriteHeader(http.StatusInternalServerError) @@ -105,7 +111,7 @@ func (h *Handler) ListUsers(w http.ResponseWriter, r *http.Request) { func (h *Handler) GetUser(w http.ResponseWriter, r *http.Request) { id := mux.Vars(r)["id"] - user, rgwError, err := rgw.GetUser(h.context, id, h.config.ClusterHandler.GetClusterInfo) + user, rgwError, err := rgw.GetUser(h.objectContext(), id) if err != nil { logger.Errorf("Error getting user (%s): %+v", id, err) @@ -133,7 +139,7 @@ func (h *Handler) CreateUser(w http.ResponseWriter, r *http.Request) { return } - createdUser, rgwError, err := rgw.CreateUser(h.context, user, h.config.ClusterHandler.GetClusterInfo) + createdUser, rgwError, err := rgw.CreateUser(h.objectContext(), user) if err != nil { logger.Errorf("Error creating user: %+v", err) @@ -167,7 +173,7 @@ func (h *Handler) UpdateUser(w http.ResponseWriter, r *http.Request) { user.UserID = id - updatedUser, rgwError, err := rgw.UpdateUser(h.context, user, h.config.ClusterHandler.GetClusterInfo) + updatedUser, rgwError, err := rgw.UpdateUser(h.objectContext(), user) if err != nil { logger.Errorf("Error updating user: %+v", err) @@ -188,7 +194,7 @@ func (h *Handler) UpdateUser(w http.ResponseWriter, r *http.Request) { func (h *Handler) DeleteUser(w http.ResponseWriter, r *http.Request) { id := mux.Vars(r)["id"] - _, rgwError, err := rgw.DeleteUser(h.context, id, h.config.ClusterHandler.GetClusterInfo) + _, rgwError, err := rgw.DeleteUser(h.objectContext(), id) if err != nil { if rgwError == rgw.RGWErrorNotFound { w.WriteHeader(http.StatusNotFound) @@ -207,7 +213,7 @@ func (h *Handler) DeleteUser(w http.ResponseWriter, r *http.Request) { // GET // /objectstore/buckets func (h *Handler) ListBuckets(w http.ResponseWriter, r *http.Request) { - buckets, err := rgw.ListBuckets(h.context, h.config.ClusterHandler.GetClusterInfo) + buckets, err := rgw.ListBuckets(h.objectContext()) if err != nil { logger.Errorf("Error listing buckets: %+v", err) w.WriteHeader(http.StatusInternalServerError) @@ -226,7 +232,7 @@ func (h *Handler) ListBuckets(w http.ResponseWriter, r *http.Request) { func (h *Handler) GetBucket(w http.ResponseWriter, r *http.Request) { bucketName := mux.Vars(r)["bucketName"] - user, rgwError, err := rgw.GetBucket(h.context, bucketName, h.config.ClusterHandler.GetClusterInfo) + user, rgwError, err := rgw.GetBucket(h.objectContext(), bucketName) if err != nil { logger.Errorf("Error getting bucket (%s): %+v", bucketName, err) @@ -250,7 +256,7 @@ func (h *Handler) DeleteBucket(w http.ResponseWriter, r *http.Request) { purgeParams, found := r.URL.Query()["purge"] purge := found && len(purgeParams) == 1 && purgeParams[0] == "true" - rgwError, err := rgw.DeleteBucket(h.context, bucketName, purge, h.config.ClusterHandler.GetClusterInfo) + rgwError, err := rgw.DeleteBucket(h.objectContext(), bucketName, purge) if err != nil { if rgwError == rgw.RGWErrorNotFound { w.WriteHeader(http.StatusNotFound) diff --git a/pkg/api/object_test.go b/pkg/api/object_test.go index 6359344d1..60a56afc0 100644 --- a/pkg/api/object_test.go +++ b/pkg/api/object_test.go @@ -142,8 +142,8 @@ func TestListUsers(t *testing.T) { expectedConfigArg := getExectedConfigArg(configSubDir) expectedKeyringArg := getExpectedKeyringArg(configSubDir) - expectedListArgs := []string{"user", "list", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg} - expectedInfoArgs := []string{"user", "info", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid"} + expectedListArgs := []string{"user", "list", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg} + expectedInfoArgs := []string{"user", "info", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid"} // Empty list w := runTest(func(args ...string) (string, error) { @@ -247,7 +247,7 @@ func TestGetUser(t *testing.T) { expectedConfigArg := getExectedConfigArg(configSubDir) expectedKeyringArg := getExpectedKeyringArg(configSubDir) - expectedArgs := []string{"user", "info", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "someuser"} + expectedArgs := []string{"user", "info", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "someuser"} // Error getting user w := runTest("", fmt.Errorf("some error"), expectedArgs...) @@ -316,7 +316,7 @@ func TestCreateUser(t *testing.T) { expectedConfigArg := getExectedConfigArg(configSubDir) expectedKeyringArg := getExpectedKeyringArg(configSubDir) - expectedDisplayNameArgs := []string{"user", "create", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "foo", "--display-name", "the foo"} + expectedDisplayNameArgs := []string{"user", "create", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "foo", "--display-name", "the foo"} expectedDisplayNameAndEmailArgs := append(expectedDisplayNameArgs, "--email", "test@example.com") // Empty body @@ -396,7 +396,7 @@ func TestUpdateUser(t *testing.T) { expectedConfigArg := getExectedConfigArg(configSubDir) expectedKeyringArg := getExpectedKeyringArg(configSubDir) - expectedArgs := []string{"user", "modify", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "foo"} + expectedArgs := []string{"user", "modify", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "foo"} // Empty body w := runTest("", "", nil, "", "") @@ -479,7 +479,7 @@ func TestDeleteUser(t *testing.T) { expectedConfigArg := getExectedConfigArg(configSubDir) expectedKeyringArg := getExpectedKeyringArg(configSubDir) - expectedArgs := []string{"user", "rm", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "foo"} + expectedArgs := []string{"user", "rm", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "foo"} // Some error w := runTest("", fmt.Errorf("some error"), expectedArgs...) @@ -528,8 +528,8 @@ func TestListBuckets(t *testing.T) { expectedConfigArg := getExectedConfigArg(configSubDir) expectedKeyringArg := getExpectedKeyringArg(configSubDir) - expectedStatsArgs := []string{"bucket", "stats", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg} - expectedMetadataArgs := []string{"metadata", "get", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "bucket:foo"} + expectedStatsArgs := []string{"bucket", "stats", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg} + expectedMetadataArgs := []string{"metadata", "get", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "bucket:foo"} // List error w := runTest(func(args ...string) (string, error) { @@ -676,8 +676,8 @@ func TestGetBucket(t *testing.T) { expectedConfigArg := getExectedConfigArg(configSubDir) expectedKeyringArg := getExpectedKeyringArg(configSubDir) - expectedStatsArgs := []string{"bucket", "stats", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--bucket", "test"} - expectedMetadataArgs := []string{"metadata", "get", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "bucket:test"} + expectedStatsArgs := []string{"bucket", "stats", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--bucket", "test"} + expectedMetadataArgs := []string{"metadata", "get", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "bucket:test"} // Stats fails w := runTest(func(args ...string) (string, error) { @@ -795,7 +795,7 @@ func TestBucketDelete(t *testing.T) { expectedConfigArg := getExectedConfigArg(configSubDir) expectedKeyringArg := getExpectedKeyringArg(configSubDir) - expectedArgs := []string{"bucket", "rm", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--bucket", "test"} + expectedArgs := []string{"bucket", "rm", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--bucket", "test"} // errors w := runTest("", fmt.Errorf("some error"), expectedArgs...) diff --git a/pkg/ceph/rgw/admin.go b/pkg/ceph/rgw/admin.go index 41c03555d..fc9a112b2 100644 --- a/pkg/ceph/rgw/admin.go +++ b/pkg/ceph/rgw/admin.go @@ -23,8 +23,18 @@ import ( "github.com/rook/rook/pkg/clusterd" ) -func RunAdminCommand(context *clusterd.Context, getClusterInfo func() (*mon.ClusterInfo, error), command, subcommand string, args ...string) (string, error) { - cluster, err := getClusterInfo() +type Context struct { + context *clusterd.Context + objectStore string + getClusterInfo func() (*mon.ClusterInfo, error) +} + +func NewContext(context *clusterd.Context, objectStore string, getClusterInfo func() (*mon.ClusterInfo, error)) *Context { + return &Context{context: context, objectStore: objectStore, getClusterInfo: getClusterInfo} +} + +func RunAdminCommand(c *Context, command, subcommand string, args ...string) (string, error) { + cluster, err := c.getClusterInfo() if err != nil { return "", fmt.Errorf("failed to get cluster info. %+v", err) } @@ -32,12 +42,14 @@ func RunAdminCommand(context *clusterd.Context, getClusterInfo func() (*mon.Clus options := []string{ command, subcommand, + fmt.Sprintf("--rgw-realm=%s", c.objectStore), + fmt.Sprintf("--rgw-zonegroup=%s", c.objectStore), } - options = client.AppendAdminConnectionArgs(options, context.ConfigDir, cluster.Name) + options = client.AppendAdminConnectionArgs(options, c.context.ConfigDir, cluster.Name) options = append(options, args...) // start the rgw admin command - output, err := context.Executor.ExecuteCommandWithCombinedOutput(false, "", "radosgw-admin", options...) + output, err := c.context.Executor.ExecuteCommandWithCombinedOutput(false, "", "radosgw-admin", options...) if err != nil { return "", fmt.Errorf("failed to run radosgw-admin: %+v", err) } diff --git a/pkg/ceph/rgw/bucket.go b/pkg/ceph/rgw/bucket.go index 8ba9ddfc5..1f0cd3320 100644 --- a/pkg/ceph/rgw/bucket.go +++ b/pkg/ceph/rgw/bucket.go @@ -21,8 +21,6 @@ import ( "strings" "time" - "github.com/rook/rook/pkg/ceph/mon" - "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/model" ) @@ -43,8 +41,8 @@ func bucketStatsFromRGW(stats rgwBucketStats) model.ObjectBucketStats { return s } -func GetBucketStats(context *clusterd.Context, bucketName string, getClusterInfo func() (*mon.ClusterInfo, error)) (*model.ObjectBucketStats, bool, error) { - result, err := RunAdminCommand(context, getClusterInfo, +func GetBucketStats(c *Context, bucketName string) (*model.ObjectBucketStats, bool, error) { + result, err := RunAdminCommand(c, "bucket", "stats", "--bucket", bucketName) @@ -66,8 +64,8 @@ func GetBucketStats(context *clusterd.Context, bucketName string, getClusterInfo return &stat, false, nil } -func GetBucketsStats(context *clusterd.Context, getClusterInfo func() (*mon.ClusterInfo, error)) (map[string]model.ObjectBucketStats, error) { - result, err := RunAdminCommand(context, getClusterInfo, +func GetBucketsStats(c *Context) (map[string]model.ObjectBucketStats, error) { + result, err := RunAdminCommand(c, "bucket", "stats") if err != nil { @@ -88,8 +86,8 @@ func GetBucketsStats(context *clusterd.Context, getClusterInfo func() (*mon.Clus return stats, nil } -func getBucketMetadata(context *clusterd.Context, bucket string, getClusterInfo func() (*mon.ClusterInfo, error)) (*model.ObjectBucketMetadata, bool, error) { - result, err := RunAdminCommand(context, getClusterInfo, +func getBucketMetadata(c *Context, bucket string) (*model.ObjectBucketMetadata, bool, error) { + result, err := RunAdminCommand(c, "metadata", "get", "bucket:"+bucket) @@ -119,10 +117,10 @@ func getBucketMetadata(context *clusterd.Context, bucket string, getClusterInfo return &model.ObjectBucketMetadata{Owner: s.Data.Owner, CreatedAt: createdAt}, false, nil } -func ListBuckets(context *clusterd.Context, getClusterInfo func() (*mon.ClusterInfo, error)) ([]model.ObjectBucket, error) { +func ListBuckets(c *Context) ([]model.ObjectBucket, error) { logger.Infof("Listing buckets") - stats, err := GetBucketsStats(context, getClusterInfo) + stats, err := GetBucketsStats(c) if err != nil { return nil, fmt.Errorf("Failed to get bucket stats: %+v", err) } @@ -130,7 +128,7 @@ func ListBuckets(context *clusterd.Context, getClusterInfo func() (*mon.ClusterI buckets := []model.ObjectBucket{} for bucket, stat := range stats { - metadata, _, err := getBucketMetadata(context, bucket, getClusterInfo) + metadata, _, err := getBucketMetadata(c, bucket) if err != nil { return nil, err } @@ -141,8 +139,8 @@ func ListBuckets(context *clusterd.Context, getClusterInfo func() (*mon.ClusterI return buckets, nil } -func GetBucket(context *clusterd.Context, bucket string, getClusterInfo func() (*mon.ClusterInfo, error)) (*model.ObjectBucket, int, error) { - stat, notFound, err := GetBucketStats(context, bucket, getClusterInfo) +func GetBucket(c *Context, bucket string) (*model.ObjectBucket, int, error) { + stat, notFound, err := GetBucketStats(c, bucket) if notFound { return nil, RGWErrorNotFound, fmt.Errorf("Bucket not found") } @@ -151,7 +149,7 @@ func GetBucket(context *clusterd.Context, bucket string, getClusterInfo func() ( return nil, RGWErrorUnknown, fmt.Errorf("Failed to get bucket stats: %+v", err) } - metadata, notFound, err := getBucketMetadata(context, bucket, getClusterInfo) + metadata, notFound, err := getBucketMetadata(c, bucket) if notFound { return nil, RGWErrorNotFound, fmt.Errorf("Bucket not found") } @@ -163,16 +161,13 @@ func GetBucket(context *clusterd.Context, bucket string, getClusterInfo func() ( return &model.ObjectBucket{Name: bucket, ObjectBucketMetadata: model.ObjectBucketMetadata{Owner: metadata.Owner, CreatedAt: metadata.CreatedAt}, ObjectBucketStats: *stat}, RGWErrorNone, nil } -func DeleteBucket(context *clusterd.Context, bucketName string, purge bool, getClusterInfo func() (*mon.ClusterInfo, error)) (int, error) { +func DeleteBucket(c *Context, bucketName string, purge bool) (int, error) { options := []string{"--bucket", bucketName} if purge { options = append(options, "--purge-objects") } - result, err := RunAdminCommand(context, getClusterInfo, - "bucket", - "rm", - options...) + result, err := RunAdminCommand(c, "bucket", "rm", options...) if err != nil { return RGWErrorUnknown, fmt.Errorf("failed to delete bucket: %+v", err) } diff --git a/pkg/ceph/rgw/daemon.go b/pkg/ceph/rgw/daemon.go index 0c3feae35..24d9d9e11 100644 --- a/pkg/ceph/rgw/daemon.go +++ b/pkg/ceph/rgw/daemon.go @@ -31,6 +31,7 @@ import ( ) type Config struct { + Name string Host string Port int Keyring string @@ -73,6 +74,8 @@ func generateConfigFiles(context *clusterd.Context, config *Config) error { "rgw log nonexistent bucket": "true", "rgw intent log object name utc": "true", "rgw enable usage log": "true", + "rgw_zone": config.Name, + "rgw_zonegroup": config.Name, } _, err := mon.GenerateConfigFile(context, config.ClusterInfo, getRGWConfDir(context.ConfigDir), "client.radosgw.gateway", getRGWKeyringPath(context.ConfigDir), false, nil, settings) diff --git a/pkg/ceph/rgw/user.go b/pkg/ceph/rgw/user.go index 27bce5960..0dff4b8dd 100644 --- a/pkg/ceph/rgw/user.go +++ b/pkg/ceph/rgw/user.go @@ -21,8 +21,6 @@ import ( "strings" - "github.com/rook/rook/pkg/ceph/mon" - "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/model" ) @@ -34,8 +32,8 @@ const ( RGWErrorParse = iota ) -func ListUsers(context *clusterd.Context, getClusterInfo func() (*mon.ClusterInfo, error)) ([]string, int, error) { - result, err := RunAdminCommand(context, getClusterInfo, "user", "list") +func ListUsers(c *Context) ([]string, int, error) { + result, err := RunAdminCommand(c, "user", "list") if err != nil { return nil, RGWErrorUnknown, fmt.Errorf("failed to list users: %+v", err) } @@ -75,10 +73,10 @@ func decodeUser(data string) (*model.ObjectUser, int, error) { return &rookUser, RGWErrorNone, nil } -func GetUser(context *clusterd.Context, id string, getClusterInfo func() (*mon.ClusterInfo, error)) (*model.ObjectUser, int, error) { +func GetUser(c *Context, id string) (*model.ObjectUser, int, error) { logger.Infof("Getting user: %s", id) - result, err := RunAdminCommand(context, getClusterInfo, "user", "info", "--uid", id) + result, err := RunAdminCommand(c, "user", "info", "--uid", id) if err != nil { return nil, RGWErrorUnknown, fmt.Errorf("failed to get users: %+v", err) } @@ -88,7 +86,7 @@ func GetUser(context *clusterd.Context, id string, getClusterInfo func() (*mon.C return decodeUser(result) } -func CreateUser(context *clusterd.Context, user model.ObjectUser, getClusterInfo func() (*mon.ClusterInfo, error)) (*model.ObjectUser, int, error) { +func CreateUser(c *Context, user model.ObjectUser) (*model.ObjectUser, int, error) { logger.Infof("Creating user: %s", user.UserID) if strings.TrimSpace(user.UserID) == "" { @@ -108,7 +106,7 @@ func CreateUser(context *clusterd.Context, user model.ObjectUser, getClusterInfo args = append(args, "--email", *user.Email) } - result, err := RunAdminCommand(context, getClusterInfo, "user", "create", args...) + result, err := RunAdminCommand(c, "user", "create", args...) if err != nil { return nil, RGWErrorUnknown, fmt.Errorf("failed to create user: %+v", err) } @@ -124,7 +122,7 @@ func CreateUser(context *clusterd.Context, user model.ObjectUser, getClusterInfo return decodeUser(result) } -func UpdateUser(context *clusterd.Context, user model.ObjectUser, getClusterInfo func() (*mon.ClusterInfo, error)) (*model.ObjectUser, int, error) { +func UpdateUser(c *Context, user model.ObjectUser) (*model.ObjectUser, int, error) { logger.Infof("Updating user: %s", user.UserID) args := []string{"--uid", user.UserID} @@ -136,7 +134,7 @@ func UpdateUser(context *clusterd.Context, user model.ObjectUser, getClusterInfo args = append(args, "--email", *user.Email) } - body, err := RunAdminCommand(context, getClusterInfo, "user", "modify", args...) + body, err := RunAdminCommand(c, "user", "modify", args...) if err != nil { return nil, RGWErrorUnknown, fmt.Errorf("failed to update user: %+v", err) } @@ -148,9 +146,9 @@ func UpdateUser(context *clusterd.Context, user model.ObjectUser, getClusterInfo return decodeUser(body) } -func DeleteUser(context *clusterd.Context, id string, getClusterInfo func() (*mon.ClusterInfo, error)) (string, int, error) { +func DeleteUser(c *Context, id string) (string, int, error) { logger.Infof("Deleting user: %s", id) - result, err := RunAdminCommand(context, getClusterInfo, "user", "rm", "--uid", id) + result, err := RunAdminCommand(c, "user", "rm", "--uid", id) if err != nil { return "", RGWErrorUnknown, fmt.Errorf("failed to delete user: %+v", err) } diff --git a/pkg/operator/rgw/rgw.go b/pkg/operator/rgw/rgw.go index 37f804d14..0fdf69c5e 100644 --- a/pkg/operator/rgw/rgw.go +++ b/pkg/operator/rgw/rgw.go @@ -18,9 +18,11 @@ limitations under the License. package rgw import ( + "encoding/json" "fmt" "github.com/coreos/pkg/capnslog" + "github.com/rook/rook/pkg/ceph/client" cephrgw "github.com/rook/rook/pkg/ceph/rgw" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/operator/k8sutil" @@ -42,6 +44,7 @@ const ( // Cluster for rgw management type Cluster struct { context *clusterd.Context + Name string Namespace string placement k8sutil.Placement Version string @@ -49,9 +52,10 @@ type Cluster struct { } // New creates an instance of an rgw manager -func New(context *clusterd.Context, namespace, version string, placement k8sutil.Placement) *Cluster { +func New(context *clusterd.Context, name, namespace, version string, placement k8sutil.Placement) *Cluster { return &Cluster{ context: context, + Name: name, Namespace: namespace, placement: placement, Version: version, @@ -69,11 +73,13 @@ func (c *Cluster) Start() error { } // start the service - err = c.startService() + serviceIP, err := c.startService() if err != nil { return fmt.Errorf("failed to start rgw service. %+v", err) } + err = c.createRealm(serviceIP) + // start the deployment deployment := c.makeDeployment() _, err = c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment) @@ -89,8 +95,75 @@ func (c *Cluster) Start() error { return nil } +type idType struct { + ID string `json:"id"` +} + +func (c *Cluster) createRealm(serviceIP string) error { + output, err := c.runRGWCommand("realm", "create", fmt.Sprintf("--rgw-realm=%s", c.Name)) + if err != nil { + return fmt.Errorf("failed to create rgw realm %s. %+v", c.Name, err) + } + + realmID, err := decodeID(output) + if err != nil { + return fmt.Errorf("failed to parse realm id. %+v", err) + } + + output, err = c.runRGWCommand("zonegroup", "create", "--master", + fmt.Sprintf("--endpoints=%s:%d", serviceIP, cephrgw.RGWPort), + fmt.Sprintf("--rgw-zonegroup=%s", c.Name), + fmt.Sprintf("--rgw-realm=%s", c.Name)) + if err != nil { + return fmt.Errorf("failed to create rgw zonegroup for %s. %+v", c.Name, err) + } + + zoneGroupID, err := decodeID(output) + if err != nil { + return fmt.Errorf("failed to parse realm id. %+v", err) + } + + output, err = c.runRGWCommand("zone", "create", "--master", + fmt.Sprintf("--endpoints=%s:%d", serviceIP, cephrgw.RGWPort), + fmt.Sprintf("--rgw-zone=%s", c.Name), + fmt.Sprintf("--rgw-zonegroup=%s", c.Name), + fmt.Sprintf("--rgw-realm=%s", c.Name)) + if err != nil { + return fmt.Errorf("failed to create rgw zonegroup for %s. %+v", c.Name, err) + } + + zoneID, err := decodeID(output) + if err != nil { + return fmt.Errorf("failed to parse zone id. %+v", err) + } + + logger.Infof("RGW: realm=%s, zonegroup=%s, zone=%s", realmID, zoneGroupID, zoneID) + return nil +} + +func decodeID(data string) (string, error) { + var id idType + err := json.Unmarshal([]byte(data), &id) + if err != nil { + return "", fmt.Errorf("Failed to unmarshal json: %+v", err) + } + + return id.ID, err +} + +func (c *Cluster) runRGWCommand(args ...string) (string, error) { + options := client.AppendAdminConnectionArgs(args, c.context.ConfigDir, c.Namespace) + + // start the rgw admin command + output, err := c.context.Executor.ExecuteCommandWithCombinedOutput(false, "", "radosgw-admin", options...) + if err != nil { + return "", fmt.Errorf("failed to run radosgw-admin: %+v", err) + } + return output, nil +} + func (c *Cluster) createKeyring() error { - _, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(appName, metav1.GetOptions{}) + _, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(c.instanceName(), metav1.GetOptions{}) if err == nil { logger.Infof("the rgw keyring was already generated") return nil @@ -111,7 +184,7 @@ func (c *Cluster) createKeyring() error { keyringName: keyring, } secret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: appName, Namespace: c.Namespace}, + ObjectMeta: metav1.ObjectMeta{Name: c.instanceName(), Namespace: c.Namespace}, StringData: secrets, Type: k8sutil.RookType, } @@ -123,9 +196,17 @@ func (c *Cluster) createKeyring() error { return nil } +func (c *Cluster) instanceName() string { + return InstanceName(c.Name) +} + +func InstanceName(name string) string { + return fmt.Sprintf("%s-%s", appName, name) +} + func (c *Cluster) makeDeployment() *extensions.Deployment { deployment := &extensions.Deployment{} - deployment.Name = appName + deployment.Name = c.instanceName() deployment.Namespace = c.Namespace podSpec := v1.PodSpec{ @@ -158,17 +239,18 @@ func (c *Cluster) rgwContainer() v1.Container { Args: []string{ "rgw", fmt.Sprintf("--config-dir=%s", k8sutil.DataDir), + fmt.Sprintf("--rgw-name=%s", c.Name), fmt.Sprintf("--rgw-port=%d", cephrgw.RGWPort), fmt.Sprintf("--rgw-host=%s", cephrgw.DNSName), }, - Name: appName, + Name: c.instanceName(), Image: k8sutil.MakeRookImage(c.Version), VolumeMounts: []v1.VolumeMount{ {Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir}, k8sutil.ConfigOverrideMount(), }, Env: []v1.EnvVar{ - {Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: appName}, Key: keyringName}}}, + {Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: c.instanceName()}, Key: keyringName}}}, k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar), k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar), opmon.ClusterNameEnvVar(c.Namespace), @@ -180,18 +262,18 @@ func (c *Cluster) rgwContainer() v1.Container { } } -func (c *Cluster) startService() error { +func (c *Cluster) startService() (string, error) { labels := c.getLabels() s := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: appName, + Name: c.instanceName(), Namespace: c.Namespace, Labels: labels, }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ { - Name: appName, + Name: c.instanceName(), Port: cephrgw.RGWPort, TargetPort: intstr.FromInt(int(cephrgw.RGWPort)), Protocol: v1.ProtocolTCP, @@ -204,19 +286,20 @@ func (c *Cluster) startService() error { s, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(s) if err != nil { if !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create mon service. %+v", err) + return "", fmt.Errorf("failed to create mon service. %+v", err) } logger.Infof("RGW service already running") - return nil + return "", nil } logger.Infof("RGW service running at %s:%d", s.Spec.ClusterIP, cephrgw.RGWPort) - return nil + return s.Spec.ClusterIP, nil } func (c *Cluster) getLabels() map[string]string { return map[string]string{ k8sutil.AppAttr: appName, k8sutil.ClusterAttr: c.Namespace, + "rook_object_store": c.Name, } } diff --git a/pkg/operator/rgw/rgw_test.go b/pkg/operator/rgw/rgw_test.go index d4e1cd3bf..536c20de0 100644 --- a/pkg/operator/rgw/rgw_test.go +++ b/pkg/operator/rgw/rgw_test.go @@ -42,7 +42,7 @@ func TestStartRGW(t *testing.T) { configDir, _ := ioutil.TempDir("", "") defer os.RemoveAll(configDir) - c := New(&clusterd.Context{Clientset: clientset, Executor: executor, ConfigDir: configDir}, "ns", "version", k8sutil.Placement{}) + c := New(&clusterd.Context{Clientset: clientset, Executor: executor, ConfigDir: configDir}, "default", "ns", "version", k8sutil.Placement{}) // start a basic cluster err := c.Start() @@ -59,35 +59,36 @@ func TestStartRGW(t *testing.T) { func validateStart(t *testing.T, c *Cluster, clientset *fake.Clientset) { - r, err := clientset.ExtensionsV1beta1().Deployments(c.Namespace).Get(appName, metav1.GetOptions{}) + r, err := clientset.ExtensionsV1beta1().Deployments(c.Namespace).Get(c.instanceName(), metav1.GetOptions{}) assert.Nil(t, err) - assert.Equal(t, appName, r.Name) + assert.Equal(t, c.instanceName(), r.Name) - s, err := clientset.CoreV1().Services(c.Namespace).Get(appName, metav1.GetOptions{}) + s, err := clientset.CoreV1().Services(c.Namespace).Get(c.instanceName(), metav1.GetOptions{}) assert.Nil(t, err) - assert.Equal(t, appName, s.Name) + assert.Equal(t, c.instanceName(), s.Name) - secret, err := clientset.CoreV1().Secrets(c.Namespace).Get(appName, metav1.GetOptions{}) + secret, err := clientset.CoreV1().Secrets(c.Namespace).Get(c.instanceName(), metav1.GetOptions{}) assert.Nil(t, err) - assert.Equal(t, appName, secret.Name) + assert.Equal(t, c.instanceName(), secret.Name) assert.Equal(t, 1, len(secret.StringData)) } func TestPodSpecs(t *testing.T) { - c := New(nil, "ns", "myversion", k8sutil.Placement{}) + c := New(nil, "default", "ns", "myversion", k8sutil.Placement{}) d := c.makeDeployment() assert.NotNil(t, d) - assert.Equal(t, appName, d.Name) + assert.Equal(t, c.instanceName(), d.Name) assert.Equal(t, v1.RestartPolicyAlways, d.Spec.Template.Spec.RestartPolicy) assert.Equal(t, 2, len(d.Spec.Template.Spec.Volumes)) assert.Equal(t, "rook-data", d.Spec.Template.Spec.Volumes[0].Name) assert.Equal(t, k8sutil.ConfigOverrideName, d.Spec.Template.Spec.Volumes[1].Name) - assert.Equal(t, appName, d.ObjectMeta.Name) + assert.Equal(t, c.instanceName(), d.ObjectMeta.Name) assert.Equal(t, appName, d.Spec.Template.ObjectMeta.Labels["app"]) assert.Equal(t, c.Namespace, d.Spec.Template.ObjectMeta.Labels["rook_cluster"]) + assert.Equal(t, c.Name, d.Spec.Template.ObjectMeta.Labels["rook_object_store"]) assert.Equal(t, 0, len(d.ObjectMeta.Annotations)) cont := d.Spec.Template.Spec.Containers[0] @@ -96,6 +97,7 @@ func TestPodSpecs(t *testing.T) { assert.Equal(t, "rgw", cont.Args[0]) assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[1]) - assert.Equal(t, fmt.Sprintf("--rgw-port=%d", cephrgw.RGWPort), cont.Args[2]) - assert.Equal(t, fmt.Sprintf("--rgw-host=%s", cephrgw.DNSName), cont.Args[3]) + assert.Equal(t, fmt.Sprintf("--rgw-name=%s", "default"), cont.Args[2]) + assert.Equal(t, fmt.Sprintf("--rgw-port=%d", cephrgw.RGWPort), cont.Args[3]) + assert.Equal(t, fmt.Sprintf("--rgw-host=%s", cephrgw.DNSName), cont.Args[4]) }