rgw: support alternate object store realms

This commit is contained in:
Travis Nielsen committed 2017-09-06 10:59:32 -07:00
1 parent 6fe912b5cc
commit bb6e05ec57
11 files changed
+198 -96

No files matched your search

+4 -1
View File
@@ -32,12 +32,14 @@ var rgwCmd = &cobra.Command{
}
var (
rgwName string
rgwKeyring string
rgwHost string
rgwPort int
)
func init() {
rgwCmd.Flags().StringVar(&rgwName, "rgw-name", "", "name of the object store")
rgwCmd.Flags().StringVar(&rgwKeyring, "rgw-keyring", "", "the rgw keyring")
rgwCmd.Flags().StringVar(&rgwHost, "rgw-host", "", "dns host name")
rgwCmd.Flags().IntVar(&rgwPort, "rgw-port", 0, "rgw port number")
@@ -49,7 +51,7 @@ func init() {
}
func startRGW(cmd *cobra.Command, args []string) error {
required := []string{"mon-endpoints", "cluster-name", "mon-secret", "admin-secret", "rgw-host", "rgw-keyring", "public-ipv4", "private-ipv4"}
required := []string{"mon-endpoints", "cluster-name", "mon-secret", "admin-secret", "rgw-name", "rgw-host", "rgw-keyring", "public-ipv4", "private-ipv4"}
if err := flags.VerifyRequiredFlags(rgwCmd, required); err != nil {
return err
}
@@ -65,6 +67,7 @@ func startRGW(cmd *cobra.Command, args []string) error {
clusterInfo.Monitors = mon.ParseMonEndpoints(cfg.monEndpoints)
config := &rgw.Config{
ClusterInfo: &clusterInfo,
Name: rgwName,
Keyring: rgwKeyring,
Host: rgwHost,
Port: rgwPort,
+6 -6
View File
@@ -29,9 +29,9 @@ import (
type ClusterHandler interface {
GetClusterInfo() (*mon.ClusterInfo, error)
EnableObjectStore() error
RemoveObjectStore() error
GetObjectStoreConnectionInfo() (s3info *model.ObjectStoreConnectInfo, found bool, err error)
EnableObjectStore(name string) error
RemoveObjectStore(name string) error
GetObjectStoreConnectionInfo(name string) (s3info *model.ObjectStoreConnectInfo, found bool, err error)
StartFileSystem(fs *model.FilesystemRequest) error
RemoveFileSystem(fs *model.FilesystemRequest) error
GetMonitors() (map[string]*mon.CephMonitorConfig, error)
@@ -50,15 +50,15 @@ func (e *etcdHandler) GetClusterInfo() (*mon.ClusterInfo, error) {
return mon.LoadClusterInfo(e.context.EtcdClient)
}
func (e *etcdHandler) EnableObjectStore() error {
func (e *etcdHandler) EnableObjectStore(name string) error {
return rgw.EnableObjectStore(e.context.EtcdClient)
}
func (e *etcdHandler) RemoveObjectStore() error {
func (e *etcdHandler) RemoveObjectStore(name string) error {
return rgw.RemoveObjectStore(e.context.EtcdClient)
}
func (e *etcdHandler) GetObjectStoreConnectionInfo() (*model.ObjectStoreConnectInfo, bool, error) {
func (e *etcdHandler) GetObjectStoreConnectionInfo(name string) (*model.ObjectStoreConnectInfo, bool, error) {
clusterInventory, err := inventory.LoadDiscoveredNodes(e.context.EtcdClient)
if err != nil {
+6 -6
View File
@@ -43,11 +43,11 @@ func (s *clusterHandler) GetClusterInfo() (*mon.ClusterInfo, error) {
return s.clusterInfo, nil
}
func (s *clusterHandler) EnableObjectStore() error {
func (s *clusterHandler) EnableObjectStore(name string) error {
logger.Infof("Starting the Object store")
// Passing an empty Placement{} as the api doesn't know about placement
// information. This should be resolved with the transition to CRD (TPR).
r := k8srgw.New(s.context, s.namespace, s.versionTag, k8sutil.Placement{})
r := k8srgw.New(s.context, name, s.namespace, s.versionTag, k8sutil.Placement{})
err := r.Start()
if err != nil {
return fmt.Errorf("failed to start rgw. %+v", err)
@@ -55,20 +55,20 @@ func (s *clusterHandler) EnableObjectStore() error {
return nil
}
func (s *clusterHandler) RemoveObjectStore() error {
func (s *clusterHandler) RemoveObjectStore(name string) error {
logger.Infof("TODO: Remove the object store")
return nil
}
func (s *clusterHandler) GetObjectStoreConnectionInfo() (*model.ObjectStoreConnectInfo, bool, error) {
func (s *clusterHandler) GetObjectStoreConnectionInfo(name string) (*model.ObjectStoreConnectInfo, bool, error) {
logger.Infof("Getting the object store connection info")
service, err := s.context.Clientset.CoreV1().Services(s.namespace).Get("rook-ceph-rgw", metav1.GetOptions{})
service, err := s.context.Clientset.CoreV1().Services(s.namespace).Get(k8srgw.InstanceName(name), metav1.GetOptions{})
if err != nil {
return nil, false, fmt.Errorf("failed to get rgw service. %+v", err)
}
info := &model.ObjectStoreConnectInfo{
Host: "rook-ceph-rgw",
Host: k8srgw.InstanceName(name),
IPEndpoint: rgw.GetRGWEndpoint(service.Spec.ClusterIP),
}
logger.Infof("Object store connection: %+v", info)
+18 -12
View File
@@ -25,12 +25,18 @@ import (
"github.com/rook/rook/pkg/model"
)
const objectStoreName = "default"
func (h *Handler) objectContext() *rgw.Context {
return rgw.NewContext(h.context, objectStoreName, h.config.ClusterHandler.GetClusterInfo)
}
// CreateObjectStore creates a new object store in this cluster.
// POST
// /objectstore
func (h *Handler) CreateObjectStore(w http.ResponseWriter, r *http.Request) {
if err := h.config.ClusterHandler.EnableObjectStore(); err != nil {
if err := h.config.ClusterHandler.EnableObjectStore(objectStoreName); err != nil {
logger.Errorf("failed to create object store: %+v", err)
w.WriteHeader(http.StatusInternalServerError)
return
@@ -44,7 +50,7 @@ func (h *Handler) CreateObjectStore(w http.ResponseWriter, r *http.Request) {
// DELETE
// /objectstore
func (h *Handler) RemoveObjectStore(w http.ResponseWriter, r *http.Request) {
if err := h.config.ClusterHandler.RemoveObjectStore(); err != nil {
if err := h.config.ClusterHandler.RemoveObjectStore(objectStoreName); err != nil {
logger.Errorf("failed to remove object store: %+v", err)
w.WriteHeader(http.StatusInternalServerError)
return
@@ -59,7 +65,7 @@ func (h *Handler) RemoveObjectStore(w http.ResponseWriter, r *http.Request) {
// /objectstore/connectioninfo
func (h *Handler) GetObjectStoreConnectionInfo(w http.ResponseWriter, r *http.Request) {
s3Info, found, err := h.config.ClusterHandler.GetObjectStoreConnectionInfo()
s3Info, found, err := h.config.ClusterHandler.GetObjectStoreConnectionInfo(objectStoreName)
if err != nil {
logger.Errorf("failed get object store info. %+v", err)
if found {
@@ -77,7 +83,7 @@ func (h *Handler) GetObjectStoreConnectionInfo(w http.ResponseWriter, r *http.Re
// GET
// /objectstore/users
func (h *Handler) ListUsers(w http.ResponseWriter, r *http.Request) {
userNames, _, err := rgw.ListUsers(h.context, h.config.ClusterHandler.GetClusterInfo)
userNames, _, err := rgw.ListUsers(h.objectContext())
if err != nil {
logger.Errorf("Error listing users: %+v", err)
w.WriteHeader(http.StatusInternalServerError)
@@ -86,7 +92,7 @@ func (h *Handler) ListUsers(w http.ResponseWriter, r *http.Request) {
users := []model.ObjectUser{}
for _, userName := range userNames {
user, _, err := rgw.GetUser(h.context, userName, h.config.ClusterHandler.GetClusterInfo)
user, _, err := rgw.GetUser(h.objectContext(), userName)
if err != nil {
logger.Errorf("Error listing users: %+v", err)
w.WriteHeader(http.StatusInternalServerError)
@@ -105,7 +111,7 @@ func (h *Handler) ListUsers(w http.ResponseWriter, r *http.Request) {
func (h *Handler) GetUser(w http.ResponseWriter, r *http.Request) {
id := mux.Vars(r)["id"]
user, rgwError, err := rgw.GetUser(h.context, id, h.config.ClusterHandler.GetClusterInfo)
user, rgwError, err := rgw.GetUser(h.objectContext(), id)
if err != nil {
logger.Errorf("Error getting user (%s): %+v", id, err)
@@ -133,7 +139,7 @@ func (h *Handler) CreateUser(w http.ResponseWriter, r *http.Request) {
return
}
createdUser, rgwError, err := rgw.CreateUser(h.context, user, h.config.ClusterHandler.GetClusterInfo)
createdUser, rgwError, err := rgw.CreateUser(h.objectContext(), user)
if err != nil {
logger.Errorf("Error creating user: %+v", err)
@@ -167,7 +173,7 @@ func (h *Handler) UpdateUser(w http.ResponseWriter, r *http.Request) {
user.UserID = id
updatedUser, rgwError, err := rgw.UpdateUser(h.context, user, h.config.ClusterHandler.GetClusterInfo)
updatedUser, rgwError, err := rgw.UpdateUser(h.objectContext(), user)
if err != nil {
logger.Errorf("Error updating user: %+v", err)
@@ -188,7 +194,7 @@ func (h *Handler) UpdateUser(w http.ResponseWriter, r *http.Request) {
func (h *Handler) DeleteUser(w http.ResponseWriter, r *http.Request) {
id := mux.Vars(r)["id"]
_, rgwError, err := rgw.DeleteUser(h.context, id, h.config.ClusterHandler.GetClusterInfo)
_, rgwError, err := rgw.DeleteUser(h.objectContext(), id)
if err != nil {
if rgwError == rgw.RGWErrorNotFound {
w.WriteHeader(http.StatusNotFound)
@@ -207,7 +213,7 @@ func (h *Handler) DeleteUser(w http.ResponseWriter, r *http.Request) {
// GET
// /objectstore/buckets
func (h *Handler) ListBuckets(w http.ResponseWriter, r *http.Request) {
buckets, err := rgw.ListBuckets(h.context, h.config.ClusterHandler.GetClusterInfo)
buckets, err := rgw.ListBuckets(h.objectContext())
if err != nil {
logger.Errorf("Error listing buckets: %+v", err)
w.WriteHeader(http.StatusInternalServerError)
@@ -226,7 +232,7 @@ func (h *Handler) ListBuckets(w http.ResponseWriter, r *http.Request) {
func (h *Handler) GetBucket(w http.ResponseWriter, r *http.Request) {
bucketName := mux.Vars(r)["bucketName"]
user, rgwError, err := rgw.GetBucket(h.context, bucketName, h.config.ClusterHandler.GetClusterInfo)
user, rgwError, err := rgw.GetBucket(h.objectContext(), bucketName)
if err != nil {
logger.Errorf("Error getting bucket (%s): %+v", bucketName, err)
@@ -250,7 +256,7 @@ func (h *Handler) DeleteBucket(w http.ResponseWriter, r *http.Request) {
purgeParams, found := r.URL.Query()["purge"]
purge := found && len(purgeParams) == 1 && purgeParams[0] == "true"
rgwError, err := rgw.DeleteBucket(h.context, bucketName, purge, h.config.ClusterHandler.GetClusterInfo)
rgwError, err := rgw.DeleteBucket(h.objectContext(), bucketName, purge)
if err != nil {
if rgwError == rgw.RGWErrorNotFound {
w.WriteHeader(http.StatusNotFound)
+11 -11
View File
@@ -142,8 +142,8 @@ func TestListUsers(t *testing.T) {
expectedConfigArg := getExectedConfigArg(configSubDir)
expectedKeyringArg := getExpectedKeyringArg(configSubDir)
expectedListArgs := []string{"user", "list", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg}
expectedInfoArgs := []string{"user", "info", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid"}
expectedListArgs := []string{"user", "list", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg}
expectedInfoArgs := []string{"user", "info", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid"}
// Empty list
w := runTest(func(args ...string) (string, error) {
@@ -247,7 +247,7 @@ func TestGetUser(t *testing.T) {
expectedConfigArg := getExectedConfigArg(configSubDir)
expectedKeyringArg := getExpectedKeyringArg(configSubDir)
expectedArgs := []string{"user", "info", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "someuser"}
expectedArgs := []string{"user", "info", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "someuser"}
// Error getting user
w := runTest("", fmt.Errorf("some error"), expectedArgs...)
@@ -316,7 +316,7 @@ func TestCreateUser(t *testing.T) {
expectedConfigArg := getExectedConfigArg(configSubDir)
expectedKeyringArg := getExpectedKeyringArg(configSubDir)
expectedDisplayNameArgs := []string{"user", "create", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "foo", "--display-name", "the foo"}
expectedDisplayNameArgs := []string{"user", "create", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "foo", "--display-name", "the foo"}
expectedDisplayNameAndEmailArgs := append(expectedDisplayNameArgs, "--email", "test@example.com")
// Empty body
@@ -396,7 +396,7 @@ func TestUpdateUser(t *testing.T) {
expectedConfigArg := getExectedConfigArg(configSubDir)
expectedKeyringArg := getExpectedKeyringArg(configSubDir)
expectedArgs := []string{"user", "modify", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "foo"}
expectedArgs := []string{"user", "modify", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "foo"}
// Empty body
w := runTest("", "", nil, "", "")
@@ -479,7 +479,7 @@ func TestDeleteUser(t *testing.T) {
expectedConfigArg := getExectedConfigArg(configSubDir)
expectedKeyringArg := getExpectedKeyringArg(configSubDir)
expectedArgs := []string{"user", "rm", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "foo"}
expectedArgs := []string{"user", "rm", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--uid", "foo"}
// Some error
w := runTest("", fmt.Errorf("some error"), expectedArgs...)
@@ -528,8 +528,8 @@ func TestListBuckets(t *testing.T) {
expectedConfigArg := getExectedConfigArg(configSubDir)
expectedKeyringArg := getExpectedKeyringArg(configSubDir)
expectedStatsArgs := []string{"bucket", "stats", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg}
expectedMetadataArgs := []string{"metadata", "get", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "bucket:foo"}
expectedStatsArgs := []string{"bucket", "stats", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg}
expectedMetadataArgs := []string{"metadata", "get", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "bucket:foo"}
// List error
w := runTest(func(args ...string) (string, error) {
@@ -676,8 +676,8 @@ func TestGetBucket(t *testing.T) {
expectedConfigArg := getExectedConfigArg(configSubDir)
expectedKeyringArg := getExpectedKeyringArg(configSubDir)
expectedStatsArgs := []string{"bucket", "stats", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--bucket", "test"}
expectedMetadataArgs := []string{"metadata", "get", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "bucket:test"}
expectedStatsArgs := []string{"bucket", "stats", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--bucket", "test"}
expectedMetadataArgs := []string{"metadata", "get", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "bucket:test"}
// Stats fails
w := runTest(func(args ...string) (string, error) {
@@ -795,7 +795,7 @@ func TestBucketDelete(t *testing.T) {
expectedConfigArg := getExectedConfigArg(configSubDir)
expectedKeyringArg := getExpectedKeyringArg(configSubDir)
expectedArgs := []string{"bucket", "rm", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--bucket", "test"}
expectedArgs := []string{"bucket", "rm", "--rgw-realm=default", "--rgw-zonegroup=default", "--cluster=rookcluster", expectedConfigArg, expectedKeyringArg, "--bucket", "test"}
// errors
w := runTest("", fmt.Errorf("some error"), expectedArgs...)
+16 -4
View File
@@ -23,8 +23,18 @@ import (
"github.com/rook/rook/pkg/clusterd"
)
func RunAdminCommand(context *clusterd.Context, getClusterInfo func() (*mon.ClusterInfo, error), command, subcommand string, args ...string) (string, error) {
cluster, err := getClusterInfo()
type Context struct {
context *clusterd.Context
objectStore string
getClusterInfo func() (*mon.ClusterInfo, error)
}
func NewContext(context *clusterd.Context, objectStore string, getClusterInfo func() (*mon.ClusterInfo, error)) *Context {
return &Context{context: context, objectStore: objectStore, getClusterInfo: getClusterInfo}
}
func RunAdminCommand(c *Context, command, subcommand string, args ...string) (string, error) {
cluster, err := c.getClusterInfo()
if err != nil {
return "", fmt.Errorf("failed to get cluster info. %+v", err)
}
@@ -32,12 +42,14 @@ func RunAdminCommand(context *clusterd.Context, getClusterInfo func() (*mon.Clus
options := []string{
command,
subcommand,
fmt.Sprintf("--rgw-realm=%s", c.objectStore),
fmt.Sprintf("--rgw-zonegroup=%s", c.objectStore),
}
options = client.AppendAdminConnectionArgs(options, context.ConfigDir, cluster.Name)
options = client.AppendAdminConnectionArgs(options, c.context.ConfigDir, cluster.Name)
options = append(options, args...)
// start the rgw admin command
output, err := context.Executor.ExecuteCommandWithCombinedOutput(false, "", "radosgw-admin", options...)
output, err := c.context.Executor.ExecuteCommandWithCombinedOutput(false, "", "radosgw-admin", options...)
if err != nil {
return "", fmt.Errorf("failed to run radosgw-admin: %+v", err)
}
+14 -19
View File
@@ -21,8 +21,6 @@ import (
"strings"
"time"
"github.com/rook/rook/pkg/ceph/mon"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/model"
)
@@ -43,8 +41,8 @@ func bucketStatsFromRGW(stats rgwBucketStats) model.ObjectBucketStats {
return s
}
func GetBucketStats(context *clusterd.Context, bucketName string, getClusterInfo func() (*mon.ClusterInfo, error)) (*model.ObjectBucketStats, bool, error) {
result, err := RunAdminCommand(context, getClusterInfo,
func GetBucketStats(c *Context, bucketName string) (*model.ObjectBucketStats, bool, error) {
result, err := RunAdminCommand(c,
"bucket",
"stats",
"--bucket", bucketName)
@@ -66,8 +64,8 @@ func GetBucketStats(context *clusterd.Context, bucketName string, getClusterInfo
return &stat, false, nil
}
func GetBucketsStats(context *clusterd.Context, getClusterInfo func() (*mon.ClusterInfo, error)) (map[string]model.ObjectBucketStats, error) {
result, err := RunAdminCommand(context, getClusterInfo,
func GetBucketsStats(c *Context) (map[string]model.ObjectBucketStats, error) {
result, err := RunAdminCommand(c,
"bucket",
"stats")
if err != nil {
@@ -88,8 +86,8 @@ func GetBucketsStats(context *clusterd.Context, getClusterInfo func() (*mon.Clus
return stats, nil
}
func getBucketMetadata(context *clusterd.Context, bucket string, getClusterInfo func() (*mon.ClusterInfo, error)) (*model.ObjectBucketMetadata, bool, error) {
result, err := RunAdminCommand(context, getClusterInfo,
func getBucketMetadata(c *Context, bucket string) (*model.ObjectBucketMetadata, bool, error) {
result, err := RunAdminCommand(c,
"metadata",
"get",
"bucket:"+bucket)
@@ -119,10 +117,10 @@ func getBucketMetadata(context *clusterd.Context, bucket string, getClusterInfo
return &model.ObjectBucketMetadata{Owner: s.Data.Owner, CreatedAt: createdAt}, false, nil
}
func ListBuckets(context *clusterd.Context, getClusterInfo func() (*mon.ClusterInfo, error)) ([]model.ObjectBucket, error) {
func ListBuckets(c *Context) ([]model.ObjectBucket, error) {
logger.Infof("Listing buckets")
stats, err := GetBucketsStats(context, getClusterInfo)
stats, err := GetBucketsStats(c)
if err != nil {
return nil, fmt.Errorf("Failed to get bucket stats: %+v", err)
}
@@ -130,7 +128,7 @@ func ListBuckets(context *clusterd.Context, getClusterInfo func() (*mon.ClusterI
buckets := []model.ObjectBucket{}
for bucket, stat := range stats {
metadata, _, err := getBucketMetadata(context, bucket, getClusterInfo)
metadata, _, err := getBucketMetadata(c, bucket)
if err != nil {
return nil, err
}
@@ -141,8 +139,8 @@ func ListBuckets(context *clusterd.Context, getClusterInfo func() (*mon.ClusterI
return buckets, nil
}
func GetBucket(context *clusterd.Context, bucket string, getClusterInfo func() (*mon.ClusterInfo, error)) (*model.ObjectBucket, int, error) {
stat, notFound, err := GetBucketStats(context, bucket, getClusterInfo)
func GetBucket(c *Context, bucket string) (*model.ObjectBucket, int, error) {
stat, notFound, err := GetBucketStats(c, bucket)
if notFound {
return nil, RGWErrorNotFound, fmt.Errorf("Bucket not found")
}
@@ -151,7 +149,7 @@ func GetBucket(context *clusterd.Context, bucket string, getClusterInfo func() (
return nil, RGWErrorUnknown, fmt.Errorf("Failed to get bucket stats: %+v", err)
}
metadata, notFound, err := getBucketMetadata(context, bucket, getClusterInfo)
metadata, notFound, err := getBucketMetadata(c, bucket)
if notFound {
return nil, RGWErrorNotFound, fmt.Errorf("Bucket not found")
}
@@ -163,16 +161,13 @@ func GetBucket(context *clusterd.Context, bucket string, getClusterInfo func() (
return &model.ObjectBucket{Name: bucket, ObjectBucketMetadata: model.ObjectBucketMetadata{Owner: metadata.Owner, CreatedAt: metadata.CreatedAt}, ObjectBucketStats: *stat}, RGWErrorNone, nil
}
func DeleteBucket(context *clusterd.Context, bucketName string, purge bool, getClusterInfo func() (*mon.ClusterInfo, error)) (int, error) {
func DeleteBucket(c *Context, bucketName string, purge bool) (int, error) {
options := []string{"--bucket", bucketName}
if purge {
options = append(options, "--purge-objects")
}
result, err := RunAdminCommand(context, getClusterInfo,
"bucket",
"rm",
options...)
result, err := RunAdminCommand(c, "bucket", "rm", options...)
if err != nil {
return RGWErrorUnknown, fmt.Errorf("failed to delete bucket: %+v", err)
}
+3
View File
@@ -31,6 +31,7 @@ import (
)
type Config struct {
Name string
Host string
Port int
Keyring string
@@ -73,6 +74,8 @@ func generateConfigFiles(context *clusterd.Context, config *Config) error {
"rgw log nonexistent bucket": "true",
"rgw intent log object name utc": "true",
"rgw enable usage log": "true",
"rgw_zone": config.Name,
"rgw_zonegroup": config.Name,
}
_, err := mon.GenerateConfigFile(context, config.ClusterInfo, getRGWConfDir(context.ConfigDir),
"client.radosgw.gateway", getRGWKeyringPath(context.ConfigDir), false, nil, settings)
+10 -12
View File
@@ -21,8 +21,6 @@ import (
"strings"
"github.com/rook/rook/pkg/ceph/mon"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/model"
)
@@ -34,8 +32,8 @@ const (
RGWErrorParse = iota
)
func ListUsers(context *clusterd.Context, getClusterInfo func() (*mon.ClusterInfo, error)) ([]string, int, error) {
result, err := RunAdminCommand(context, getClusterInfo, "user", "list")
func ListUsers(c *Context) ([]string, int, error) {
result, err := RunAdminCommand(c, "user", "list")
if err != nil {
return nil, RGWErrorUnknown, fmt.Errorf("failed to list users: %+v", err)
}
@@ -75,10 +73,10 @@ func decodeUser(data string) (*model.ObjectUser, int, error) {
return &rookUser, RGWErrorNone, nil
}
func GetUser(context *clusterd.Context, id string, getClusterInfo func() (*mon.ClusterInfo, error)) (*model.ObjectUser, int, error) {
func GetUser(c *Context, id string) (*model.ObjectUser, int, error) {
logger.Infof("Getting user: %s", id)
result, err := RunAdminCommand(context, getClusterInfo, "user", "info", "--uid", id)
result, err := RunAdminCommand(c, "user", "info", "--uid", id)
if err != nil {
return nil, RGWErrorUnknown, fmt.Errorf("failed to get users: %+v", err)
}
@@ -88,7 +86,7 @@ func GetUser(context *clusterd.Context, id string, getClusterInfo func() (*mon.C
return decodeUser(result)
}
func CreateUser(context *clusterd.Context, user model.ObjectUser, getClusterInfo func() (*mon.ClusterInfo, error)) (*model.ObjectUser, int, error) {
func CreateUser(c *Context, user model.ObjectUser) (*model.ObjectUser, int, error) {
logger.Infof("Creating user: %s", user.UserID)
if strings.TrimSpace(user.UserID) == "" {
@@ -108,7 +106,7 @@ func CreateUser(context *clusterd.Context, user model.ObjectUser, getClusterInfo
args = append(args, "--email", *user.Email)
}
result, err := RunAdminCommand(context, getClusterInfo, "user", "create", args...)
result, err := RunAdminCommand(c, "user", "create", args...)
if err != nil {
return nil, RGWErrorUnknown, fmt.Errorf("failed to create user: %+v", err)
}
@@ -124,7 +122,7 @@ func CreateUser(context *clusterd.Context, user model.ObjectUser, getClusterInfo
return decodeUser(result)
}
func UpdateUser(context *clusterd.Context, user model.ObjectUser, getClusterInfo func() (*mon.ClusterInfo, error)) (*model.ObjectUser, int, error) {
func UpdateUser(c *Context, user model.ObjectUser) (*model.ObjectUser, int, error) {
logger.Infof("Updating user: %s", user.UserID)
args := []string{"--uid", user.UserID}
@@ -136,7 +134,7 @@ func UpdateUser(context *clusterd.Context, user model.ObjectUser, getClusterInfo
args = append(args, "--email", *user.Email)
}
body, err := RunAdminCommand(context, getClusterInfo, "user", "modify", args...)
body, err := RunAdminCommand(c, "user", "modify", args...)
if err != nil {
return nil, RGWErrorUnknown, fmt.Errorf("failed to update user: %+v", err)
}
@@ -148,9 +146,9 @@ func UpdateUser(context *clusterd.Context, user model.ObjectUser, getClusterInfo
return decodeUser(body)
}
func DeleteUser(context *clusterd.Context, id string, getClusterInfo func() (*mon.ClusterInfo, error)) (string, int, error) {
func DeleteUser(c *Context, id string) (string, int, error) {
logger.Infof("Deleting user: %s", id)
result, err := RunAdminCommand(context, getClusterInfo, "user", "rm", "--uid", id)
result, err := RunAdminCommand(c, "user", "rm", "--uid", id)
if err != nil {
return "", RGWErrorUnknown, fmt.Errorf("failed to delete user: %+v", err)
}
+96 -13
View File
@@ -18,9 +18,11 @@ limitations under the License.
package rgw
import (
"encoding/json"
"fmt"
"github.com/coreos/pkg/capnslog"
"github.com/rook/rook/pkg/ceph/client"
cephrgw "github.com/rook/rook/pkg/ceph/rgw"
"github.com/rook/rook/pkg/clusterd"
"github.com/rook/rook/pkg/operator/k8sutil"
@@ -42,6 +44,7 @@ const (
// Cluster for rgw management
type Cluster struct {
context *clusterd.Context
Name string
Namespace string
placement k8sutil.Placement
Version string
@@ -49,9 +52,10 @@ type Cluster struct {
}
// New creates an instance of an rgw manager
func New(context *clusterd.Context, namespace, version string, placement k8sutil.Placement) *Cluster {
func New(context *clusterd.Context, name, namespace, version string, placement k8sutil.Placement) *Cluster {
return &Cluster{
context: context,
Name: name,
Namespace: namespace,
placement: placement,
Version: version,
@@ -69,11 +73,13 @@ func (c *Cluster) Start() error {
}
// start the service
err = c.startService()
serviceIP, err := c.startService()
if err != nil {
return fmt.Errorf("failed to start rgw service. %+v", err)
}
err = c.createRealm(serviceIP)
// start the deployment
deployment := c.makeDeployment()
_, err = c.context.Clientset.ExtensionsV1beta1().Deployments(c.Namespace).Create(deployment)
@@ -89,8 +95,75 @@ func (c *Cluster) Start() error {
return nil
}
type idType struct {
ID string `json:"id"`
}
func (c *Cluster) createRealm(serviceIP string) error {
output, err := c.runRGWCommand("realm", "create", fmt.Sprintf("--rgw-realm=%s", c.Name))
if err != nil {
return fmt.Errorf("failed to create rgw realm %s. %+v", c.Name, err)
}
realmID, err := decodeID(output)
if err != nil {
return fmt.Errorf("failed to parse realm id. %+v", err)
}
output, err = c.runRGWCommand("zonegroup", "create", "--master",
fmt.Sprintf("--endpoints=%s:%d", serviceIP, cephrgw.RGWPort),
fmt.Sprintf("--rgw-zonegroup=%s", c.Name),
fmt.Sprintf("--rgw-realm=%s", c.Name))
if err != nil {
return fmt.Errorf("failed to create rgw zonegroup for %s. %+v", c.Name, err)
}
zoneGroupID, err := decodeID(output)
if err != nil {
return fmt.Errorf("failed to parse realm id. %+v", err)
}
output, err = c.runRGWCommand("zone", "create", "--master",
fmt.Sprintf("--endpoints=%s:%d", serviceIP, cephrgw.RGWPort),
fmt.Sprintf("--rgw-zone=%s", c.Name),
fmt.Sprintf("--rgw-zonegroup=%s", c.Name),
fmt.Sprintf("--rgw-realm=%s", c.Name))
if err != nil {
return fmt.Errorf("failed to create rgw zonegroup for %s. %+v", c.Name, err)
}
zoneID, err := decodeID(output)
if err != nil {
return fmt.Errorf("failed to parse zone id. %+v", err)
}
logger.Infof("RGW: realm=%s, zonegroup=%s, zone=%s", realmID, zoneGroupID, zoneID)
return nil
}
func decodeID(data string) (string, error) {
var id idType
err := json.Unmarshal([]byte(data), &id)
if err != nil {
return "", fmt.Errorf("Failed to unmarshal json: %+v", err)
}
return id.ID, err
}
func (c *Cluster) runRGWCommand(args ...string) (string, error) {
options := client.AppendAdminConnectionArgs(args, c.context.ConfigDir, c.Namespace)
// start the rgw admin command
output, err := c.context.Executor.ExecuteCommandWithCombinedOutput(false, "", "radosgw-admin", options...)
if err != nil {
return "", fmt.Errorf("failed to run radosgw-admin: %+v", err)
}
return output, nil
}
func (c *Cluster) createKeyring() error {
_, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(appName, metav1.GetOptions{})
_, err := c.context.Clientset.CoreV1().Secrets(c.Namespace).Get(c.instanceName(), metav1.GetOptions{})
if err == nil {
logger.Infof("the rgw keyring was already generated")
return nil
@@ -111,7 +184,7 @@ func (c *Cluster) createKeyring() error {
keyringName: keyring,
}
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: appName, Namespace: c.Namespace},
ObjectMeta: metav1.ObjectMeta{Name: c.instanceName(), Namespace: c.Namespace},
StringData: secrets,
Type: k8sutil.RookType,
}
@@ -123,9 +196,17 @@ func (c *Cluster) createKeyring() error {
return nil
}
func (c *Cluster) instanceName() string {
return InstanceName(c.Name)
}
func InstanceName(name string) string {
return fmt.Sprintf("%s-%s", appName, name)
}
func (c *Cluster) makeDeployment() *extensions.Deployment {
deployment := &extensions.Deployment{}
deployment.Name = appName
deployment.Name = c.instanceName()
deployment.Namespace = c.Namespace
podSpec := v1.PodSpec{
@@ -158,17 +239,18 @@ func (c *Cluster) rgwContainer() v1.Container {
Args: []string{
"rgw",
fmt.Sprintf("--config-dir=%s", k8sutil.DataDir),
fmt.Sprintf("--rgw-name=%s", c.Name),
fmt.Sprintf("--rgw-port=%d", cephrgw.RGWPort),
fmt.Sprintf("--rgw-host=%s", cephrgw.DNSName),
},
Name: appName,
Name: c.instanceName(),
Image: k8sutil.MakeRookImage(c.Version),
VolumeMounts: []v1.VolumeMount{
{Name: k8sutil.DataDirVolume, MountPath: k8sutil.DataDir},
k8sutil.ConfigOverrideMount(),
},
Env: []v1.EnvVar{
{Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: appName}, Key: keyringName}}},
{Name: "ROOK_RGW_KEYRING", ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{LocalObjectReference: v1.LocalObjectReference{Name: c.instanceName()}, Key: keyringName}}},
k8sutil.PodIPEnvVar(k8sutil.PrivateIPEnvVar),
k8sutil.PodIPEnvVar(k8sutil.PublicIPEnvVar),
opmon.ClusterNameEnvVar(c.Namespace),
@@ -180,18 +262,18 @@ func (c *Cluster) rgwContainer() v1.Container {
}
}
func (c *Cluster) startService() error {
func (c *Cluster) startService() (string, error) {
labels := c.getLabels()
s := &v1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: appName,
Name: c.instanceName(),
Namespace: c.Namespace,
Labels: labels,
},
Spec: v1.ServiceSpec{
Ports: []v1.ServicePort{
{
Name: appName,
Name: c.instanceName(),
Port: cephrgw.RGWPort,
TargetPort: intstr.FromInt(int(cephrgw.RGWPort)),
Protocol: v1.ProtocolTCP,
@@ -204,19 +286,20 @@ func (c *Cluster) startService() error {
s, err := c.context.Clientset.CoreV1().Services(c.Namespace).Create(s)
if err != nil {
if !errors.IsAlreadyExists(err) {
return fmt.Errorf("failed to create mon service. %+v", err)
return "", fmt.Errorf("failed to create mon service. %+v", err)
}
logger.Infof("RGW service already running")
return nil
return "", nil
}
logger.Infof("RGW service running at %s:%d", s.Spec.ClusterIP, cephrgw.RGWPort)
return nil
return s.Spec.ClusterIP, nil
}
func (c *Cluster) getLabels() map[string]string {
return map[string]string{
k8sutil.AppAttr: appName,
k8sutil.ClusterAttr: c.Namespace,
"rook_object_store": c.Name,
}
}
+14 -12
View File
@@ -42,7 +42,7 @@ func TestStartRGW(t *testing.T) {
configDir, _ := ioutil.TempDir("", "")
defer os.RemoveAll(configDir)
c := New(&clusterd.Context{Clientset: clientset, Executor: executor, ConfigDir: configDir}, "ns", "version", k8sutil.Placement{})
c := New(&clusterd.Context{Clientset: clientset, Executor: executor, ConfigDir: configDir}, "default", "ns", "version", k8sutil.Placement{})
// start a basic cluster
err := c.Start()
@@ -59,35 +59,36 @@ func TestStartRGW(t *testing.T) {
func validateStart(t *testing.T, c *Cluster, clientset *fake.Clientset) {
r, err := clientset.ExtensionsV1beta1().Deployments(c.Namespace).Get(appName, metav1.GetOptions{})
r, err := clientset.ExtensionsV1beta1().Deployments(c.Namespace).Get(c.instanceName(), metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, appName, r.Name)
assert.Equal(t, c.instanceName(), r.Name)
s, err := clientset.CoreV1().Services(c.Namespace).Get(appName, metav1.GetOptions{})
s, err := clientset.CoreV1().Services(c.Namespace).Get(c.instanceName(), metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, appName, s.Name)
assert.Equal(t, c.instanceName(), s.Name)
secret, err := clientset.CoreV1().Secrets(c.Namespace).Get(appName, metav1.GetOptions{})
secret, err := clientset.CoreV1().Secrets(c.Namespace).Get(c.instanceName(), metav1.GetOptions{})
assert.Nil(t, err)
assert.Equal(t, appName, secret.Name)
assert.Equal(t, c.instanceName(), secret.Name)
assert.Equal(t, 1, len(secret.StringData))
}
func TestPodSpecs(t *testing.T) {
c := New(nil, "ns", "myversion", k8sutil.Placement{})
c := New(nil, "default", "ns", "myversion", k8sutil.Placement{})
d := c.makeDeployment()
assert.NotNil(t, d)
assert.Equal(t, appName, d.Name)
assert.Equal(t, c.instanceName(), d.Name)
assert.Equal(t, v1.RestartPolicyAlways, d.Spec.Template.Spec.RestartPolicy)
assert.Equal(t, 2, len(d.Spec.Template.Spec.Volumes))
assert.Equal(t, "rook-data", d.Spec.Template.Spec.Volumes[0].Name)
assert.Equal(t, k8sutil.ConfigOverrideName, d.Spec.Template.Spec.Volumes[1].Name)
assert.Equal(t, appName, d.ObjectMeta.Name)
assert.Equal(t, c.instanceName(), d.ObjectMeta.Name)
assert.Equal(t, appName, d.Spec.Template.ObjectMeta.Labels["app"])
assert.Equal(t, c.Namespace, d.Spec.Template.ObjectMeta.Labels["rook_cluster"])
assert.Equal(t, c.Name, d.Spec.Template.ObjectMeta.Labels["rook_object_store"])
assert.Equal(t, 0, len(d.ObjectMeta.Annotations))
cont := d.Spec.Template.Spec.Containers[0]
@@ -96,6 +97,7 @@ func TestPodSpecs(t *testing.T) {
assert.Equal(t, "rgw", cont.Args[0])
assert.Equal(t, "--config-dir=/var/lib/rook", cont.Args[1])
assert.Equal(t, fmt.Sprintf("--rgw-port=%d", cephrgw.RGWPort), cont.Args[2])
assert.Equal(t, fmt.Sprintf("--rgw-host=%s", cephrgw.DNSName), cont.Args[3])
assert.Equal(t, fmt.Sprintf("--rgw-name=%s", "default"), cont.Args[2])
assert.Equal(t, fmt.Sprintf("--rgw-port=%d", cephrgw.RGWPort), cont.Args[3])
assert.Equal(t, fmt.Sprintf("--rgw-host=%s", cephrgw.DNSName), cont.Args[4])
}