ceph: minor fixes + add realm/zg/zone to object context

- Add realm/zone group/zone to Object Context,
so that any call to runAdminCommand has the
same realm, zone group, and zone as the object
store that the call is going to.

- Also modify the delete code for the object-store
using multisite to remove the endpoints of an
object store from a zone instead of the normal
clean-up

- Added more debug logging all around the object-store
code related to multisite

- files generated by rerun of `make codegen`

- change back the edit on the Copyright in object.go

Signed-off-by: Ali Maredia <amaredia@redhat.com>

pkg/apis/ceph.rook.io/v1/object.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The Rook Authors. All rights reserved.
+Copyright 2018 The Rook Authors. All rights reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go

@@ -562,7 +562,7 @@ func (in *CephObjectStoreUser) DeepCopyInto(out *CephObjectStoreUser) {
 	if in.Status != nil {
 		in, out := &in.Status, &out.Status
 		*out = new(ObjectStoreUserStatus)
-		**out = **in
+		(*in).DeepCopyInto(*out)
 	}
 	return
 }
@@ -623,7 +623,7 @@ func (in *CephObjectZone) DeepCopyInto(out *CephObjectZone) {
 	*out = *in
 	out.TypeMeta = in.TypeMeta
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	out.Spec = in.Spec
+	in.Spec.DeepCopyInto(&out.Spec)
 	if in.Status != nil {
 		in, out := &in.Status, &out.Status
 		*out = new(Status)
@@ -1509,6 +1509,29 @@ func (in *ObjectStoreUserSpec) DeepCopy() *ObjectStoreUserSpec {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ObjectStoreUserStatus) DeepCopyInto(out *ObjectStoreUserStatus) {
+	*out = *in
+	if in.Info != nil {
+		in, out := &in.Info, &out.Info
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreUserStatus.
+func (in *ObjectStoreUserStatus) DeepCopy() *ObjectStoreUserStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ObjectStoreUserStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ObjectZoneGroupSpec) DeepCopyInto(out *ObjectZoneGroupSpec) {
 	*out = *in
@@ -1528,6 +1551,8 @@ func (in *ObjectZoneGroupSpec) DeepCopy() *ObjectZoneGroupSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ObjectZoneSpec) DeepCopyInto(out *ObjectZoneSpec) {
 	*out = *in
+	in.MetadataPool.DeepCopyInto(&out.MetadataPool)
+	in.DataPool.DeepCopyInto(&out.DataPool)
 	return
 }
 

pkg/operator/ceph/object/admin.go

@@ -19,6 +19,8 @@ package object
 import (
 	"fmt"
 
+	"github.com/pkg/errors"
+	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
 	"github.com/rook/rook/pkg/clusterd"
 	"github.com/rook/rook/pkg/daemon/ceph/client"
 )
@@ -30,6 +32,9 @@ type Context struct {
 	Name        string
 	UID         string
 	Endpoint    string
+	Realm       string
+	ZoneGroup   string
+	Zone        string
 }
 
 // NewContext creates a new object store context.
@@ -37,7 +42,21 @@ func NewContext(context *clusterd.Context, clusterInfo *client.ClusterInfo, name
 	return &Context{Context: context, Name: name, clusterInfo: clusterInfo}
 }
 
-func RunAdminCommandNoRealm(c *Context, args ...string) (string, error) {
+func NewMultisiteContext(context *clusterd.Context, clusterInfo *client.ClusterInfo, store *cephv1.CephObjectStore) (*Context, error) {
+	objContext := &Context{Context: context, Name: store.Name, clusterInfo: clusterInfo}
+	realmName, zoneGroupName, zoneName, err := getMultisiteForObjectStore(context, store)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to get realm/zone group/zone for object store %q", store.Name)
+	}
+
+	objContext.Realm = realmName
+	objContext.ZoneGroup = zoneGroupName
+	objContext.Zone = zoneName
+	return objContext, nil
+}
+
+// This function is for running radosgw-admin commands in scenarios where an object-store has not been created yet or for commands on the realm or zonegroup (ex: radosgw-admin zonegroup get)
+func RunAdminCommandNoMultisite(c *Context, args ...string) (string, error) {
 	command, args := client.FinalizeCephCommandArgs("radosgw-admin", c.clusterInfo, args, c.Context.ConfigDir)
 
 	// start the rgw admin command
@@ -49,6 +68,7 @@ func RunAdminCommandNoRealm(c *Context, args ...string) (string, error) {
 	return output, nil
 }
 
+// This function is for running radosgw-admin commands in scenarios where an object-store has been created and the Context has been updated with the appropriate realm, zone group, and zone.
 func runAdminCommand(c *Context, args ...string) (string, error) {
 	// If the objectStoreName is not passed in the storage class
 	// This means we are pointing to an external cluster so these commands are not needed
@@ -59,12 +79,13 @@ func runAdminCommand(c *Context, args ...string) (string, error) {
 	// This is not perfect though since "client.admin" is somehow supported...
 	if c.Name != "" && c.clusterInfo.CephCred.Username == client.AdminUsername {
 		options := []string{
-			fmt.Sprintf("--rgw-realm=%s", c.Name),
-			fmt.Sprintf("--rgw-zonegroup=%s", c.Name),
+			fmt.Sprintf("--rgw-realm=%s", c.Realm),
+			fmt.Sprintf("--rgw-zonegroup=%s", c.ZoneGroup),
+			fmt.Sprintf("--rgw-zone=%s", c.Zone),
 		}
 
-		return RunAdminCommandNoRealm(c, append(args, options...)...)
+		return RunAdminCommandNoMultisite(c, append(args, options...)...)
 	}
 
-	return RunAdminCommandNoRealm(c, args...)
+	return RunAdminCommandNoMultisite(c, args...)
 }

pkg/operator/ceph/object/bucket/provisioner.go

@@ -395,7 +395,16 @@ func (p *Provisioner) setObjectContext() error {
 	if p.objectStoreName == "" && p.endpoint == "" {
 		return errors.Errorf(msg, "name")
 	}
-	p.objectContext = cephObject.NewContext(p.context, p.clusterInfo, p.objectStoreName)
+
+	store, err := p.getObjectStore()
+	if err != nil {
+		return errors.Wrap(err, "failed to get cephObjectStore")
+	}
+
+	p.objectContext, err = cephObject.NewMultisiteContext(p.context, p.clusterInfo, store)
+	if err != nil {
+		return errors.Wrapf(err, "failed to set multisite on provisioner's objectContext")
+	}
 
 	return nil
 }
@@ -407,7 +416,7 @@ func (p *Provisioner) setObjectStoreDomainName(sc *storagev1.StorageClass) error
 	name := getObjectStoreName(sc)
 	namespace := getObjectStoreNameSpace(sc)
 	// make sure the object store actually exists
-	_, err := getObjectStore(p.context.RookClientset.CephV1(), p.clusterInfo.Namespace, p.objectStoreName)
+	_, err := p.getObjectStore()
 	if err != nil {
 		return err
 	}

pkg/operator/ceph/object/bucket/provisioner_test.go

@@ -37,6 +37,7 @@ func TestPopulateDomainAndPort(t *testing.T) {
 	namespace := "ns"
 	clusterInfo := client.AdminClusterInfo(namespace)
 	p := NewProvisioner(&clusterd.Context{RookClientset: rookclient.NewSimpleClientset(), Clientset: test.New(t, 1)}, clusterInfo)
+	p.objectContext = object.NewContext(p.context, clusterInfo, store)
 	sc := &storagev1.StorageClass{
 		Parameters: map[string]string{
 			"foo": "bar",

pkg/operator/ceph/object/bucket/util.go

@@ -32,7 +32,6 @@ import (
 	"k8s.io/client-go/rest"
 
 	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
-	cephclientset "github.com/rook/rook/pkg/client/clientset/versioned/typed/ceph.rook.io/v1"
 	cephObject "github.com/rook/rook/pkg/operator/ceph/object"
 )
 
@@ -100,9 +99,9 @@ func getCephUser(ob *bktv1alpha1.ObjectBucket) string {
 	return ob.Spec.AdditionalState[cephUser]
 }
 
-func getObjectStore(c cephclientset.CephV1Interface, namespace, name string) (*cephv1.CephObjectStore, error) {
+func (p *Provisioner) getObjectStore() (*cephv1.CephObjectStore, error) {
 	// Verify the object store API object actually exists
-	store, err := c.CephObjectStores(namespace).Get(name, metav1.GetOptions{})
+	store, err := p.context.RookClientset.CephV1().CephObjectStores(p.clusterInfo.Namespace).Get(p.objectStoreName, metav1.GetOptions{})
 	if err != nil {
 		if kerrors.IsNotFound(err) {
 			return nil, errors.Wrap(err, "cephObjectStore not found")

pkg/operator/ceph/object/controller.go

@@ -333,6 +333,11 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
 			}
 		}
 
+		objContext.Realm = realmName
+		objContext.ZoneGroup = zoneGroupName
+		objContext.Zone = zoneName
+		logger.Debugf("realm for object-store is %q, zone group for object-store is %q, zone for object-store is %q", objContext.Realm, objContext.ZoneGroup, objContext.Zone)
+
 		// RECONCILE SERVICE
 		logger.Debug("reconciling object store service")
 		serviceIP, err = cfg.reconcileService(cephObjectStore)
@@ -351,7 +356,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
 
 		// Reconcile Multisite Creation
 		logger.Infof("setting multisite settings for object store %q", cephObjectStore.Name)
-		err = setMultisite(objContext, cephObjectStore.Spec, serviceIP, realmName, zoneGroupName, zoneName, cephObjectStore.Namespace)
+		err = setMultisite(objContext, cephObjectStore, serviceIP)
 		if err != nil {
 			return r.setFailedStatus(namespacedName, "failed to configure multisite for object store", err)
 		}
@@ -377,7 +382,7 @@ func (r *ReconcileCephObjectStore) reconcileCephZone(store *cephv1.CephObjectSto
 	zoneArg := fmt.Sprintf("--rgw-zone=%s", store.Spec.Zone.Name)
 	objContext := NewContext(r.context, r.clusterInfo, store.Name)
 
-	_, err := RunAdminCommandNoRealm(objContext, "zone", "get", realmArg, zoneGroupArg, zoneArg)
+	_, err := RunAdminCommandNoMultisite(objContext, "zone", "get", realmArg, zoneGroupArg, zoneArg)
 	if err != nil {
 		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
 			return waitForRequeueIfObjectStoreNotReady, errors.Wrapf(err, "ceph zone %q not found", store.Spec.Zone.Name)
@@ -400,7 +405,7 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1
 			}
 			return "", "", "", waitForRequeueIfObjectStoreNotReady, errors.Wrapf(err, "error getting CephObjectZone %q", cephObjectStore.Spec.Zone.Name)
 		}
-		logger.Infof("CephObjectZone resource %s found", zone.Name)
+		logger.Debugf("CephObjectZone resource %s found", zone.Name)
 
 		zonegroup, err := r.context.RookClientset.CephV1().CephObjectZoneGroups(cephObjectStore.Namespace).Get(zone.Spec.ZoneGroup, metav1.GetOptions{})
 		if err != nil {
@@ -409,7 +414,7 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1
 			}
 			return "", "", "", waitForRequeueIfObjectStoreNotReady, errors.Wrapf(err, "error getting CephObjectZoneGroup %q", zone.Spec.ZoneGroup)
 		}
-		logger.Infof("CephObjectZoneGroup resource %s found", zonegroup.Name)
+		logger.Debugf("CephObjectZoneGroup resource %s found", zonegroup.Name)
 
 		realm, err := r.context.RookClientset.CephV1().CephObjectRealms(cephObjectStore.Namespace).Get(zonegroup.Spec.Realm, metav1.GetOptions{})
 		if err != nil {
@@ -418,7 +423,7 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1
 			}
 			return "", "", "", waitForRequeueIfObjectStoreNotReady, errors.Wrapf(err, "error getting CephObjectRealm %q", zonegroup.Spec.Realm)
 		}
-		logger.Infof("CephObjectRealm resource %s found", realm.Name)
+		logger.Debugf("CephObjectRealm resource %s found", realm.Name)
 
 		return realm.Name, zonegroup.Name, zone.Name, reconcile.Result{}, nil
 	}

pkg/operator/ceph/object/objectstore.go

@@ -77,45 +77,138 @@ type realmType struct {
 	Realms []string `json:"realms"`
 }
 
-func deleteRealmAndPools(context *Context, spec cephv1.ObjectStoreSpec) error {
-	stores, err := getObjectStores(context)
+func deleteRealmAndPools(objContext *Context, spec cephv1.ObjectStoreSpec) error {
+	if spec.IsMultisite() {
+		// since pools for object store are created by the zone, the object store only needs to be removed from the zone
+		err := removeObjectStoreFromMultisite(objContext, spec)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	return deleteSingleSiteRealmAndPools(objContext, spec)
+}
+
+func removeObjectStoreFromMultisite(objContext *Context, spec cephv1.ObjectStoreSpec) error {
+	// get list of endpoints not including the endpoint of the object-store for the zone
+	zoneEndpointsList, err := getZoneEndpoints(objContext, objContext.Endpoint)
+	if err != nil {
+		return err
+	}
+
+	realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm)
+	zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup)
+	zoneEndpoints := strings.Join(zoneEndpointsList, ",")
+	endpointArg := fmt.Sprintf("--endpoints=%s", zoneEndpoints)
+
+	zoneIsMaster, err := checkZoneIsMaster(objContext)
+	zoneGroupIsMaster := false
+	if zoneIsMaster {
+		_, err = RunAdminCommandNoMultisite(objContext, "zonegroup", "modify", realmArg, zoneGroupArg, endpointArg)
+		if err != nil {
+			return errors.Wrapf(err, "failed to remove object store %q endpoint from rgw zone group %q", objContext.Name, objContext.ZoneGroup)
+		}
+		logger.Debugf("endpoint %q was removed from zone group %q. the remaining endpoints in the zone group are %q", objContext.Endpoint, objContext.ZoneGroup, zoneEndpoints)
+
+		// check if zone group is master only if zone is master for creating the system user
+		zoneGroupIsMaster, err = checkZoneGroupIsMaster(objContext)
+		if err != nil {
+			return errors.Wrapf(err, "failed to find out whether zone group %q in is the master zone group", objContext.ZoneGroup)
+		}
+	}
+
+	_, err = runAdminCommand(objContext, "zone", "modify", endpointArg)
+	if err != nil {
+		return errors.Wrapf(err, "failed to remove object store %q endpoint from rgw zone %q", objContext.Name, spec.Zone.Name)
+	}
+	logger.Debugf("endpoint %q was removed from zone %q. the remaining endpoints in the zone are %q", objContext.Endpoint, objContext.Zone, zoneEndpoints)
+
+	if zoneIsMaster && zoneGroupIsMaster && zoneEndpoints == "" {
+		logger.Infof("WARNING: No other zone in realm %q can commit to the period or pull the realm until you create another object-store in zone %q", objContext.Realm, objContext.Zone)
+	}
+
+	// the period will help notify other zones of changes if there are multi-zones
+	_, err = runAdminCommand(objContext, "period", "update", "--commit")
+	if err != nil {
+		return errors.Wrap(err, "failed to update period after removing an endpoint from the zone")
+	}
+	logger.Infof("successfully updated period for realm %v after removal of object-store %v", objContext.Realm, objContext.Name)
+
+	return nil
+}
+
+func deleteSingleSiteRealmAndPools(objContext *Context, spec cephv1.ObjectStoreSpec) error {
+	stores, err := getObjectStores(objContext)
 	if err != nil {
 		return errors.Wrap(err, "failed to detect object stores during deletion")
 	}
-	logger.Infof("Found stores %v when deleting store %s", stores, context.Name)
+	logger.Infof("Found stores %v when deleting store %s", stores, objContext.Name)
 
-	err = deleteRealm(context)
+	err = deleteRealm(objContext)
 	if err != nil {
 		return errors.Wrap(err, "failed to delete realm")
 	}
 
 	lastStore := false
-	if len(stores) == 1 && stores[0] == context.Name {
+	if len(stores) == 1 && stores[0] == objContext.Name {
 		lastStore = true
 	}
 
 	if !spec.PreservePoolsOnDelete {
-		err = deletePools(context, spec, lastStore)
+		err = deletePools(objContext, spec, lastStore)
 		if err != nil {
 			return errors.Wrap(err, "failed to delete object store pools")
 		}
 	} else {
-		logger.Infof("PreservePoolsOnDelete is set in object store %s. Pools not deleted", context.Name)
+		logger.Infof("PreservePoolsOnDelete is set in object store %s. Pools not deleted", objContext.Name)
 	}
+
 	return nil
 }
 
-func checkZoneIsMaster(context *Context, realmArg, zoneGroupArg, zoneArg string) (bool, error) {
-	zoneGroupOutput, err := RunAdminCommandNoRealm(context, "zonegroup", "get", realmArg, zoneGroupArg)
+// This is used for quickly getting the name of the realm, zone group, and zone for an object-store to pass into a Context
+func getMultisiteForObjectStore(context *clusterd.Context, store *cephv1.CephObjectStore) (string, string, string, error) {
+	if store.Spec.IsMultisite() {
+		zone, err := context.RookClientset.CephV1().CephObjectZones(store.Namespace).Get(store.Spec.Zone.Name, metav1.GetOptions{})
+		if err != nil {
+			return "", "", "", errors.Wrapf(err, "failed to find zone for object-store %q", store.Name)
+		}
+
+		zonegroup, err := context.RookClientset.CephV1().CephObjectZoneGroups(store.Namespace).Get(zone.Spec.ZoneGroup, metav1.GetOptions{})
+		if err != nil {
+			return "", "", "", errors.Wrapf(err, "failed to find zone group for object-store %q", store.Name)
+		}
+
+		realm, err := context.RookClientset.CephV1().CephObjectRealms(store.Namespace).Get(zonegroup.Spec.Realm, metav1.GetOptions{})
+		if err != nil {
+			return "", "", "", errors.Wrapf(err, "failed to find realm for object-store %q", store.Name)
+		}
+
+		return realm.Name, zonegroup.Name, zone.Name, nil
+	}
+
+	return store.Name, store.Name, store.Name, nil
+}
+
+func checkZoneIsMaster(objContext *Context) (bool, error) {
+	logger.Debugf("checking if zone %v is the master zone", objContext.Zone)
+	realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm)
+	zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup)
+	zoneArg := fmt.Sprintf("--rgw-zone=%s", objContext.Zone)
+
+	zoneGroupJson, err := RunAdminCommandNoMultisite(objContext, "zonegroup", "get", realmArg, zoneGroupArg)
 	if err != nil {
 		return false, errors.Wrap(err, "failed to get rgw zone group")
 	}
-	zoneGroupJson, err := DecodeZoneGroupConfig(zoneGroupOutput)
+	zoneGroupOutput, err := DecodeZoneGroupConfig(zoneGroupJson)
 	if err != nil {
 		return false, errors.Wrap(err, "failed to parse zonegroup get json")
 	}
+	logger.Debugf("got master zone ID for zone group %v", objContext.ZoneGroup)
 
-	zoneOutput, err := RunAdminCommandNoRealm(context, "zone", "get", realmArg, zoneGroupArg, zoneArg)
+	zoneOutput, err := RunAdminCommandNoMultisite(objContext, "zone", "get", realmArg, zoneGroupArg, zoneArg)
 	if err != nil {
 		return false, errors.Wrap(err, "failed to get rgw zone")
 	}
@@ -123,19 +216,27 @@ func checkZoneIsMaster(context *Context, realmArg, zoneGroupArg, zoneArg string)
 	if err != nil {
 		return false, errors.Wrap(err, "failed to parse zone id")
 	}
+	logger.Debugf("got zone ID for zone %v", objContext.Zone)
 
-	if zoneID == zoneGroupJson.MasterZoneID {
+	if zoneID == zoneGroupOutput.MasterZoneID {
+		logger.Debugf("zone is master")
 		return true, nil
 	}
 
+	logger.Debugf("zone is not master")
 	return false, nil
 }
 
-func checkZoneGroupIsMaster(context *Context, realmArg, zoneGroupArg string) (bool, error) {
-	zoneGroupOutput, err := RunAdminCommandNoRealm(context, "zonegroup", "get", realmArg, zoneGroupArg)
+func checkZoneGroupIsMaster(objContext *Context) (bool, error) {
+	logger.Debugf("checking if zone group %v is the master zone group", objContext.ZoneGroup)
+	realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm)
+	zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup)
+
+	zoneGroupOutput, err := RunAdminCommandNoMultisite(objContext, "zonegroup", "get", realmArg, zoneGroupArg)
 	if err != nil {
 		return false, errors.Wrap(err, "failed to get rgw zone group")
 	}
+
 	zoneGroupJson, err := DecodeZoneGroupConfig(zoneGroupOutput)
 	if err != nil {
 		return false, errors.Wrap(err, "failed to parse master zone id")
@@ -160,12 +261,14 @@ func DecodeSecret(secret *v1.Secret, keyName string) (string, error) {
 }
 
 func GetRealmKeyArgs(context *clusterd.Context, realmName, namespace string) (string, string, error) {
+	logger.Debugf("getting keys for realm %v", realmName)
 	// get realm's access and secret keys
 	realmSecretName := realmName + "-keys"
 	realmSecret, err := context.Clientset.CoreV1().Secrets(namespace).Get(realmSecretName, metav1.GetOptions{})
 	if err != nil {
 		return "", "", errors.Wrapf(err, "failed to get realm %q keys secret", realmName)
 	}
+	logger.Debugf("found keys secret for realm %v", realmName)
 
 	accessKey, err := DecodeSecret(realmSecret, AccessKeyName)
 	if err != nil {
@@ -175,6 +278,7 @@ func GetRealmKeyArgs(context *clusterd.Context, realmName, namespace string) (st
 	if err != nil {
 		return "", "", errors.Wrapf(err, "failed to decode realm %q access key", realmName)
 	}
+	logger.Debugf("decoded keys for realm %v", realmName)
 
 	accessKeyArg := fmt.Sprintf("--access-key=%s", accessKey)
 	secretKeyArg := fmt.Sprintf("--secret-key=%s", secretKey)
@@ -182,96 +286,109 @@ func GetRealmKeyArgs(context *clusterd.Context, realmName, namespace string) (st
 	return accessKeyArg, secretKeyArg, nil
 }
 
-func getZoneEndpoints(context *Context, realmArg, zoneGroupArg, zoneName, serviceEndpoint string) (string, error) {
-	zoneGroupOutput, err := RunAdminCommandNoRealm(context, "zonegroup", "get", realmArg, zoneGroupArg)
+func getZoneEndpoints(objContext *Context, serviceEndpoint string) ([]string, error) {
+	logger.Debugf("getting current endpoints for zone %v", objContext.Zone)
+	realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm)
+	zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup)
+
+	zoneGroupOutput, err := RunAdminCommandNoMultisite(objContext, "zonegroup", "get", realmArg, zoneGroupArg)
 	if err != nil {
-		return "", errors.Wrap(err, "failed to get rgw zone group")
+		return []string{}, errors.Wrap(err, "failed to get rgw zone group")
 	}
 	zoneGroupJson, err := DecodeZoneGroupConfig(zoneGroupOutput)
 	if err != nil {
-		return "", errors.Wrap(err, "failed to parse zones list")
+		return []string{}, errors.Wrap(err, "failed to parse zones list")
 	}
 
-	zoneEndpoints := []string{}
+	zoneEndpointsList := []string{}
 	for _, zone := range zoneGroupJson.Zones {
-		if zone.Name == zoneName {
+		if zone.Name == objContext.Zone {
 			for _, endpoint := range zone.Endpoints {
 				// in case object-store operator code is rereconciled, zone modify could get run again with serviceEndpoint added again
 				if endpoint != serviceEndpoint {
-					zoneEndpoints = append(zoneEndpoints, endpoint)
+					zoneEndpointsList = append(zoneEndpointsList, endpoint)
 				}
 			}
 			break
 		}
 	}
 
-	zoneEndpoints = append(zoneEndpoints, serviceEndpoint)
-
-	return strings.Join(zoneEndpoints, ","), nil
+	return zoneEndpointsList, nil
 }
 
-func createMultisite(context *Context, endpointArg, realmArg, zoneGroupArg, zoneArg string) error {
+func createMultisite(objContext *Context, endpointArg string) error {
+	logger.Debugf("creating realm, zone group, zone for object-store %v", objContext.Name)
+
+	realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm)
+	zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup)
+
 	updatePeriod := false
 	// create the realm if it doesn't exist yet
-	output, err := runAdminCommand(context, "realm", "get")
+	output, err := RunAdminCommandNoMultisite(objContext, "realm", "get", realmArg)
 	if err != nil {
-		updatePeriod = true
-		output, err = runAdminCommand(context, "realm", "create")
-		if err != nil {
-			return errors.Wrapf(err, "failed to create rgw realm %q", context.Name)
+		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
+			updatePeriod = true
+			output, err = RunAdminCommandNoMultisite(objContext, "realm", "create", realmArg)
+			if err != nil {
+				return errors.Wrapf(err, "failed to create ceph realm %q, for reason %q", objContext.ZoneGroup, output)
+			}
+			logger.Debugf("created realm %v", objContext.Realm)
+		} else {
+			return errors.Wrapf(err, "radosgw-admin realm get failed with code %d, for reason %q", code, output)
 		}
 	}
 
-	realmID, err := decodeID(output)
-	if err != nil {
-		return errors.Wrap(err, "failed to parse realm id")
-	}
-
 	// create the zonegroup if it doesn't exist yet
-	output, err = runAdminCommand(context, "zonegroup", "get")
+	output, err = RunAdminCommandNoMultisite(objContext, "zonegroup", "get", realmArg, zoneGroupArg)
 	if err != nil {
-		updatePeriod = true
-		output, err = runAdminCommand(context, "zonegroup", "create", "--master", endpointArg)
-		if err != nil {
-			return errors.Wrapf(err, "failed to create rgw zonegroup for %q", context.Name)
+		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
+			updatePeriod = true
+			output, err = RunAdminCommandNoMultisite(objContext, "zonegroup", "create", "--master", realmArg, zoneGroupArg, endpointArg)
+			if err != nil {
+				return errors.Wrapf(err, "failed to create ceph zone group %q, for reason %q", objContext.ZoneGroup, output)
+			}
+			logger.Debugf("created zone group %v", objContext.ZoneGroup)
+		} else {
+			return errors.Wrapf(err, "radosgw-admin zonegroup get failed with code %d, for reason %q", code, output)
 		}
 	}
 
-	zoneGroupID, err := decodeID(output)
-	if err != nil {
-		return errors.Wrap(err, "failed to parse zone group id")
-	}
-
 	// create the zone if it doesn't exist yet
-	output, err = runAdminCommand(context, "zone", "get", zoneArg)
+	output, err = runAdminCommand(objContext, "zone", "get")
 	if err != nil {
-		updatePeriod = true
-		output, err = runAdminCommand(context, "zone", "create", "--master", endpointArg, zoneArg)
-		if err != nil {
-			return errors.Wrapf(err, "failed to create rgw zonegroup for %q", context.Name)
+		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
+			updatePeriod = true
+			output, err = runAdminCommand(objContext, "zone", "create", "--master", endpointArg)
+			if err != nil {
+				return errors.Wrapf(err, "failed to create ceph zone %q, for reason %q", objContext.Zone, output)
+			}
+			logger.Debugf("created zone %v", objContext.Zone)
+		} else {
+			return errors.Wrapf(err, "radosgw-admin zone get failed with code %d, for reason %q", code, output)
 		}
 	}
 
-	zoneID, err := decodeID(output)
-	if err != nil {
-		return errors.Wrap(err, "failed to parse zone id")
-	}
-
-	logger.Infof("RGW: realm=%s, zonegroup=%s, zone=%s", realmID, zoneGroupID, zoneID)
-
 	if updatePeriod {
 		// the period will help notify other zones of changes if there are multi-zones
-		_, err := RunAdminCommandNoRealm(context, "period", "update", "--commit", realmArg, zoneGroupArg, zoneArg)
+		_, err := runAdminCommand(objContext, "period", "update", "--commit")
 		if err != nil {
 			return errors.Wrap(err, "failed to update period")
 		}
+		logger.Debugf("updated period for realm %v", objContext.Realm)
 	}
 
+	logger.Infof("Multisite for object-store: realm=%s, zonegroup=%s, zone=%s", objContext.Realm, objContext.ZoneGroup, objContext.Zone)
+
 	return nil
 }
 
-func joinMultisite(context *Context, endpointArg, realmArg, zoneGroupArg, zoneArg, realmName, zoneGroupName, zoneName, zoneEndpoints, namespace string) error {
-	zoneIsMaster, err := checkZoneIsMaster(context, realmArg, zoneGroupArg, zoneArg)
+func joinMultisite(objContext *Context, endpointArg, zoneEndpoints, namespace string) error {
+	logger.Debugf("joining zone %v", objContext.Zone)
+	realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm)
+	zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup)
+	zoneArg := fmt.Sprintf("--rgw-zone=%s", objContext.Zone)
+
+	zoneIsMaster, err := checkZoneIsMaster(objContext)
 	if err != nil {
 		return err
 	}
@@ -279,104 +396,125 @@ func joinMultisite(context *Context, endpointArg, realmArg, zoneGroupArg, zoneAr
 
 	if zoneIsMaster {
 		// endpoints that are part of a master zone are supposed to be the endpoints for a zone group
-		_, err := RunAdminCommandNoRealm(context, "zonegroup", "modify", realmArg, zoneGroupArg, endpointArg)
+		_, err := RunAdminCommandNoMultisite(objContext, "zonegroup", "modify", realmArg, zoneGroupArg, endpointArg)
 		if err != nil {
-			return errors.Wrapf(err, "failed to add object store %q in rgw zone group %q", context.Name, zoneGroupName)
+			return errors.Wrapf(err, "failed to add object store %q in rgw zone group %q", objContext.Name, objContext.ZoneGroup)
 		}
-		logger.Debugf("endpoints for zonegroup %q are now %q", zoneGroupName, zoneEndpoints)
+		logger.Debugf("endpoints for zonegroup %q are now %q", objContext.ZoneGroup, zoneEndpoints)
 
 		// check if zone group is master only if zone is master for creating the system user
-		zoneGroupIsMaster, err = checkZoneGroupIsMaster(context, realmArg, zoneGroupArg)
+		zoneGroupIsMaster, err = checkZoneGroupIsMaster(objContext)
 		if err != nil {
-			return errors.Wrapf(err, "failed to find out whether zone group %q in is the master zone group", zoneGroupName)
+			return errors.Wrapf(err, "failed to find out whether zone group %q in is the master zone group", objContext.ZoneGroup)
 		}
 	}
-	_, err = RunAdminCommandNoRealm(context, "zone", "modify", realmArg, zoneGroupArg, zoneArg, endpointArg)
+	_, err = RunAdminCommandNoMultisite(objContext, "zone", "modify", realmArg, zoneGroupArg, zoneArg, endpointArg)
 	if err != nil {
-		return errors.Wrapf(err, "failed to add object store %q in rgw zone %q", context.Name, zoneName)
+		return errors.Wrapf(err, "failed to add object store %q in rgw zone %q", objContext.Name, objContext.Zone)
 	}
+	logger.Debugf("endpoints for zone %q are now %q", objContext.Zone, zoneEndpoints)
 
 	// the period will help notify other zones of changes if there are multi-zones
-	_, err = RunAdminCommandNoRealm(context, "period", "update", "--commit", realmArg, zoneGroupArg, zoneArg)
+	_, err = RunAdminCommandNoMultisite(objContext, "period", "update", "--commit", realmArg, zoneGroupArg, zoneArg)
 	if err != nil {
 		return errors.Wrap(err, "failed to update period")
 	}
-	logger.Infof("added object store %q to realm %q, zonegroup %q, zone %q", context.Name, realmName, zoneGroupName, zoneName)
+	logger.Infof("added object store %q to realm %q, zonegroup %q, zone %q", objContext.Name, objContext.Realm, objContext.ZoneGroup, objContext.Zone)
 
 	// create system user for realm for master zone in master zonegorup for multisite scenario
 	if zoneIsMaster && zoneGroupIsMaster {
-		uid := realmName + "-system-user"
-		uidArg := fmt.Sprintf("--uid=%s", uid)
-		output, err := RunAdminCommandNoRealm(context, "user", "info", uidArg)
+		err = createSystemUser(objContext, namespace)
 		if err != nil {
-			if code, ok := exec.ExitStatus(err); ok && code == int(syscall.EINVAL) {
-				logger.Debugf("realm system user %q not found, running `radosgw-admin user create`", uid)
-				accessKeyArg, secretKeyArg, err := GetRealmKeyArgs(context.Context, realmName, namespace)
-				if err != nil {
-					return errors.Wrap(err, "failed to get keys for realm")
-				}
-				systemArg := "--system"
-				displayNameArg := fmt.Sprintf("--display-name=%s.user", realmName)
-				output, err = RunAdminCommandNoRealm(context, "user", "create", realmArg, zoneGroupArg, zoneArg, uidArg, displayNameArg, accessKeyArg, secretKeyArg, systemArg)
-				if err != nil {
-					return errors.Wrapf(err, "failed to create realm system user %q for reason: %q", uid, output)
-				}
-			} else {
-				return errors.Wrapf(err, "radosgw-admin user info failed with code %d and output %q", code, output)
-			}
-		} else {
-			logger.Debugf("realm system user %q has already been created", uid)
+			return err
 		}
 	}
 
 	return nil
 }
 
-func setMultisite(context *Context, spec cephv1.ObjectStoreSpec, serviceIP, realmName, zoneGroupName, zoneName, namespace string) error {
-	realmArg := fmt.Sprintf("--rgw-realm=%s", realmName)
-	zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", zoneGroupName)
-	zoneArg := fmt.Sprintf("--rgw-zone=%s", zoneName)
+func createSystemUser(objContext *Context, namespace string) error {
+	uid := objContext.Realm + "-system-user"
+	uidArg := fmt.Sprintf("--uid=%s", uid)
+	realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm)
+	zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup)
+	zoneArg := fmt.Sprintf("--rgw-zone=%s", objContext.Zone)
 
-	serviceEndpoint := fmt.Sprintf("http://%s:%d", serviceIP, spec.Gateway.Port)
-	if spec.Gateway.SecurePort != 0 {
-		serviceEndpoint = fmt.Sprintf("https://%s:%d", serviceIP, spec.Gateway.SecurePort)
+	output, err := RunAdminCommandNoMultisite(objContext, "user", "info", uidArg)
+	if err == nil {
+		logger.Debugf("realm system user %q has already been created", uid)
+		return nil
 	}
 
-	zoneEndpoints, err := getZoneEndpoints(context, realmArg, zoneGroupArg, zoneName, serviceEndpoint)
-	if err != nil {
-		return err
-	}
-	logger.Debugf("Endpoints for zone %q are: %q", zoneName, zoneEndpoints)
-	endpointArg := fmt.Sprintf("--endpoints=%s", zoneEndpoints)
-
-	if spec.IsMultisite() {
-		err = joinMultisite(context, endpointArg, realmArg, zoneGroupArg, zoneArg, realmName, zoneGroupName, zoneName, zoneEndpoints, namespace)
+	if code, ok := exec.ExitStatus(err); ok && code == int(syscall.EINVAL) {
+		logger.Debugf("realm system user %q not found, running `radosgw-admin user create`", uid)
+		accessKeyArg, secretKeyArg, err := GetRealmKeyArgs(objContext.Context, objContext.Realm, namespace)
 		if err != nil {
-			return errors.Wrapf(err, "failed join ceph multisite in zone %q", zoneName)
+			return errors.Wrap(err, "failed to get keys for realm")
+		}
+		logger.Debugf("found keys to create realm system user %v", uid)
+		systemArg := "--system"
+		displayNameArg := fmt.Sprintf("--display-name=%s.user", objContext.Realm)
+		output, err = RunAdminCommandNoMultisite(objContext, "user", "create", realmArg, zoneGroupArg, zoneArg, uidArg, displayNameArg, accessKeyArg, secretKeyArg, systemArg)
+		if err != nil {
+			return errors.Wrapf(err, "failed to create realm system user %q for reason: %q", uid, output)
+		}
+		logger.Debugf("created realm system user %v", uid)
+	} else {
+		return errors.Wrapf(err, "radosgw-admin user info for system user failed with code %d and output %q", code, output)
+	}
+
+	return nil
+}
+
+func setMultisite(objContext *Context, store *cephv1.CephObjectStore, serviceIP string) error {
+	logger.Debugf("setting multisite configuration for object-store %v", store.Name)
+	serviceEndpoint := fmt.Sprintf("http://%s:%d", serviceIP, store.Spec.Gateway.Port)
+	if store.Spec.Gateway.SecurePort != 0 {
+		serviceEndpoint = fmt.Sprintf("https://%s:%d", serviceIP, store.Spec.Gateway.SecurePort)
+	}
+
+	if store.Spec.IsMultisite() {
+		zoneEndpointsList, err := getZoneEndpoints(objContext, serviceEndpoint)
+		if err != nil {
+			return err
+		}
+		zoneEndpointsList = append(zoneEndpointsList, serviceEndpoint)
+
+		zoneEndpoints := strings.Join(zoneEndpointsList, ",")
+		logger.Debugf("Endpoints for zone %q are: %q", objContext.Zone, zoneEndpoints)
+		endpointArg := fmt.Sprintf("--endpoints=%s", zoneEndpoints)
+
+		err = joinMultisite(objContext, endpointArg, zoneEndpoints, store.Namespace)
+		if err != nil {
+			return errors.Wrapf(err, "failed join ceph multisite in zone %q", objContext.Zone)
 		}
 	} else {
-		err = createMultisite(context, endpointArg, realmArg, zoneGroupArg, zoneArg)
+		endpointArg := fmt.Sprintf("--endpoints=%s", serviceEndpoint)
+		err := createMultisite(objContext, endpointArg)
 		if err != nil {
-			return errors.Wrapf(err, "failed create ceph multisite for object-store %q", context.Name)
+			return errors.Wrapf(err, "failed create ceph multisite for object-store %q", objContext.Name)
 		}
 	}
 
+	logger.Infof("multisite configuration for object-store %v is complete", store.Name)
 	return nil
 }
 
 func deleteRealm(context *Context) error {
 	//  <name>
-	_, err := runAdminCommand(context, "realm", "delete", "--rgw-realm", context.Name)
+	realmArg := fmt.Sprintf("--rgw-realm=%s", context.Name)
+	zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", context.Name)
+	_, err := RunAdminCommandNoMultisite(context, "realm", "delete", realmArg)
 	if err != nil {
 		logger.Warningf("failed to delete rgw realm %q. %v", context.Name, err)
 	}
 
-	_, err = runAdminCommand(context, "zonegroup", "delete", "--rgw-zonegroup", context.Name)
+	_, err = RunAdminCommandNoMultisite(context, "zonegroup", "delete", realmArg, zoneGroupArg)
 	if err != nil {
 		logger.Warningf("failed to delete rgw zonegroup %q. %v", context.Name, err)
 	}
 
-	_, err = runAdminCommand(context, "zone", "delete", "--rgw-zone", context.Name)
+	_, err = runAdminCommand(context, "zone", "delete")
 	if err != nil {
 		logger.Warningf("failed to delete rgw zone %q. %v", context.Name, err)
 	}
@@ -405,7 +543,7 @@ func DecodeZoneGroupConfig(data string) (zoneGroupType, error) {
 }
 
 func getObjectStores(context *Context) ([]string, error) {
-	output, err := RunAdminCommandNoRealm(context, "realm", "list")
+	output, err := RunAdminCommandNoMultisite(context, "realm", "list")
 	if err != nil {
 		if strings.Index(err.Error(), "exit status 2") != 0 {
 			return []string{}, nil

pkg/operator/ceph/object/objectstore_test.go

@@ -45,16 +45,15 @@ func TestReconcileRealm(t *testing.T) {
 	}
 
 	storeName := "myobject"
-	testNamespace := "test-namespace"
 	context := &clusterd.Context{Executor: executor}
 	objContext := NewContext(context, &client.ClusterInfo{Namespace: "mycluster"}, storeName)
 	// create the first realm, marked as default
-	spec := cephv1.ObjectStoreSpec{}
-	err := setMultisite(objContext, spec, "1.2.3.4", storeName, storeName, storeName, testNamespace)
+	store := cephv1.CephObjectStore{}
+	err := setMultisite(objContext, &store, "1.2.3.4")
 	assert.Nil(t, err)
 
 	// create the second realm, not marked as default
-	err = setMultisite(objContext, spec, "2.3.4.5", storeName, storeName, storeName, testNamespace)
+	err = setMultisite(objContext, &store, "2.3.4.5")
 	assert.Nil(t, err)
 }
 

pkg/operator/ceph/object/realm/controller.go

@@ -56,7 +56,7 @@ const (
 	secretKeyLength = 28
 )
 
-var waitForRequeueIfRealmKeysNotReady = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}
+var waitForRequeueIfRealmNotReady = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}
 
 var logger = capnslog.NewPackageLogger("github.com/rook/rook", controllerName)
 
@@ -184,9 +184,8 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R
 		logger.Debug("pull section in spec found")
 		reconcileResponse, err = r.pullCephRealm(cephObjectRealm)
 		if err != nil {
-			return r.setFailedStatus(request.NamespacedName, "failed to pull ceph realm", err)
+			return reconcile.Result{}, err
 		}
-
 	} else {
 		reconcileResponse, err = r.createRealmKeys(cephObjectRealm)
 		if err != nil {
@@ -214,18 +213,17 @@ func (r *ReconcileObjectRealm) pullCephRealm(realm *cephv1.CephObjectRealm) (rec
 	accessKeyArg, secretKeyArg, err := object.GetRealmKeyArgs(r.context, realm.Name, realm.Namespace)
 	if err != nil {
 		if kerrors.IsNotFound(err) {
-			return waitForRequeueIfRealmKeysNotReady, err
+			return waitForRequeueIfRealmNotReady, err
 		}
-		return waitForRequeueIfRealmKeysNotReady, errors.Wrap(err, "failed to get keys for realm")
+		return waitForRequeueIfRealmNotReady, errors.Wrap(err, "failed to get keys for realm")
 	}
 	logger.Debugf("keys found to pull realm, getting ready to pull from endpoint %q", realm.Spec.Pull.Endpoint)
 
-	objContext := object.NewContext(r.context, realm.Name, realm.Namespace)
-
-	output, err := object.RunAdminCommandNoRealm(objContext, "realm", "pull", realmArg, urlArg, accessKeyArg, secretKeyArg)
+	objContext := object.NewContext(r.context, r.clusterInfo, realm.Name)
+	output, err := object.RunAdminCommandNoMultisite(objContext, "realm", "pull", realmArg, urlArg, accessKeyArg, secretKeyArg)
 
 	if err != nil {
-		return reconcile.Result{}, errors.Wrapf(err, "realm pull failed for reason: %v", output)
+		return waitForRequeueIfRealmNotReady, errors.Wrapf(err, "realm pull failed for reason: %v", output)
 	}
 	logger.Debugf("realm pull for %q from endpoint %q succeeded", realm.Name, realm.Spec.Pull.Endpoint)
 
@@ -236,12 +234,12 @@ func (r *ReconcileObjectRealm) createCephRealm(realm *cephv1.CephObjectRealm) (r
 	realmArg := fmt.Sprintf("--rgw-realm=%s", realm.Name)
 	objContext := object.NewContext(r.context, r.clusterInfo, realm.Namespace)
 
-	_, err := object.RunAdminCommandNoRealm(objContext, "realm", "get", realmArg)
+	_, err := object.RunAdminCommandNoMultisite(objContext, "realm", "get", realmArg)
 
 	if err != nil {
 		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
 			logger.Debugf("ceph realm %q not found, running `radosgw-admin realm create`", realm.Name)
-			_, err := object.RunAdminCommandNoRealm(objContext, "realm", "create", realmArg)
+			_, err := object.RunAdminCommandNoMultisite(objContext, "realm", "create", realmArg)
 			if err != nil {
 				return reconcile.Result{}, errors.Wrapf(err, "failed to create ceph realm %s", realm.Name)
 			}
@@ -266,7 +264,7 @@ func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (r
 
 	secretKey, err := mgr.GeneratePassword(secretKeyLength)
 	if err != nil {
-		return reconcile.Result{}, errors.Wrapf(err, "secret key failed to generate")
+		return reconcile.Result{}, errors.Wrapf(err, "failed to generate secret key")
 	}
 	secretKey = base64.StdEncoding.EncodeToString([]byte(secretKey))
 
@@ -296,6 +294,7 @@ func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (r
 	if _, err = r.context.Clientset.CoreV1().Secrets(realm.Namespace).Create(secret); err != nil {
 		return reconcile.Result{}, errors.Wrap(err, "failed to save rgw secrets")
 	}
+	logger.Infof("secrets for keys have been created for realm %q", realm.Name)
 
 	return reconcile.Result{}, nil
 }
@@ -308,11 +307,6 @@ func validateRealmCR(u *cephv1.CephObjectRealm) error {
 	if u.Namespace == "" {
 		return errors.New("missing namespace")
 	}
-	if u.Spec.IsPullRealm() {
-		if u.Spec.Pull.Endpoint == "" {
-			return errors.New("missing endpoint for pulling realm")
-		}
-	}
 	return nil
 }
 

pkg/operator/ceph/object/rgw.go

@@ -277,8 +277,14 @@ func (c *clusterConfig) deleteStore() error {
 		}
 
 		// Delete the realm and pools
-		objContext := NewContext(c.context, c.clusterInfo, c.store.Name)
-		err := deleteRealmAndPools(objContext, c.store.Spec)
+		objContext, err := NewMultisiteContext(c.context, c.clusterInfo, c.store)
+		if err != nil {
+			return errors.Wrapf(err, "failed to set multisite on object store %q", c.store.Name)
+		}
+
+		objContext.Endpoint = c.store.Status.Info["endpoint"]
+
+		err = deleteRealmAndPools(objContext, c.store.Spec)
 		if err != nil {
 			return errors.Wrap(err, "failed to delete the realm and pools")
 		}
@@ -347,10 +353,6 @@ func (r *ReconcileCephObjectStore) validateStore(s *cephv1.CephObjectStore) erro
 		}
 	}
 
-	if s.Spec.IsMultisite() && !s.Spec.PreservePoolsOnDelete {
-		return errors.New("preservePoolsOnDelete must be true when zone is set")
-	}
-
 	return nil
 }
 

pkg/operator/ceph/object/user/controller.go

@@ -186,8 +186,8 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci
 		return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info")
 	}
 
-	// validate isObjectStoreInitialized
-	err = r.isObjectStoreInitialized(cephObjectStoreUser)
+	// validate initializeObjectStoreContext
+	err = r.initializeObjectStoreContext(cephObjectStoreUser)
 	if err != nil {
 		if !cephObjectStoreUser.GetDeletionTimestamp().IsZero() {
 			// Remove finalizer
@@ -293,15 +293,24 @@ func (r *ReconcileObjectStoreUser) createorUpdateCephUser(u *cephv1.CephObjectSt
 	return nil
 }
 
-func (r *ReconcileObjectStoreUser) isObjectStoreInitialized(u *cephv1.CephObjectStoreUser) error {
-	objContext := object.NewContext(r.context, r.clusterInfo, u.Spec.Store)
-	r.objContext = objContext
-
+func (r *ReconcileObjectStoreUser) initializeObjectStoreContext(u *cephv1.CephObjectStoreUser) error {
 	err := r.objectStoreInitialized(u)
 	if err != nil {
-		return errors.Wrap(err, "failed to detect if object store is initialized")
+		return errors.Wrapf(err, "failed to detect if object store %q is initialized", u.Spec.Store)
 	}
 
+	store, err := r.getObjectStore(u.Spec.Store)
+	if err != nil {
+		return errors.Wrapf(err, "failed to get object store %q", u.Spec.Store)
+	}
+
+	objContext, err := object.NewMultisiteContext(r.context, r.clusterInfo, store)
+	if err != nil {
+		return errors.Wrapf(err, "Multisite failed to set on object context for object store user")
+	}
+	r.objContext = objContext
+	r.objContext.Endpoint = store.Status.Info["endpoint"]
+
 	return nil
 }
 
@@ -376,7 +385,7 @@ func (r *ReconcileObjectStoreUser) reconcileCephUserSecret(cephObjectStoreUser *
 }
 
 func (r *ReconcileObjectStoreUser) objectStoreInitialized(cephObjectStoreUser *cephv1.CephObjectStoreUser) error {
-	err := r.getObjectStore(cephObjectStoreUser.Spec.Store)
+	_, err := r.getObjectStore(cephObjectStoreUser.Spec.Store)
 	if err != nil {
 		return err
 	}
@@ -404,26 +413,25 @@ func (r *ReconcileObjectStoreUser) objectStoreInitialized(cephObjectStoreUser *c
 	return errors.New("no rgw pod found")
 }
 
-func (r *ReconcileObjectStoreUser) getObjectStore(storeName string) error {
+func (r *ReconcileObjectStoreUser) getObjectStore(storeName string) (*cephv1.CephObjectStore, error) {
 	// check if CephObjectStore CR is created
 	objectStores := &cephv1.CephObjectStoreList{}
 	err := r.client.List(context.TODO(), objectStores)
 	if err != nil {
 		if kerrors.IsNotFound(err) {
-			return errors.Wrapf(err, "CephObjectStore %q could not be found", storeName)
+			return nil, errors.Wrapf(err, "CephObjectStore %q could not be found", storeName)
 		}
-		return errors.Wrap(err, "failed to get CephObjectStore")
+		return nil, errors.Wrap(err, "failed to get CephObjectStore")
 	}
 
 	for _, store := range objectStores.Items {
 		if store.Name == storeName {
 			logger.Infof("CephObjectStore %q found", storeName)
-			r.objContext.Endpoint = store.Status.Info["endpoint"]
-			return nil
+			return &store, nil
 		}
 	}
 
-	return errors.Errorf("CephObjectStore %q could not be found", storeName)
+	return nil, errors.Errorf("CephObjectStore %q could not be found", storeName)
 }
 
 func (r *ReconcileObjectStoreUser) getRgwPodList(cephObjectStoreUser *cephv1.CephObjectStoreUser) (*corev1.PodList, error) {

pkg/operator/ceph/object/zone/controller.go

@@ -188,14 +188,6 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re
 		return reconcileResponse, err
 	}
 
-	// RECONCILE POOLS
-	logger.Info("reconciling object zone pools")
-	objContext := object.NewContext(r.context, cephObjectZone.Name, cephObjectZone.Namespace)
-	err = object.CreatePools(objContext, cephObjectZone.Spec.MetadataPool, cephObjectZone.Spec.DataPool)
-	if err != nil {
-		return r.setFailedStatus(request.NamespacedName, "failed to create zone pools", err)
-	}
-
 	// Create Ceph Zone
 	reconcileResponse, err = r.createCephZone(cephObjectZone, realmName)
 	if err != nil {
@@ -219,7 +211,7 @@ func (r *ReconcileObjectZone) createCephZone(zone *cephv1.CephObjectZone, realmN
 	objContext := object.NewContext(r.context, r.clusterInfo, zone.Name)
 
 	// get zone group to see if master zone exists yet
-	output, err := object.RunAdminCommandNoRealm(objContext, "zonegroup", "get", realmArg, zoneGroupArg)
+	output, err := object.RunAdminCommandNoMultisite(objContext, "zonegroup", "get", realmArg, zoneGroupArg)
 	if err != nil {
 		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
 			return reconcile.Result{}, errors.Wrapf(err, "ceph zone group %q not found", zone.Spec.ZoneGroup)
@@ -234,41 +226,65 @@ func (r *ReconcileObjectZone) createCephZone(zone *cephv1.CephObjectZone, realmN
 		return reconcile.Result{}, errors.Wrap(err, "failed to parse `radosgw-admin zonegroup get` output")
 	}
 
-	zoneIsMaster := false
-	if zoneGroupJson.MasterZoneID == "" {
-		zoneIsMaster = true
+	// create zone
+	_, err = object.RunAdminCommandNoMultisite(objContext, "zone", "get", realmArg, zoneGroupArg, zoneArg)
+	if err == nil {
+		logger.Debugf("ceph zone %q already exists, new zone and pools will not be created", zone.Name)
+		return reconcile.Result{}, nil
 	}
 
-	// create zone
-	_, err = object.RunAdminCommandNoRealm(objContext, "zone", "get", realmArg, zoneGroupArg, zoneArg)
-	if err != nil {
-		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
-			logger.Debugf("ceph zone %q not found, running `radosgw-admin zone create`", zone.Name)
-			accessKeyArg, secretKeyArg, err := object.GetRealmKeyArgs(r.context, realmName, zone.Namespace)
-			if err != nil {
-				return reconcile.Result{}, errors.Wrap(err, "failed to get keys for realm")
-			}
+	if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
+		logger.Debugf("ceph zone %q not found, running `radosgw-admin zone create`", zone.Name)
 
-			if zoneIsMaster {
-				// master zone does not exist yet for zone group
-				masterArg := "--master"
-				_, err = object.RunAdminCommandNoRealm(objContext, "zone", "create", realmArg, zoneGroupArg, zoneArg, accessKeyArg, secretKeyArg, masterArg)
-			} else {
-				_, err = object.RunAdminCommandNoRealm(objContext, "zone", "create", realmArg, zoneGroupArg, zoneArg, accessKeyArg, secretKeyArg)
-			}
-
-			if err != nil {
-				return reconcile.Result{}, errors.Wrapf(err, "failed to create ceph zone %q", zone.Name)
-			}
-
-		} else {
-			return reconcile.Result{}, errors.Wrapf(err, "radosgw-admin zone get failed with code %d", code)
+		zoneIsMaster := false
+		if zoneGroupJson.MasterZoneID == "" {
+			zoneIsMaster = true
 		}
+
+		err = r.createPoolsAndZone(objContext, zone, realmName, zoneIsMaster)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
+	} else {
+		return reconcile.Result{}, errors.Wrapf(err, "radosgw-admin zone get failed with code %d for reason %q", code, output)
 	}
 
 	return reconcile.Result{}, nil
 }
 
+func (r *ReconcileObjectZone) createPoolsAndZone(objContext *object.Context, zone *cephv1.CephObjectZone, realmName string, zoneIsMaster bool) error {
+	// create pools for zone
+	logger.Debugf("creating pools ceph zone %q", zone.Name)
+	realmArg := fmt.Sprintf("--rgw-realm=%s", realmName)
+	zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", zone.Spec.ZoneGroup)
+	zoneArg := fmt.Sprintf("--rgw-zone=%s", zone.Name)
+
+	err := object.CreatePools(objContext, zone.Spec.MetadataPool, zone.Spec.DataPool)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create pools for zone %v", zone.Name)
+	}
+	logger.Debugf("created pools ceph zone %q", zone.Name)
+
+	accessKeyArg, secretKeyArg, err := object.GetRealmKeyArgs(r.context, realmName, zone.Namespace)
+	if err != nil {
+		return errors.Wrap(err, "failed to get keys for realm")
+	}
+	args := []string{"zone", "create", realmArg, zoneGroupArg, zoneArg, accessKeyArg, secretKeyArg}
+
+	if zoneIsMaster {
+		// master zone does not exist yet for zone group
+		args = append(args, "--master")
+	}
+
+	output, err := object.RunAdminCommandNoMultisite(objContext, args...)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create ceph zone %q for reason %q", zone.Name, output)
+	}
+	logger.Debugf("created ceph zone %q", zone.Name)
+
+	return nil
+}
+
 func (r *ReconcileObjectZone) reconcileObjectZoneGroup(zone *cephv1.CephObjectZone) (string, reconcile.Result, error) {
 	// Verify the object zone API object actually exists
 	zoneGroup, err := r.context.RookClientset.CephV1().CephObjectZoneGroups(zone.Namespace).Get(zone.Spec.ZoneGroup, metav1.GetOptions{})
@@ -288,7 +304,7 @@ func (r *ReconcileObjectZone) reconcileCephZoneGroup(zone *cephv1.CephObjectZone
 	zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", zone.Spec.ZoneGroup)
 	objContext := object.NewContext(r.context, r.clusterInfo, zone.Name)
 
-	_, err := object.RunAdminCommandNoRealm(objContext, "zonegroup", "get", realmArg, zoneGroupArg)
+	_, err := object.RunAdminCommandNoMultisite(objContext, "zonegroup", "get", realmArg, zoneGroupArg)
 	if err != nil {
 		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
 			return waitForRequeueIfObjectZoneGroupNotReady, errors.Wrapf(err, "ceph zone group %q not found", zone.Spec.ZoneGroup)
@@ -302,20 +318,20 @@ func (r *ReconcileObjectZone) reconcileCephZoneGroup(zone *cephv1.CephObjectZone
 }
 
 // validateZoneCR validates the zone arguments
-func (r *ReconcileObjectZone) validateZoneCR(u *cephv1.CephObjectZone) error {
-	if u.Name == "" {
+func (r *ReconcileObjectZone) validateZoneCR(z *cephv1.CephObjectZone) error {
+	if z.Name == "" {
 		return errors.New("missing name")
 	}
-	if u.Namespace == "" {
+	if z.Namespace == "" {
 		return errors.New("missing namespace")
 	}
-	if u.Spec.ZoneGroup == "" {
+	if z.Spec.ZoneGroup == "" {
 		return errors.New("missing zonegroup")
 	}
-	if err := pool.ValidatePoolSpec(r.context, u.Namespace, &u.Spec.MetadataPool); err != nil {
+	if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, &z.Spec.MetadataPool); err != nil {
 		return errors.Wrap(err, "invalid metadata pool spec")
 	}
-	if err := pool.ValidatePoolSpec(r.context, u.Namespace, &u.Spec.DataPool); err != nil {
+	if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, &z.Spec.DataPool); err != nil {
 		return errors.Wrap(err, "invalid data pool spec")
 	}
 	return nil

pkg/operator/ceph/object/zonegroup/controller.go

@@ -208,7 +208,7 @@ func (r *ReconcileObjectZoneGroup) createCephZoneGroup(zoneGroup *cephv1.CephObj
 	objContext := object.NewContext(r.context, r.clusterInfo, zoneGroup.Name)
 
 	// get period to see if master zone group exists yet
-	output, err := object.RunAdminCommandNoRealm(objContext, "period", "get", realmArg)
+	output, err := object.RunAdminCommandNoMultisite(objContext, "period", "get", realmArg)
 	if err != nil {
 		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
 			return reconcile.Result{}, errors.Wrapf(err, "ceph period %q not found", zoneGroup.Spec.Realm)
@@ -229,24 +229,31 @@ func (r *ReconcileObjectZoneGroup) createCephZoneGroup(zoneGroup *cephv1.CephObj
 	}
 
 	// create zone group
-	_, err = object.RunAdminCommandNoRealm(objContext, "zonegroup", "get", realmArg, zoneGroupArg)
-	if err != nil {
-		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
-			logger.Debugf("ceph zone group %q not found, running `radosgw-admin zonegroup create`", zoneGroup.Name)
+	output, err = object.RunAdminCommandNoMultisite(objContext, "zonegroup", "get", realmArg, zoneGroupArg)
+	if err == nil {
+		return reconcile.Result{}, nil
+	}
 
-			if zoneGroupIsMaster {
-				masterArg := "--master"
-				_, err = object.RunAdminCommandNoRealm(objContext, "zonegroup", "create", realmArg, zoneGroupArg, masterArg)
-			} else {
-				_, err = object.RunAdminCommandNoRealm(objContext, "zonegroup", "create", realmArg, zoneGroupArg)
-			}
-
-			if err != nil {
-				return reconcile.Result{}, errors.Wrapf(err, "failed to create ceph zone group %q", zoneGroup.Name)
-			}
-		} else {
-			return reconcile.Result{}, errors.Wrapf(err, "radosgw-admin zonegroup get failed with code %d", code)
+	if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
+		logger.Debugf("ceph zone group %q not found, running `radosgw-admin zonegroup create`", zoneGroup.Name)
+		args := []string{
+			"zonegroup",
+			"create",
+			realmArg,
+			zoneGroupArg,
 		}
+
+		if zoneGroupIsMaster {
+			// master zone group does not exist yet for realm
+			args = append(args, "--master")
+		}
+
+		output, err = object.RunAdminCommandNoMultisite(objContext, args...)
+		if err != nil {
+			return reconcile.Result{}, errors.Wrapf(err, "failed to create ceph zone group %q for reason %q", zoneGroup.Name, output)
+		}
+	} else {
+		return reconcile.Result{}, errors.Wrapf(err, "radosgw-admin zonegroup get failed with code %d for reason %q", code, output)
 	}
 
 	return reconcile.Result{}, nil
@@ -270,7 +277,7 @@ func (r *ReconcileObjectZoneGroup) reconcileCephRealm(zoneGroup *cephv1.CephObje
 	realmArg := fmt.Sprintf("--rgw-realm=%s", zoneGroup.Spec.Realm)
 	objContext := object.NewContext(r.context, r.clusterInfo, zoneGroup.Name)
 
-	_, err := object.RunAdminCommandNoRealm(objContext, "realm", "get", realmArg)
+	_, err := object.RunAdminCommandNoMultisite(objContext, "realm", "get", realmArg)
 	if err != nil {
 		if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) {
 			return waitForRequeueIfObjectRealmNotReady, errors.Wrapf(err, "ceph realm %q not found", zoneGroup.Spec.Realm)