diff --git a/pkg/apis/ceph.rook.io/v1/object.go b/pkg/apis/ceph.rook.io/v1/object.go index 11f33cc89..5c3c3e31e 100644 --- a/pkg/apis/ceph.rook.io/v1/object.go +++ b/pkg/apis/ceph.rook.io/v1/object.go @@ -1,5 +1,5 @@ /* -Copyright 2020 The Rook Authors. All rights reserved. +Copyright 2018 The Rook Authors. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go index 0dbb20f76..5de6f9a2d 100644 --- a/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/ceph.rook.io/v1/zz_generated.deepcopy.go @@ -562,7 +562,7 @@ func (in *CephObjectStoreUser) DeepCopyInto(out *CephObjectStoreUser) { if in.Status != nil { in, out := &in.Status, &out.Status *out = new(ObjectStoreUserStatus) - **out = **in + (*in).DeepCopyInto(*out) } return } @@ -623,7 +623,7 @@ func (in *CephObjectZone) DeepCopyInto(out *CephObjectZone) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) if in.Status != nil { in, out := &in.Status, &out.Status *out = new(Status) @@ -1509,6 +1509,29 @@ func (in *ObjectStoreUserSpec) DeepCopy() *ObjectStoreUserSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectStoreUserStatus) DeepCopyInto(out *ObjectStoreUserStatus) { + *out = *in + if in.Info != nil { + in, out := &in.Info, &out.Info + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectStoreUserStatus. +func (in *ObjectStoreUserStatus) DeepCopy() *ObjectStoreUserStatus { + if in == nil { + return nil + } + out := new(ObjectStoreUserStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectZoneGroupSpec) DeepCopyInto(out *ObjectZoneGroupSpec) { *out = *in @@ -1528,6 +1551,8 @@ func (in *ObjectZoneGroupSpec) DeepCopy() *ObjectZoneGroupSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectZoneSpec) DeepCopyInto(out *ObjectZoneSpec) { *out = *in + in.MetadataPool.DeepCopyInto(&out.MetadataPool) + in.DataPool.DeepCopyInto(&out.DataPool) return } diff --git a/pkg/operator/ceph/object/admin.go b/pkg/operator/ceph/object/admin.go index a9233bc05..f875068df 100644 --- a/pkg/operator/ceph/object/admin.go +++ b/pkg/operator/ceph/object/admin.go @@ -19,6 +19,8 @@ package object import ( "fmt" + "github.com/pkg/errors" + cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" "github.com/rook/rook/pkg/daemon/ceph/client" ) @@ -30,6 +32,9 @@ type Context struct { Name string UID string Endpoint string + Realm string + ZoneGroup string + Zone string } // NewContext creates a new object store context. @@ -37,7 +42,21 @@ func NewContext(context *clusterd.Context, clusterInfo *client.ClusterInfo, name return &Context{Context: context, Name: name, clusterInfo: clusterInfo} } -func RunAdminCommandNoRealm(c *Context, args ...string) (string, error) { +func NewMultisiteContext(context *clusterd.Context, clusterInfo *client.ClusterInfo, store *cephv1.CephObjectStore) (*Context, error) { + objContext := &Context{Context: context, Name: store.Name, clusterInfo: clusterInfo} + realmName, zoneGroupName, zoneName, err := getMultisiteForObjectStore(context, store) + if err != nil { + return nil, errors.Wrapf(err, "failed to get realm/zone group/zone for object store %q", store.Name) + } + + objContext.Realm = realmName + objContext.ZoneGroup = zoneGroupName + objContext.Zone = zoneName + return objContext, nil +} + +// This function is for running radosgw-admin commands in scenarios where an object-store has not been created yet or for commands on the realm or zonegroup (ex: radosgw-admin zonegroup get) +func RunAdminCommandNoMultisite(c *Context, args ...string) (string, error) { command, args := client.FinalizeCephCommandArgs("radosgw-admin", c.clusterInfo, args, c.Context.ConfigDir) // start the rgw admin command @@ -49,6 +68,7 @@ func RunAdminCommandNoRealm(c *Context, args ...string) (string, error) { return output, nil } +// This function is for running radosgw-admin commands in scenarios where an object-store has been created and the Context has been updated with the appropriate realm, zone group, and zone. func runAdminCommand(c *Context, args ...string) (string, error) { // If the objectStoreName is not passed in the storage class // This means we are pointing to an external cluster so these commands are not needed @@ -59,12 +79,13 @@ func runAdminCommand(c *Context, args ...string) (string, error) { // This is not perfect though since "client.admin" is somehow supported... if c.Name != "" && c.clusterInfo.CephCred.Username == client.AdminUsername { options := []string{ - fmt.Sprintf("--rgw-realm=%s", c.Name), - fmt.Sprintf("--rgw-zonegroup=%s", c.Name), + fmt.Sprintf("--rgw-realm=%s", c.Realm), + fmt.Sprintf("--rgw-zonegroup=%s", c.ZoneGroup), + fmt.Sprintf("--rgw-zone=%s", c.Zone), } - return RunAdminCommandNoRealm(c, append(args, options...)...) + return RunAdminCommandNoMultisite(c, append(args, options...)...) } - return RunAdminCommandNoRealm(c, args...) + return RunAdminCommandNoMultisite(c, args...) } diff --git a/pkg/operator/ceph/object/bucket/provisioner.go b/pkg/operator/ceph/object/bucket/provisioner.go index 522b9496a..af81323d1 100644 --- a/pkg/operator/ceph/object/bucket/provisioner.go +++ b/pkg/operator/ceph/object/bucket/provisioner.go @@ -395,7 +395,16 @@ func (p *Provisioner) setObjectContext() error { if p.objectStoreName == "" && p.endpoint == "" { return errors.Errorf(msg, "name") } - p.objectContext = cephObject.NewContext(p.context, p.clusterInfo, p.objectStoreName) + + store, err := p.getObjectStore() + if err != nil { + return errors.Wrap(err, "failed to get cephObjectStore") + } + + p.objectContext, err = cephObject.NewMultisiteContext(p.context, p.clusterInfo, store) + if err != nil { + return errors.Wrapf(err, "failed to set multisite on provisioner's objectContext") + } return nil } @@ -407,7 +416,7 @@ func (p *Provisioner) setObjectStoreDomainName(sc *storagev1.StorageClass) error name := getObjectStoreName(sc) namespace := getObjectStoreNameSpace(sc) // make sure the object store actually exists - _, err := getObjectStore(p.context.RookClientset.CephV1(), p.clusterInfo.Namespace, p.objectStoreName) + _, err := p.getObjectStore() if err != nil { return err } diff --git a/pkg/operator/ceph/object/bucket/provisionner_test.go b/pkg/operator/ceph/object/bucket/provisioner_test.go similarity index 97% rename from pkg/operator/ceph/object/bucket/provisionner_test.go rename to pkg/operator/ceph/object/bucket/provisioner_test.go index 26d1219ac..6e2f660e0 100644 --- a/pkg/operator/ceph/object/bucket/provisionner_test.go +++ b/pkg/operator/ceph/object/bucket/provisioner_test.go @@ -37,6 +37,7 @@ func TestPopulateDomainAndPort(t *testing.T) { namespace := "ns" clusterInfo := client.AdminClusterInfo(namespace) p := NewProvisioner(&clusterd.Context{RookClientset: rookclient.NewSimpleClientset(), Clientset: test.New(t, 1)}, clusterInfo) + p.objectContext = object.NewContext(p.context, clusterInfo, store) sc := &storagev1.StorageClass{ Parameters: map[string]string{ "foo": "bar", diff --git a/pkg/operator/ceph/object/bucket/util.go b/pkg/operator/ceph/object/bucket/util.go index b865dc76b..536c269e6 100644 --- a/pkg/operator/ceph/object/bucket/util.go +++ b/pkg/operator/ceph/object/bucket/util.go @@ -32,7 +32,6 @@ import ( "k8s.io/client-go/rest" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - cephclientset "github.com/rook/rook/pkg/client/clientset/versioned/typed/ceph.rook.io/v1" cephObject "github.com/rook/rook/pkg/operator/ceph/object" ) @@ -100,9 +99,9 @@ func getCephUser(ob *bktv1alpha1.ObjectBucket) string { return ob.Spec.AdditionalState[cephUser] } -func getObjectStore(c cephclientset.CephV1Interface, namespace, name string) (*cephv1.CephObjectStore, error) { +func (p *Provisioner) getObjectStore() (*cephv1.CephObjectStore, error) { // Verify the object store API object actually exists - store, err := c.CephObjectStores(namespace).Get(name, metav1.GetOptions{}) + store, err := p.context.RookClientset.CephV1().CephObjectStores(p.clusterInfo.Namespace).Get(p.objectStoreName, metav1.GetOptions{}) if err != nil { if kerrors.IsNotFound(err) { return nil, errors.Wrap(err, "cephObjectStore not found") diff --git a/pkg/operator/ceph/object/controller.go b/pkg/operator/ceph/object/controller.go index 02b6c1db4..2075dc31f 100644 --- a/pkg/operator/ceph/object/controller.go +++ b/pkg/operator/ceph/object/controller.go @@ -333,6 +333,11 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c } } + objContext.Realm = realmName + objContext.ZoneGroup = zoneGroupName + objContext.Zone = zoneName + logger.Debugf("realm for object-store is %q, zone group for object-store is %q, zone for object-store is %q", objContext.Realm, objContext.ZoneGroup, objContext.Zone) + // RECONCILE SERVICE logger.Debug("reconciling object store service") serviceIP, err = cfg.reconcileService(cephObjectStore) @@ -351,7 +356,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c // Reconcile Multisite Creation logger.Infof("setting multisite settings for object store %q", cephObjectStore.Name) - err = setMultisite(objContext, cephObjectStore.Spec, serviceIP, realmName, zoneGroupName, zoneName, cephObjectStore.Namespace) + err = setMultisite(objContext, cephObjectStore, serviceIP) if err != nil { return r.setFailedStatus(namespacedName, "failed to configure multisite for object store", err) } @@ -377,7 +382,7 @@ func (r *ReconcileCephObjectStore) reconcileCephZone(store *cephv1.CephObjectSto zoneArg := fmt.Sprintf("--rgw-zone=%s", store.Spec.Zone.Name) objContext := NewContext(r.context, r.clusterInfo, store.Name) - _, err := RunAdminCommandNoRealm(objContext, "zone", "get", realmArg, zoneGroupArg, zoneArg) + _, err := RunAdminCommandNoMultisite(objContext, "zone", "get", realmArg, zoneGroupArg, zoneArg) if err != nil { if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) { return waitForRequeueIfObjectStoreNotReady, errors.Wrapf(err, "ceph zone %q not found", store.Spec.Zone.Name) @@ -400,7 +405,7 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1 } return "", "", "", waitForRequeueIfObjectStoreNotReady, errors.Wrapf(err, "error getting CephObjectZone %q", cephObjectStore.Spec.Zone.Name) } - logger.Infof("CephObjectZone resource %s found", zone.Name) + logger.Debugf("CephObjectZone resource %s found", zone.Name) zonegroup, err := r.context.RookClientset.CephV1().CephObjectZoneGroups(cephObjectStore.Namespace).Get(zone.Spec.ZoneGroup, metav1.GetOptions{}) if err != nil { @@ -409,7 +414,7 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1 } return "", "", "", waitForRequeueIfObjectStoreNotReady, errors.Wrapf(err, "error getting CephObjectZoneGroup %q", zone.Spec.ZoneGroup) } - logger.Infof("CephObjectZoneGroup resource %s found", zonegroup.Name) + logger.Debugf("CephObjectZoneGroup resource %s found", zonegroup.Name) realm, err := r.context.RookClientset.CephV1().CephObjectRealms(cephObjectStore.Namespace).Get(zonegroup.Spec.Realm, metav1.GetOptions{}) if err != nil { @@ -418,7 +423,7 @@ func (r *ReconcileCephObjectStore) reconcileMultisiteCRs(cephObjectStore *cephv1 } return "", "", "", waitForRequeueIfObjectStoreNotReady, errors.Wrapf(err, "error getting CephObjectRealm %q", zonegroup.Spec.Realm) } - logger.Infof("CephObjectRealm resource %s found", realm.Name) + logger.Debugf("CephObjectRealm resource %s found", realm.Name) return realm.Name, zonegroup.Name, zone.Name, reconcile.Result{}, nil } diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go index 12090084a..c3cacfd2e 100644 --- a/pkg/operator/ceph/object/objectstore.go +++ b/pkg/operator/ceph/object/objectstore.go @@ -77,45 +77,138 @@ type realmType struct { Realms []string `json:"realms"` } -func deleteRealmAndPools(context *Context, spec cephv1.ObjectStoreSpec) error { - stores, err := getObjectStores(context) +func deleteRealmAndPools(objContext *Context, spec cephv1.ObjectStoreSpec) error { + if spec.IsMultisite() { + // since pools for object store are created by the zone, the object store only needs to be removed from the zone + err := removeObjectStoreFromMultisite(objContext, spec) + if err != nil { + return err + } + + return nil + } + + return deleteSingleSiteRealmAndPools(objContext, spec) +} + +func removeObjectStoreFromMultisite(objContext *Context, spec cephv1.ObjectStoreSpec) error { + // get list of endpoints not including the endpoint of the object-store for the zone + zoneEndpointsList, err := getZoneEndpoints(objContext, objContext.Endpoint) + if err != nil { + return err + } + + realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm) + zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup) + zoneEndpoints := strings.Join(zoneEndpointsList, ",") + endpointArg := fmt.Sprintf("--endpoints=%s", zoneEndpoints) + + zoneIsMaster, err := checkZoneIsMaster(objContext) + zoneGroupIsMaster := false + if zoneIsMaster { + _, err = RunAdminCommandNoMultisite(objContext, "zonegroup", "modify", realmArg, zoneGroupArg, endpointArg) + if err != nil { + return errors.Wrapf(err, "failed to remove object store %q endpoint from rgw zone group %q", objContext.Name, objContext.ZoneGroup) + } + logger.Debugf("endpoint %q was removed from zone group %q. the remaining endpoints in the zone group are %q", objContext.Endpoint, objContext.ZoneGroup, zoneEndpoints) + + // check if zone group is master only if zone is master for creating the system user + zoneGroupIsMaster, err = checkZoneGroupIsMaster(objContext) + if err != nil { + return errors.Wrapf(err, "failed to find out whether zone group %q in is the master zone group", objContext.ZoneGroup) + } + } + + _, err = runAdminCommand(objContext, "zone", "modify", endpointArg) + if err != nil { + return errors.Wrapf(err, "failed to remove object store %q endpoint from rgw zone %q", objContext.Name, spec.Zone.Name) + } + logger.Debugf("endpoint %q was removed from zone %q. the remaining endpoints in the zone are %q", objContext.Endpoint, objContext.Zone, zoneEndpoints) + + if zoneIsMaster && zoneGroupIsMaster && zoneEndpoints == "" { + logger.Infof("WARNING: No other zone in realm %q can commit to the period or pull the realm until you create another object-store in zone %q", objContext.Realm, objContext.Zone) + } + + // the period will help notify other zones of changes if there are multi-zones + _, err = runAdminCommand(objContext, "period", "update", "--commit") + if err != nil { + return errors.Wrap(err, "failed to update period after removing an endpoint from the zone") + } + logger.Infof("successfully updated period for realm %v after removal of object-store %v", objContext.Realm, objContext.Name) + + return nil +} + +func deleteSingleSiteRealmAndPools(objContext *Context, spec cephv1.ObjectStoreSpec) error { + stores, err := getObjectStores(objContext) if err != nil { return errors.Wrap(err, "failed to detect object stores during deletion") } - logger.Infof("Found stores %v when deleting store %s", stores, context.Name) + logger.Infof("Found stores %v when deleting store %s", stores, objContext.Name) - err = deleteRealm(context) + err = deleteRealm(objContext) if err != nil { return errors.Wrap(err, "failed to delete realm") } lastStore := false - if len(stores) == 1 && stores[0] == context.Name { + if len(stores) == 1 && stores[0] == objContext.Name { lastStore = true } if !spec.PreservePoolsOnDelete { - err = deletePools(context, spec, lastStore) + err = deletePools(objContext, spec, lastStore) if err != nil { return errors.Wrap(err, "failed to delete object store pools") } } else { - logger.Infof("PreservePoolsOnDelete is set in object store %s. Pools not deleted", context.Name) + logger.Infof("PreservePoolsOnDelete is set in object store %s. Pools not deleted", objContext.Name) } + return nil } -func checkZoneIsMaster(context *Context, realmArg, zoneGroupArg, zoneArg string) (bool, error) { - zoneGroupOutput, err := RunAdminCommandNoRealm(context, "zonegroup", "get", realmArg, zoneGroupArg) +// This is used for quickly getting the name of the realm, zone group, and zone for an object-store to pass into a Context +func getMultisiteForObjectStore(context *clusterd.Context, store *cephv1.CephObjectStore) (string, string, string, error) { + if store.Spec.IsMultisite() { + zone, err := context.RookClientset.CephV1().CephObjectZones(store.Namespace).Get(store.Spec.Zone.Name, metav1.GetOptions{}) + if err != nil { + return "", "", "", errors.Wrapf(err, "failed to find zone for object-store %q", store.Name) + } + + zonegroup, err := context.RookClientset.CephV1().CephObjectZoneGroups(store.Namespace).Get(zone.Spec.ZoneGroup, metav1.GetOptions{}) + if err != nil { + return "", "", "", errors.Wrapf(err, "failed to find zone group for object-store %q", store.Name) + } + + realm, err := context.RookClientset.CephV1().CephObjectRealms(store.Namespace).Get(zonegroup.Spec.Realm, metav1.GetOptions{}) + if err != nil { + return "", "", "", errors.Wrapf(err, "failed to find realm for object-store %q", store.Name) + } + + return realm.Name, zonegroup.Name, zone.Name, nil + } + + return store.Name, store.Name, store.Name, nil +} + +func checkZoneIsMaster(objContext *Context) (bool, error) { + logger.Debugf("checking if zone %v is the master zone", objContext.Zone) + realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm) + zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup) + zoneArg := fmt.Sprintf("--rgw-zone=%s", objContext.Zone) + + zoneGroupJson, err := RunAdminCommandNoMultisite(objContext, "zonegroup", "get", realmArg, zoneGroupArg) if err != nil { return false, errors.Wrap(err, "failed to get rgw zone group") } - zoneGroupJson, err := DecodeZoneGroupConfig(zoneGroupOutput) + zoneGroupOutput, err := DecodeZoneGroupConfig(zoneGroupJson) if err != nil { return false, errors.Wrap(err, "failed to parse zonegroup get json") } + logger.Debugf("got master zone ID for zone group %v", objContext.ZoneGroup) - zoneOutput, err := RunAdminCommandNoRealm(context, "zone", "get", realmArg, zoneGroupArg, zoneArg) + zoneOutput, err := RunAdminCommandNoMultisite(objContext, "zone", "get", realmArg, zoneGroupArg, zoneArg) if err != nil { return false, errors.Wrap(err, "failed to get rgw zone") } @@ -123,19 +216,27 @@ func checkZoneIsMaster(context *Context, realmArg, zoneGroupArg, zoneArg string) if err != nil { return false, errors.Wrap(err, "failed to parse zone id") } + logger.Debugf("got zone ID for zone %v", objContext.Zone) - if zoneID == zoneGroupJson.MasterZoneID { + if zoneID == zoneGroupOutput.MasterZoneID { + logger.Debugf("zone is master") return true, nil } + logger.Debugf("zone is not master") return false, nil } -func checkZoneGroupIsMaster(context *Context, realmArg, zoneGroupArg string) (bool, error) { - zoneGroupOutput, err := RunAdminCommandNoRealm(context, "zonegroup", "get", realmArg, zoneGroupArg) +func checkZoneGroupIsMaster(objContext *Context) (bool, error) { + logger.Debugf("checking if zone group %v is the master zone group", objContext.ZoneGroup) + realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm) + zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup) + + zoneGroupOutput, err := RunAdminCommandNoMultisite(objContext, "zonegroup", "get", realmArg, zoneGroupArg) if err != nil { return false, errors.Wrap(err, "failed to get rgw zone group") } + zoneGroupJson, err := DecodeZoneGroupConfig(zoneGroupOutput) if err != nil { return false, errors.Wrap(err, "failed to parse master zone id") @@ -160,12 +261,14 @@ func DecodeSecret(secret *v1.Secret, keyName string) (string, error) { } func GetRealmKeyArgs(context *clusterd.Context, realmName, namespace string) (string, string, error) { + logger.Debugf("getting keys for realm %v", realmName) // get realm's access and secret keys realmSecretName := realmName + "-keys" realmSecret, err := context.Clientset.CoreV1().Secrets(namespace).Get(realmSecretName, metav1.GetOptions{}) if err != nil { return "", "", errors.Wrapf(err, "failed to get realm %q keys secret", realmName) } + logger.Debugf("found keys secret for realm %v", realmName) accessKey, err := DecodeSecret(realmSecret, AccessKeyName) if err != nil { @@ -175,6 +278,7 @@ func GetRealmKeyArgs(context *clusterd.Context, realmName, namespace string) (st if err != nil { return "", "", errors.Wrapf(err, "failed to decode realm %q access key", realmName) } + logger.Debugf("decoded keys for realm %v", realmName) accessKeyArg := fmt.Sprintf("--access-key=%s", accessKey) secretKeyArg := fmt.Sprintf("--secret-key=%s", secretKey) @@ -182,96 +286,109 @@ func GetRealmKeyArgs(context *clusterd.Context, realmName, namespace string) (st return accessKeyArg, secretKeyArg, nil } -func getZoneEndpoints(context *Context, realmArg, zoneGroupArg, zoneName, serviceEndpoint string) (string, error) { - zoneGroupOutput, err := RunAdminCommandNoRealm(context, "zonegroup", "get", realmArg, zoneGroupArg) +func getZoneEndpoints(objContext *Context, serviceEndpoint string) ([]string, error) { + logger.Debugf("getting current endpoints for zone %v", objContext.Zone) + realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm) + zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup) + + zoneGroupOutput, err := RunAdminCommandNoMultisite(objContext, "zonegroup", "get", realmArg, zoneGroupArg) if err != nil { - return "", errors.Wrap(err, "failed to get rgw zone group") + return []string{}, errors.Wrap(err, "failed to get rgw zone group") } zoneGroupJson, err := DecodeZoneGroupConfig(zoneGroupOutput) if err != nil { - return "", errors.Wrap(err, "failed to parse zones list") + return []string{}, errors.Wrap(err, "failed to parse zones list") } - zoneEndpoints := []string{} + zoneEndpointsList := []string{} for _, zone := range zoneGroupJson.Zones { - if zone.Name == zoneName { + if zone.Name == objContext.Zone { for _, endpoint := range zone.Endpoints { // in case object-store operator code is rereconciled, zone modify could get run again with serviceEndpoint added again if endpoint != serviceEndpoint { - zoneEndpoints = append(zoneEndpoints, endpoint) + zoneEndpointsList = append(zoneEndpointsList, endpoint) } } break } } - zoneEndpoints = append(zoneEndpoints, serviceEndpoint) - - return strings.Join(zoneEndpoints, ","), nil + return zoneEndpointsList, nil } -func createMultisite(context *Context, endpointArg, realmArg, zoneGroupArg, zoneArg string) error { +func createMultisite(objContext *Context, endpointArg string) error { + logger.Debugf("creating realm, zone group, zone for object-store %v", objContext.Name) + + realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm) + zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup) + updatePeriod := false // create the realm if it doesn't exist yet - output, err := runAdminCommand(context, "realm", "get") + output, err := RunAdminCommandNoMultisite(objContext, "realm", "get", realmArg) if err != nil { - updatePeriod = true - output, err = runAdminCommand(context, "realm", "create") - if err != nil { - return errors.Wrapf(err, "failed to create rgw realm %q", context.Name) + if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) { + updatePeriod = true + output, err = RunAdminCommandNoMultisite(objContext, "realm", "create", realmArg) + if err != nil { + return errors.Wrapf(err, "failed to create ceph realm %q, for reason %q", objContext.ZoneGroup, output) + } + logger.Debugf("created realm %v", objContext.Realm) + } else { + return errors.Wrapf(err, "radosgw-admin realm get failed with code %d, for reason %q", code, output) } } - realmID, err := decodeID(output) - if err != nil { - return errors.Wrap(err, "failed to parse realm id") - } - // create the zonegroup if it doesn't exist yet - output, err = runAdminCommand(context, "zonegroup", "get") + output, err = RunAdminCommandNoMultisite(objContext, "zonegroup", "get", realmArg, zoneGroupArg) if err != nil { - updatePeriod = true - output, err = runAdminCommand(context, "zonegroup", "create", "--master", endpointArg) - if err != nil { - return errors.Wrapf(err, "failed to create rgw zonegroup for %q", context.Name) + if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) { + updatePeriod = true + output, err = RunAdminCommandNoMultisite(objContext, "zonegroup", "create", "--master", realmArg, zoneGroupArg, endpointArg) + if err != nil { + return errors.Wrapf(err, "failed to create ceph zone group %q, for reason %q", objContext.ZoneGroup, output) + } + logger.Debugf("created zone group %v", objContext.ZoneGroup) + } else { + return errors.Wrapf(err, "radosgw-admin zonegroup get failed with code %d, for reason %q", code, output) } } - zoneGroupID, err := decodeID(output) - if err != nil { - return errors.Wrap(err, "failed to parse zone group id") - } - // create the zone if it doesn't exist yet - output, err = runAdminCommand(context, "zone", "get", zoneArg) + output, err = runAdminCommand(objContext, "zone", "get") if err != nil { - updatePeriod = true - output, err = runAdminCommand(context, "zone", "create", "--master", endpointArg, zoneArg) - if err != nil { - return errors.Wrapf(err, "failed to create rgw zonegroup for %q", context.Name) + if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) { + updatePeriod = true + output, err = runAdminCommand(objContext, "zone", "create", "--master", endpointArg) + if err != nil { + return errors.Wrapf(err, "failed to create ceph zone %q, for reason %q", objContext.Zone, output) + } + logger.Debugf("created zone %v", objContext.Zone) + } else { + return errors.Wrapf(err, "radosgw-admin zone get failed with code %d, for reason %q", code, output) } } - zoneID, err := decodeID(output) - if err != nil { - return errors.Wrap(err, "failed to parse zone id") - } - - logger.Infof("RGW: realm=%s, zonegroup=%s, zone=%s", realmID, zoneGroupID, zoneID) - if updatePeriod { // the period will help notify other zones of changes if there are multi-zones - _, err := RunAdminCommandNoRealm(context, "period", "update", "--commit", realmArg, zoneGroupArg, zoneArg) + _, err := runAdminCommand(objContext, "period", "update", "--commit") if err != nil { return errors.Wrap(err, "failed to update period") } + logger.Debugf("updated period for realm %v", objContext.Realm) } + logger.Infof("Multisite for object-store: realm=%s, zonegroup=%s, zone=%s", objContext.Realm, objContext.ZoneGroup, objContext.Zone) + return nil } -func joinMultisite(context *Context, endpointArg, realmArg, zoneGroupArg, zoneArg, realmName, zoneGroupName, zoneName, zoneEndpoints, namespace string) error { - zoneIsMaster, err := checkZoneIsMaster(context, realmArg, zoneGroupArg, zoneArg) +func joinMultisite(objContext *Context, endpointArg, zoneEndpoints, namespace string) error { + logger.Debugf("joining zone %v", objContext.Zone) + realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm) + zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup) + zoneArg := fmt.Sprintf("--rgw-zone=%s", objContext.Zone) + + zoneIsMaster, err := checkZoneIsMaster(objContext) if err != nil { return err } @@ -279,104 +396,125 @@ func joinMultisite(context *Context, endpointArg, realmArg, zoneGroupArg, zoneAr if zoneIsMaster { // endpoints that are part of a master zone are supposed to be the endpoints for a zone group - _, err := RunAdminCommandNoRealm(context, "zonegroup", "modify", realmArg, zoneGroupArg, endpointArg) + _, err := RunAdminCommandNoMultisite(objContext, "zonegroup", "modify", realmArg, zoneGroupArg, endpointArg) if err != nil { - return errors.Wrapf(err, "failed to add object store %q in rgw zone group %q", context.Name, zoneGroupName) + return errors.Wrapf(err, "failed to add object store %q in rgw zone group %q", objContext.Name, objContext.ZoneGroup) } - logger.Debugf("endpoints for zonegroup %q are now %q", zoneGroupName, zoneEndpoints) + logger.Debugf("endpoints for zonegroup %q are now %q", objContext.ZoneGroup, zoneEndpoints) // check if zone group is master only if zone is master for creating the system user - zoneGroupIsMaster, err = checkZoneGroupIsMaster(context, realmArg, zoneGroupArg) + zoneGroupIsMaster, err = checkZoneGroupIsMaster(objContext) if err != nil { - return errors.Wrapf(err, "failed to find out whether zone group %q in is the master zone group", zoneGroupName) + return errors.Wrapf(err, "failed to find out whether zone group %q in is the master zone group", objContext.ZoneGroup) } } - _, err = RunAdminCommandNoRealm(context, "zone", "modify", realmArg, zoneGroupArg, zoneArg, endpointArg) + _, err = RunAdminCommandNoMultisite(objContext, "zone", "modify", realmArg, zoneGroupArg, zoneArg, endpointArg) if err != nil { - return errors.Wrapf(err, "failed to add object store %q in rgw zone %q", context.Name, zoneName) + return errors.Wrapf(err, "failed to add object store %q in rgw zone %q", objContext.Name, objContext.Zone) } + logger.Debugf("endpoints for zone %q are now %q", objContext.Zone, zoneEndpoints) // the period will help notify other zones of changes if there are multi-zones - _, err = RunAdminCommandNoRealm(context, "period", "update", "--commit", realmArg, zoneGroupArg, zoneArg) + _, err = RunAdminCommandNoMultisite(objContext, "period", "update", "--commit", realmArg, zoneGroupArg, zoneArg) if err != nil { return errors.Wrap(err, "failed to update period") } - logger.Infof("added object store %q to realm %q, zonegroup %q, zone %q", context.Name, realmName, zoneGroupName, zoneName) + logger.Infof("added object store %q to realm %q, zonegroup %q, zone %q", objContext.Name, objContext.Realm, objContext.ZoneGroup, objContext.Zone) // create system user for realm for master zone in master zonegorup for multisite scenario if zoneIsMaster && zoneGroupIsMaster { - uid := realmName + "-system-user" - uidArg := fmt.Sprintf("--uid=%s", uid) - output, err := RunAdminCommandNoRealm(context, "user", "info", uidArg) + err = createSystemUser(objContext, namespace) if err != nil { - if code, ok := exec.ExitStatus(err); ok && code == int(syscall.EINVAL) { - logger.Debugf("realm system user %q not found, running `radosgw-admin user create`", uid) - accessKeyArg, secretKeyArg, err := GetRealmKeyArgs(context.Context, realmName, namespace) - if err != nil { - return errors.Wrap(err, "failed to get keys for realm") - } - systemArg := "--system" - displayNameArg := fmt.Sprintf("--display-name=%s.user", realmName) - output, err = RunAdminCommandNoRealm(context, "user", "create", realmArg, zoneGroupArg, zoneArg, uidArg, displayNameArg, accessKeyArg, secretKeyArg, systemArg) - if err != nil { - return errors.Wrapf(err, "failed to create realm system user %q for reason: %q", uid, output) - } - } else { - return errors.Wrapf(err, "radosgw-admin user info failed with code %d and output %q", code, output) - } - } else { - logger.Debugf("realm system user %q has already been created", uid) + return err } } return nil } -func setMultisite(context *Context, spec cephv1.ObjectStoreSpec, serviceIP, realmName, zoneGroupName, zoneName, namespace string) error { - realmArg := fmt.Sprintf("--rgw-realm=%s", realmName) - zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", zoneGroupName) - zoneArg := fmt.Sprintf("--rgw-zone=%s", zoneName) +func createSystemUser(objContext *Context, namespace string) error { + uid := objContext.Realm + "-system-user" + uidArg := fmt.Sprintf("--uid=%s", uid) + realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm) + zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup) + zoneArg := fmt.Sprintf("--rgw-zone=%s", objContext.Zone) - serviceEndpoint := fmt.Sprintf("http://%s:%d", serviceIP, spec.Gateway.Port) - if spec.Gateway.SecurePort != 0 { - serviceEndpoint = fmt.Sprintf("https://%s:%d", serviceIP, spec.Gateway.SecurePort) + output, err := RunAdminCommandNoMultisite(objContext, "user", "info", uidArg) + if err == nil { + logger.Debugf("realm system user %q has already been created", uid) + return nil } - zoneEndpoints, err := getZoneEndpoints(context, realmArg, zoneGroupArg, zoneName, serviceEndpoint) - if err != nil { - return err - } - logger.Debugf("Endpoints for zone %q are: %q", zoneName, zoneEndpoints) - endpointArg := fmt.Sprintf("--endpoints=%s", zoneEndpoints) - - if spec.IsMultisite() { - err = joinMultisite(context, endpointArg, realmArg, zoneGroupArg, zoneArg, realmName, zoneGroupName, zoneName, zoneEndpoints, namespace) + if code, ok := exec.ExitStatus(err); ok && code == int(syscall.EINVAL) { + logger.Debugf("realm system user %q not found, running `radosgw-admin user create`", uid) + accessKeyArg, secretKeyArg, err := GetRealmKeyArgs(objContext.Context, objContext.Realm, namespace) if err != nil { - return errors.Wrapf(err, "failed join ceph multisite in zone %q", zoneName) + return errors.Wrap(err, "failed to get keys for realm") + } + logger.Debugf("found keys to create realm system user %v", uid) + systemArg := "--system" + displayNameArg := fmt.Sprintf("--display-name=%s.user", objContext.Realm) + output, err = RunAdminCommandNoMultisite(objContext, "user", "create", realmArg, zoneGroupArg, zoneArg, uidArg, displayNameArg, accessKeyArg, secretKeyArg, systemArg) + if err != nil { + return errors.Wrapf(err, "failed to create realm system user %q for reason: %q", uid, output) + } + logger.Debugf("created realm system user %v", uid) + } else { + return errors.Wrapf(err, "radosgw-admin user info for system user failed with code %d and output %q", code, output) + } + + return nil +} + +func setMultisite(objContext *Context, store *cephv1.CephObjectStore, serviceIP string) error { + logger.Debugf("setting multisite configuration for object-store %v", store.Name) + serviceEndpoint := fmt.Sprintf("http://%s:%d", serviceIP, store.Spec.Gateway.Port) + if store.Spec.Gateway.SecurePort != 0 { + serviceEndpoint = fmt.Sprintf("https://%s:%d", serviceIP, store.Spec.Gateway.SecurePort) + } + + if store.Spec.IsMultisite() { + zoneEndpointsList, err := getZoneEndpoints(objContext, serviceEndpoint) + if err != nil { + return err + } + zoneEndpointsList = append(zoneEndpointsList, serviceEndpoint) + + zoneEndpoints := strings.Join(zoneEndpointsList, ",") + logger.Debugf("Endpoints for zone %q are: %q", objContext.Zone, zoneEndpoints) + endpointArg := fmt.Sprintf("--endpoints=%s", zoneEndpoints) + + err = joinMultisite(objContext, endpointArg, zoneEndpoints, store.Namespace) + if err != nil { + return errors.Wrapf(err, "failed join ceph multisite in zone %q", objContext.Zone) } } else { - err = createMultisite(context, endpointArg, realmArg, zoneGroupArg, zoneArg) + endpointArg := fmt.Sprintf("--endpoints=%s", serviceEndpoint) + err := createMultisite(objContext, endpointArg) if err != nil { - return errors.Wrapf(err, "failed create ceph multisite for object-store %q", context.Name) + return errors.Wrapf(err, "failed create ceph multisite for object-store %q", objContext.Name) } } + logger.Infof("multisite configuration for object-store %v is complete", store.Name) return nil } func deleteRealm(context *Context) error { // - _, err := runAdminCommand(context, "realm", "delete", "--rgw-realm", context.Name) + realmArg := fmt.Sprintf("--rgw-realm=%s", context.Name) + zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", context.Name) + _, err := RunAdminCommandNoMultisite(context, "realm", "delete", realmArg) if err != nil { logger.Warningf("failed to delete rgw realm %q. %v", context.Name, err) } - _, err = runAdminCommand(context, "zonegroup", "delete", "--rgw-zonegroup", context.Name) + _, err = RunAdminCommandNoMultisite(context, "zonegroup", "delete", realmArg, zoneGroupArg) if err != nil { logger.Warningf("failed to delete rgw zonegroup %q. %v", context.Name, err) } - _, err = runAdminCommand(context, "zone", "delete", "--rgw-zone", context.Name) + _, err = runAdminCommand(context, "zone", "delete") if err != nil { logger.Warningf("failed to delete rgw zone %q. %v", context.Name, err) } @@ -405,7 +543,7 @@ func DecodeZoneGroupConfig(data string) (zoneGroupType, error) { } func getObjectStores(context *Context) ([]string, error) { - output, err := RunAdminCommandNoRealm(context, "realm", "list") + output, err := RunAdminCommandNoMultisite(context, "realm", "list") if err != nil { if strings.Index(err.Error(), "exit status 2") != 0 { return []string{}, nil diff --git a/pkg/operator/ceph/object/objectstore_test.go b/pkg/operator/ceph/object/objectstore_test.go index 00816d155..1a56a4a5a 100644 --- a/pkg/operator/ceph/object/objectstore_test.go +++ b/pkg/operator/ceph/object/objectstore_test.go @@ -45,16 +45,15 @@ func TestReconcileRealm(t *testing.T) { } storeName := "myobject" - testNamespace := "test-namespace" context := &clusterd.Context{Executor: executor} objContext := NewContext(context, &client.ClusterInfo{Namespace: "mycluster"}, storeName) // create the first realm, marked as default - spec := cephv1.ObjectStoreSpec{} - err := setMultisite(objContext, spec, "1.2.3.4", storeName, storeName, storeName, testNamespace) + store := cephv1.CephObjectStore{} + err := setMultisite(objContext, &store, "1.2.3.4") assert.Nil(t, err) // create the second realm, not marked as default - err = setMultisite(objContext, spec, "2.3.4.5", storeName, storeName, storeName, testNamespace) + err = setMultisite(objContext, &store, "2.3.4.5") assert.Nil(t, err) } diff --git a/pkg/operator/ceph/object/realm/controller.go b/pkg/operator/ceph/object/realm/controller.go index f572484b8..a5e32ac6c 100644 --- a/pkg/operator/ceph/object/realm/controller.go +++ b/pkg/operator/ceph/object/realm/controller.go @@ -56,7 +56,7 @@ const ( secretKeyLength = 28 ) -var waitForRequeueIfRealmKeysNotReady = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second} +var waitForRequeueIfRealmNotReady = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second} var logger = capnslog.NewPackageLogger("github.com/rook/rook", controllerName) @@ -184,9 +184,8 @@ func (r *ReconcileObjectRealm) reconcile(request reconcile.Request) (reconcile.R logger.Debug("pull section in spec found") reconcileResponse, err = r.pullCephRealm(cephObjectRealm) if err != nil { - return r.setFailedStatus(request.NamespacedName, "failed to pull ceph realm", err) + return reconcile.Result{}, err } - } else { reconcileResponse, err = r.createRealmKeys(cephObjectRealm) if err != nil { @@ -214,18 +213,17 @@ func (r *ReconcileObjectRealm) pullCephRealm(realm *cephv1.CephObjectRealm) (rec accessKeyArg, secretKeyArg, err := object.GetRealmKeyArgs(r.context, realm.Name, realm.Namespace) if err != nil { if kerrors.IsNotFound(err) { - return waitForRequeueIfRealmKeysNotReady, err + return waitForRequeueIfRealmNotReady, err } - return waitForRequeueIfRealmKeysNotReady, errors.Wrap(err, "failed to get keys for realm") + return waitForRequeueIfRealmNotReady, errors.Wrap(err, "failed to get keys for realm") } logger.Debugf("keys found to pull realm, getting ready to pull from endpoint %q", realm.Spec.Pull.Endpoint) - objContext := object.NewContext(r.context, realm.Name, realm.Namespace) - - output, err := object.RunAdminCommandNoRealm(objContext, "realm", "pull", realmArg, urlArg, accessKeyArg, secretKeyArg) + objContext := object.NewContext(r.context, r.clusterInfo, realm.Name) + output, err := object.RunAdminCommandNoMultisite(objContext, "realm", "pull", realmArg, urlArg, accessKeyArg, secretKeyArg) if err != nil { - return reconcile.Result{}, errors.Wrapf(err, "realm pull failed for reason: %v", output) + return waitForRequeueIfRealmNotReady, errors.Wrapf(err, "realm pull failed for reason: %v", output) } logger.Debugf("realm pull for %q from endpoint %q succeeded", realm.Name, realm.Spec.Pull.Endpoint) @@ -236,12 +234,12 @@ func (r *ReconcileObjectRealm) createCephRealm(realm *cephv1.CephObjectRealm) (r realmArg := fmt.Sprintf("--rgw-realm=%s", realm.Name) objContext := object.NewContext(r.context, r.clusterInfo, realm.Namespace) - _, err := object.RunAdminCommandNoRealm(objContext, "realm", "get", realmArg) + _, err := object.RunAdminCommandNoMultisite(objContext, "realm", "get", realmArg) if err != nil { if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) { logger.Debugf("ceph realm %q not found, running `radosgw-admin realm create`", realm.Name) - _, err := object.RunAdminCommandNoRealm(objContext, "realm", "create", realmArg) + _, err := object.RunAdminCommandNoMultisite(objContext, "realm", "create", realmArg) if err != nil { return reconcile.Result{}, errors.Wrapf(err, "failed to create ceph realm %s", realm.Name) } @@ -266,7 +264,7 @@ func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (r secretKey, err := mgr.GeneratePassword(secretKeyLength) if err != nil { - return reconcile.Result{}, errors.Wrapf(err, "secret key failed to generate") + return reconcile.Result{}, errors.Wrapf(err, "failed to generate secret key") } secretKey = base64.StdEncoding.EncodeToString([]byte(secretKey)) @@ -296,6 +294,7 @@ func (r *ReconcileObjectRealm) createRealmKeys(realm *cephv1.CephObjectRealm) (r if _, err = r.context.Clientset.CoreV1().Secrets(realm.Namespace).Create(secret); err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to save rgw secrets") } + logger.Infof("secrets for keys have been created for realm %q", realm.Name) return reconcile.Result{}, nil } @@ -308,11 +307,6 @@ func validateRealmCR(u *cephv1.CephObjectRealm) error { if u.Namespace == "" { return errors.New("missing namespace") } - if u.Spec.IsPullRealm() { - if u.Spec.Pull.Endpoint == "" { - return errors.New("missing endpoint for pulling realm") - } - } return nil } diff --git a/pkg/operator/ceph/object/rgw.go b/pkg/operator/ceph/object/rgw.go index 872b4fb76..c8708f787 100644 --- a/pkg/operator/ceph/object/rgw.go +++ b/pkg/operator/ceph/object/rgw.go @@ -277,8 +277,14 @@ func (c *clusterConfig) deleteStore() error { } // Delete the realm and pools - objContext := NewContext(c.context, c.clusterInfo, c.store.Name) - err := deleteRealmAndPools(objContext, c.store.Spec) + objContext, err := NewMultisiteContext(c.context, c.clusterInfo, c.store) + if err != nil { + return errors.Wrapf(err, "failed to set multisite on object store %q", c.store.Name) + } + + objContext.Endpoint = c.store.Status.Info["endpoint"] + + err = deleteRealmAndPools(objContext, c.store.Spec) if err != nil { return errors.Wrap(err, "failed to delete the realm and pools") } @@ -347,10 +353,6 @@ func (r *ReconcileCephObjectStore) validateStore(s *cephv1.CephObjectStore) erro } } - if s.Spec.IsMultisite() && !s.Spec.PreservePoolsOnDelete { - return errors.New("preservePoolsOnDelete must be true when zone is set") - } - return nil } diff --git a/pkg/operator/ceph/object/user/controller.go b/pkg/operator/ceph/object/user/controller.go index 44ef9b883..fb51e7049 100644 --- a/pkg/operator/ceph/object/user/controller.go +++ b/pkg/operator/ceph/object/user/controller.go @@ -186,8 +186,8 @@ func (r *ReconcileObjectStoreUser) reconcile(request reconcile.Request) (reconci return reconcile.Result{}, errors.Wrap(err, "failed to populate cluster info") } - // validate isObjectStoreInitialized - err = r.isObjectStoreInitialized(cephObjectStoreUser) + // validate initializeObjectStoreContext + err = r.initializeObjectStoreContext(cephObjectStoreUser) if err != nil { if !cephObjectStoreUser.GetDeletionTimestamp().IsZero() { // Remove finalizer @@ -293,15 +293,24 @@ func (r *ReconcileObjectStoreUser) createorUpdateCephUser(u *cephv1.CephObjectSt return nil } -func (r *ReconcileObjectStoreUser) isObjectStoreInitialized(u *cephv1.CephObjectStoreUser) error { - objContext := object.NewContext(r.context, r.clusterInfo, u.Spec.Store) - r.objContext = objContext - +func (r *ReconcileObjectStoreUser) initializeObjectStoreContext(u *cephv1.CephObjectStoreUser) error { err := r.objectStoreInitialized(u) if err != nil { - return errors.Wrap(err, "failed to detect if object store is initialized") + return errors.Wrapf(err, "failed to detect if object store %q is initialized", u.Spec.Store) } + store, err := r.getObjectStore(u.Spec.Store) + if err != nil { + return errors.Wrapf(err, "failed to get object store %q", u.Spec.Store) + } + + objContext, err := object.NewMultisiteContext(r.context, r.clusterInfo, store) + if err != nil { + return errors.Wrapf(err, "Multisite failed to set on object context for object store user") + } + r.objContext = objContext + r.objContext.Endpoint = store.Status.Info["endpoint"] + return nil } @@ -376,7 +385,7 @@ func (r *ReconcileObjectStoreUser) reconcileCephUserSecret(cephObjectStoreUser * } func (r *ReconcileObjectStoreUser) objectStoreInitialized(cephObjectStoreUser *cephv1.CephObjectStoreUser) error { - err := r.getObjectStore(cephObjectStoreUser.Spec.Store) + _, err := r.getObjectStore(cephObjectStoreUser.Spec.Store) if err != nil { return err } @@ -404,26 +413,25 @@ func (r *ReconcileObjectStoreUser) objectStoreInitialized(cephObjectStoreUser *c return errors.New("no rgw pod found") } -func (r *ReconcileObjectStoreUser) getObjectStore(storeName string) error { +func (r *ReconcileObjectStoreUser) getObjectStore(storeName string) (*cephv1.CephObjectStore, error) { // check if CephObjectStore CR is created objectStores := &cephv1.CephObjectStoreList{} err := r.client.List(context.TODO(), objectStores) if err != nil { if kerrors.IsNotFound(err) { - return errors.Wrapf(err, "CephObjectStore %q could not be found", storeName) + return nil, errors.Wrapf(err, "CephObjectStore %q could not be found", storeName) } - return errors.Wrap(err, "failed to get CephObjectStore") + return nil, errors.Wrap(err, "failed to get CephObjectStore") } for _, store := range objectStores.Items { if store.Name == storeName { logger.Infof("CephObjectStore %q found", storeName) - r.objContext.Endpoint = store.Status.Info["endpoint"] - return nil + return &store, nil } } - return errors.Errorf("CephObjectStore %q could not be found", storeName) + return nil, errors.Errorf("CephObjectStore %q could not be found", storeName) } func (r *ReconcileObjectStoreUser) getRgwPodList(cephObjectStoreUser *cephv1.CephObjectStoreUser) (*corev1.PodList, error) { diff --git a/pkg/operator/ceph/object/zone/controller.go b/pkg/operator/ceph/object/zone/controller.go index 152c69506..f338a3eb1 100644 --- a/pkg/operator/ceph/object/zone/controller.go +++ b/pkg/operator/ceph/object/zone/controller.go @@ -188,14 +188,6 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re return reconcileResponse, err } - // RECONCILE POOLS - logger.Info("reconciling object zone pools") - objContext := object.NewContext(r.context, cephObjectZone.Name, cephObjectZone.Namespace) - err = object.CreatePools(objContext, cephObjectZone.Spec.MetadataPool, cephObjectZone.Spec.DataPool) - if err != nil { - return r.setFailedStatus(request.NamespacedName, "failed to create zone pools", err) - } - // Create Ceph Zone reconcileResponse, err = r.createCephZone(cephObjectZone, realmName) if err != nil { @@ -219,7 +211,7 @@ func (r *ReconcileObjectZone) createCephZone(zone *cephv1.CephObjectZone, realmN objContext := object.NewContext(r.context, r.clusterInfo, zone.Name) // get zone group to see if master zone exists yet - output, err := object.RunAdminCommandNoRealm(objContext, "zonegroup", "get", realmArg, zoneGroupArg) + output, err := object.RunAdminCommandNoMultisite(objContext, "zonegroup", "get", realmArg, zoneGroupArg) if err != nil { if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) { return reconcile.Result{}, errors.Wrapf(err, "ceph zone group %q not found", zone.Spec.ZoneGroup) @@ -234,41 +226,65 @@ func (r *ReconcileObjectZone) createCephZone(zone *cephv1.CephObjectZone, realmN return reconcile.Result{}, errors.Wrap(err, "failed to parse `radosgw-admin zonegroup get` output") } - zoneIsMaster := false - if zoneGroupJson.MasterZoneID == "" { - zoneIsMaster = true + // create zone + _, err = object.RunAdminCommandNoMultisite(objContext, "zone", "get", realmArg, zoneGroupArg, zoneArg) + if err == nil { + logger.Debugf("ceph zone %q already exists, new zone and pools will not be created", zone.Name) + return reconcile.Result{}, nil } - // create zone - _, err = object.RunAdminCommandNoRealm(objContext, "zone", "get", realmArg, zoneGroupArg, zoneArg) - if err != nil { - if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) { - logger.Debugf("ceph zone %q not found, running `radosgw-admin zone create`", zone.Name) - accessKeyArg, secretKeyArg, err := object.GetRealmKeyArgs(r.context, realmName, zone.Namespace) - if err != nil { - return reconcile.Result{}, errors.Wrap(err, "failed to get keys for realm") - } + if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) { + logger.Debugf("ceph zone %q not found, running `radosgw-admin zone create`", zone.Name) - if zoneIsMaster { - // master zone does not exist yet for zone group - masterArg := "--master" - _, err = object.RunAdminCommandNoRealm(objContext, "zone", "create", realmArg, zoneGroupArg, zoneArg, accessKeyArg, secretKeyArg, masterArg) - } else { - _, err = object.RunAdminCommandNoRealm(objContext, "zone", "create", realmArg, zoneGroupArg, zoneArg, accessKeyArg, secretKeyArg) - } - - if err != nil { - return reconcile.Result{}, errors.Wrapf(err, "failed to create ceph zone %q", zone.Name) - } - - } else { - return reconcile.Result{}, errors.Wrapf(err, "radosgw-admin zone get failed with code %d", code) + zoneIsMaster := false + if zoneGroupJson.MasterZoneID == "" { + zoneIsMaster = true } + + err = r.createPoolsAndZone(objContext, zone, realmName, zoneIsMaster) + if err != nil { + return reconcile.Result{}, err + } + } else { + return reconcile.Result{}, errors.Wrapf(err, "radosgw-admin zone get failed with code %d for reason %q", code, output) } return reconcile.Result{}, nil } +func (r *ReconcileObjectZone) createPoolsAndZone(objContext *object.Context, zone *cephv1.CephObjectZone, realmName string, zoneIsMaster bool) error { + // create pools for zone + logger.Debugf("creating pools ceph zone %q", zone.Name) + realmArg := fmt.Sprintf("--rgw-realm=%s", realmName) + zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", zone.Spec.ZoneGroup) + zoneArg := fmt.Sprintf("--rgw-zone=%s", zone.Name) + + err := object.CreatePools(objContext, zone.Spec.MetadataPool, zone.Spec.DataPool) + if err != nil { + return errors.Wrapf(err, "failed to create pools for zone %v", zone.Name) + } + logger.Debugf("created pools ceph zone %q", zone.Name) + + accessKeyArg, secretKeyArg, err := object.GetRealmKeyArgs(r.context, realmName, zone.Namespace) + if err != nil { + return errors.Wrap(err, "failed to get keys for realm") + } + args := []string{"zone", "create", realmArg, zoneGroupArg, zoneArg, accessKeyArg, secretKeyArg} + + if zoneIsMaster { + // master zone does not exist yet for zone group + args = append(args, "--master") + } + + output, err := object.RunAdminCommandNoMultisite(objContext, args...) + if err != nil { + return errors.Wrapf(err, "failed to create ceph zone %q for reason %q", zone.Name, output) + } + logger.Debugf("created ceph zone %q", zone.Name) + + return nil +} + func (r *ReconcileObjectZone) reconcileObjectZoneGroup(zone *cephv1.CephObjectZone) (string, reconcile.Result, error) { // Verify the object zone API object actually exists zoneGroup, err := r.context.RookClientset.CephV1().CephObjectZoneGroups(zone.Namespace).Get(zone.Spec.ZoneGroup, metav1.GetOptions{}) @@ -288,7 +304,7 @@ func (r *ReconcileObjectZone) reconcileCephZoneGroup(zone *cephv1.CephObjectZone zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", zone.Spec.ZoneGroup) objContext := object.NewContext(r.context, r.clusterInfo, zone.Name) - _, err := object.RunAdminCommandNoRealm(objContext, "zonegroup", "get", realmArg, zoneGroupArg) + _, err := object.RunAdminCommandNoMultisite(objContext, "zonegroup", "get", realmArg, zoneGroupArg) if err != nil { if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) { return waitForRequeueIfObjectZoneGroupNotReady, errors.Wrapf(err, "ceph zone group %q not found", zone.Spec.ZoneGroup) @@ -302,20 +318,20 @@ func (r *ReconcileObjectZone) reconcileCephZoneGroup(zone *cephv1.CephObjectZone } // validateZoneCR validates the zone arguments -func (r *ReconcileObjectZone) validateZoneCR(u *cephv1.CephObjectZone) error { - if u.Name == "" { +func (r *ReconcileObjectZone) validateZoneCR(z *cephv1.CephObjectZone) error { + if z.Name == "" { return errors.New("missing name") } - if u.Namespace == "" { + if z.Namespace == "" { return errors.New("missing namespace") } - if u.Spec.ZoneGroup == "" { + if z.Spec.ZoneGroup == "" { return errors.New("missing zonegroup") } - if err := pool.ValidatePoolSpec(r.context, u.Namespace, &u.Spec.MetadataPool); err != nil { + if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, &z.Spec.MetadataPool); err != nil { return errors.Wrap(err, "invalid metadata pool spec") } - if err := pool.ValidatePoolSpec(r.context, u.Namespace, &u.Spec.DataPool); err != nil { + if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, &z.Spec.DataPool); err != nil { return errors.Wrap(err, "invalid data pool spec") } return nil diff --git a/pkg/operator/ceph/object/zonegroup/controller.go b/pkg/operator/ceph/object/zonegroup/controller.go index 148c9bc4b..d8e1ea8eb 100644 --- a/pkg/operator/ceph/object/zonegroup/controller.go +++ b/pkg/operator/ceph/object/zonegroup/controller.go @@ -208,7 +208,7 @@ func (r *ReconcileObjectZoneGroup) createCephZoneGroup(zoneGroup *cephv1.CephObj objContext := object.NewContext(r.context, r.clusterInfo, zoneGroup.Name) // get period to see if master zone group exists yet - output, err := object.RunAdminCommandNoRealm(objContext, "period", "get", realmArg) + output, err := object.RunAdminCommandNoMultisite(objContext, "period", "get", realmArg) if err != nil { if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) { return reconcile.Result{}, errors.Wrapf(err, "ceph period %q not found", zoneGroup.Spec.Realm) @@ -229,24 +229,31 @@ func (r *ReconcileObjectZoneGroup) createCephZoneGroup(zoneGroup *cephv1.CephObj } // create zone group - _, err = object.RunAdminCommandNoRealm(objContext, "zonegroup", "get", realmArg, zoneGroupArg) - if err != nil { - if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) { - logger.Debugf("ceph zone group %q not found, running `radosgw-admin zonegroup create`", zoneGroup.Name) + output, err = object.RunAdminCommandNoMultisite(objContext, "zonegroup", "get", realmArg, zoneGroupArg) + if err == nil { + return reconcile.Result{}, nil + } - if zoneGroupIsMaster { - masterArg := "--master" - _, err = object.RunAdminCommandNoRealm(objContext, "zonegroup", "create", realmArg, zoneGroupArg, masterArg) - } else { - _, err = object.RunAdminCommandNoRealm(objContext, "zonegroup", "create", realmArg, zoneGroupArg) - } - - if err != nil { - return reconcile.Result{}, errors.Wrapf(err, "failed to create ceph zone group %q", zoneGroup.Name) - } - } else { - return reconcile.Result{}, errors.Wrapf(err, "radosgw-admin zonegroup get failed with code %d", code) + if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) { + logger.Debugf("ceph zone group %q not found, running `radosgw-admin zonegroup create`", zoneGroup.Name) + args := []string{ + "zonegroup", + "create", + realmArg, + zoneGroupArg, } + + if zoneGroupIsMaster { + // master zone group does not exist yet for realm + args = append(args, "--master") + } + + output, err = object.RunAdminCommandNoMultisite(objContext, args...) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "failed to create ceph zone group %q for reason %q", zoneGroup.Name, output) + } + } else { + return reconcile.Result{}, errors.Wrapf(err, "radosgw-admin zonegroup get failed with code %d for reason %q", code, output) } return reconcile.Result{}, nil @@ -270,7 +277,7 @@ func (r *ReconcileObjectZoneGroup) reconcileCephRealm(zoneGroup *cephv1.CephObje realmArg := fmt.Sprintf("--rgw-realm=%s", zoneGroup.Spec.Realm) objContext := object.NewContext(r.context, r.clusterInfo, zoneGroup.Name) - _, err := object.RunAdminCommandNoRealm(objContext, "realm", "get", realmArg) + _, err := object.RunAdminCommandNoMultisite(objContext, "realm", "get", realmArg) if err != nil { if code, ok := exec.ExitStatus(err); ok && code == int(syscall.ENOENT) { return waitForRequeueIfObjectRealmNotReady, errors.Wrapf(err, "ceph realm %q not found", zoneGroup.Spec.Realm)