ceph: validate pools for stretch clusters
Pools in stretch clusters will all use the same crush rule that is generated by the operator when configuring the cluster for stretch mode, with replica 4 and two replicas per failure domain. Pools cannot create new crush rules, so we require that replica be 4 when the pools is created, EC is not allowed, and no new crush rule will be created. Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
16 files changed
+133
-102
No files matched your search
@@ -79,7 +79,7 @@ type ReconcileCephObjectStore struct {
|
||||
bktclient bktclient.Interface
|
||||
scheme *runtime.Scheme
|
||||
context *clusterd.Context
|
||||
cephClusterSpec *cephv1.ClusterSpec
|
||||
clusterSpec *cephv1.ClusterSpec
|
||||
clusterInfo *cephclient.ClusterInfo
|
||||
objectStoreChannels map[string]*objectStoreHealth
|
||||
}
|
||||
@@ -194,7 +194,7 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
|
||||
|
||||
return reconcileResponse, nil
|
||||
}
|
||||
r.cephClusterSpec = &cephCluster.Spec
|
||||
r.clusterSpec = &cephCluster.Spec
|
||||
|
||||
// Initialize the channel for this object store
|
||||
// This allows us to track multiple ObjectStores in the same namespace
|
||||
@@ -240,7 +240,7 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
|
||||
cfg := clusterConfig{
|
||||
context: r.context,
|
||||
store: cephObjectStore,
|
||||
clusterSpec: r.cephClusterSpec,
|
||||
clusterSpec: r.clusterSpec,
|
||||
clusterInfo: r.clusterInfo,
|
||||
}
|
||||
err = cfg.deleteStore()
|
||||
@@ -289,9 +289,9 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
|
||||
context: r.context,
|
||||
clusterInfo: r.clusterInfo,
|
||||
store: cephObjectStore,
|
||||
rookVersion: r.cephClusterSpec.CephVersion.Image,
|
||||
clusterSpec: r.cephClusterSpec,
|
||||
DataPathMap: opconfig.NewStatelessDaemonDataPathMap(opconfig.RgwType, cephObjectStore.Name, cephObjectStore.Namespace, r.cephClusterSpec.DataDirHostPath),
|
||||
rookVersion: r.clusterSpec.CephVersion.Image,
|
||||
clusterSpec: r.clusterSpec,
|
||||
DataPathMap: opconfig.NewStatelessDaemonDataPathMap(opconfig.RgwType, cephObjectStore.Name, cephObjectStore.Namespace, r.clusterSpec.DataDirHostPath),
|
||||
client: r.client,
|
||||
scheme: r.scheme,
|
||||
}
|
||||
@@ -301,7 +301,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
|
||||
var serviceIP string
|
||||
var err error
|
||||
|
||||
if r.cephClusterSpec.External.Enable {
|
||||
if r.clusterSpec.External.Enable {
|
||||
logger.Info("reconciling external object store")
|
||||
|
||||
// RECONCILE SERVICE
|
||||
@@ -351,7 +351,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
|
||||
// Reconcile Pool Creation
|
||||
if !cephObjectStore.Spec.IsMultisite() {
|
||||
logger.Info("reconciling object store pools")
|
||||
err = CreatePools(objContext, cephObjectStore.Spec.MetadataPool, cephObjectStore.Spec.DataPool, cluster)
|
||||
err = CreatePools(objContext, r.clusterSpec, cephObjectStore.Spec.MetadataPool, cephObjectStore.Spec.DataPool)
|
||||
if err != nil {
|
||||
return r.setFailedStatus(namespacedName, "failed to create object pools", err)
|
||||
}
|
||||
@@ -482,7 +482,7 @@ func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjec
|
||||
return
|
||||
}
|
||||
|
||||
rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck, r.cephClusterSpec.External.Enable)
|
||||
rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck, r.clusterSpec.External.Enable)
|
||||
logger.Info("starting rgw healthcheck")
|
||||
go rgwChecker.checkObjectStore(r.objectStoreChannels[objectstore.Name].stopChan)
|
||||
}
|
||||
|
||||
@@ -605,7 +605,7 @@ func deletePools(context *Context, spec cephv1.ObjectStoreSpec, lastStore bool)
|
||||
return nil
|
||||
}
|
||||
|
||||
func CreatePools(context *Context, metadataPool, dataPool cephv1.PoolSpec, cluster cephv1.ClusterSpec) error {
|
||||
func CreatePools(context *Context, clusterSpec *cephv1.ClusterSpec, metadataPool, dataPool cephv1.PoolSpec) error {
|
||||
if emptyPool(dataPool) && emptyPool(metadataPool) {
|
||||
logger.Info("no pools specified for the CR, checking for their existence...")
|
||||
pools := append(metadataPools, dataPoolName)
|
||||
@@ -631,7 +631,7 @@ func CreatePools(context *Context, metadataPool, dataPool cephv1.PoolSpec, clust
|
||||
metadataPoolPGs = ceph.DefaultPGCount
|
||||
}
|
||||
|
||||
if err := createSimilarPools(context, append(metadataPools, rootPool), metadataPool, metadataPoolPGs, "", cluster); err != nil {
|
||||
if err := createSimilarPools(context, append(metadataPools, rootPool), clusterSpec, metadataPool, metadataPoolPGs, ""); err != nil {
|
||||
return errors.Wrap(err, "failed to create metadata pools")
|
||||
}
|
||||
|
||||
@@ -644,14 +644,14 @@ func CreatePools(context *Context, metadataPool, dataPool cephv1.PoolSpec, clust
|
||||
}
|
||||
}
|
||||
|
||||
if err := createSimilarPools(context, []string{dataPoolName}, dataPool, ceph.DefaultPGCount, ecProfileName, cluster); err != nil {
|
||||
if err := createSimilarPools(context, []string{dataPoolName}, clusterSpec, dataPool, ceph.DefaultPGCount, ecProfileName); err != nil {
|
||||
return errors.Wrap(err, "failed to create data pool")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func createSimilarPools(context *Context, pools []string, poolSpec cephv1.PoolSpec, pgCount, ecProfileName string, cluster cephv1.ClusterSpec) error {
|
||||
func createSimilarPools(context *Context, pools []string, clusterSpec *cephv1.ClusterSpec, poolSpec cephv1.PoolSpec, pgCount, ecProfileName string) error {
|
||||
for _, pool := range pools {
|
||||
// create the pool if it doesn't exist yet
|
||||
name := poolName(context.Name, pool)
|
||||
@@ -664,7 +664,7 @@ func createSimilarPools(context *Context, pools []string, poolSpec cephv1.PoolSp
|
||||
// created with that property disabled to avoid unnecessary performance impact.
|
||||
err = ceph.CreateECPoolForApp(context.Context, context.clusterInfo, name, ecProfileName, poolSpec, pgCount, AppName, false /* enableECOverwrite */)
|
||||
} else {
|
||||
err = ceph.CreateReplicatedPoolForApp(context.Context, context.clusterInfo, name, poolSpec, pgCount, AppName, cluster)
|
||||
err = ceph.CreateReplicatedPoolForApp(context.Context, context.clusterInfo, clusterSpec, name, poolSpec, pgCount, AppName)
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create pool %s for object store %s.", name, context.Name)
|
||||
|
||||
@@ -344,18 +344,18 @@ func (r *ReconcileCephObjectStore) validateStore(s *cephv1.CephObjectStore) erro
|
||||
// Validate the pool settings, but allow for empty pools specs in case they have already been created
|
||||
// such as by the ceph mgr
|
||||
if !emptyPool(s.Spec.MetadataPool) {
|
||||
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, &s.Spec.MetadataPool); err != nil {
|
||||
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, r.clusterSpec, &s.Spec.MetadataPool); err != nil {
|
||||
return errors.Wrap(err, "invalid metadata pool spec")
|
||||
}
|
||||
}
|
||||
if !emptyPool(s.Spec.DataPool) {
|
||||
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, &s.Spec.DataPool); err != nil {
|
||||
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, r.clusterSpec, &s.Spec.DataPool); err != nil {
|
||||
return errors.Wrap(err, "invalid data pool spec")
|
||||
}
|
||||
}
|
||||
|
||||
// Fail if we detected an external CephCluster CR and the list of endpoints is empty
|
||||
if r.cephClusterSpec.External.Enable && r.clusterInfo.CephCred.Username != cephclient.AdminUsername {
|
||||
if r.clusterSpec.External.Enable && r.clusterInfo.CephCred.Username != cephclient.AdminUsername {
|
||||
if len(s.Spec.Gateway.ExternalRgwEndpoints) == 0 {
|
||||
return errors.New("ceph cluster is external but externalRgwEndpoints list is empty")
|
||||
}
|
||||
|
||||
@@ -148,7 +148,7 @@ func TestValidateSpec(t *testing.T) {
|
||||
|
||||
r := &ReconcileCephObjectStore{
|
||||
context: context,
|
||||
cephClusterSpec: &cephv1.ClusterSpec{
|
||||
clusterSpec: &cephv1.ClusterSpec{
|
||||
External: cephv1.ExternalSpec{
|
||||
Enable: false,
|
||||
},
|
||||
@@ -190,7 +190,7 @@ func TestValidateSpec(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
|
||||
// external with no endpoints but ok since client.admin is used
|
||||
r.cephClusterSpec.External.Enable = true
|
||||
r.clusterSpec.External.Enable = true
|
||||
err = r.validateStore(s)
|
||||
assert.NoError(t, err)
|
||||
|
||||
|
||||
@@ -70,6 +70,7 @@ type ReconcileObjectZone struct {
|
||||
scheme *runtime.Scheme
|
||||
context *clusterd.Context
|
||||
clusterInfo *cephclient.ClusterInfo
|
||||
clusterSpec *cephv1.ClusterSpec
|
||||
}
|
||||
|
||||
// Add creates a new CephObjectZone Controller and adds it to the Manager. The Manager will set fields on the Controller
|
||||
@@ -152,6 +153,7 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re
|
||||
}
|
||||
return reconcileResponse, nil
|
||||
}
|
||||
r.clusterSpec = &cephCluster.Spec
|
||||
|
||||
// DELETE: the CR was deleted
|
||||
if !cephObjectZone.GetDeletionTimestamp().IsZero() {
|
||||
@@ -190,7 +192,7 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re
|
||||
}
|
||||
|
||||
// Create Ceph Zone
|
||||
_, err = r.createCephZone(cephObjectZone, realmName, cephCluster.Spec)
|
||||
_, err = r.createCephZone(cephObjectZone, realmName)
|
||||
if err != nil {
|
||||
return r.setFailedStatus(request.NamespacedName, "failed to create ceph zone", err)
|
||||
}
|
||||
@@ -203,7 +205,7 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
func (r *ReconcileObjectZone) createCephZone(zone *cephv1.CephObjectZone, realmName string, cluster cephv1.ClusterSpec) (reconcile.Result, error) {
|
||||
func (r *ReconcileObjectZone) createCephZone(zone *cephv1.CephObjectZone, realmName string) (reconcile.Result, error) {
|
||||
logger.Infof("creating object zone %q in zonegroup %q in realm %q", zone.Name, zone.Spec.ZoneGroup, realmName)
|
||||
|
||||
realmArg := fmt.Sprintf("--rgw-realm=%s", realmName)
|
||||
@@ -242,7 +244,7 @@ func (r *ReconcileObjectZone) createCephZone(zone *cephv1.CephObjectZone, realmN
|
||||
zoneIsMaster = true
|
||||
}
|
||||
|
||||
err = r.createPoolsAndZone(objContext, zone, realmName, zoneIsMaster, cluster)
|
||||
err = r.createPoolsAndZone(objContext, zone, realmName, zoneIsMaster)
|
||||
if err != nil {
|
||||
return reconcile.Result{}, err
|
||||
}
|
||||
@@ -253,14 +255,14 @@ func (r *ReconcileObjectZone) createCephZone(zone *cephv1.CephObjectZone, realmN
|
||||
return reconcile.Result{}, nil
|
||||
}
|
||||
|
||||
func (r *ReconcileObjectZone) createPoolsAndZone(objContext *object.Context, zone *cephv1.CephObjectZone, realmName string, zoneIsMaster bool, cluster cephv1.ClusterSpec) error {
|
||||
func (r *ReconcileObjectZone) createPoolsAndZone(objContext *object.Context, zone *cephv1.CephObjectZone, realmName string, zoneIsMaster bool) error {
|
||||
// create pools for zone
|
||||
logger.Debugf("creating pools ceph zone %q", zone.Name)
|
||||
realmArg := fmt.Sprintf("--rgw-realm=%s", realmName)
|
||||
zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", zone.Spec.ZoneGroup)
|
||||
zoneArg := fmt.Sprintf("--rgw-zone=%s", zone.Name)
|
||||
|
||||
err := object.CreatePools(objContext, zone.Spec.MetadataPool, zone.Spec.DataPool, cluster)
|
||||
err := object.CreatePools(objContext, r.clusterSpec, zone.Spec.MetadataPool, zone.Spec.DataPool)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "failed to create pools for zone %v", zone.Name)
|
||||
}
|
||||
@@ -330,10 +332,10 @@ func (r *ReconcileObjectZone) validateZoneCR(z *cephv1.CephObjectZone) error {
|
||||
if z.Spec.ZoneGroup == "" {
|
||||
return errors.New("missing zonegroup")
|
||||
}
|
||||
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, &z.Spec.MetadataPool); err != nil {
|
||||
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, r.clusterSpec, &z.Spec.MetadataPool); err != nil {
|
||||
return errors.Wrap(err, "invalid metadata pool spec")
|
||||
}
|
||||
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, &z.Spec.DataPool); err != nil {
|
||||
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, r.clusterSpec, &z.Spec.DataPool); err != nil {
|
||||
return errors.Wrap(err, "invalid data pool spec")
|
||||
}
|
||||
return nil
|
||||
|
||||
Reference in new issue
Block a user