ceph: validate pools for stretch clusters

Pools in stretch clusters will all use the same crush rule that
is generated by the operator when configuring the cluster for
stretch mode, with replica 4 and two replicas per failure domain.
Pools cannot create new crush rules, so we require that replica be
4 when the pools is created, EC is not allowed, and no new
crush rule will be created.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
Travis Nielsen committed 2020-11-05 09:23:57 -07:00
1 parent b91f4211c9
commit e72b69f8e0
16 files changed
+133 -102

No files matched your search

+9 -9
View File
@@ -79,7 +79,7 @@ type ReconcileCephObjectStore struct {
bktclient bktclient.Interface
scheme *runtime.Scheme
context *clusterd.Context
cephClusterSpec *cephv1.ClusterSpec
clusterSpec *cephv1.ClusterSpec
clusterInfo *cephclient.ClusterInfo
objectStoreChannels map[string]*objectStoreHealth
}
@@ -194,7 +194,7 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
return reconcileResponse, nil
}
r.cephClusterSpec = &cephCluster.Spec
r.clusterSpec = &cephCluster.Spec
// Initialize the channel for this object store
// This allows us to track multiple ObjectStores in the same namespace
@@ -240,7 +240,7 @@ func (r *ReconcileCephObjectStore) reconcile(request reconcile.Request) (reconci
cfg := clusterConfig{
context: r.context,
store: cephObjectStore,
clusterSpec: r.cephClusterSpec,
clusterSpec: r.clusterSpec,
clusterInfo: r.clusterInfo,
}
err = cfg.deleteStore()
@@ -289,9 +289,9 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
context: r.context,
clusterInfo: r.clusterInfo,
store: cephObjectStore,
rookVersion: r.cephClusterSpec.CephVersion.Image,
clusterSpec: r.cephClusterSpec,
DataPathMap: opconfig.NewStatelessDaemonDataPathMap(opconfig.RgwType, cephObjectStore.Name, cephObjectStore.Namespace, r.cephClusterSpec.DataDirHostPath),
rookVersion: r.clusterSpec.CephVersion.Image,
clusterSpec: r.clusterSpec,
DataPathMap: opconfig.NewStatelessDaemonDataPathMap(opconfig.RgwType, cephObjectStore.Name, cephObjectStore.Namespace, r.clusterSpec.DataDirHostPath),
client: r.client,
scheme: r.scheme,
}
@@ -301,7 +301,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
var serviceIP string
var err error
if r.cephClusterSpec.External.Enable {
if r.clusterSpec.External.Enable {
logger.Info("reconciling external object store")
// RECONCILE SERVICE
@@ -351,7 +351,7 @@ func (r *ReconcileCephObjectStore) reconcileCreateObjectStore(cephObjectStore *c
// Reconcile Pool Creation
if !cephObjectStore.Spec.IsMultisite() {
logger.Info("reconciling object store pools")
err = CreatePools(objContext, cephObjectStore.Spec.MetadataPool, cephObjectStore.Spec.DataPool, cluster)
err = CreatePools(objContext, r.clusterSpec, cephObjectStore.Spec.MetadataPool, cephObjectStore.Spec.DataPool)
if err != nil {
return r.setFailedStatus(namespacedName, "failed to create object pools", err)
}
@@ -482,7 +482,7 @@ func (r *ReconcileCephObjectStore) startMonitoring(objectstore *cephv1.CephObjec
return
}
rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck, r.cephClusterSpec.External.Enable)
rgwChecker := newBucketChecker(r.context, objContext, serviceIP, port, r.client, namespacedName, &objectstore.Spec.HealthCheck, r.clusterSpec.External.Enable)
logger.Info("starting rgw healthcheck")
go rgwChecker.checkObjectStore(r.objectStoreChannels[objectstore.Name].stopChan)
}
+5 -5
View File
@@ -605,7 +605,7 @@ func deletePools(context *Context, spec cephv1.ObjectStoreSpec, lastStore bool)
return nil
}
func CreatePools(context *Context, metadataPool, dataPool cephv1.PoolSpec, cluster cephv1.ClusterSpec) error {
func CreatePools(context *Context, clusterSpec *cephv1.ClusterSpec, metadataPool, dataPool cephv1.PoolSpec) error {
if emptyPool(dataPool) && emptyPool(metadataPool) {
logger.Info("no pools specified for the CR, checking for their existence...")
pools := append(metadataPools, dataPoolName)
@@ -631,7 +631,7 @@ func CreatePools(context *Context, metadataPool, dataPool cephv1.PoolSpec, clust
metadataPoolPGs = ceph.DefaultPGCount
}
if err := createSimilarPools(context, append(metadataPools, rootPool), metadataPool, metadataPoolPGs, "", cluster); err != nil {
if err := createSimilarPools(context, append(metadataPools, rootPool), clusterSpec, metadataPool, metadataPoolPGs, ""); err != nil {
return errors.Wrap(err, "failed to create metadata pools")
}
@@ -644,14 +644,14 @@ func CreatePools(context *Context, metadataPool, dataPool cephv1.PoolSpec, clust
}
}
if err := createSimilarPools(context, []string{dataPoolName}, dataPool, ceph.DefaultPGCount, ecProfileName, cluster); err != nil {
if err := createSimilarPools(context, []string{dataPoolName}, clusterSpec, dataPool, ceph.DefaultPGCount, ecProfileName); err != nil {
return errors.Wrap(err, "failed to create data pool")
}
return nil
}
func createSimilarPools(context *Context, pools []string, poolSpec cephv1.PoolSpec, pgCount, ecProfileName string, cluster cephv1.ClusterSpec) error {
func createSimilarPools(context *Context, pools []string, clusterSpec *cephv1.ClusterSpec, poolSpec cephv1.PoolSpec, pgCount, ecProfileName string) error {
for _, pool := range pools {
// create the pool if it doesn't exist yet
name := poolName(context.Name, pool)
@@ -664,7 +664,7 @@ func createSimilarPools(context *Context, pools []string, poolSpec cephv1.PoolSp
// created with that property disabled to avoid unnecessary performance impact.
err = ceph.CreateECPoolForApp(context.Context, context.clusterInfo, name, ecProfileName, poolSpec, pgCount, AppName, false /* enableECOverwrite */)
} else {
err = ceph.CreateReplicatedPoolForApp(context.Context, context.clusterInfo, name, poolSpec, pgCount, AppName, cluster)
err = ceph.CreateReplicatedPoolForApp(context.Context, context.clusterInfo, clusterSpec, name, poolSpec, pgCount, AppName)
}
if err != nil {
return errors.Wrapf(err, "failed to create pool %s for object store %s.", name, context.Name)
+3 -3
View File
@@ -344,18 +344,18 @@ func (r *ReconcileCephObjectStore) validateStore(s *cephv1.CephObjectStore) erro
// Validate the pool settings, but allow for empty pools specs in case they have already been created
// such as by the ceph mgr
if !emptyPool(s.Spec.MetadataPool) {
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, &s.Spec.MetadataPool); err != nil {
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, r.clusterSpec, &s.Spec.MetadataPool); err != nil {
return errors.Wrap(err, "invalid metadata pool spec")
}
}
if !emptyPool(s.Spec.DataPool) {
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, &s.Spec.DataPool); err != nil {
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, r.clusterSpec, &s.Spec.DataPool); err != nil {
return errors.Wrap(err, "invalid data pool spec")
}
}
// Fail if we detected an external CephCluster CR and the list of endpoints is empty
if r.cephClusterSpec.External.Enable && r.clusterInfo.CephCred.Username != cephclient.AdminUsername {
if r.clusterSpec.External.Enable && r.clusterInfo.CephCred.Username != cephclient.AdminUsername {
if len(s.Spec.Gateway.ExternalRgwEndpoints) == 0 {
return errors.New("ceph cluster is external but externalRgwEndpoints list is empty")
}
+2 -2
View File
@@ -148,7 +148,7 @@ func TestValidateSpec(t *testing.T) {
r := &ReconcileCephObjectStore{
context: context,
cephClusterSpec: &cephv1.ClusterSpec{
clusterSpec: &cephv1.ClusterSpec{
External: cephv1.ExternalSpec{
Enable: false,
},
@@ -190,7 +190,7 @@ func TestValidateSpec(t *testing.T) {
assert.Nil(t, err)
// external with no endpoints but ok since client.admin is used
r.cephClusterSpec.External.Enable = true
r.clusterSpec.External.Enable = true
err = r.validateStore(s)
assert.NoError(t, err)
+9 -7
View File
@@ -70,6 +70,7 @@ type ReconcileObjectZone struct {
scheme *runtime.Scheme
context *clusterd.Context
clusterInfo *cephclient.ClusterInfo
clusterSpec *cephv1.ClusterSpec
}
// Add creates a new CephObjectZone Controller and adds it to the Manager. The Manager will set fields on the Controller
@@ -152,6 +153,7 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re
}
return reconcileResponse, nil
}
r.clusterSpec = &cephCluster.Spec
// DELETE: the CR was deleted
if !cephObjectZone.GetDeletionTimestamp().IsZero() {
@@ -190,7 +192,7 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re
}
// Create Ceph Zone
_, err = r.createCephZone(cephObjectZone, realmName, cephCluster.Spec)
_, err = r.createCephZone(cephObjectZone, realmName)
if err != nil {
return r.setFailedStatus(request.NamespacedName, "failed to create ceph zone", err)
}
@@ -203,7 +205,7 @@ func (r *ReconcileObjectZone) reconcile(request reconcile.Request) (reconcile.Re
return reconcile.Result{}, nil
}
func (r *ReconcileObjectZone) createCephZone(zone *cephv1.CephObjectZone, realmName string, cluster cephv1.ClusterSpec) (reconcile.Result, error) {
func (r *ReconcileObjectZone) createCephZone(zone *cephv1.CephObjectZone, realmName string) (reconcile.Result, error) {
logger.Infof("creating object zone %q in zonegroup %q in realm %q", zone.Name, zone.Spec.ZoneGroup, realmName)
realmArg := fmt.Sprintf("--rgw-realm=%s", realmName)
@@ -242,7 +244,7 @@ func (r *ReconcileObjectZone) createCephZone(zone *cephv1.CephObjectZone, realmN
zoneIsMaster = true
}
err = r.createPoolsAndZone(objContext, zone, realmName, zoneIsMaster, cluster)
err = r.createPoolsAndZone(objContext, zone, realmName, zoneIsMaster)
if err != nil {
return reconcile.Result{}, err
}
@@ -253,14 +255,14 @@ func (r *ReconcileObjectZone) createCephZone(zone *cephv1.CephObjectZone, realmN
return reconcile.Result{}, nil
}
func (r *ReconcileObjectZone) createPoolsAndZone(objContext *object.Context, zone *cephv1.CephObjectZone, realmName string, zoneIsMaster bool, cluster cephv1.ClusterSpec) error {
func (r *ReconcileObjectZone) createPoolsAndZone(objContext *object.Context, zone *cephv1.CephObjectZone, realmName string, zoneIsMaster bool) error {
// create pools for zone
logger.Debugf("creating pools ceph zone %q", zone.Name)
realmArg := fmt.Sprintf("--rgw-realm=%s", realmName)
zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", zone.Spec.ZoneGroup)
zoneArg := fmt.Sprintf("--rgw-zone=%s", zone.Name)
err := object.CreatePools(objContext, zone.Spec.MetadataPool, zone.Spec.DataPool, cluster)
err := object.CreatePools(objContext, r.clusterSpec, zone.Spec.MetadataPool, zone.Spec.DataPool)
if err != nil {
return errors.Wrapf(err, "failed to create pools for zone %v", zone.Name)
}
@@ -330,10 +332,10 @@ func (r *ReconcileObjectZone) validateZoneCR(z *cephv1.CephObjectZone) error {
if z.Spec.ZoneGroup == "" {
return errors.New("missing zonegroup")
}
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, &z.Spec.MetadataPool); err != nil {
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, r.clusterSpec, &z.Spec.MetadataPool); err != nil {
return errors.Wrap(err, "invalid metadata pool spec")
}
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, &z.Spec.DataPool); err != nil {
if err := pool.ValidatePoolSpec(r.context, r.clusterInfo, r.clusterSpec, &z.Spec.DataPool); err != nil {
return errors.Wrap(err, "invalid data pool spec")
}
return nil