diff --git a/pkg/daemon/ceph/client/command.go b/pkg/daemon/ceph/client/command.go index e0f5d1ff0..1d53b4a1d 100644 --- a/pkg/daemon/ceph/client/command.go +++ b/pkg/daemon/ceph/client/command.go @@ -42,6 +42,8 @@ const ( CrushTool = "crushtool" CmdExecuteTimeout = 1 * time.Minute cephConnectionTimeout = "15" // in seconds + // DefaultPGCount will cause Ceph to use the internal default PG count + DefaultPGCount = "0" ) // CephConfFilePath returns the location to the cluster's config file in the operator container. diff --git a/pkg/daemon/ceph/client/pool.go b/pkg/daemon/ceph/client/pool.go index a8af573e5..2ce427d94 100644 --- a/pkg/daemon/ceph/client/pool.go +++ b/pkg/daemon/ceph/client/pool.go @@ -144,7 +144,7 @@ func GetPoolDetails(context *clusterd.Context, namespace, name string) (CephStor func CreatePoolWithProfile(context *clusterd.Context, namespace, poolName string, pool cephv1.PoolSpec, appName string) error { if pool.IsReplicated() { - return CreateReplicatedPoolForApp(context, namespace, poolName, pool, appName) + return CreateReplicatedPoolForApp(context, namespace, poolName, pool, DefaultPGCount, appName) } if !pool.IsErasureCoded() { @@ -165,6 +165,7 @@ func CreatePoolWithProfile(context *clusterd.Context, namespace, poolName string poolName, ecProfileName, pool, + DefaultPGCount, appName, true /* enableECOverwrite */) } @@ -233,9 +234,8 @@ func GetErasureCodeProfileForPool(baseName string) string { return fmt.Sprintf("%s_ecprofile", baseName) } -func CreateECPoolForApp(context *clusterd.Context, namespace, poolName, ecProfileName string, pool cephv1.PoolSpec, appName string, enableECOverwrite bool) error { - args := []string{"osd", "pool", "create", poolName, "0", "erasure", ecProfileName} - +func CreateECPoolForApp(context *clusterd.Context, namespace, poolName, ecProfileName string, pool cephv1.PoolSpec, pgCount, appName string, enableECOverwrite bool) error { + args := []string{"osd", "pool", "create", poolName, pgCount, "erasure", ecProfileName} output, err := NewCephCommand(context, namespace, args).Run() if err != nil { return errors.Wrapf(err, "failed to create EC pool %s. %s", poolName, string(output)) @@ -258,13 +258,13 @@ func CreateECPoolForApp(context *clusterd.Context, namespace, poolName, ecProfil return nil } -func CreateReplicatedPoolForApp(context *clusterd.Context, namespace, poolName string, pool cephv1.PoolSpec, appName string) error { +func CreateReplicatedPoolForApp(context *clusterd.Context, namespace, poolName string, pool cephv1.PoolSpec, pgCount, appName string) error { // create a crush rule for a replicated pool, if a failure domain is specified if err := createReplicationCrushRule(context, namespace, poolName, pool); err != nil { return err } - args := []string{"osd", "pool", "create", poolName, "0", "replicated", poolName} + args := []string{"osd", "pool", "create", poolName, pgCount, "replicated", poolName} output, err := NewCephCommand(context, namespace, args).Run() if err != nil { return errors.Wrapf(err, "failed to create replicated pool %s. %s", poolName, string(output)) diff --git a/pkg/daemon/ceph/client/pool_test.go b/pkg/daemon/ceph/client/pool_test.go index 9fdccf6f4..cd44c13a1 100644 --- a/pkg/daemon/ceph/client/pool_test.go +++ b/pkg/daemon/ceph/client/pool_test.go @@ -61,7 +61,7 @@ func TestCreateECPoolWithOverwrites(t *testing.T) { return "", errors.Errorf("unexpected ceph command %q", args) } - err := CreateECPoolForApp(context, "myns", poolName, "mypoolprofile", p, "myapp", true) + err := CreateECPoolForApp(context, "myns", poolName, "mypoolprofile", p, DefaultPGCount, "myapp", true) assert.Nil(t, err) } @@ -97,7 +97,7 @@ func TestCreateECPoolWithoutOverwrites(t *testing.T) { return "", errors.Errorf("unexpected ceph command %q", args) } - err := CreateECPoolForApp(context, "myns", poolName, "mypoolprofile", p, "myapp", false) + err := CreateECPoolForApp(context, "myns", poolName, "mypoolprofile", p, DefaultPGCount, "myapp", false) assert.Nil(t, err) } @@ -166,7 +166,7 @@ func testCreateReplicaPool(t *testing.T, failureDomain, crushRoot, deviceClass s FailureDomain: failureDomain, CrushRoot: crushRoot, DeviceClass: deviceClass, Replicated: cephv1.ReplicatedSpec{Size: 12345}, } - err := CreateReplicatedPoolForApp(context, "myns", "mypool", p, "myapp") + err := CreateReplicatedPoolForApp(context, "myns", "mypool", p, DefaultPGCount, "myapp") assert.Nil(t, err) assert.True(t, crushRuleCreated) } diff --git a/pkg/operator/ceph/config/monstore.go b/pkg/operator/ceph/config/monstore.go index fae214fc3..c1fe4fe2b 100644 --- a/pkg/operator/ceph/config/monstore.go +++ b/pkg/operator/ceph/config/monstore.go @@ -62,6 +62,18 @@ func (m *MonStore) Set(who, option, value string) error { return nil } +// Get retrieves a config in the centralized mon configuration database. +// https://docs.ceph.com/docs/master/rados/configuration/ceph-conf/#monitor-configuration-database +func (m *MonStore) Get(who, option string) (string, error) { + args := []string{"config", "get", who, normalizeKey(option)} + cephCmd := client.NewCephCommand(m.context, m.namespace, args) + out, err := cephCmd.Run() + if err != nil { + return "", errors.Wrapf(err, "failed to get config setting %q for user %q", option, who) + } + return string(out), nil +} + // SetAll sets all configs from the overrides in the centralized mon configuration database. // See MonStore.Set for more. func (m *MonStore) SetAll(options ...Option) error { diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go index 24bc54d94..e0b03065d 100644 --- a/pkg/operator/ceph/object/objectstore.go +++ b/pkg/operator/ceph/object/objectstore.go @@ -26,6 +26,7 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/daemon/ceph/client" ceph "github.com/rook/rook/pkg/daemon/ceph/client" + "github.com/rook/rook/pkg/operator/ceph/config" ) const ( @@ -258,7 +259,14 @@ func createPools(context *Context, spec cephv1.ObjectStoreSpec) error { } } - if err := createSimilarPools(context, append(metadataPools, rootPool), spec.MetadataPool, ""); err != nil { + // get the default PG count for rgw metadata pools + metadataPoolPGs, err := config.GetMonStore(context.Context, context.ClusterName).Get("mon.", "rgw_rados_pool_pg_num_min") + if err != nil { + logger.Warningf("failed to adjust the PG count for rgw metadata pools. using the general default. %v", err) + metadataPoolPGs = ceph.DefaultPGCount + } + + if err := createSimilarPools(context, append(metadataPools, rootPool), spec.MetadataPool, metadataPoolPGs, ""); err != nil { return errors.Wrapf(err, "failed to create metadata pools") } @@ -271,14 +279,14 @@ func createPools(context *Context, spec cephv1.ObjectStoreSpec) error { } } - if err := createSimilarPools(context, []string{dataPoolName}, spec.DataPool, ecProfileName); err != nil { + if err := createSimilarPools(context, []string{dataPoolName}, spec.DataPool, ceph.DefaultPGCount, ecProfileName); err != nil { return errors.Wrapf(err, "failed to create data pool") } return nil } -func createSimilarPools(context *Context, pools []string, poolSpec cephv1.PoolSpec, ecProfileName string) error { +func createSimilarPools(context *Context, pools []string, poolSpec cephv1.PoolSpec, pgCount, ecProfileName string) error { for _, pool := range pools { // create the pool if it doesn't exist yet name := poolName(context.Name, pool) @@ -289,9 +297,9 @@ func createSimilarPools(context *Context, pools []string, poolSpec cephv1.PoolSp if poolSpec.IsErasureCoded() { // An EC pool backing an object store does not need to enable EC overwrites, so the pool is // created with that property disabled to avoid unnecessary performance impact. - err = ceph.CreateECPoolForApp(context.Context, context.ClusterName, name, ecProfileName, poolSpec, AppName, false /* enableECOverwrite */) + err = ceph.CreateECPoolForApp(context.Context, context.ClusterName, name, ecProfileName, poolSpec, pgCount, AppName, false /* enableECOverwrite */) } else { - err = ceph.CreateReplicatedPoolForApp(context.Context, context.ClusterName, name, poolSpec, AppName) + err = ceph.CreateReplicatedPoolForApp(context.Context, context.ClusterName, name, poolSpec, pgCount, AppName) } if err != nil { return errors.Wrapf(err, "failed to create pool %s for object store %s.", name, context.Name)