feedc0de/my-rook-config
1
0
Fork 0
forked from rook/rook
Code Pull Requests Activity

ceph: set pg_num for rgw metadata pools to lower default

Browse Source 
The PG count on metadata pools should default to rgw_rados_pool_pg_num_min
instead of the more general default pg count. This means rgw pools
will default to 8 PGs instead of 32 PGs, which means a lot more pools
can be created before hitting the default PG limit.

Signed-off-by: Travis Nielsen <tnielsen@redhat.com>
This commit is contained in:
Travis Nielsen
2020-03-27 09:06:59 -06:00
parent 9c343ae94e
commit e39637930d
5 changed files with 36 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/daemon/ceph/client/command.go
+2
View File
@@ -42,6 +42,8 @@ const (
CrushTool = "crushtool"
CmdExecuteTimeout = 1 * time.Minute
cephConnectionTimeout = "15" // in seconds
// DefaultPGCount will cause Ceph to use the internal default PG count
DefaultPGCount = "0"
)
// CephConfFilePath returns the location to the cluster's config file in the operator container.
pkg/daemon/ceph/client/pool.go
+6 -6
View File
@@ -144,7 +144,7 @@ func GetPoolDetails(context *clusterd.Context, namespace, name string) (CephStor
func CreatePoolWithProfile(context *clusterd.Context, namespace, poolName string, pool cephv1.PoolSpec, appName string) error {
if pool.IsReplicated() {
return CreateReplicatedPoolForApp(context, namespace, poolName, pool, appName)
return CreateReplicatedPoolForApp(context, namespace, poolName, pool, DefaultPGCount, appName)
}
if !pool.IsErasureCoded() {
@@ -165,6 +165,7 @@ func CreatePoolWithProfile(context *clusterd.Context, namespace, poolName string
poolName,
ecProfileName,
pool,
DefaultPGCount,
appName,
true /* enableECOverwrite */)
}
@@ -233,9 +234,8 @@ func GetErasureCodeProfileForPool(baseName string) string {
return fmt.Sprintf("%s_ecprofile", baseName)
}
func CreateECPoolForApp(context *clusterd.Context, namespace, poolName, ecProfileName string, pool cephv1.PoolSpec, appName string, enableECOverwrite bool) error {
args := []string{"osd", "pool", "create", poolName, "0", "erasure", ecProfileName}
func CreateECPoolForApp(context *clusterd.Context, namespace, poolName, ecProfileName string, pool cephv1.PoolSpec, pgCount, appName string, enableECOverwrite bool) error {
args := []string{"osd", "pool", "create", poolName, pgCount, "erasure", ecProfileName}
output, err := NewCephCommand(context, namespace, args).Run()
if err != nil {
return errors.Wrapf(err, "failed to create EC pool %s. %s", poolName, string(output))
@@ -258,13 +258,13 @@ func CreateECPoolForApp(context *clusterd.Context, namespace, poolName, ecProfil
return nil
}
func CreateReplicatedPoolForApp(context *clusterd.Context, namespace, poolName string, pool cephv1.PoolSpec, appName string) error {
func CreateReplicatedPoolForApp(context *clusterd.Context, namespace, poolName string, pool cephv1.PoolSpec, pgCount, appName string) error {
// create a crush rule for a replicated pool, if a failure domain is specified
if err := createReplicationCrushRule(context, namespace, poolName, pool); err != nil {
return err
}
args := []string{"osd", "pool", "create", poolName, "0", "replicated", poolName}
args := []string{"osd", "pool", "create", poolName, pgCount, "replicated", poolName}
output, err := NewCephCommand(context, namespace, args).Run()
if err != nil {
return errors.Wrapf(err, "failed to create replicated pool %s. %s", poolName, string(output))
pkg/daemon/ceph/client/pool_test.go
+3 -3
View File
@@ -61,7 +61,7 @@ func TestCreateECPoolWithOverwrites(t *testing.T) {
return "", errors.Errorf("unexpected ceph command %q", args)
}
err := CreateECPoolForApp(context, "myns", poolName, "mypoolprofile", p, "myapp", true)
err := CreateECPoolForApp(context, "myns", poolName, "mypoolprofile", p, DefaultPGCount, "myapp", true)
assert.Nil(t, err)
}
@@ -97,7 +97,7 @@ func TestCreateECPoolWithoutOverwrites(t *testing.T) {
return "", errors.Errorf("unexpected ceph command %q", args)
}
err := CreateECPoolForApp(context, "myns", poolName, "mypoolprofile", p, "myapp", false)
err := CreateECPoolForApp(context, "myns", poolName, "mypoolprofile", p, DefaultPGCount, "myapp", false)
assert.Nil(t, err)
}
@@ -166,7 +166,7 @@ func testCreateReplicaPool(t *testing.T, failureDomain, crushRoot, deviceClass s
FailureDomain: failureDomain, CrushRoot: crushRoot, DeviceClass: deviceClass,
Replicated: cephv1.ReplicatedSpec{Size: 12345},
}
err := CreateReplicatedPoolForApp(context, "myns", "mypool", p, "myapp")
err := CreateReplicatedPoolForApp(context, "myns", "mypool", p, DefaultPGCount, "myapp")
assert.Nil(t, err)
assert.True(t, crushRuleCreated)
}
pkg/operator/ceph/config/monstore.go
+12
View File
@@ -62,6 +62,18 @@ func (m *MonStore) Set(who, option, value string) error {
return nil
}
// Get retrieves a config in the centralized mon configuration database.
// https://docs.ceph.com/docs/master/rados/configuration/ceph-conf/#monitor-configuration-database
func (m *MonStore) Get(who, option string) (string, error) {
args := []string{"config", "get", who, normalizeKey(option)}
cephCmd := client.NewCephCommand(m.context, m.namespace, args)
out, err := cephCmd.Run()
if err != nil {
return "", errors.Wrapf(err, "failed to get config setting %q for user %q", option, who)
}
return string(out), nil
}
// SetAll sets all configs from the overrides in the centralized mon configuration database.
// See MonStore.Set for more.
func (m *MonStore) SetAll(options ...Option) error {
pkg/operator/ceph/object/objectstore.go
+13 -5
View File
@@ -26,6 +26,7 @@ import (
cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
"github.com/rook/rook/pkg/daemon/ceph/client"
ceph "github.com/rook/rook/pkg/daemon/ceph/client"
"github.com/rook/rook/pkg/operator/ceph/config"
)
const (
@@ -258,7 +259,14 @@ func createPools(context *Context, spec cephv1.ObjectStoreSpec) error {
}
}
if err := createSimilarPools(context, append(metadataPools, rootPool), spec.MetadataPool, ""); err != nil {
// get the default PG count for rgw metadata pools
metadataPoolPGs, err := config.GetMonStore(context.Context, context.ClusterName).Get("mon.", "rgw_rados_pool_pg_num_min")
if err != nil {
logger.Warningf("failed to adjust the PG count for rgw metadata pools. using the general default. %v", err)
metadataPoolPGs = ceph.DefaultPGCount
}
if err := createSimilarPools(context, append(metadataPools, rootPool), spec.MetadataPool, metadataPoolPGs, ""); err != nil {
return errors.Wrapf(err, "failed to create metadata pools")
}
@@ -271,14 +279,14 @@ func createPools(context *Context, spec cephv1.ObjectStoreSpec) error {
}
}
if err := createSimilarPools(context, []string{dataPoolName}, spec.DataPool, ecProfileName); err != nil {
if err := createSimilarPools(context, []string{dataPoolName}, spec.DataPool, ceph.DefaultPGCount, ecProfileName); err != nil {
return errors.Wrapf(err, "failed to create data pool")
}
return nil
}
func createSimilarPools(context *Context, pools []string, poolSpec cephv1.PoolSpec, ecProfileName string) error {
func createSimilarPools(context *Context, pools []string, poolSpec cephv1.PoolSpec, pgCount, ecProfileName string) error {
for _, pool := range pools {
// create the pool if it doesn't exist yet
name := poolName(context.Name, pool)
@@ -289,9 +297,9 @@ func createSimilarPools(context *Context, pools []string, poolSpec cephv1.PoolSp
if poolSpec.IsErasureCoded() {
// An EC pool backing an object store does not need to enable EC overwrites, so the pool is
// created with that property disabled to avoid unnecessary performance impact.
err = ceph.CreateECPoolForApp(context.Context, context.ClusterName, name, ecProfileName, poolSpec, AppName, false /* enableECOverwrite */)
err = ceph.CreateECPoolForApp(context.Context, context.ClusterName, name, ecProfileName, poolSpec, pgCount, AppName, false /* enableECOverwrite */)
} else {
err = ceph.CreateReplicatedPoolForApp(context.Context, context.ClusterName, name, poolSpec, AppName)
err = ceph.CreateReplicatedPoolForApp(context.Context, context.ClusterName, name, poolSpec, pgCount, AppName)
}
if err != nil {
return errors.Wrapf(err, "failed to create pool %s for object store %s.", name, context.Name)