Merge pull request #5124 from ShyamsundarR/ceph-pool-compression

ceph: Add ability to specify compression mode for pools
This commit is contained in:
Travis Nielsen authored and GitHub committed 2020-04-01 14:29:00 -06:00
commit 14fcbcb151
22 files changed
+312 -55

No files matched your search

+1
View File
@@ -83,6 +83,7 @@ When creating an erasure-coded pool, it is highly recommended to create the pool
> **NOTE**: Neither Rook, nor Ceph, prevent the creation of a cluster where the replicated data (or Erasure Coded chunks) can be written safely. By design, Ceph will delay checking for suitable OSDs until a write request is made and this write can hang if there are not sufficient OSDs to satisfy the request.
* `deviceClass`: Sets up the CRUSH rule for the pool to distribute data only on the specified device class. If left empty or unspecified, the pool will use the cluster's default CRUSH root, which usually distributes data over all OSDs, regardless of their class.
* `crushRoot`: The root in the crush map to be used by the pool. If left empty or unspecified, the default root will be used. Creating a crush hierarchy for the OSDs currently requires the Rook toolbox to run the Ceph tools described [here](http://docs.ceph.com/docs/master/rados/operations/crush-map/#modifying-the-crush-map).
* `compressionMode`: Sets up the pool for inline compression when using a Bluestore OSD. If left unspecified does not setup any compression mode for the pool. Values supported are the same as Bluestore inline compression [modes](https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#inline-compression), such as `none`, `passive`, `aggressive`, and `force`.
### Erasure Coding
+1
View File
@@ -20,6 +20,7 @@
ConfigMap can be used in mix with the already existing Env Vars defined in operator deployment manifest. Precedence will be given to ConfigMap in case of conflicting configurations.
- Rook Ceph cleanupPolicy will clean up the dataDirHostPath only after user confirmation. For more info about CleanUpPolicy [read the design](https://github.com/rook/rook/blob/master/design/ceph/ceph-cluster-cleanup.md) as well as the documentation [cleanupPolicy](Documentation/ceph-cluster-crd.md#cluster-settings)
- Rook monitor, mds and osd now have liveness probe checks on their respective sockets
- Pools can now be configured to inline compress the data using the `compressionMode` parameter. Support added [here](https://github.com/rook/rook/pull/5124)
### EdgeFS
@@ -225,6 +225,14 @@ spec:
minimum: 0
maximum: 10
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
dataPools:
type: array
items:
@@ -247,6 +255,14 @@ spec:
minimum: 0
maximum: 10
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
preservePoolsOnDelete:
type: boolean
additionalPrinterColumns:
@@ -339,6 +355,14 @@ spec:
type: integer
codingChunks:
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
dataPool:
properties:
failureDomain:
@@ -353,6 +377,14 @@ spec:
type: integer
codingChunks:
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
preservePoolsOnDelete:
type: boolean
---
@@ -413,6 +445,14 @@ spec:
type: integer
minimum: 0
maximum: 9
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
@@ -286,6 +286,14 @@ spec:
minimum: 0
maximum: 10
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
dataPools:
type: array
items:
@@ -310,6 +318,14 @@ spec:
minimum: 0
maximum: 10
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
preservePoolsOnDelete:
type: boolean
additionalPrinterColumns:
@@ -410,6 +426,14 @@ spec:
type: integer
codingChunks:
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
dataPool:
properties:
failureDomain:
@@ -424,6 +448,14 @@ spec:
type: integer
codingChunks:
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
preservePoolsOnDelete:
type: boolean
subresources:
@@ -492,6 +524,14 @@ spec:
type: integer
minimum: 0
maximum: 9
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
subresources:
status: {}
# OLM: END CEPH BLOCK POOL CRD
@@ -21,6 +21,8 @@ spec:
- erasureCoded:
dataChunks: 2
codingChunks: 1
# Inline compression mode for the data pool
compressionMode: none
# Whether to preserve metadata and data pools on filesystem deletion
preservePoolsOnDelete: true
# The metadata service (mds) configuration
@@ -18,6 +18,7 @@ spec:
replicated:
size: 1
requireSafeReplicaSize: false
compressionMode: none
preservePoolsOnDelete: false
metadataServer:
activeCount: 1
@@ -26,6 +26,9 @@ spec:
# gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
# for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size
#targetSizeRatio: .5
# Inline compression mode for the data pool
# Further reference: https://docs.ceph.com/docs/nautilus/rados/configuration/bluestore-config-ref/#inline-compression
compressionMode: none
# Whether to preserve metadata and data pools on filesystem deletion
preservePoolsOnDelete: true
# The metadata service (mds) configuration
@@ -21,6 +21,8 @@ spec:
erasureCoded:
dataChunks: 2
codingChunks: 1
# Inline compression mode for the data pool
compressionMode: none
# Whether to preserve metadata and data pools on object store deletion
preservePoolsOnDelete: true
# The gateway service configuration
@@ -20,6 +20,8 @@ spec:
failureDomain: host
replicated:
size: 3
# Inline compression mode for the data pool
compressionMode: none
# Whether to preserve metadata and data pools on object store deletion
preservePoolsOnDelete: true
# The gateway service configuration
@@ -15,6 +15,7 @@ spec:
dataPool:
replicated:
size: 1
compressionMode: none
preservePoolsOnDelete: false
gateway:
type: s3
@@ -32,6 +32,9 @@ spec:
# gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
# for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size
#targetSizeRatio: .5
# Inline compression mode for the data pool
# Further reference: https://docs.ceph.com/docs/nautilus/rados/configuration/bluestore-config-ref/#inline-compression
compressionMode: none
# Whether to preserve metadata and data pools on object store deletion
preservePoolsOnDelete: false
# The gateway service configuration
@@ -16,6 +16,8 @@ spec:
erasureCoded:
dataChunks: 2
codingChunks: 1
# Inline compression mode for the data pool
compressionMode: none
# A key/value list of annotations
annotations:
# key: value
@@ -11,3 +11,4 @@ metadata:
spec:
replicated:
size: 1
compressionMode: none
@@ -27,6 +27,9 @@ spec:
# The Ceph CRUSH device class associated with the CRUSH replicated rule
# For reference: https://docs.ceph.com/docs/nautilus/rados/operations/crush-map/#device-classes
#deviceClass: my-class
# Inline compression mode for the data pool
# Further reference: https://docs.ceph.com/docs/nautilus/rados/configuration/bluestore-config-ref/#inline-compression
compressionMode: none
# A key/value list of annotations
annotations:
# key: value
@@ -265,6 +265,14 @@ spec:
minimum: 0
maximum: 10
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
dataPools:
type: array
items:
@@ -289,6 +297,14 @@ spec:
minimum: 0
maximum: 10
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
preservePoolsOnDelete:
type: boolean
additionalPrinterColumns:
@@ -385,6 +401,14 @@ spec:
type: integer
codingChunks:
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
dataPool:
properties:
failureDomain:
@@ -399,6 +423,14 @@ spec:
type: integer
codingChunks:
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
preservePoolsOnDelete:
type: boolean
subresources:
@@ -463,6 +495,14 @@ spec:
type: integer
minimum: 0
maximum: 9
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
subresources:
status: {}
---
+2
View File
@@ -17,7 +17,9 @@
override GOOS=linux
ifeq ($(origin DOCKERCMD),undefined)
DOCKERCMD := docker
endif
# include the common make file
SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
+3
View File
@@ -268,6 +268,9 @@ type PoolSpec struct {
// The device class the OSD should set to (options are: hdd, ssd, or nvme)
DeviceClass string `json:"deviceClass"`
// The inline compression mode in Bluestore OSD to set to (options are: none, passive, aggressive, force)
CompressionMode string `json:"compressionMode"`
// The replication settings
Replicated ReplicatedSpec `json:"replicated"`
+25 -11
View File
@@ -45,6 +45,7 @@ type CephStoragePoolDetails struct {
FailureDomain string `json:"failureDomain"`
CrushRoot string `json:"crushRoot"`
DeviceClass string `json:"deviceClass"`
CompressionMode string `json:"compression_mode"`
TargetSizeRatio float64 `json:"target_size_ratio,omitempty"`
RequireSafeReplicaSize bool `json:"requireSafeReplicaSize,omitempty"`
}
@@ -230,6 +231,26 @@ func givePoolAppTag(context *clusterd.Context, namespace string, poolName string
return nil
}
func setCommonPoolProperties(context *clusterd.Context, pool cephv1.PoolSpec, namespace, poolName, appName string) error {
compressionModeProperty := "compression_mode"
if pool.CompressionMode != "" {
err := SetPoolProperty(context, namespace, poolName, compressionModeProperty, pool.CompressionMode)
if err != nil {
return errors.Wrapf(err, "failed to set property %q on pool %q to %q", compressionModeProperty, poolName, pool.CompressionMode)
}
}
// ensure that the newly created pool gets an application tag
if appName != "" {
err := givePoolAppTag(context, namespace, poolName, appName)
if err != nil {
return errors.Wrapf(err, "failed to tag pool %q for application %q", poolName, appName)
}
}
return nil
}
func GetErasureCodeProfileForPool(baseName string) string {
return fmt.Sprintf("%s_ecprofile", baseName)
}
@@ -247,11 +268,8 @@ func CreateECPoolForApp(context *clusterd.Context, namespace, poolName, ecProfil
}
}
if appName != "" {
err = givePoolAppTag(context, namespace, poolName, appName)
if err != nil {
return err
}
if err = setCommonPoolProperties(context, pool, namespace, poolName, appName); err != nil {
return err
}
logger.Infof("creating EC pool %s succeeded", poolName)
@@ -283,12 +301,8 @@ func CreateReplicatedPoolForApp(context *clusterd.Context, namespace, poolName s
}
}
// ensure that the newly created pool gets an application tag
if appName != "" {
err = givePoolAppTag(context, namespace, poolName, appName)
if err != nil {
return err
}
if err = setCommonPoolProperties(context, pool, namespace, poolName, appName); err != nil {
return err
}
logger.Infof("creating replicated pool %s succeeded", poolName)
+54 -43
View File
@@ -28,11 +28,28 @@ import (
)
func TestCreateECPoolWithOverwrites(t *testing.T) {
testCreateECPool(t, true, "")
}
func TestCreateECPoolWithoutOverwrites(t *testing.T) {
testCreateECPool(t, false, "")
}
func TestCreateECPoolWithCompression(t *testing.T) {
testCreateECPool(t, false, "aggressive")
testCreateECPool(t, true, "none")
}
func testCreateECPool(t *testing.T, overwrite bool, compressionMode string) {
poolName := "mypool"
compressionModeCreated := false
p := cephv1.PoolSpec{
FailureDomain: "host",
ErasureCoded: cephv1.ErasureCodedSpec{},
}
if compressionMode != "" {
p.CompressionMode = compressionMode
}
executor := &exectest.MockExecutor{}
context := &clusterd.Context{Executor: executor}
executor.MockExecuteCommandWithOutputFile = func(command, outputFile string, args ...string) (string, error) {
@@ -45,11 +62,17 @@ func TestCreateECPoolWithOverwrites(t *testing.T) {
return "", nil
}
if args[2] == "set" {
assert.Equal(t, "mypool", args[3])
if args[4] == "allow_ec_overwrites" {
assert.Equal(t, "mypool", args[3])
assert.Equal(t, true, overwrite)
assert.Equal(t, "true", args[5])
return "", nil
}
if args[4] == "compression_mode" {
assert.Equal(t, compressionMode, args[5])
compressionModeCreated = true
return "", nil
}
}
if args[2] == "application" {
assert.Equal(t, "enable", args[3])
@@ -61,59 +84,34 @@ func TestCreateECPoolWithOverwrites(t *testing.T) {
return "", errors.Errorf("unexpected ceph command %q", args)
}
err := CreateECPoolForApp(context, "myns", poolName, "mypoolprofile", p, DefaultPGCount, "myapp", true)
err := CreateECPoolForApp(context, "myns", poolName, "mypoolprofile", p, DefaultPGCount, "myapp", overwrite)
assert.Nil(t, err)
}
func TestCreateECPoolWithoutOverwrites(t *testing.T) {
poolName := "mypool"
p := cephv1.PoolSpec{
FailureDomain: "host",
ErasureCoded: cephv1.ErasureCodedSpec{},
if compressionMode != "" {
assert.True(t, compressionModeCreated)
} else {
assert.False(t, compressionModeCreated)
}
executor := &exectest.MockExecutor{}
context := &clusterd.Context{Executor: executor}
executor.MockExecuteCommandWithOutputFile = func(command, outputFile string, args ...string) (string, error) {
logger.Infof("Command: %s %v", command, args)
if args[1] == "pool" {
if args[2] == "create" {
assert.Equal(t, "mypool", args[3])
assert.Equal(t, "erasure", args[5])
assert.Equal(t, "mypoolprofile", args[6])
return "", nil
}
if args[2] == "set" {
assert.Equal(t, "mypool", args[3])
assert.Equal(t, "1", args[4])
return "", nil
}
if args[2] == "application" {
assert.Equal(t, "enable", args[3])
assert.Equal(t, "mypool", args[4])
assert.Equal(t, "myapp", args[5])
return "", nil
}
}
return "", errors.Errorf("unexpected ceph command %q", args)
}
err := CreateECPoolForApp(context, "myns", poolName, "mypoolprofile", p, DefaultPGCount, "myapp", false)
assert.Nil(t, err)
}
func TestCreateReplicaPool(t *testing.T) {
testCreateReplicaPool(t, "", "", "")
testCreateReplicaPool(t, "", "", "", "")
}
func TestCreateReplicaPoolWithFailureDomain(t *testing.T) {
testCreateReplicaPool(t, "osd", "mycrushroot", "")
testCreateReplicaPool(t, "osd", "mycrushroot", "", "")
}
func TestCreateReplicaPoolWithDeviceClass(t *testing.T) {
testCreateReplicaPool(t, "osd", "mycrushroot", "hdd")
testCreateReplicaPool(t, "osd", "mycrushroot", "hdd", "")
}
func testCreateReplicaPool(t *testing.T, failureDomain, crushRoot, deviceClass string) {
func TestCreateReplicaPoolWithCompression(t *testing.T) {
testCreateReplicaPool(t, "osd", "mycrushroot", "hdd", "passive")
testCreateReplicaPool(t, "osd", "mycrushroot", "hdd", "force")
}
func testCreateReplicaPool(t *testing.T, failureDomain, crushRoot, deviceClass, compressionMode string) {
crushRuleCreated := false
compressionModeCreated := false
executor := &exectest.MockExecutor{}
context := &clusterd.Context{Executor: executor}
executor.MockExecuteCommandWithOutputFile = func(command, outputFile string, args ...string) (string, error) {
@@ -126,8 +124,13 @@ func testCreateReplicaPool(t *testing.T, failureDomain, crushRoot, deviceClass s
}
if args[2] == "set" {
assert.Equal(t, "mypool", args[3])
assert.Equal(t, "size", args[4])
assert.Equal(t, "12345", args[5])
if args[4] == "size" {
assert.Equal(t, "12345", args[5])
}
if args[4] == "compression_mode" {
assert.Equal(t, compressionMode, args[5])
compressionModeCreated = true
}
return "", nil
}
if args[2] == "application" {
@@ -166,9 +169,17 @@ func testCreateReplicaPool(t *testing.T, failureDomain, crushRoot, deviceClass s
FailureDomain: failureDomain, CrushRoot: crushRoot, DeviceClass: deviceClass,
Replicated: cephv1.ReplicatedSpec{Size: 12345},
}
if compressionMode != "" {
p.CompressionMode = compressionMode
}
err := CreateReplicatedPoolForApp(context, "myns", "mypool", p, DefaultPGCount, "myapp")
assert.Nil(t, err)
assert.True(t, crushRuleCreated)
if compressionMode != "" {
assert.True(t, compressionModeCreated)
} else {
assert.False(t, compressionModeCreated)
}
}
func testIsStringInSlice(a string, list []string) bool {
+32
View File
@@ -84,6 +84,38 @@ func TestValidatePool(t *testing.T) {
err = ValidatePool(context, &p)
assert.Nil(t, err)
// Tests with various compression modes
// succeed with compression mode "none"
p = cephv1.CephBlockPool{ObjectMeta: metav1.ObjectMeta{Name: "mypool", Namespace: "myns"}}
p.Spec.Replicated.Size = 1
p.Spec.Replicated.RequireSafeReplicaSize = false
p.Spec.CompressionMode = "none"
err = ValidatePool(context, &p)
assert.Nil(t, err)
// succeed with compression mode "aggressive"
p = cephv1.CephBlockPool{ObjectMeta: metav1.ObjectMeta{Name: "mypool", Namespace: "myns"}}
p.Spec.Replicated.Size = 1
p.Spec.Replicated.RequireSafeReplicaSize = false
p.Spec.CompressionMode = "aggressive"
err = ValidatePool(context, &p)
assert.Nil(t, err)
// fail with compression mode "unsupported"
p = cephv1.CephBlockPool{ObjectMeta: metav1.ObjectMeta{Name: "mypool", Namespace: "myns"}}
p.Spec.Replicated.Size = 1
p.Spec.Replicated.RequireSafeReplicaSize = false
p.Spec.CompressionMode = "unsupported"
err = ValidatePool(context, &p)
assert.Error(t, err)
// succeed with ec pool and valid compression mode
p = cephv1.CephBlockPool{ObjectMeta: metav1.ObjectMeta{Name: "mypool", Namespace: "myns"}}
p.Spec.ErasureCoded.CodingChunks = 1
p.Spec.ErasureCoded.DataChunks = 2
p.Spec.CompressionMode = "passive"
err = ValidatePool(context, &p)
assert.Nil(t, err)
}
func TestValidateCrushProperties(t *testing.T) {
+10
View File
@@ -87,5 +87,15 @@ func ValidatePoolSpec(context *clusterd.Context, namespace string, p *cephv1.Poo
return errors.Errorf("error pool size is %d and requireSafeReplicaSize is %t, must be false", p.Replicated.Size, p.Replicated.RequireSafeReplicaSize)
}
// validate pool compression mode if specified
if p.CompressionMode != "" {
switch p.CompressionMode {
case "none", "passive", "aggressive", "force":
break
default:
return errors.Errorf("unrecognized compression mode %q", p.CompressionMode)
}
}
return nil
}
+44 -1
View File
@@ -311,6 +311,14 @@ spec:
minimum: 0
maximum: 10
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
dataPools:
type: array
items:
@@ -333,6 +341,14 @@ spec:
minimum: 0
maximum: 10
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
preservePoolsOnDelete:
type: boolean
additionalPrinterColumns:
@@ -431,6 +447,14 @@ spec:
type: integer
codingChunks:
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
dataPool:
properties:
failureDomain:
@@ -447,6 +471,14 @@ spec:
type: integer
codingChunks:
type: integer
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
preservePoolsOnDelete:
type: boolean
subresources:
@@ -511,6 +543,14 @@ spec:
type: integer
minimum: 0
maximum: 9
compressionMode:
type: string
enum:
- ""
- none
- passive
- aggressive
- force
subresources:
status: {}
---
@@ -2024,7 +2064,8 @@ spec:
replicated:
size: ` + replicaSize + `
targetSizeRatio: .5
requireSafeReplicaSize: false`
requireSafeReplicaSize: false
compressionMode: aggressive`
}
func (m *CephManifestsMaster) GetBlockStorageClassDef(csi bool, poolName, storageClassName, reclaimPolicy, namespace, systemNamespace string) string {
@@ -2110,6 +2151,7 @@ spec:
- replicated:
size: 1
requireSafeReplicaSize: false
compressionMode: none
metadataServer:
activeCount: ` + strconv.Itoa(activeCount) + `
activeStandby: true`
@@ -2145,6 +2187,7 @@ spec:
replicated:
size: 1
requireSafeReplicaSize: false
compressionMode: passive
gateway:
type: s3
sslCertificateRef: