Merge pull request #4895 from leseb/pool-size-1

ceph: add CRD setting for pool size 1
This commit is contained in:
Sébastien Han authored and GitHub committed 2020-02-25 18:03:52 +01:00
commit 18671a0dce
22 files changed
+182 -39

No files matched your search

+1
View File
@@ -71,6 +71,7 @@ When creating an erasure-coded pool, it is highly recommended to create the pool
* `replicated`: Settings for a replicated pool. If specified, `erasureCoded` settings must not be specified.
* `size`: The desired number of copies to make of the data in the pool.
* `targetSizeRatio:` gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool, for more info see the [ceph documentation](https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size)
* `requireSafeReplicaSize`: set to false if you want to create a pool with size 1, setting pool size 1 could lead to data loss without recovery. Make sure you are *ABSOLUTELY CERTAIN* that is what you want.
* `erasureCoded`: Settings for an erasure-coded pool. If specified, `replicated` settings must not be specified. See below for more details on [erasure coding](#erasure-coding).
* `dataChunks`: Number of chunks to divide the original object into
* `codingChunks`: Number of coding chunks to generate
+1 -1
View File
@@ -15,7 +15,7 @@
- OSD on PVC doesn't use LVM anymore to configure OSD, but solely relies on the entire block device, done [here](https://github.com/rook/rook/pull/4435).
- Specific devices for OSDs can now be specified using the full udev path (e.g. /dev/disk/by-id/ata-ST4000DM004-XXXX) instead of the device name.
- OSD on PVC CRUSH device storage class can now be changed by setting an annotation "crushDeviceClass" on the "data" volume template. See "cluster-on-pvc.yaml" for example.
- Rook will now refuse to create pools with replica size 1 unless `requireSafeReplicaSize` is set to false.
### EdgeFS
@@ -396,6 +396,8 @@ spec:
maximum: 9
targetSizeRatio:
type: number
requireSafeReplicaSize:
type: boolean
erasureCoded:
properties:
dataChunks:
@@ -264,6 +264,8 @@ spec:
minimum: 0
maximum: 10
type: integer
requireSafeReplicaSize:
type: boolean
erasureCoded:
properties:
dataChunks:
@@ -286,6 +288,8 @@ spec:
minimum: 0
maximum: 10
type: integer
requireSafeReplicaSize:
type: boolean
erasureCoded:
properties:
dataChunks:
@@ -458,6 +462,8 @@ spec:
maximum: 9
targetSizeRatio:
type: number
requireSafeReplicaSize:
type: boolean
erasureCoded:
properties:
dataChunks:
@@ -7,6 +7,12 @@ spec:
failureDomain: host
replicated:
size: 1
# Disallow setting pool with replica 1, this could lead to data loss without recovery.
# Make sure you're *ABSOLUTELY CERTAIN* that is what you want
requireSafeReplicaSize: false
# gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
# for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size
#targetSizeRatio: .5
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
@@ -7,6 +7,12 @@ spec:
failureDomain: host
replicated:
size: 3
# Disallow setting pool with replica 1, this could lead to data loss without recovery.
# Make sure you're *ABSOLUTELY CERTAIN* that is what you want
requireSafeReplicaSize: true
# gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
# for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size
#targetSizeRatio: .5
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
@@ -12,10 +12,12 @@ spec:
metadataPool:
replicated:
size: 1
requireSafeReplicaSize: false
dataPools:
- failureDomain: osd
replicated:
size: 1
requireSafeReplicaSize: false
preservePoolsOnDelete: false
metadataServer:
activeCount: 1
@@ -14,11 +14,15 @@ spec:
metadataPool:
replicated:
size: 3
requireSafeReplicaSize: true
# The list of data pool specs. Can use replication or erasure coding.
dataPools:
- failureDomain: host
replicated:
size: 3
# Disallow setting pool with replica 1, this could lead to data loss without recovery.
# Make sure you're *ABSOLUTELY CERTAIN* that is what you want
requireSafeReplicaSize: true
# gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
# for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size
#targetSizeRatio: .5
@@ -15,11 +15,23 @@ spec:
failureDomain: host
replicated:
size: 3
# Disallow setting pool with replica 1, this could lead to data loss without recovery.
# Make sure you're *ABSOLUTELY CERTAIN* that is what you want
requireSafeReplicaSize: true
# gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
# for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size
#targetSizeRatio: .5
# The pool spec used to create the data pool. Can use replication or erasure coding.
dataPool:
failureDomain: host
replicated:
size: 3
# Disallow setting pool with replica 1, this could lead to data loss without recovery.
# Make sure you're *ABSOLUTELY CERTAIN* that is what you want
requireSafeReplicaSize: true
# gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
# for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size
#targetSizeRatio: .5
# Whether to preserve metadata and data pools on object store deletion
preservePoolsOnDelete: false
# The gateway service configuration
+3 -1
View File
@@ -15,10 +15,12 @@ spec:
# For a pool based on raw copies, specify the number of copies. A size of 1 indicates no redundancy.
replicated:
size: 3
# Disallow setting pool with replica 1, this could lead to data loss without recovery.
# Make sure you're *ABSOLUTELY CERTAIN* that is what you want
requireSafeReplicaSize: true
# gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool
# for more info: https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size
#targetSizeRatio: .5
# A key/value list of annotations
annotations:
# key: value
+2
View File
@@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1
import "github.com/rook/rook/pkg/daemon/ceph/model"
@@ -25,6 +26,7 @@ func (p *PoolSpec) ToModel(name string) *model.Pool {
pool.ReplicatedConfig.Size = r.Size
pool.Type = model.Replicated
pool.ReplicatedConfig.TargetSizeRatio = r.TargetSizeRatio
pool.ReplicatedConfig.RequireSafeReplicaSize = r.RequireSafeReplicaSize
} else {
ec := p.ErasureCode()
if ec != nil {
+3
View File
@@ -270,6 +270,9 @@ type ReplicatedSpec struct {
// TargetSizeRatio gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity
TargetSizeRatio float64 `json:"targetSizeRatio"`
// RequireSafeReplicaSize if false allows you to set replica 1
RequireSafeReplicaSize bool `json:"requireSafeReplicaSize"`
}
// ErasureCodeSpec represents the spec for erasure code in a pool
@@ -125,6 +125,7 @@ func ModelPoolToCephPool(modelPool model.Pool) CephStoragePoolDetails {
if modelPool.Type == model.Replicated {
pool.Size = modelPool.ReplicatedConfig.Size
pool.TargetSizeRatio = modelPool.ReplicatedConfig.TargetSizeRatio
pool.RequireSafeReplicaSize = modelPool.ReplicatedConfig.RequireSafeReplicaSize
} else if modelPool.Type == model.ErasureCoded {
pool.ErasureCodeProfile = GetErasureCodeProfileForPool(modelPool.Name)
}
+30 -13
View File
@@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
@@ -38,15 +39,16 @@ type CephStoragePoolSummary struct {
}
type CephStoragePoolDetails struct {
Name string `json:"pool"`
Number int `json:"pool_id"`
Size uint `json:"size"`
ErasureCodeProfile string `json:"erasure_code_profile"`
FailureDomain string `json:"failureDomain"`
CrushRoot string `json:"crushRoot"`
DeviceClass string `json:"deviceClass"`
NotEnableAppPool bool `json:"notEnableAppPool"`
TargetSizeRatio float64 `json:"target_size_ratio,omitempty"`
Name string `json:"pool"`
Number int `json:"pool_id"`
Size uint `json:"size"`
ErasureCodeProfile string `json:"erasure_code_profile"`
FailureDomain string `json:"failureDomain"`
CrushRoot string `json:"crushRoot"`
DeviceClass string `json:"deviceClass"`
NotEnableAppPool bool `json:"notEnableAppPool"`
TargetSizeRatio float64 `json:"target_size_ratio,omitempty"`
RequireSafeReplicaSize bool `json:"requireSafeReplicaSize,omitempty"`
}
type CephStoragePoolStats struct {
@@ -265,9 +267,8 @@ func CreateReplicatedPoolForApp(context *clusterd.Context, clusterName string, n
}
// the pool is type replicated, set the size for the pool now that it's been created
sizeProperty := "size"
if err = SetPoolProperty(context, clusterName, newPool.Name, sizeProperty, strconv.FormatUint(uint64(newPool.Size), 10)); err != nil {
return errors.Wrapf(err, "failed to set property %q to replicated pool %q to %d", sizeProperty, newPool.Name, newPool.Size)
if err = SetPoolReplicatedSizeProperty(context, clusterName, newPool.Name, strconv.FormatUint(uint64(newPool.Size), 10), newPool.RequireSafeReplicaSize); err != nil {
return errors.Wrapf(err, "failed to set size property to replicated pool %q to %d", newPool.Name, newPool.Size)
}
targetSizeRatioProperty := "target_size_ratio"
@@ -318,7 +319,7 @@ func createReplicationCrushRule(context *clusterd.Context, clusterName string, n
}
// SetPoolProperty sets a property to a given pool
func SetPoolProperty(context *clusterd.Context, clusterName, name, propName string, propVal string) error {
func SetPoolProperty(context *clusterd.Context, clusterName, name, propName, propVal string) error {
args := []string{"osd", "pool", "set", name, propName, propVal}
_, err := NewCephCommand(context, clusterName, args).Run()
if err != nil {
@@ -327,6 +328,22 @@ func SetPoolProperty(context *clusterd.Context, clusterName, name, propName stri
return nil
}
// SetPoolReplicatedSizeProperty sets the replica size of a pool
func SetPoolReplicatedSizeProperty(context *clusterd.Context, clusterName, poolName, propVal string, requireSafeReplicaSize bool) error {
propName := "size"
args := []string{"osd", "pool", "set", poolName, propName, propVal}
if propVal == "1" {
args = append(args, "--yes-i-really-mean-it")
}
_, err := NewCephCommand(context, clusterName, args).Run()
if err != nil {
return errors.Wrapf(err, "failed to set pool property %q on pool %q", propName, poolName)
}
return nil
}
func GetPoolStats(context *clusterd.Context, clusterName string) (*CephStoragePoolStats, error) {
args := []string{"df", "detail"}
buf, err := NewCephCommand(context, clusterName, args).Run()
+44
View File
@@ -17,6 +17,7 @@ package client
import (
"reflect"
"strconv"
"testing"
"github.com/pkg/errors"
@@ -203,3 +204,46 @@ func TestGetPoolStatistics(t *testing.T) {
assert.NotNil(t, err)
assert.Nil(t, stats)
}
func TestSetPoolReplicatedSizeProperty(t *testing.T) {
// TEST POOL SIZE > 1
p := CephStoragePoolDetails{Name: "mypool", Size: 12345}
executor := &exectest.MockExecutor{}
context := &clusterd.Context{Executor: executor}
executor.MockExecuteCommandWithOutputFile = func(debug bool, actionName, command, outputFile string, args ...string) (string, error) {
logger.Infof("Command: %s %v", command, args)
if args[2] == "set" {
assert.Equal(t, "mypool", args[3])
assert.Equal(t, "size", args[4])
assert.Equal(t, "12345", args[5])
return "", nil
}
return "", errors.Errorf("unexpected ceph command %q", args)
}
size := strconv.FormatUint(uint64(p.Size), 10)
err := SetPoolReplicatedSizeProperty(context, "myns", p.Name, size, p.RequireSafeReplicaSize)
assert.NoError(t, err)
// TEST POOL SIZE 1 AND RequireSafeReplicaSize True
executor.MockExecuteCommandWithOutputFile = func(debug bool, actionName, command, outputFile string, args ...string) (string, error) {
logger.Infof("Command: %s %v", command, args)
if args[2] == "set" {
assert.Equal(t, "mypool", args[3])
assert.Equal(t, "size", args[4])
assert.Equal(t, "1", args[5])
assert.Equal(t, "--yes-i-really-mean-it", args[6])
return "", nil
}
return "", errors.Errorf("unexpected ceph command %q", args)
}
p.RequireSafeReplicaSize = true
p.Size = 1
size = strconv.FormatUint(uint64(p.Size), 10)
err = SetPoolReplicatedSizeProperty(context, "myns", p.Name, size, p.RequireSafeReplicaSize)
assert.NoError(t, err)
}
+18 -16
View File
@@ -57,14 +57,15 @@ var (
// GlobalConfig represents the [global] sections of Ceph's config file.
type GlobalConfig struct {
FSID string `ini:"fsid,omitempty"`
MonMembers string `ini:"mon initial members,omitempty"`
MonHost string `ini:"mon host"`
PublicAddr string `ini:"public addr,omitempty"`
PublicNetwork string `ini:"public network,omitempty"`
ClusterAddr string `ini:"cluster addr,omitempty"`
ClusterNetwork string `ini:"cluster network,omitempty"`
MonAllowPoolDelete bool `ini:"mon_allow_pool_delete"`
FSID string `ini:"fsid,omitempty"`
MonMembers string `ini:"mon initial members,omitempty"`
MonHost string `ini:"mon host"`
PublicAddr string `ini:"public addr,omitempty"`
PublicNetwork string `ini:"public network,omitempty"`
ClusterAddr string `ini:"cluster addr,omitempty"`
ClusterNetwork string `ini:"cluster network,omitempty"`
MonAllowPoolDelete bool `ini:"mon_allow_pool_delete"`
MonAllowPoolSizeOne bool `ini:"mon_allow_pool_size_one"`
}
// CephConfig represents an entire Ceph config including all sections.
@@ -162,14 +163,15 @@ func CreateDefaultCephConfig(context *clusterd.Context, cluster *ClusterInfo) (*
conf := &CephConfig{
GlobalConfig: &GlobalConfig{
FSID: cluster.FSID,
MonMembers: strings.Join(monMembers, " "),
MonHost: strings.Join(monHosts, ","),
PublicAddr: context.NetworkInfo.PublicAddr,
PublicNetwork: context.NetworkInfo.PublicNetwork,
ClusterAddr: context.NetworkInfo.ClusterAddr,
ClusterNetwork: context.NetworkInfo.ClusterNetwork,
MonAllowPoolDelete: true,
FSID: cluster.FSID,
MonMembers: strings.Join(monMembers, " "),
MonHost: strings.Join(monHosts, ","),
PublicAddr: context.NetworkInfo.PublicAddr,
PublicNetwork: context.NetworkInfo.PublicNetwork,
ClusterAddr: context.NetworkInfo.ClusterAddr,
ClusterNetwork: context.NetworkInfo.ClusterNetwork,
MonAllowPoolDelete: true,
MonAllowPoolSizeOne: true,
},
}
+4 -2
View File
@@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package model
const (
@@ -24,8 +25,9 @@ const (
type PoolType int
type ReplicatedPoolConfig struct {
Size uint `json:"size"`
TargetSizeRatio float64 `json:"targetSizeRatio"`
Size uint `json:"size"`
TargetSizeRatio float64 `json:"targetSizeRatio"`
RequireSafeReplicaSize bool `json:"requireSafeReplicaSize"`
}
type ErasureCodedPoolConfig struct {
+3 -3
View File
@@ -53,7 +53,7 @@ func TestValidateSpec(t *testing.T) {
// missing data pools
assert.NotNil(t, validateFilesystem(context, fs))
p := cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1}}
p := cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}}
fs.Spec.DataPools = append(fs.Spec.DataPools, p)
// missing metadata pool
@@ -98,8 +98,8 @@ func TestCreateFilesystem(t *testing.T) {
fs := cephv1.CephFilesystem{
ObjectMeta: metav1.ObjectMeta{Name: "myfs", Namespace: "ns"},
Spec: cephv1.FilesystemSpec{
MetadataPool: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1}},
DataPools: []cephv1.PoolSpec{{Replicated: cephv1.ReplicatedSpec{Size: 1}}},
MetadataPool: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}},
DataPools: []cephv1.PoolSpec{{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}}},
MetadataServer: cephv1.MetadataServerSpec{
ActiveCount: 1,
Resources: v1.ResourceRequirements{
+1 -1
View File
@@ -103,7 +103,7 @@ func simpleStore() cephv1.CephObjectStore {
return cephv1.CephObjectStore{
ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: "mycluster"},
Spec: cephv1.ObjectStoreSpec{
MetadataPool: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1}},
MetadataPool: cephv1.PoolSpec{Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false}},
DataPool: cephv1.PoolSpec{ErasureCoded: cephv1.ErasureCodedSpec{CodingChunks: 1, DataChunks: 2}},
Gateway: cephv1.GatewaySpec{Port: 123},
},
+5
View File
@@ -278,6 +278,11 @@ func ValidatePoolSpec(context *clusterd.Context, namespace string, p *cephv1.Poo
}
}
// validate pool replica size
if p.Replicated.Size == 1 && p.Replicated.RequireSafeReplicaSize {
return errors.Errorf("error pool size is %d and requireSafeReplicaSize is %t, must be false", p.Replicated.Size, p.Replicated.RequireSafeReplicaSize)
}
return nil
}
+16 -1
View File
@@ -48,6 +48,7 @@ func TestValidatePool(t *testing.T) {
// must not specify both replication and EC settings
p = cephv1.CephBlockPool{ObjectMeta: metav1.ObjectMeta{Name: "mypool", Namespace: "myns"}}
p.Spec.Replicated.Size = 1
p.Spec.Replicated.RequireSafeReplicaSize = false
p.Spec.ErasureCoded.CodingChunks = 2
p.Spec.ErasureCoded.DataChunks = 3
err = ValidatePool(context, &p)
@@ -56,15 +57,24 @@ func TestValidatePool(t *testing.T) {
// succeed with replication settings
p = cephv1.CephBlockPool{ObjectMeta: metav1.ObjectMeta{Name: "mypool", Namespace: "myns"}}
p.Spec.Replicated.Size = 1
p.Spec.Replicated.RequireSafeReplicaSize = false
err = ValidatePool(context, &p)
assert.Nil(t, err)
// size is 1 and RequireSafeReplicaSize is true
p = cephv1.CephBlockPool{ObjectMeta: metav1.ObjectMeta{Name: "mypool", Namespace: "myns"}}
p.Spec.Replicated.Size = 1
p.Spec.Replicated.RequireSafeReplicaSize = true
err = ValidatePool(context, &p)
assert.Error(t, err)
// succeed with ec settings
p = cephv1.CephBlockPool{ObjectMeta: metav1.ObjectMeta{Name: "mypool", Namespace: "myns"}}
p.Spec.ErasureCoded.CodingChunks = 1
p.Spec.ErasureCoded.DataChunks = 2
err = ValidatePool(context, &p)
assert.Nil(t, err)
}
func TestValidateCrushProperties(t *testing.T) {
@@ -82,7 +92,7 @@ func TestValidateCrushProperties(t *testing.T) {
p := &cephv1.CephBlockPool{
ObjectMeta: metav1.ObjectMeta{Name: "mypool", Namespace: "myns"},
Spec: cephv1.PoolSpec{
Replicated: cephv1.ReplicatedSpec{Size: 1},
Replicated: cephv1.ReplicatedSpec{Size: 1, RequireSafeReplicaSize: false},
FailureDomain: "osd",
},
}
@@ -119,12 +129,17 @@ func TestCreatePool(t *testing.T) {
p := &cephv1.CephBlockPool{ObjectMeta: metav1.ObjectMeta{Name: "mypool", Namespace: "myns"}}
p.Spec.Replicated.Size = 1
p.Spec.Replicated.RequireSafeReplicaSize = false
exists, err := poolExists(context, p)
assert.False(t, exists)
err = createPool(context, p)
assert.Nil(t, err)
p.Spec.Replicated.RequireSafeReplicaSize = true
err = createPool(context, p)
assert.Error(t, err)
// fail if both replication and EC are specified
p.Spec.ErasureCoded.CodingChunks = 2
p.Spec.ErasureCoded.DataChunks = 2
+12 -1
View File
@@ -414,6 +414,8 @@ spec:
properties:
size:
type: integer
requireSafeReplicaSize:
type: boolean
erasureCoded:
properties:
dataChunks:
@@ -428,6 +430,8 @@ spec:
properties:
size:
type: integer
requireSafeReplicaSize:
type: boolean
erasureCoded:
properties:
dataChunks:
@@ -482,6 +486,8 @@ spec:
maximum: 9
targetSizeRatio:
type: number
requireSafeReplicaSize:
type: boolean
erasureCoded:
properties:
dataChunks:
@@ -1992,7 +1998,8 @@ metadata:
spec:
replicated:
size: ` + replicaSize + `
targetSizeRatio: .5`
targetSizeRatio: .5
requireSafeReplicaSize: false`
}
func (m *CephManifestsMaster) GetBlockStorageClassDef(csi bool, poolName, storageClassName, reclaimPolicy, namespace, systemNamespace string) string {
@@ -2073,9 +2080,11 @@ spec:
metadataPool:
replicated:
size: 1
requireSafeReplicaSize: false
dataPools:
- replicated:
size: 1
requireSafeReplicaSize: false
metadataServer:
activeCount: ` + strconv.Itoa(activeCount) + `
activeStandby: true`
@@ -2106,9 +2115,11 @@ spec:
metadataPool:
replicated:
size: 1
requireSafeReplicaSize: false
dataPool:
replicated:
size: 1
requireSafeReplicaSize: false
gateway:
type: s3
sslCertificateRef: