From 10b398cc02b8091f53edad334b299c7ae865171e Mon Sep 17 00:00:00 2001 From: Anthony D'Atri Date: Sun, 31 Jul 2022 01:48:47 -0700 Subject: [PATCH] docs: add additional RBD image features to docs and YAML files Signed-off-by: Anthony D'Atri --- .../CRDs/Cluster/external-cluster.md | 2 ++ .../Prerequisites/prerequisites.md | 22 ++++++++++++++----- .../Block-Storage-RBD/block-storage.md | 12 +++++++++- deploy/charts/rook-ceph-cluster/values.yaml | 11 +++++++--- deploy/examples/csi/rbd/storageclass-ec.yaml | 6 ++++- deploy/examples/csi/rbd/storageclass.yaml | 12 +++++++++- deploy/examples/import-external-cluster.sh | 11 ++++++++-- design/ceph/ceph-config-updates.md | 10 ++++----- 8 files changed, 67 insertions(+), 19 deletions(-) diff --git a/Documentation/CRDs/Cluster/external-cluster.md b/Documentation/CRDs/Cluster/external-cluster.md index 5cee1ef63..39c44cc0e 100644 --- a/Documentation/CRDs/Cluster/external-cluster.md +++ b/Documentation/CRDs/Cluster/external-cluster.md @@ -120,6 +120,8 @@ export RGW_POOL_PREFIX=default 3. Paste the above output from `create-external-cluster-resources.py` into your current shell to allow importing the source data. 4. Run the [import](https://github.com/rook/rook/blob/master/deploy/examples/import-external-cluster.sh) script. + Note that if your Rook cluster nodes are running a kernel earlier than 5.4 or equivalent you may need to + remove `fast-diff,object-map,deep-flatten,exclusive-lock` from the `imageFeatures` line. ```console . import-external-cluster.sh diff --git a/Documentation/Getting-Started/Prerequisites/prerequisites.md b/Documentation/Getting-Started/Prerequisites/prerequisites.md index fd637321e..6dd4b476a 100644 --- a/Documentation/Getting-Started/Prerequisites/prerequisites.md +++ b/Documentation/Getting-Started/Prerequisites/prerequisites.md @@ -11,18 +11,18 @@ Kubernetes **v1.19** or higher is supported for the Ceph operator. ## CPU Architecture -Architectures released are `amd64 / x86_64` and `arm64`. +Architectures supported are `amd64 / x86_64` and `arm64`. ## Ceph Prerequisites -In order to configure the Ceph storage cluster, at least one of these local storage options are required: +In order to configure the Ceph storage cluster, at least one of these local storage types is required: * Raw devices (no partitions or formatted filesystems) * Raw partitions (no formatted filesystem) * LVM Logical Volumes (no formatted filesystem) * Persistent Volumes available from a storage class in `block` mode -You can confirm whether your partitions or devices are formatted with filesystems with the following command. +You can confirm whether your partitions or devices are formatted with filesystems with the following command: ```console $ lsblk -f @@ -89,12 +89,22 @@ runcmd: ### RBD -Ceph requires a Linux kernel built with the RBD module. Many Linux distributions have this module, but not all distributions. +Ceph requires a Linux kernel built with the RBD module. Many Linux distributions +have this module, but not all. For example, the GKE Container-Optimised OS (COS) does not have RBD. You can test your Kubernetes nodes by running `modprobe rbd`. -If it says 'not found', you may have to rebuild your kernel and include at least the `rbd` module -or choose a different Linux distribution. +If it says 'not found', you may have to rebuild your kernel and include at least +the `rbd` module, install a newer kernel, or choose a different Linux distribution. + +Rook's default RBD configuration specifies only the `layering` feature, for +broad compatibility with older kernels. If your Kubernetes nodes run a 5.4 +or later kernel you may wish to enable additional feature flags. The `fast-diff` +and `object-map` features are especially useful. + +```yaml +imageFeatures: layering,fast-diff,object-map,deep-flatten,exclusive-lock +``` ### CephFS diff --git a/Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md b/Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md index c3ab4868f..3ea209b4d 100644 --- a/Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md +++ b/Documentation/Storage-Configuration/Block-Storage-RBD/block-storage.md @@ -59,7 +59,17 @@ parameters: # RBD image format. Defaults to "2". imageFormat: "2" - # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. + # RBD image features + # Available for imageFormat: "2". Older releases of CSI RBD + # support only the `layering` feature. The Linux kernel (KRBD) supports the + # full complement of features as of 5.4 + # `layering` alone corresponds to Ceph's bitfield value of "2" ; + # `layering` + `fast-diff` + `object-map` + `deep-flatten` + `exclusive-lock` together + # correspond to Ceph's OR'd bitfield value of "63". Here we use + # a symbolic, comma-separated format: + # For 5.4 or later kernels: + #imageFeatures: layering,fast-diff,object-map,deep-flatten,exclusive-lock + # For 5.3 or earlier kernels: imageFeatures: layering # The secrets contain Ceph admin credentials. diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml index d7366666e..e04fb961b 100644 --- a/deploy/charts/rook-ceph-cluster/values.yaml +++ b/deploy/charts/rook-ceph-cluster/values.yaml @@ -313,7 +313,7 @@ cephClusterSpec: # osdsPerDevice: "1" # this value can be overridden at the node or device level # encryptedDevice: "true" # the default value for this option is "false" # # Individual nodes and their config can be specified as well, but 'useAllNodes' above must be set to false. Then, only the named - # # nodes below will be used as storage resources. Each node's 'name' field should match their 'kubernetes.io/hostname' label. + # # nodes below will be used as storage resources. Each node's 'name' field should match their 'kubernetes.io/hostname' label. # nodes: # - name: "172.17.4.201" # devices: # specific devices to use for storage can be specified for each node @@ -424,9 +424,14 @@ cephBlockPools: # RBD image format. Defaults to "2". imageFormat: "2" - # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. + + # RBD image features, equivalent to OR'd bitfield value: 63 + # Available for imageFormat: "2". Older releases of CSI RBD + # support only the `layering` feature. The Linux kernel (KRBD) supports the + # full feature complement as of 5.4 imageFeatures: layering - # The secrets contain Ceph admin credentials. + + # These secrets contain Ceph admin credentials. csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner diff --git a/deploy/examples/csi/rbd/storageclass-ec.yaml b/deploy/examples/csi/rbd/storageclass-ec.yaml index c62507ffc..6804b0b23 100644 --- a/deploy/examples/csi/rbd/storageclass-ec.yaml +++ b/deploy/examples/csi/rbd/storageclass-ec.yaml @@ -62,7 +62,11 @@ parameters: # RBD image format. Defaults to "2". imageFormat: "2" - # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. + # RBD image features, equivalent to OR'd bitfield value: 63 + # Available for imageFormat: "2". Older releases of CSI RBD + # support only the `layering` feature. The Linux kernel (KRBD) supports the + # full feature complement as of 5.4 + # imageFeatures: layering,fast-diff,object-map,deep-flatten,exclusive-lock imageFeatures: layering # The secrets contain Ceph admin credentials. These are generated automatically by the operator diff --git a/deploy/examples/csi/rbd/storageclass.yaml b/deploy/examples/csi/rbd/storageclass.yaml index f0002c4c8..c1218982c 100644 --- a/deploy/examples/csi/rbd/storageclass.yaml +++ b/deploy/examples/csi/rbd/storageclass.yaml @@ -59,7 +59,17 @@ parameters: # RBD image format. Defaults to "2". imageFormat: "2" - # RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature. + # RBD image features + # Available for imageFormat: "2". Older releases of CSI RBD + # support only the `layering` feature. The Linux kernel (KRBD) supports the + # full complement of features as of 5.4 + # `layering` alone corresponds to Ceph's bitfield value of "2" ; + # `layering` + `fast-diff` + `object-map` + `deep-flatten` + `exclusive-lock` together + # correspond to Ceph's OR'd bitfield value of "63". Here we use + # a symbolic, comma-separated format: + # For 5.4 or later kernels: + #imageFeatures: layering,fast-diff,object-map,deep-flatten,exclusive-lock + # For 5.3 or earlier kernels: imageFeatures: layering # The secrets contain Ceph admin credentials. These are generated automatically by the operator diff --git a/deploy/examples/import-external-cluster.sh b/deploy/examples/import-external-cluster.sh index ccaf24ae6..15f3479dc 100644 --- a/deploy/examples/import-external-cluster.sh +++ b/deploy/examples/import-external-cluster.sh @@ -14,6 +14,7 @@ MON_SECRET_CEPH_USERNAME_KEYNAME=ceph-username MON_SECRET_CEPH_SECRET_KEYNAME=ceph-secret MON_ENDPOINT_CONFIGMAP_NAME=rook-ceph-mon-endpoints ROOK_EXTERNAL_CLUSTER_NAME=$NAMESPACE +ROOK_RBD_FEATURES=${ROOK_RBD_FEATURES:-"layering"} ROOK_EXTERNAL_MAX_MON_ID=2 ROOK_EXTERNAL_MAPPING={} RBD_STORAGE_CLASS_NAME=ceph-rbd @@ -34,6 +35,12 @@ function checkEnvVars() { if [ -z "$NAMESPACE" ]; then echo "Please populate the environment variable NAMESPACE" exit 1 + fi + if [ -z "$ROOK_RBD_FEATURES" ] || [[ ! "$ROOK_RBD_FEATURES" =~ .*"layering".* ]]; then + echo "Please populate the environment variable ROOK_RBD_FEATURES" + echo "For a kernel earlier than 5.4 use a value of 'layering'; for 5.4 or later" + echo "use 'layering,fast-diff,object-map,deep-flatten,exclusive-lock'" + exit 1 fi if [ -z "$RBD_POOL_NAME" ]; then echo "Please populate the environment variable RBD_POOL_NAME" @@ -196,7 +203,7 @@ parameters: pool: $RBD_POOL_NAME dataPool: $RBD_METADATA_EC_POOL_NAME imageFormat: "2" - imageFeatures: layering + imageFeatures: $ROOK_RBD_FEATURES csi.storage.k8s.io/provisioner-secret-name: "rook-$CSI_RBD_PROVISIONER_SECRET_NAME" csi.storage.k8s.io/provisioner-secret-namespace: $NAMESPACE csi.storage.k8s.io/controller-expand-secret-name: "rook-$CSI_RBD_PROVISIONER_SECRET_NAME" @@ -220,7 +227,7 @@ parameters: clusterID: $CLUSTER_ID_RBD pool: $RBD_POOL_NAME imageFormat: "2" - imageFeatures: layering + imageFeatures: $ROOK_RBD_FEATURES csi.storage.k8s.io/provisioner-secret-name: "rook-$CSI_RBD_PROVISIONER_SECRET_NAME" csi.storage.k8s.io/provisioner-secret-namespace: $NAMESPACE csi.storage.k8s.io/controller-expand-secret-name: "rook-$CSI_RBD_PROVISIONER_SECRET_NAME" diff --git a/design/ceph/ceph-config-updates.md b/design/ceph/ceph-config-updates.md index 1e3c4b6da..3d00f79ff 100644 --- a/design/ceph/ceph-config-updates.md +++ b/design/ceph/ceph-config-updates.md @@ -5,12 +5,12 @@ Starting with Ceph Mimic, Ceph is able to store the vast majority of config options for all daemons in the Ceph mons' key-value store. This ability to centrally manage 99% of its configuration -is Ceph's preferred way of managing config. This conveniently allows Ceph to fit better into the -containerized application space than before. +is Ceph's preferred way of managing configuration and option settings. This allows Ceph to fit +more cleanly into the containerized application space than before. -However, for it's own backwards compatibility, Ceph options set in the config file or via command -line flags will override the centrally-configured settings. To make the most of this functionality -within Ceph, it will be necessary to limit the configuration options specified by Rook in either +However, for backwards compatibility, Ceph options set in the config file or via command +line arguments override the centrally-configured settings. To make the most of this functionality +within Ceph, it is necessary to limit the configuration options specified by Rook in either config files or on the command line to a minimum.