diff --git a/.markdownlint-cli2.cjs b/.markdownlint-cli2.cjs new file mode 100644 index 000000000..954aedd77 --- /dev/null +++ b/.markdownlint-cli2.cjs @@ -0,0 +1,48 @@ +module.exports = { + "config": { + "default": false, // no default rules enabled + "extends": null, // no default rules enabled + "list-indent": true, // all list items must be indented at the same level + "ul-indent": { + "indent": 4, // mkdocs requires 4 spaces for tabs + }, + "no-hard-tabs": { + "spaces_per_tab": 4, // mkdocs requires 4 spaces for tabs + }, + "ol-prefix": { + // require fully-numbered lists. this rule helps ensure that code blocks in between ordered + // list items (which require surrounding spaces) don't break lists + "style": "ordered", + }, + "blanks-around-lists": true, // mkdocs requires blank lines around lists + "blanks-around-fences": { // mkdocs requires blank lines around code blocks (fences) + "list_items": true, /// ... including in lists + }, + "code-block-style": { + // do not allow implicit code blocks. ensure code blocks have fences so we know explicitly + // what will be rendered + "style": "fenced", + }, + "fenced-code-language": { + // enforce code blocks must have language specified + // this helps ensure rendering is as intended, and it helps doc-wide searches for code blocks + language_only: true, + }, + "no-duplicate-heading": true, // do not allow duplicate headings + "link-fragments": true, // validate links to headings within a doc + "single-trailing-newline": true, // require single trailing newline in docs + "no-multiple-blanks": { + // allow max 2 blank lines in markdown docs + "maximum": 2, + }, + + // custom rules for Rook! + "mkdocs-admonitions": true, // checking mkdocs admonitions format + "strict-tab-spacing": true, // enforce strict 4-space tabs for mkdocs + + }, + "customRules": [ + "./tests/scripts/markdownlint-admonitions.js", + "./tests/scripts/markdownlint-tab-spacing.js", + ], +}; diff --git a/.vscode/settings.json b/.vscode/settings.json index baee5aee6..5e689d22b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -16,4 +16,13 @@ "editor.defaultFormatter": "ms-python.black-formatter", "editor.formatOnSave": true }, + "[markdown]": { + "editor.tabSize": 4, + "editor.detectIndentation": false, + }, + "markdown.extension.list.indentationSize": "inherit", + "markdownlint.lintWorkspaceGlobs": [ + "Documentation/**/*.md", + "!Documentation/Helm-Charts", + ], } diff --git a/Documentation/CRDs/Block-Storage/ceph-block-pool-crd.md b/Documentation/CRDs/Block-Storage/ceph-block-pool-crd.md index b8a22aad4..ff7dabb1e 100644 --- a/Documentation/CRDs/Block-Storage/ceph-block-pool-crd.md +++ b/Documentation/CRDs/Block-Storage/ceph-block-pool-crd.md @@ -60,7 +60,7 @@ spec: This sample will lower the overall storage capacity requirement, while also adding redundancy by using [erasure coding](#erasure-coding). !!! note - This sample requires *at least 3 bluestore OSDs*. + This sample requires **at least 3 bluestore OSDs**. The OSDs can be located on a single Ceph node or spread across multiple nodes, because the [`failureDomain`](ceph-block-pool-crd.md#spec) is set to `osd` and the `erasureCoded` chunk settings require at least 3 different OSDs (2 `dataChunks` + 1 `codingChunks`). @@ -202,12 +202,12 @@ stretched) then you will have 2 replicas per datacenter where each replica ends * `crushRoot`: The root in the crush map to be used by the pool. If left empty or unspecified, the default root will be used. Creating a crush hierarchy for the OSDs currently requires the Rook toolbox to run the Ceph tools described [here](http://docs.ceph.com/docs/master/rados/operations/crush-map/#modifying-the-crush-map). * `enableRBDStats`: Enables collecting RBD per-image IO statistics by enabling dynamic OSD performance counters. Defaults to false. For more info see the [ceph documentation](https://docs.ceph.com/docs/master/mgr/prometheus/#rbd-io-statistics). * `name`: The name of Ceph pools is based on the `metadata.name` of the CephBlockPool CR. Some built-in Ceph pools - require names that are incompatible with K8s resource names. These special pools can be configured - by setting this `name` to override the name of the Ceph pool that is created instead of using the `metadata.name` for the pool. - Only the following pool names are supported: `.nfs`, `.mgr`, and `.rgw.root`. See the example - [builtin mgr pool](https://github.com/rook/rook/blob/master/deploy/examples/pool-builtin-mgr.yaml). + require names that are incompatible with K8s resource names. These special pools can be configured + by setting this `name` to override the name of the Ceph pool that is created instead of using the `metadata.name` for the pool. + Only the following pool names are supported: `.nfs`, `.mgr`, and `.rgw.root`. See the example + [builtin mgr pool](https://github.com/rook/rook/blob/master/deploy/examples/pool-builtin-mgr.yaml). * `application`: The type of application set on the pool. By default, Ceph pools for CephBlockPools will be `rbd`, - CephObjectStore pools will be `rgw`, and CephFilesystem pools will be `cephfs`. + CephObjectStore pools will be `rgw`, and CephFilesystem pools will be `cephfs`. * `parameters`: Sets any [parameters](https://docs.ceph.com/docs/master/rados/operations/pools/#set-pool-values) listed to the given pool * `target_size_ratio:` gives a hint (%) to Ceph in terms of expected consumption of the total cluster capacity of a given pool, for more info see the [ceph documentation](https://docs.ceph.com/docs/master/rados/operations/placement-groups/#specifying-expected-pool-size) diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md index 72b0fbc70..b0eaf8a76 100755 --- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md +++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md @@ -27,10 +27,10 @@ Settings can be specified at the global level to apply to the cluster as a whole * `enable`: if `true`, the cluster will not be managed by Rook but via an external entity. This mode is intended to connect to an existing cluster. In this case, Rook will only consume the external cluster. However, Rook will be able to deploy various daemons in Kubernetes such as object gateways, mds and nfs if an image is provided and will refuse otherwise. If this setting is enabled **all** the other options will be ignored except `cephVersion.image` and `dataDirHostPath`. See [external cluster configuration](external-cluster/external-cluster.md). If `cephVersion.image` is left blank, Rook will refuse the creation of extra CRs like object, file and nfs. * `cephVersion`: The version information for launching the ceph daemons. * `image`: The image used for running the ceph daemons. For example, `quay.io/ceph/ceph:v18.2.2`. For more details read the [container images section](#ceph-container-images). - For the latest ceph images, see the [Ceph DockerHub](https://hub.docker.com/r/ceph/ceph/tags/). - To ensure a consistent version of the image is running across all nodes in the cluster, it is recommended to use a very specific image version. - Tags also exist that would give the latest version, but they are only recommended for test environments. For example, the tag `v17` will be updated each time a new Quincy build is released. - Using the `v17` tag is not recommended in production because it may lead to inconsistent versions of the image running across different nodes in the cluster. + For the latest ceph images, see the [Ceph DockerHub](https://hub.docker.com/r/ceph/ceph/tags/). + To ensure a consistent version of the image is running across all nodes in the cluster, it is recommended to use a very specific image version. + Tags also exist that would give the latest version, but they are only recommended for test environments. For example, the tag `v17` will be updated each time a new Quincy build is released. + Using the `v17` tag is not recommended in production because it may lead to inconsistent versions of the image running across different nodes in the cluster. * `allowUnsupported`: If `true`, allow an unsupported major version of the Ceph release. Currently `quincy` and `reef` are supported. Future versions such as `squid` (v19) would require this to be set to `true`. Should be set to `false` in production. * `imagePullPolicy`: The image pull policy for the ceph daemon pods. Possible values are `Always`, `IfNotPresent`, and `Never`. The default is `IfNotPresent`. * `dataDirHostPath`: The path on the host ([hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath)) where config and data should be stored for each of the services. If the directory does not exist, it will be created. Because this directory persists on the host, it will remain after pods are deleted. Following paths and any of their subpaths **must not be used**: `/etc/ceph`, `/rook` or `/var/log/ceph`. @@ -49,7 +49,7 @@ If this value is empty, each pod will get an ephemeral directory to store their * `metricsDisabled`: Whether to disable the metrics reported by Ceph. If false, the prometheus mgr module and Ceph exporter are enabled. If true, the prometheus mgr module and Ceph exporter are both disabled. Default is false. * `externalMgrEndpoints`: external cluster manager endpoints - * `externalMgrPrometheusPort`: external prometheus manager module port. See [external cluster configuration](#external-cluster) for more details. + * `externalMgrPrometheusPort`: external prometheus manager module port. See [external cluster configuration](./external-cluster/external-cluster.md) for more details. * `port`: The internal prometheus manager module port where the prometheus mgr module listens. The port may need to be configured when host networking is enabled. * `interval`: The interval for the prometheus module to to scrape targets. * `network`: For the network settings for the cluster, refer to the [network configuration settings](#network-configuration-settings) @@ -57,10 +57,10 @@ If this value is empty, each pod will get an ephemeral directory to store their For more details on the mons and when to choose a number other than `3`, see the [mon health doc](../../Storage-Configuration/Advanced/ceph-mon-health.md). * `mgr`: manager top level section * `count`: set number of ceph managers between `1` to `2`. The default value is 2. - If there are two managers, it is important for all mgr services point to the active mgr and not the standby mgr. Rook automatically - updates the label `mgr_role` on the mgr pods to be either `active` or `standby`. Therefore, services need just to add the label - `mgr_role=active` to their selector to point to the active mgr. This applies to all services that rely on the ceph mgr such as - the dashboard or the prometheus metrics collector. + If there are two managers, it is important for all mgr services point to the active mgr and not the standby mgr. Rook automatically + updates the label `mgr_role` on the mgr pods to be either `active` or `standby`. Therefore, services need just to add the label + `mgr_role=active` to their selector to point to the active mgr. This applies to all services that rely on the ceph mgr such as + the dashboard or the prometheus metrics collector. * `modules`: A list of Ceph manager modules to enable or disable. Note the "dashboard" and "monitoring" modules are already configured by other settings. * `crashCollector`: The settings for crash collector daemon(s). * `disable`: is set to `true`, the crash collector will not run on any node where a Ceph daemon runs @@ -75,10 +75,10 @@ For more details on the mons and when to choose a number other than `3`, see the * `priorityClassNames`: [priority class names configuration settings](#priority-class-names) * `storage`: Storage selection and configuration that will be used across the cluster. Note that these settings can be overridden for specific nodes. * `useAllNodes`: `true` or `false`, indicating if all nodes in the cluster should be used for storage according to the cluster level storage selection and configuration values. - If individual nodes are specified under the `nodes` field, then `useAllNodes` must be set to `false`. + If individual nodes are specified under the `nodes` field, then `useAllNodes` must be set to `false`. * `nodes`: Names of individual nodes in the cluster that should have their storage included in accordance with either the cluster level configuration specified above or any node specific overrides described in the next section below. - `useAllNodes` must be set to `false` to use specific nodes and their config. - See [node settings](#node-settings) below. + `useAllNodes` must be set to `false` to use specific nodes and their config. + See [node settings](#node-settings) below. * `config`: Config settings applied to all OSDs on the node unless overridden by `devices`. See the [config settings](#osd-configuration-settings) below. * [storage selection settings](#storage-selection-settings) * [Storage Class Device Sets](#storage-class-device-sets) @@ -109,7 +109,7 @@ These are general purpose Ceph container with all necessary daemons and dependen | TAG | MEANING | | -------------------- | --------------------------------------------------------- | -| vRELNUM | Latest release in this series (e.g., *v17* = Quincy) | +| vRELNUM | Latest release in this series (e.g., **v17** = Quincy) | | vRELNUM.Y | Latest stable release in this stable series (e.g., v17.2) | | vRELNUM.Y.Z | A specific release (e.g., v18.2.2) | | vRELNUM.Y.Z-YYYYMMDD | A specific build (e.g., v18.2.2-20240311) | @@ -119,37 +119,37 @@ A specific will contain a specific release of Ceph as well as security fixes fro ### Mon Settings * `count`: Set the number of mons to be started. The number must be between `1` and `9`. The recommended value is most commonly `3`. - For highest availability, an odd number of mons should be specified. - For higher durability in case of mon loss, an even number can be specified although availability may be lower. - To maintain quorum a majority of mons must be up. For example, if there are three mons, two must be up. - If there are four mons, three must be up. If there are two mons, both must be up. - If quorum is lost, see the [disaster recovery guide](../../Troubleshooting/disaster-recovery.md#restoring-mon-quorum) to restore quorum from a single mon. + For highest availability, an odd number of mons should be specified. + For higher durability in case of mon loss, an even number can be specified although availability may be lower. + To maintain quorum a majority of mons must be up. For example, if there are three mons, two must be up. + If there are four mons, three must be up. If there are two mons, both must be up. + If quorum is lost, see the [disaster recovery guide](../../Troubleshooting/disaster-recovery.md#restoring-mon-quorum) to restore quorum from a single mon. * `allowMultiplePerNode`: Whether to allow the placement of multiple mons on a single node. Default is `false` for production. Should only be set to `true` in test environments. * `volumeClaimTemplate`: A `PersistentVolumeSpec` used by Rook to create PVCs - for monitor storage. This field is optional, and when not provided, HostPath - volume mounts are used. The current set of fields from template that are used - are `storageClassName` and the `storage` resource request and limit. The - default storage size request for new PVCs is `10Gi`. Ensure that associated - storage class is configured to use `volumeBindingMode: WaitForFirstConsumer`. - This setting only applies to new monitors that are created when the requested - number of monitors increases, or when a monitor fails and is recreated. An - [example CRD configuration is provided below](#using-pvc-storage-for-monitors). + for monitor storage. This field is optional, and when not provided, HostPath + volume mounts are used. The current set of fields from template that are used + are `storageClassName` and the `storage` resource request and limit. The + default storage size request for new PVCs is `10Gi`. Ensure that associated + storage class is configured to use `volumeBindingMode: WaitForFirstConsumer`. + This setting only applies to new monitors that are created when the requested + number of monitors increases, or when a monitor fails and is recreated. An + [example CRD configuration is provided below](./pvc-cluster.md). * `failureDomainLabel`: The label that is expected on each node where the mons - are expected to be deployed. The labels must be found in the list of - well-known [topology labels](#osd-topology). + are expected to be deployed. The labels must be found in the list of + well-known [topology labels](#osd-topology). * `zones`: The failure domain names where the Mons are expected to be deployed. - There must be **at least three zones** specified in the list. Each zone can be - backed by a different storage class by specifying the `volumeClaimTemplate`. - * `name`: The name of the zone, which is the value of the domain label. - * `volumeClaimTemplate`: A `PersistentVolumeSpec` used by Rook to create PVCs - for monitor storage. This field is optional, and when not provided, HostPath - volume mounts are used. The current set of fields from template that are used - are `storageClassName` and the `storage` resource request and limit. The - default storage size request for new PVCs is `10Gi`. Ensure that associated - storage class is configured to use `volumeBindingMode: WaitForFirstConsumer`. - This setting only applies to new monitors that are created when the requested - number of monitors increases, or when a monitor fails and is recreated. An - [example CRD configuration is provided below](#using-pvc-storage-for-monitors). + There must be **at least three zones** specified in the list. Each zone can be + backed by a different storage class by specifying the `volumeClaimTemplate`. + * `name`: The name of the zone, which is the value of the domain label. + * `volumeClaimTemplate`: A `PersistentVolumeSpec` used by Rook to create PVCs + for monitor storage. This field is optional, and when not provided, HostPath + volume mounts are used. The current set of fields from template that are used + are `storageClassName` and the `storage` resource request and limit. The + default storage size request for new PVCs is `10Gi`. Ensure that associated + storage class is configured to use `volumeBindingMode: WaitForFirstConsumer`. + This setting only applies to new monitors that are created when the requested + number of monitors increases, or when a monitor fails and is recreated. An + [example CRD configuration is provided below](./pvc-cluster.md). * `stretchCluster`: The stretch cluster settings that define the zones (or other failure domain labels) across which to configure the cluster. * `failureDomainLabel`: The label that is expected on each node where the cluster is expected to be deployed. The labels must be found @@ -160,14 +160,14 @@ A specific will contain a specific release of Ceph as well as security fixes fro * `name`: The name of the zone, which is the value of the domain label. * `arbiter`: Whether the zone is expected to be the arbiter zone which only runs a single mon. Exactly one zone must be labeled `true`. * `volumeClaimTemplate`: A `PersistentVolumeSpec` used by Rook to create PVCs - for monitor storage. This field is optional, and when not provided, HostPath - volume mounts are used. The current set of fields from template that are used - are `storageClassName` and the `storage` resource request and limit. The - default storage size request for new PVCs is `10Gi`. Ensure that associated - storage class is configured to use `volumeBindingMode: WaitForFirstConsumer`. - This setting only applies to new monitors that are created when the requested - number of monitors increases, or when a monitor fails and is recreated. An - [example CRD configuration is provided below](#using-pvc-storage-for-monitors). + for monitor storage. This field is optional, and when not provided, HostPath + volume mounts are used. The current set of fields from template that are used + are `storageClassName` and the `storage` resource request and limit. The + default storage size request for new PVCs is `10Gi`. Ensure that associated + storage class is configured to use `volumeBindingMode: WaitForFirstConsumer`. + This setting only applies to new monitors that are created when the requested + number of monitors increases, or when a monitor fails and is recreated. An + [example CRD configuration is provided below](./pvc-cluster.md). The two zones that are not the arbiter zone are expected to have OSDs deployed. If these settings are changed in the CRD the operator will update the number of mons during a periodic check of the mon health, which by default is every 45 seconds. @@ -209,20 +209,20 @@ Configure the network that will be enabled for the cluster and services. Requires a kernel that supports msgr2 (kernel 5.11 or CentOS 8.4 or newer). Default is false. * `encryption`: Settings for encryption on the wire to Ceph daemons * `enabled`: Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. - The default is false. When encryption is enabled, all communication between clients and Ceph daemons, or between - Ceph daemons will be encrypted. When encryption is not enabled, clients still establish a strong initial authentication - and data integrity is still validated with a crc check. - **IMPORTANT**: Encryption requires the 5.11 kernel for the latest nbd and cephfs drivers. Alternatively for testing only, - set "mounter: rbd-nbd" in the rbd storage class, or "mounter: fuse" in the cephfs storage class. - The nbd and fuse drivers are *not* recommended in production since restarting the csi driver pod will disconnect the volumes. - If this setting is enabled, CephFS volumes also require setting `CSI_CEPHFS_KERNEL_MOUNT_OPTIONS` to `"ms_mode=secure"` in operator.yaml. + The default is false. When encryption is enabled, all communication between clients and Ceph daemons, or between + Ceph daemons will be encrypted. When encryption is not enabled, clients still establish a strong initial authentication + and data integrity is still validated with a crc check. + **IMPORTANT**: Encryption requires the 5.11 kernel for the latest nbd and cephfs drivers. Alternatively for testing only, + set "mounter: rbd-nbd" in the rbd storage class, or "mounter: fuse" in the cephfs storage class. + The nbd and fuse drivers are **not** recommended in production since restarting the csi driver pod will disconnect the volumes. + If this setting is enabled, CephFS volumes also require setting `CSI_CEPHFS_KERNEL_MOUNT_OPTIONS` to `"ms_mode=secure"` in operator.yaml. * `compression`: * `enabled`: Whether to compress the data in transit across the wire. The default is false. - See the kernel requirements above for encryption. + See the kernel requirements above for encryption. !!! caution Changing networking configuration after a Ceph cluster has been deployed is only supported for - the network encryption settings. Changing other network settings is *NOT* supported and will + the network encryption settings. Changing other network settings is **NOT** supported and will likely result in a non-functioning cluster. #### Provider @@ -277,7 +277,7 @@ Below are the settings for host-based cluster. This type of cluster can specify * `^sd.`: Selects all devices starting with `sd` * `^sd[a-d]`: Selects devices starting with `sda`, `sdb`, `sdc`, and `sdd` if found * `^s`: Selects all devices that start with `s` - * `^[^r]`: Selects all devices that do *not* start with `r` + * `^[^r]`: Selects all devices that do **not** start with `r` * `devicePathFilter`: A regular expression for device paths (e.g. `/dev/disk/by-path/pci-0:1:2:3-scsi-1`) that allows selection of devices and partitions to be consumed by OSDs. LVM logical volumes are not picked by `devicePathFilter`.If individual devices or `deviceFilter` have been specified for a node then this filter will be ignored. This field uses [golang regular expression syntax](https://golang.org/pkg/regexp/syntax/). For example: * `^/dev/sd.`: Selects all devices starting with `sd` * `^/dev/disk/by-path/pci-.*`: Selects all devices which are connected to PCI bus @@ -301,18 +301,18 @@ The following are the settings for Storage Class Device Sets which can be config * `resources`: The CPU and RAM requests/limits for the devices. (Optional) * `placement`: The placement criteria for the devices. (Optional) Default is no placement criteria. - The syntax is the same as for [other placement configuration](#placement-configuration-settings). It supports `nodeAffinity`, `podAffinity`, `podAntiAffinity` and `tolerations` keys. + The syntax is the same as for [other placement configuration](#placement-configuration-settings). It supports `nodeAffinity`, `podAffinity`, `podAntiAffinity` and `tolerations` keys. - It is recommended to configure the placement such that the OSDs will be as evenly spread across nodes as possible. At a minimum, anti-affinity should be added so at least one OSD will be placed on each available nodes. + It is recommended to configure the placement such that the OSDs will be as evenly spread across nodes as possible. At a minimum, anti-affinity should be added so at least one OSD will be placed on each available nodes. - However, if there are more OSDs than nodes, this anti-affinity will not be effective. Another placement scheme to consider is to add labels to the nodes in such a way that the OSDs can be grouped on those nodes, create multiple storageClassDeviceSets, and add node affinity to each of the device sets that will place the OSDs in those sets of nodes. + However, if there are more OSDs than nodes, this anti-affinity will not be effective. Another placement scheme to consider is to add labels to the nodes in such a way that the OSDs can be grouped on those nodes, create multiple storageClassDeviceSets, and add node affinity to each of the device sets that will place the OSDs in those sets of nodes. - Rook will automatically add required nodeAffinity to the OSD daemons to match the topology labels that are found - on the nodes where the OSD prepare jobs ran. To ensure data durability, the OSDs are required to run in the same - topology that the Ceph CRUSH map expects. For example, if the nodes are labeled with rack topology labels, the - OSDs will be constrained to a certain rack. Without the topology labels, Rook will not constrain the OSDs beyond - what is required by the PVs, for example to run in the zone where provisioned. See the [OSD Topology](#osd-topology) - section for the related labels. + Rook will automatically add required nodeAffinity to the OSD daemons to match the topology labels that are found + on the nodes where the OSD prepare jobs ran. To ensure data durability, the OSDs are required to run in the same + topology that the Ceph CRUSH map expects. For example, if the nodes are labeled with rack topology labels, the + OSDs will be constrained to a certain rack. Without the topology labels, Rook will not constrain the OSDs beyond + what is required by the PVs, for example to run in the zone where provisioned. See the [OSD Topology](#osd-topology) + section for the related labels. * `preparePlacement`: The placement criteria for the preparation of the OSD devices. Creating OSDs is a two-step process and the prepare job may require different placement than the OSD daemons. If the `preparePlacement` is not specified, the `placement` will instead be applied for consistent placement for the OSD prepare jobs and OSD deployments. The `preparePlacement` is only useful for `portable` OSDs in the device sets. OSDs that are not portable will be tied to the host where the OSD prepare job initially runs. * For example, provisioning may require topology spread constraints across zones, but the OSD daemons may require constraints across hosts within the zones. @@ -320,12 +320,11 @@ The following are the settings for Storage Class Device Sets which can be config * `tuneDeviceClass`: For example, Ceph cannot detect AWS volumes as HDDs from the storage class "gp2", so you can improve Ceph performance by setting this to true. * `tuneFastDeviceClass`: For example, Ceph cannot detect Azure disks as SSDs from the storage class "managed-premium", so you can improve Ceph performance by setting this to true.. * `volumeClaimTemplates`: A list of PVC templates to use for provisioning the underlying storage devices. - * `metadata.name`: "data", "metadata", or "wal". If a single template is provided, the name must be "data". If the name is "metadata" or "wal", the devices are used to store the Ceph metadata or WAL respectively. In both cases, the devices must be raw devices or LVM logical volumes. - - * `resources.requests.storage`: The desired capacity for the underlying storage devices. - * `storageClassName`: The StorageClass to provision PVCs from. Default would be to use the cluster-default StorageClass. - * `volumeMode`: The volume mode to be set for the PVC. Which should be Block - * `accessModes`: The access mode for the PVC to be bound by OSD. + * `metadata.name`: "data", "metadata", or "wal". If a single template is provided, the name must be "data". If the name is "metadata" or "wal", the devices are used to store the Ceph metadata or WAL respectively. In both cases, the devices must be raw devices or LVM logical volumes. + * `resources.requests.storage`: The desired capacity for the underlying storage devices. + * `storageClassName`: The StorageClass to provision PVCs from. Default would be to use the cluster-default StorageClass. + * `volumeMode`: The volume mode to be set for the PVC. Which should be Block + * `accessModes`: The access mode for the PVC to be bound by OSD. * `schedulerName`: Scheduler name for OSD pod placement. (Optional) * `encrypted`: whether to encrypt all the OSDs in a given storageClassDeviceSet @@ -356,6 +355,7 @@ Allowed configurations are: | mpath | | | #### Limitations of metadata device + - If `metadataDevice` is specified in the global OSD configuration or in the node level OSD configuration, the metadata device will be shared between all OSDs on the same node. In other words, OSDs will be initialized by `lvm batch`. In this case, we can't use partition device. - If `metadataDevice` is specified in the device local configuration, we can use partition as metadata device. In other words, OSDs are initialized by `lvm prepare`. @@ -458,13 +458,16 @@ You can set resource requests/limits for Rook components through the [Resource R * `mon`: Set resource requests/limits for mons * `osd`: Set resource requests/limits for OSDs. - This key applies for all OSDs regardless of their device classes. In case of need to apply resource requests/limits for OSDs with particular device class use specific osd keys below. If the memory resource is declared Rook will automatically set the OSD configuration `osd_memory_target` to the same value. This aims to ensure that the actual OSD memory consumption is consistent with the OSD pods' resource declaration. -* `osd-`: Set resource requests/limits for OSDs on a specific device class. Rook will automatically detect `hdd`, - `ssd`, or `nvme` device classes. Custom device classes can also be set. + This key applies for all OSDs regardless of their device classes. + In case of need to apply resource requests/limits for OSDs with particular device class use specific osd keys below. + If the memory resource is declared Rook will automatically set the OSD configuration `osd_memory_target` to the same value. + This aims to ensure that the actual OSD memory consumption is consistent with the OSD pods' resource declaration. +* `osd-`: Set resource requests/limits for OSDs on a specific device class. + Rook will automatically detect `hdd`, `ssd`, or `nvme` device classes. Custom device classes can also be set. * `mgr`: Set resource requests/limits for MGRs * `mgr-sidecar`: Set resource requests/limits for the MGR sidecar, which is only created when `mgr.count: 2`. - The sidecar requires very few resources since it only executes every 15 seconds to query Ceph for the active - mgr and update the mgr services if the active mgr changed. + The sidecar requires very few resources since it only executes every 15 seconds to query Ceph for the active + mgr and update the mgr services if the active mgr changed. * `prepareosd`: Set resource requests/limits for OSD prepare job * `crashcollector`: Set resource requests/limits for crash. This pod runs wherever there is a Ceph pod running. It scrapes for Ceph daemon core dumps and sends them to the Ceph manager crash module so that core dumps are centralized and can be easily listed/accessed. @@ -676,25 +679,25 @@ The available space will be less that you may expect due to overhead in the OSDs The `conditions` represent the status of the Rook operator. * If the cluster is fully configured and the operator is stable, the - `Ready` condition is raised with `ClusterCreated` reason and no other conditions. The cluster - will remain in the `Ready` condition after the first successful configuration since it - is expected the storage is consumable from this point on. If there are issues preventing - the storage layer from working, they are expected to show as Ceph health errors. + `Ready` condition is raised with `ClusterCreated` reason and no other conditions. The cluster + will remain in the `Ready` condition after the first successful configuration since it + is expected the storage is consumable from this point on. If there are issues preventing + the storage layer from working, they are expected to show as Ceph health errors. * If the cluster is externally connected successfully, the `Ready` condition will have the reason `ClusterConnected`. * If the operator is currently being configured or the operator is checking for update, - there will be a `Progressing` condition. + there will be a `Progressing` condition. * If there was a failure, the condition(s) status will be `false` and the `message` will - give a summary of the error. See the operator log for more details. + give a summary of the error. See the operator log for more details. ### Other Status There are several other properties for the overall status including: * `message`, `phase`, and `state`: A summary of the overall current state of the cluster, which - is somewhat duplicated from the conditions for backward compatibility. + is somewhat duplicated from the conditions for backward compatibility. * `storage.deviceClasses`: The names of the types of storage devices that Ceph discovered - in the cluster. These types will be `ssd` or `hdd` unless they have been overridden - with the `crushDeviceClass` in the `storageClassDeviceSets`. + in the cluster. These types will be `ssd` or `hdd` unless they have been overridden + with the `crushDeviceClass` in the `storageClassDeviceSets`. * `version`: The version of the Ceph image currently deployed. ## OSD Topology @@ -771,8 +774,8 @@ the CephCluster being deleted. Rook will warn about which other resources are bl three ways until all blocking resources are deleted: 1. An event will be registered on the CephCluster resource -1. A status condition will be added to the CephCluster resource -1. An error will be added to the Rook Ceph operator log +2. A status condition will be added to the CephCluster resource +3. An error will be added to the Rook Ceph operator log ## Cleanup policy @@ -781,15 +784,15 @@ The policy settings indicate which data should be forcibly deleted and in what w The `cleanupPolicy` has several fields: * `confirmation`: Only an empty string and `yes-really-destroy-data` are valid values for this field. - If this setting is empty, the `cleanupPolicy` settings will be ignored and Rook will not cleanup any resources during cluster removal. - To reinstall the cluster, the admin would then be required to follow the [cleanup guide](../../Storage-Configuration/ceph-teardown.md) to delete the data on hosts. - If this setting is `yes-really-destroy-data`, the operator will automatically delete the data on hosts. - Because this cleanup policy is destructive, after the confirmation is set to `yes-really-destroy-data` - Rook will stop configuring the cluster as if the cluster is about to be destroyed. + If this setting is empty, the `cleanupPolicy` settings will be ignored and Rook will not cleanup any resources during cluster removal. + To reinstall the cluster, the admin would then be required to follow the [cleanup guide](../../Storage-Configuration/ceph-teardown.md) to delete the data on hosts. + If this setting is `yes-really-destroy-data`, the operator will automatically delete the data on hosts. + Because this cleanup policy is destructive, after the confirmation is set to `yes-really-destroy-data` + Rook will stop configuring the cluster as if the cluster is about to be destroyed. * `sanitizeDisks`: sanitizeDisks represents advanced settings that can be used to delete data on drives. * `method`: indicates if the entire disk should be sanitized or simply ceph's metadata. Possible choices are `quick` (default) or `complete` * `dataSource`: indicate where to get random bytes from to write on the disk. Possible choices are `zero` (default) or `random`. - Using random sources will consume entropy from the system and will take much more time then the zero source + Using random sources will consume entropy from the system and will take much more time then the zero source * `iteration`: overwrite N times instead of the default (1). Takes an integer value * `allowUninstallWithVolumes`: If set to true, then the cephCluster deletion doesn't wait for the PVCs to be deleted. Default is `false`. diff --git a/Documentation/CRDs/Cluster/external-cluster/external-cluster.md b/Documentation/CRDs/Cluster/external-cluster/external-cluster.md index 2bedb1702..da67cf44c 100644 --- a/Documentation/CRDs/Cluster/external-cluster/external-cluster.md +++ b/Documentation/CRDs/Cluster/external-cluster/external-cluster.md @@ -71,7 +71,6 @@ It will generate the secrets which you can use for creating new `Consumer cluste So you would be running different isolated consumer clusters on top of single `Source cluster`. !!! note - Restricting the csi-users per pool, and per cluster will require creating new csi-users and new secrets for that csi-users. So apply these secrets only to new `Consumer cluster` deployment while using the same `Source cluster`. @@ -112,7 +111,6 @@ Restricted users created using `--restricted-auth-permission` flag need to pass ``` !!! note - An existing non-restricted user cannot be converted to a restricted user by upgrading. The upgrade flag should only be used to append new permissions to users. It shouldn't be used for changing a csi user already applied permissions. For example, you shouldn't change the pool(s) a user has access to. @@ -121,7 +119,6 @@ Restricted users created using `--restricted-auth-permission` flag need to pass If in case the cluster needs the admin keyring to configure, update the admin key `rook-ceph-mon` secret with client.admin keyring !!! note - Sharing the admin key with the external cluster is not generally recommended 1. Get the `client.admin` keyring from the ceph cluster @@ -185,9 +182,9 @@ If not installing with Helm, here are the steps to install with manifests. 1. Paste the above output from `create-external-cluster-resources.py` into your current shell to allow importing the source data. 2. The import script in the next step uses the current kubeconfig context by - default. If you want to specify the kubernetes cluster to use without - changing the current context, you can specify the cluster name by setting - the KUBECONTEXT environment variable. + default. If you want to specify the kubernetes cluster to use without + changing the current context, you can specify the cluster name by setting + the KUBECONTEXT environment variable. ```console export KUBECONTEXT= @@ -202,7 +199,6 @@ If not installing with Helm, here are the steps to install with manifests. 4. Run the import script. !!! note - If your Rook cluster nodes are running a kernel earlier than or equivalent to 5.4, remove `fast-diff, object-map, deep-flatten,exclusive-lock` from the `imageFeatures` line. @@ -256,7 +252,6 @@ Consume the S3 Storage, in two different ways: ``` !!! hint - For more details see the [Object Store topic](../../../Storage-Configuration/Object-Storage-RGW/object-storage.md#connect-to-an-external-object-store) ### Connect to v2 mon port @@ -279,5 +274,4 @@ you can export the settings from this cluster with the following steps. 2. Exec to the toolbox pod and execute create-external-cluster-resources.py with needed options to create required [users and keys](#1-create-all-users-and-keys). !!! important - For other clusters to connect to storage in this cluster, Rook must be configured with a networking configuration that is accessible from other clusters. Most commonly this is done by enabling host networking in the CephCluster CR so the Ceph daemons will be addressable by their host IPs. diff --git a/Documentation/CRDs/Cluster/external-cluster/topology-for-external-mode.md b/Documentation/CRDs/Cluster/external-cluster/topology-for-external-mode.md index 8d7bb7195..6e379c2c8 100644 --- a/Documentation/CRDs/Cluster/external-cluster/topology-for-external-mode.md +++ b/Documentation/CRDs/Cluster/external-cluster/topology-for-external-mode.md @@ -23,7 +23,7 @@ Determine the names of the zones (or other failure domains) in the Ceph CRUSH ma Create a zone-specific CRUSH rule for each of the pools. For example, this is a CRUSH rule for `zone-a`: -``` +```console $ ceph osd crush rule create-replicated { "rule_id": 5, @@ -50,7 +50,7 @@ $ ceph osd crush rule create-replicated ping -c3 192.168.20.19 @@ -515,6 +518,7 @@ After this, `csi-*plugin-*` pods will restart, and `csi-*plugin-holder-*` pods w Check that CSI pods are using the correct host networking configuration using the example below as guidance (in the example, `CSI_ENABLE_HOST_NETWORK` is `"true"`): + ```console $ kubectl -n rook-ceph get -o yaml daemonsets.apps csi-rbdplugin | grep -i hostnetwork hostNetwork: true @@ -544,6 +548,7 @@ After this process is done for all Kubernetes nodes, it is safe to delete the `c daemonsets. Delete the holder daemonsets using the example below as guidance: + ```console $ kubectl -n rook-ceph get daemonset -o name | grep plugin-holder daemonset.apps/csi-cephfsplugin-holder-my-cluster diff --git a/Documentation/CRDs/Cluster/pvc-cluster.md b/Documentation/CRDs/Cluster/pvc-cluster.md index 1883e6fb8..f644016ef 100644 --- a/Documentation/CRDs/Cluster/pvc-cluster.md +++ b/Documentation/CRDs/Cluster/pvc-cluster.md @@ -206,141 +206,141 @@ The bluestore partition has the following reference combinations supported by th * A single "data" device. - ```yaml - storage: - storageClassDeviceSets: - - name: set1 - count: 3 - portable: false - volumeClaimTemplates: - - metadata: - name: data - spec: - resources: - requests: - storage: 10Gi - # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) - storageClassName: gp2 - volumeMode: Block - accessModes: - - ReadWriteOnce - ``` + ```yaml + storage: + storageClassDeviceSets: + - name: set1 + count: 3 + portable: false + volumeClaimTemplates: + - metadata: + name: data + spec: + resources: + requests: + storage: 10Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) + storageClassName: gp2 + volumeMode: Block + accessModes: + - ReadWriteOnce + ``` * A "data" device and a "metadata" device. - ```yaml - storage: - storageClassDeviceSets: - - name: set1 - count: 3 - portable: false - volumeClaimTemplates: - - metadata: - name: data - spec: - resources: - requests: - storage: 10Gi - # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) - storageClassName: gp2 - volumeMode: Block - accessModes: - - ReadWriteOnce - - metadata: - name: metadata - spec: - resources: - requests: - # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing - storage: 5Gi - # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) - storageClassName: io1 - volumeMode: Block - accessModes: - - ReadWriteOnce - ``` + ```yaml + storage: + storageClassDeviceSets: + - name: set1 + count: 3 + portable: false + volumeClaimTemplates: + - metadata: + name: data + spec: + resources: + requests: + storage: 10Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) + storageClassName: gp2 + volumeMode: Block + accessModes: + - ReadWriteOnce + - metadata: + name: metadata + spec: + resources: + requests: + # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing + storage: 5Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) + storageClassName: io1 + volumeMode: Block + accessModes: + - ReadWriteOnce + ``` * A "data" device and a "wal" device. A WAL device can be used for BlueStore’s internal journal or write-ahead log (block.wal), it is only useful to use a WAL device if the device is faster than the primary device (data device). There is no separate "metadata" device in this case, the data of main OSD block and block.db located in "data" device. - ```yaml - storage: - storageClassDeviceSets: - - name: set1 - count: 3 - portable: false - volumeClaimTemplates: - - metadata: - name: data - spec: - resources: - requests: - storage: 10Gi - # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) - storageClassName: gp2 - volumeMode: Block - accessModes: - - ReadWriteOnce - - metadata: - name: wal - spec: - resources: - requests: - # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing - storage: 5Gi - # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) - storageClassName: io1 - volumeMode: Block - accessModes: - - ReadWriteOnce - ``` + ```yaml + storage: + storageClassDeviceSets: + - name: set1 + count: 3 + portable: false + volumeClaimTemplates: + - metadata: + name: data + spec: + resources: + requests: + storage: 10Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) + storageClassName: gp2 + volumeMode: Block + accessModes: + - ReadWriteOnce + - metadata: + name: wal + spec: + resources: + requests: + # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing + storage: 5Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) + storageClassName: io1 + volumeMode: Block + accessModes: + - ReadWriteOnce + ``` * A "data" device, a "metadata" device and a "wal" device. - ```yaml - storage: - storageClassDeviceSets: - - name: set1 - count: 3 - portable: false - volumeClaimTemplates: - - metadata: - name: data - spec: - resources: - requests: - storage: 10Gi - # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) - storageClassName: gp2 - volumeMode: Block - accessModes: - - ReadWriteOnce - - metadata: - name: metadata - spec: - resources: - requests: - # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing - storage: 5Gi - # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) - storageClassName: io1 - volumeMode: Block - accessModes: - - ReadWriteOnce - - metadata: - name: wal - spec: - resources: - requests: - # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing - storage: 5Gi - # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) - storageClassName: io1 - volumeMode: Block - accessModes: - - ReadWriteOnce - ``` + ```yaml + storage: + storageClassDeviceSets: + - name: set1 + count: 3 + portable: false + volumeClaimTemplates: + - metadata: + name: data + spec: + resources: + requests: + storage: 10Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, gp2) + storageClassName: gp2 + volumeMode: Block + accessModes: + - ReadWriteOnce + - metadata: + name: metadata + spec: + resources: + requests: + # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing + storage: 5Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) + storageClassName: io1 + volumeMode: Block + accessModes: + - ReadWriteOnce + - metadata: + name: wal + spec: + resources: + requests: + # Find the right size https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing + storage: 5Gi + # IMPORTANT: Change the storage class depending on your environment (e.g. local-storage, io1) + storageClassName: io1 + volumeMode: Block + accessModes: + - ReadWriteOnce + ``` To determine the size of the metadata block follow the [official Ceph sizing guide](https://docs.ceph.com/docs/master/rados/configuration/bluestore-config-ref/#sizing). diff --git a/Documentation/CRDs/Cluster/stretch-cluster.md b/Documentation/CRDs/Cluster/stretch-cluster.md index f06477ec3..a4f5d483b 100644 --- a/Documentation/CRDs/Cluster/stretch-cluster.md +++ b/Documentation/CRDs/Cluster/stretch-cluster.md @@ -11,7 +11,7 @@ Two mons run in each of the two data zones, while two replicas of the data are i The third zone (arbiter) runs a single mon. No other Rook or Ceph daemons need to be run in the arbiter zone. For this example, we assume the desired failure domain is a zone. Another failure domain can also be specified with a -known [topology node label](#osd-topology) which is already being used for OSD failure domains. +known [topology node label](../../CRDs/Cluster/ceph-cluster-crd.md#osd-topology) which is already being used for OSD failure domains. ```yaml apiVersion: ceph.rook.io/v1 diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md index dd4b8ad37..0612a5e3b 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-crd.md @@ -86,12 +86,12 @@ When the `zone` section is set pools with the object stores name will not be cre * `metadataPool`: The settings used to create all of the object store metadata pools. Must use replication. * `dataPool`: The settings to create the object store data pool. Can use replication or erasure coding. * `preservePoolsOnDelete`: If it is set to 'true' the pools used to support the object store will remain when the object store - will be deleted. This is a security measure to avoid accidental loss of data. It is set to 'false' by default. If not specified - is also deemed as 'false'. + will be deleted. This is a security measure to avoid accidental loss of data. It is set to 'false' by default. If not specified + is also deemed as 'false'. * `allowUsersInNamespaces`: If a CephObjectStoreUser is created in a namespace other than the Rook cluster namespace, - the namespace must be added to this list of allowed namespaces, or specify "*" to allow all namespaces. - This is useful for applications that need object store credentials to be created in their own namespace, - where neither OBCs nor COSI is being used to create buckets. The default is empty. + the namespace must be added to this list of allowed namespaces, or specify "*" to allow all namespaces. + This is useful for applications that need object store credentials to be created in their own namespace, + where neither OBCs nor COSI is being used to create buckets. The default is empty. ## Gateway Settings @@ -99,30 +99,30 @@ The gateway settings correspond to the RGW daemon settings. * `type`: `S3` is supported * `sslCertificateRef`: If specified, this is the name of the Kubernetes secret(`opaque` or `tls` - type) that contains the TLS certificate to be used for secure connections to the object store. - If it is an opaque Kubernetes Secret, Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be - in the format expected by the [RGW - service](https://docs.ceph.com/docs/master/install/ceph-deploy/install-ceph-gateway/#using-ssl-with-civetweb): - "The server key, server certificate, and any other CA or intermediate certificates be supplied in - one file. Each of these items must be in PEM form." They are scenarios where the certificate DNS is set for a particular domain - that does not include the local Kubernetes DNS, namely the object store DNS service endpoint. If - adding the service DNS name to the certificate is not empty another key can be specified in the - secret's data: `insecureSkipVerify: true` to skip the certificate verification. It is not - recommended to enable this option since TLS is susceptible to machine-in-the-middle attacks unless - custom verification is used. + type) that contains the TLS certificate to be used for secure connections to the object store. + If it is an opaque Kubernetes Secret, Rook will look in the secret provided at the `cert` key name. The value of the `cert` key must be + in the format expected by the [RGW + service](https://docs.ceph.com/docs/master/install/ceph-deploy/install-ceph-gateway/#using-ssl-with-civetweb): + "The server key, server certificate, and any other CA or intermediate certificates be supplied in + one file. Each of these items must be in PEM form." They are scenarios where the certificate DNS is set for a particular domain + that does not include the local Kubernetes DNS, namely the object store DNS service endpoint. If + adding the service DNS name to the certificate is not empty another key can be specified in the + secret's data: `insecureSkipVerify: true` to skip the certificate verification. It is not + recommended to enable this option since TLS is susceptible to machine-in-the-middle attacks unless + custom verification is used. * `caBundleRef`: If specified, this is the name of the Kubernetes secret (type `opaque`) that - contains additional custom ca-bundle to use. The secret must be in the same namespace as the Rook - cluster. Rook will look in the secret provided at the `cabundle` key name. + contains additional custom ca-bundle to use. The secret must be in the same namespace as the Rook + cluster. Rook will look in the secret provided at the `cabundle` key name. * `hostNetwork`: Whether host networking is enabled for the rgw daemon. If not set, the network settings from the cluster CR will be applied. * `port`: The port on which the Object service will be reachable. If host networking is enabled, the RGW daemons will also listen on that port. If running on SDN, the RGW daemon listening port will be 8080 internally. * `securePort`: The secure port on which RGW pods will be listening. A TLS certificate must be specified either via `sslCerticateRef` or `service.annotations` * `instances`: The number of pods that will be started to load balance this object store. * `externalRgwEndpoints`: A list of IP addresses to connect to external existing Rados Gateways - (works with external mode). This setting will be ignored if the `CephCluster` does not have - `external` spec enabled. Refer to the [external cluster section](../Cluster/ceph-cluster-crd.md#external-cluster) - for more details. Multiple endpoints can be given, but for stability of ObjectBucketClaims, we - highly recommend that users give only a single external RGW endpoint that is a load balancer that - sends requests to the multiple RGWs. + (works with external mode). This setting will be ignored if the `CephCluster` does not have + `external` spec enabled. Refer to the [external cluster section](../Cluster/ceph-cluster-crd.md#external-cluster) + for more details. Multiple endpoints can be given, but for stability of ObjectBucketClaims, we + highly recommend that users give only a single external RGW endpoint that is a load balancer that + sends requests to the multiple RGWs. * `annotations`: Key value pair list of annotations to add. * `labels`: Key value pair list of labels to add. * `placement`: The Kubernetes placement settings to determine where the RGW pods should be started in the cluster. @@ -256,8 +256,8 @@ Kubernetes Custom Resource, but the Ceph pools which store the data will not be Rook will warn about which buckets are blocking deletion in three ways: 1. An event will be registered on the CephObjectStore resource -1. A status condition will be added to the CephObjectStore resource -1. An error will be added to the Rook Ceph Operator log +2. A status condition will be added to the CephObjectStore resource +3. An error will be added to the Rook Ceph Operator log If the CephObjectStore is configured in a [multisite setup](../../Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md) the above conditions are applicable only to stores that belong to a single master zone. Otherwise the conditions are ignored. Even if the store is removed the user can access the diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md index 4dde0d8bf..af84691b2 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md @@ -37,8 +37,8 @@ spec: * `store`: The object store in which the user will be created. This matches the name of the objectstore CRD. * `displayName`: The display name which will be passed to the `radosgw-admin user create` command. * `clusterNamespace`: The namespace where the parent CephCluster and CephObjectStore are found. If not specified, - the user must be in the same namespace as the cluster and object store. - To enable this feature, the CephObjectStore allowUsersInNamespaces must include the namespace of this user. + the user must be in the same namespace as the cluster and object store. + To enable this feature, the CephObjectStore allowUsersInNamespaces must include the namespace of this user. * `quotas`: This represents quota limitation can be set on the user. Please refer [here](https://docs.ceph.com/en/latest/radosgw/admin/#quota-management) for details. * `maxBuckets`: The maximum bucket limit for the user. * `maxSize`: Maximum size limit of all objects across all the user's buckets. diff --git a/Documentation/CRDs/Object-Storage/ceph-object-zone-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-zone-crd.md index f741cc291..f892d74a5 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-zone-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-zone-crd.md @@ -3,7 +3,7 @@ title: CephObjectZone CRD --- Rook allows creation of zones in a ceph cluster for a [Ceph Object Multisite](../../Storage-Configuration/Object-Storage-RGW/ceph-object-multisite.md) - configuration through a CRD. The following settings are available for Ceph object store zones. +configuration through a CRD. The following settings are available for Ceph object store zones. ## Example @@ -47,14 +47,14 @@ The pools allow all of the settings defined in the Pool CRD spec. For more detai * `dataPool`: The settings to create the object store data pool. Can use replication or erasure coding. * `customEndpoints`: Specify the endpoint(s) that will accept multisite replication traffic for this zone. You may include the port in the definition if necessary. For example: "https://my-object-store.my-domain.net:443". By default, Rook will set this to the DNS name of the ClusterIP Service created for the CephObjectStore that corresponds to this zone. - Most multisite configurations will not exist within the same Kubernetes cluster, meaning the default value will not be useful. In these cases, you will be required to create your own custom ingress resource for the CephObjectStore in order to make the zone available for replication. You must add the endpoint for your custom ingress resource to this list to allow the store to accept replication traffic. + Most multisite configurations will not exist within the same Kubernetes cluster, meaning the default value will not be useful. In these cases, you will be required to create your own custom ingress resource for the CephObjectStore in order to make the zone available for replication. You must add the endpoint for your custom ingress resource to this list to allow the store to accept replication traffic. - In the case of multiple stores (or multiple endpoints for a single store), you are not required to put all endpoints in this list. Only specify the endpoints that should be used for replication traffic. + In the case of multiple stores (or multiple endpoints for a single store), you are not required to put all endpoints in this list. Only specify the endpoints that should be used for replication traffic. - If you update `customEndpoints` to return to an empty list, you must the Rook operator to automatically add the CephObjectStore service endpoint to Ceph's internal configuration. + If you update `customEndpoints` to return to an empty list, you must the Rook operator to automatically add the CephObjectStore service endpoint to Ceph's internal configuration. * `preservePoolsOnDelete`: If it is set to 'true' the pools used to support the CephObjectZone will remain when it is deleted. This is a security measure to avoid accidental loss of data. It is set to 'true' by default. - It is better to check whether data synced with other peer zones before triggering the deletion to avoid accidental loss of data via steps mentioned [here](https://docs.ceph.com/en/latest/radosgw/multisite/#check-synchronization-status) + It is better to check whether data synced with other peer zones before triggering the deletion to avoid accidental loss of data via steps mentioned [here](https://docs.ceph.com/en/latest/radosgw/multisite/#check-synchronization-status) - When deleting a CephObjectZone, deletion will be blocked until all `CephObjectStores` belonging to the zone are removed. + When deleting a CephObjectZone, deletion will be blocked until all `CephObjectStores` belonging to the zone are removed. diff --git a/Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md b/Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md index 4f8fcdb38..cc87c6658 100644 --- a/Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md +++ b/Documentation/CRDs/Shared-Filesystem/ceph-filesystem-crd.md @@ -114,12 +114,12 @@ The pools allow all of the settings defined in the Pool CRD spec. For more detai * `dataPools`: The settings to create the filesystem data pools. Optionally (and we highly recommend), a pool name can be specified with the `name` field to override the default generated name; see more below. If multiple pools are specified, Rook will add the pools to the filesystem. Assigning users or files to a pool is left as an exercise for the reader with the [CephFS documentation](http://docs.ceph.com/docs/master/cephfs/file-layouts/). The data pools can use replication or erasure coding. If erasure coding pools are specified, the cluster must be running with bluestore enabled on the OSDs. * `name`: (optional, and highly recommended) Override the default generated name of the pool. The final pool name will consist of the filesystem name and pool name, e.g., `-`. We highly recommend to specify `name` to prevent issues that can arise from modifying the spec in a way that causes Rook to lose the original pool ordering. * `preserveFilesystemOnDelete`: If it is set to 'true' the filesystem will remain when the - CephFilesystem resource is deleted. This is a security measure to avoid loss of data if the - CephFilesystem resource is deleted accidentally. The default value is 'false'. This option - replaces `preservePoolsOnDelete` which should no longer be set. + CephFilesystem resource is deleted. This is a security measure to avoid loss of data if the + CephFilesystem resource is deleted accidentally. The default value is 'false'. This option + replaces `preservePoolsOnDelete` which should no longer be set. * (deprecated) `preservePoolsOnDelete`: This option is replaced by the above - `preserveFilesystemOnDelete`. For backwards compatibility and upgradeability, if this is set to - 'true', Rook will treat `preserveFilesystemOnDelete` as being set to 'true'. + `preserveFilesystemOnDelete`. For backwards compatibility and upgradeability, if this is set to + 'true', Rook will treat `preserveFilesystemOnDelete` as being set to 'true'. ## Metadata Server Settings @@ -135,9 +135,9 @@ The metadata server settings correspond to the MDS daemon settings. * `path`: filesystem source path to take the snapshot on * `interval`: frequency of the snapshots. The interval can be specified in days, hours, or minutes using d, h, m suffix respectively. * `startTime`: optional, determines at what time the snapshot process starts, specified using the ISO 8601 time format. - * `snapshotRetention`: allow to manage retention policies: - * `path`: filesystem source path to apply the retention on - * `duration`: + * `snapshotRetention`: allow to manage retention policies: + * `path`: filesystem source path to apply the retention on + * `duration`: * `annotations`: Key value pair list of annotations to add. * `labels`: Key value pair list of labels to add. * `placement`: The mds pods can be given standard Kubernetes placement restrictions with `nodeAffinity`, `tolerations`, `podAffinity`, and `podAntiAffinity` similar to placement defined for daemons configured by the [cluster CRD](https://github.com/rook/rook/blob/master/deploy/examples/cluster.yaml). diff --git a/Documentation/CRDs/ceph-client-crd.md b/Documentation/CRDs/ceph-client-crd.md index 04e9d11ab..5b86f8ea5 100644 --- a/Documentation/CRDs/ceph-client-crd.md +++ b/Documentation/CRDs/ceph-client-crd.md @@ -51,7 +51,7 @@ Extract Ceph cluster credentials from the generated secret (note that the subkey kubectl --namespace rook-ceph get secret rook-ceph-client-example -o jsonpath="{.data.example}" | base64 -d ``` -The base64 encoded value that is returned *is* the password for your ceph client. +The base64 encoded value that is returned **is** the password for your ceph client. ### 4. Retrieve the mon endpoints @@ -61,9 +61,9 @@ To send writes to the cluster, you must retrieve the mons in use: kubectl --namespace rook-ceph get configmap rook-ceph-mon-endpoints -o jsonpath='{.data.data}' | sed 's/.=//g'` ``` -This command *should* produce a line that looks somewhat like this: +This command should produce a line that looks somewhat like this: -``` +```console 10.107.72.122:6789,10.103.244.218:6789,10.99.33.227:6789 ``` @@ -77,6 +77,7 @@ If you choose to generate files for Ceph to use you will need to generate the fo Examples of the files follow: `ceph.conf` + ```ini [global] mon_host=10.107.72.122:6789,10.103.244.218:6789,10.99.33.227:6789 @@ -84,6 +85,7 @@ log file = /tmp/ceph-$pid.log ``` `ceph.keyring` + ```ini [client.example] key = < key, decoded from k8s secret> diff --git a/Documentation/CRDs/ceph-nfs-crd.md b/Documentation/CRDs/ceph-nfs-crd.md index 80b8465f6..8accbbc7c 100644 --- a/Documentation/CRDs/ceph-nfs-crd.md +++ b/Documentation/CRDs/ceph-nfs-crd.md @@ -89,83 +89,85 @@ spec: The `server` spec sets configuration for Rook-created NFS-Ganesha server pods. * `active`: The number of active NFS servers. Rook supports creating more than one active NFS - server, but cannot guarantee high availability. For values greater than 1, see the - [known issue](#serveractive-count-greater-than-1) below. + server, but cannot guarantee high availability. For values greater than 1, see the + [known issue](#serveractive-count-greater-than-1) below. * `placement`: Kubernetes placement restrictions to apply to NFS server Pod(s). This is similar to - placement defined for daemons configured by the - [CephCluster CRD](https://github.com/rook/rook/blob/master/deploy/examples/cluster.yaml). + placement defined for daemons configured by the + [CephCluster CRD](https://github.com/rook/rook/blob/master/deploy/examples/cluster.yaml). * `annotations`: Kubernetes annotations to apply to NFS server Pod(s) * `labels`: Kubernetes labels to apply to NFS server Pod(s) * `resources`: Kubernetes resource requests and limits to set on NFS server containers * `priorityClassName`: Set priority class name for the NFS server Pod(s) * `logLevel`: The log level that NFS-Ganesha servers should output.
- Default value: `NIV_INFO`
- Supported values: `NIV_NULL | NIV_FATAL | NIV_MAJ | NIV_CRIT | NIV_WARN | NIV_EVENT | NIV_INFO | NIV_DEBUG | NIV_MID_DEBUG | NIV_FULL_DEBUG | NB_LOG_LEVEL` + Default value: `NIV_INFO`
+ Supported values: `NIV_NULL | NIV_FATAL | NIV_MAJ | NIV_CRIT | NIV_WARN | NIV_EVENT | NIV_INFO | NIV_DEBUG | NIV_MID_DEBUG | NIV_FULL_DEBUG | NB_LOG_LEVEL` * `hostNetwork`: Whether host networking is enabled for the NFS server pod(s). If not set, the network - settings from the CephCluster CR will be applied. + settings from the CephCluster CR will be applied. ### Security The `security` spec sets security configuration for the NFS cluster. -* `kerberos`: Kerberos configures NFS-Ganesha to secure NFS client connections with Kerberos. - * `principalName`: this value is combined with (a) the namespace and name of the CephNFS (with a - hyphen between) and (b) the Realm configured in the user-provided kerberos config file(s) to - determine the full service principal name: `/-@`. - e.g., nfs/rook-ceph-my-nfs@example.net. For full details, see the - [NFS security doc](../Storage-Configuration/NFS/nfs-security.md#nfs-service-principals). - * `domainName`: this is the domain name used in the kerberos credentials. This is used to - configure idmap to map the kerberos credentials to uid/gid. Without this configured, NFS-Ganesha - will use the anonuid/anongid configured (default: -2) when accessing the local filesystem. - eg., DOMAIN1.EXAMPLE.COM. - [NFS security doc](../Storage-Configuration/NFS/nfs-security.md#kerberos-domain-name). - * `configFiles`: defines where the Kerberos configuration should be sourced from. Config - files will be placed into the `/etc/krb5.conf.rook/` directory. For advanced usage, see the - [NFS security doc](../Storage-Configuration/NFS/nfs-security.md#kerberos-configuration). - * `volumeSource`: this is a standard Kubernetes VolumeSource for Kerberos configuration files - like what is normally used to configure Volumes for a Pod. For example, a ConfigMap, Secret, - or HostPath. The volume may contain multiple files, all of which will be loaded. - * `keytabFile`: defines where the Kerberos keytab should be sourced from. The keytab file will be - placed into `/etc/krb5.keytab`. For advanced usage, see the - [NFS security doc](../Storage-Configuration/NFS/nfs-security.md#kerberos-configuration). - * `volumeSource`: this is a standard Kubernetes VolumeSource for the Kerberos keytab file like - what is normally used to configure Volumes for a Pod. For example, a Secret or HostPath. - There are two requirements for the source's content: - 1. The config file must be mountable via `subPath: krb5.keytab`. For example, in a Secret, the - data item must be named `krb5.keytab`, or `items` must be defined to select the key and - give it path `krb5.keytab`. A HostPath directory must have the `krb5.keytab` file. - 2. The volume or config file must have mode 0600. +* `kerberos`: Kerberos configures NFS-Ganesha to secure NFS client connections with Kerberos. + * `principalName`: this value is combined with (a) the namespace and name of the CephNFS (with a + hyphen between) and (b) the Realm configured in the user-provided kerberos config file(s) to + determine the full service principal name: `/-@`. + e.g., nfs/rook-ceph-my-nfs@example.net. For full details, see the + [NFS security doc](../Storage-Configuration/NFS/nfs-security.md#nfs-service-principals). + * `domainName`: this is the domain name used in the kerberos credentials. This is used to + configure idmap to map the kerberos credentials to uid/gid. Without this configured, NFS-Ganesha + will use the anonuid/anongid configured (default: -2) when accessing the local filesystem. + eg., DOMAIN1.EXAMPLE.COM. + [NFS security doc](../Storage-Configuration/NFS/nfs-security.md#kerberos-domain-name). + * `configFiles`: defines where the Kerberos configuration should be sourced from. Config + files will be placed into the `/etc/krb5.conf.rook/` directory. For advanced usage, see the + [NFS security doc](../Storage-Configuration/NFS/nfs-security.md#kerberos-configuration). + * `volumeSource`: this is a standard Kubernetes VolumeSource for Kerberos configuration files + like what is normally used to configure Volumes for a Pod. For example, a ConfigMap, Secret, + or HostPath. The volume may contain multiple files, all of which will be loaded. + * `keytabFile`: defines where the Kerberos keytab should be sourced from. The keytab file will be + placed into `/etc/krb5.keytab`. For advanced usage, see the + [NFS security doc](../Storage-Configuration/NFS/nfs-security.md#kerberos-configuration). + * `volumeSource`: this is a standard Kubernetes VolumeSource for the Kerberos keytab file like + what is normally used to configure Volumes for a Pod. For example, a Secret or HostPath. -* `sssd`: SSSD enables integration with System Security Services Daemon (SSSD). See also: - [ID mapping via SSSD](../Storage-Configuration/NFS/nfs-security.md#id-mapping-via-sssd). - * `sidecar`: Specifying this configuration tells Rook to run SSSD in a sidecar alongside the NFS - server in each NFS pod. - * `image`: defines the container image that should be used for the SSSD sidecar. - * `sssdConfigFile`: defines where the SSSD configuration should be sourced from. The - config file will be placed into `/etc/sssd/sssd.conf`. For advanced usage, see the - [NFS security doc](../Storage-Configuration/NFS/nfs-security.md#sssd-configuration). - * `volumeSource`: this is a standard Kubernetes + There are two requirements for the source's content: + + 1. The config file must be mountable via `subPath: krb5.keytab`. For example, in a Secret, the + data item must be named `krb5.keytab`, or `items` must be defined to select the key and + give it path `krb5.keytab`. A HostPath directory must have the `krb5.keytab` file. + 2. The volume or config file must have mode 0600. + +* `sssd`: SSSD enables integration with System Security Services Daemon (SSSD). See also: + [ID mapping via SSSD](../Storage-Configuration/NFS/nfs-security.md#id-mapping-via-sssd). + * `sidecar`: Specifying this configuration tells Rook to run SSSD in a sidecar alongside the NFS + server in each NFS pod. + * `image`: defines the container image that should be used for the SSSD sidecar. + * `sssdConfigFile`: defines where the SSSD configuration should be sourced from. The + config file will be placed into `/etc/sssd/sssd.conf`. For advanced usage, see the + [NFS security doc](../Storage-Configuration/NFS/nfs-security.md#sssd-configuration). + * `volumeSource`: this is a standard Kubernetes [VolumeSource](https://pkg.go.dev/k8s.io/api/core/v1#VolumeSource) like what is normally used to configure Volumes for a Pod. For example, a ConfigMap, Secret, or HostPath. There are two requirements for the source's content: - 1. The config file must be mountable via `subPath: sssd.conf`. For example, in a ConfigMap, - the data item must be named `sssd.conf`, or `items` must be defined to select the key and - give it path `sssd.conf`. A HostPath directory must have the `sssd.conf` file. - 2. The volume or config file must have mode 0600. - * `additionalFiles`: adds any number of additional files into the SSSD sidecar. All files will - be placed into `/etc/sssd/rook-additional/` and can be referenced by the SSSD - config file. For example, CA and/or TLS certificates to authenticate with Kerberos. - - `subPath`: the sub-path of `/etc/sssd/rook-additional` to add files into. This can - include `/` to create arbitrarily deep sub-paths if desired. If the `volumeSource` is a - file, this will refer to a file name. - - `volumeSource`: this is a standard Kubernetes VolumeSource for additional files like what is - normally used to configure Volumes for a Pod. For example, a ConfigMap, Secret, or HostPath. - The volume may contain multiple files, a single file, or may be a file on its own (e.g., a - host path with `type: File`). - * `debugLevel`: sets the debug level for SSSD. If unset or `0`, Rook does nothing. Otherwise, - this may be a value between 1 and 10. See the - [SSSD docs](https://sssd.io/troubleshooting/basics.html#sssd-debug-logs) for more info. - * `resources`: Kubernetes resource requests and limits to set on NFS server containers + 1. The config file must be mountable via `subPath: sssd.conf`. For example, in a ConfigMap, + the data item must be named `sssd.conf`, or `items` must be defined to select the key and + give it path `sssd.conf`. A HostPath directory must have the `sssd.conf` file. + 2. The volume or config file must have mode 0600. + * `additionalFiles`: adds any number of additional files into the SSSD sidecar. All files will + be placed into `/etc/sssd/rook-additional/` and can be referenced by the SSSD + config file. For example, CA and/or TLS certificates to authenticate with Kerberos. + - `subPath`: the sub-path of `/etc/sssd/rook-additional` to add files into. This can + include `/` to create arbitrarily deep sub-paths if desired. If the `volumeSource` is a + file, this will refer to a file name. + - `volumeSource`: this is a standard Kubernetes VolumeSource for additional files like what is + normally used to configure Volumes for a Pod. For example, a ConfigMap, Secret, or HostPath. + The volume may contain multiple files, a single file, or may be a file on its own (e.g., a + host path with `type: File`). + * `debugLevel`: sets the debug level for SSSD. If unset or `0`, Rook does nothing. Otherwise, + this may be a value between 1 and 10. See the + [SSSD docs](https://sssd.io/troubleshooting/basics.html#sssd-debug-logs) for more info. + * `resources`: Kubernetes resource requests and limits to set on NFS server containers ## Scaling the active server count @@ -188,19 +190,19 @@ the size of the cluster. ### server.active count greater than 1 * Active-active scale out does not work well with the NFS protocol. If one NFS server in a cluster - is offline, other servers may block client requests until the offline server returns, which may - not always happen due to the Kubernetes scheduler. - * Workaround: It is safest to run only a single NFS server, but we do not limit this if it - benefits your use case. + is offline, other servers may block client requests until the offline server returns, which may + not always happen due to the Kubernetes scheduler. + * Workaround: It is safest to run only a single NFS server, but we do not limit this if it + benefits your use case. ### Ceph v17.2.1 * Ceph NFS management with the Rook mgr module enabled has a breaking regression with the Ceph - Quincy v17.2.1 release. - * Workaround: Leave Ceph's Rook orchestrator mgr module disabled. If you have enabled it, you must - disable it using the snippet below from the toolbox. + Quincy v17.2.1 release. + * Workaround: Leave Ceph's Rook orchestrator mgr module disabled. If you have enabled it, you must + disable it using the snippet below from the toolbox. - ```console - ceph orch set backend "" - ceph mgr module disable rook - ``` + ```console + ceph orch set backend "" + ceph mgr module disable rook + ``` diff --git a/Documentation/Contributing/development-environment.md b/Documentation/Contributing/development-environment.md index 2a15fca12..40ac7985c 100644 --- a/Documentation/Contributing/development-environment.md +++ b/Documentation/Contributing/development-environment.md @@ -67,23 +67,23 @@ on their minikube cluster. 1) Set the local Docker environment to use minikube: -```console -eval $(minikube docker-env -p minikube) -``` + ```console + eval $(minikube docker-env -p minikube) + ``` 2) Build your local Rook image. The following command will generate a Rook image labeled in the format `local/ceph-`. -```console -cd -make BUILD_REGISTRY=local -``` + ```console + cd + make BUILD_REGISTRY=local + ``` 3) Tag the generated image as `rook/ceph:master` so operator will pick it. -```console -docker tag "local/ceph-$(go env GOARCH)" 'rook/ceph:master' -``` + ```console + docker tag "local/ceph-$(go env GOARCH)" 'rook/ceph:master' + ``` 4) Create a Rook cluster in minikube, or if the Rook cluster is already configured, apply the new operator image by restarting the operator. diff --git a/Documentation/Contributing/development-flow.md b/Documentation/Contributing/development-flow.md index aeab825e3..ffba3daf4 100644 --- a/Documentation/Contributing/development-flow.md +++ b/Documentation/Contributing/development-flow.md @@ -326,23 +326,23 @@ Make modifications directly in the manager module and reload: 2. Shell into the manager container: -```console -kubectl exec -n rook-ceph --stdin --tty $(kubectl get pod -n rook-ceph -l ceph_daemon_type=mgr,instance=a -o jsonpath='{.items[0].metadata.name}') -c mgr -- /bin/bash -``` + ```console + kubectl exec -n rook-ceph --stdin --tty $(kubectl get pod -n rook-ceph -l ceph_daemon_type=mgr,instance=a -o jsonpath='{.items[0].metadata.name}') -c mgr -- /bin/bash + ``` 3. Make the modifications needed in the required manager module. The manager module source code is found in `/usr/share/ceph/mgr/`. -!!! Note - If the manager pod is restarted, all modifications made in the mgr container will be lost + !!! Note + If the manager pod is restarted, all modifications made in the mgr container will be lost -1. Restart the modified manager module to test the modifications: +4. Restart the modified manager module to test the modifications: -Example for restarting the rook manager module with the [kubectl plugin](https://github.com/rook/kubectl-rook-ceph): + Example for restarting the rook manager module with the [kubectl plugin](https://github.com/rook/kubectl-rook-ceph): -```console -kubectl rook-ceph ceph mgr module disable rook -kubectl rook-ceph ceph mgr module enable rook -``` + ```console + kubectl rook-ceph ceph mgr module disable rook + kubectl rook-ceph ceph mgr module enable rook + ``` -Once the module is restarted the modifications will be running in the active manager. -View the manager pod log or other changed behavior to validate the changes. +5. Once the module is restarted the modifications will be running in the active manager. + View the manager pod log or other changed behavior to validate the changes. diff --git a/Documentation/Contributing/documentation.md b/Documentation/Contributing/documentation.md index aa68f94c2..ef9e1d370 100644 --- a/Documentation/Contributing/documentation.md +++ b/Documentation/Contributing/documentation.md @@ -27,16 +27,12 @@ make docs-preview When previewing, now you can navigate your browser to [http://127.0.0.1:8000/](http://127.0.0.1:8000/) to open the preview of the documentation. !!! hint - Should you encounter a `command not found` error while trying to preview the docs for the first time on a machine, you probably need to install the dependencies for MkDocs and extensions used. - - ```console - pip3 install -r build/release/requirements_docs.txt - ``` - - Please make sure that your Python binary path is included in your `PATH`. + Should you encounter a `command not found` error while trying to preview the docs for the first + time on a machine, you probably need to install the dependencies for MkDocs and extensions used: + `pip3 install -r build/release/requirements_docs.txt`. Make sure that your Python binary path is + included in your `PATH`. ## Making docs -[helm-docs](https://github.com/norwoodj/helm-docs) is a tool that generates the documentation for a helm chart automatically. If there are changes in the helm chart, the developer needs to run `make docs` (to run helm-docs) and check in the resulting autogenerated files. +[helm-docs](https://github.com/norwoodj/helm-docs) is a tool that generates the documentation for a helm chart automatically. If there are changes in the helm chart, the developer needs to run `make docs` (to run helm-docs) and check in the resulting autogenerated files. To make it easy to check locally for uncommitted changes generated by `make docs`, an additional `make` target exists: simply running `make check-docs` will run the docs auto-generation and will complain if this produces uncommitted changes to doc files. It is therefore a good habit to always run `make check-docs` locally before creating or updating a PR. - diff --git a/Documentation/Contributing/rook-test-framework.md b/Documentation/Contributing/rook-test-framework.md index e2f941beb..bd8402f6f 100644 --- a/Documentation/Contributing/rook-test-framework.md +++ b/Documentation/Contributing/rook-test-framework.md @@ -93,10 +93,10 @@ go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integra 3. Only the `CephSmokeSuite` is currently supported on OpenShift. 4. Set the following environment variables depending on the environment: -```console -export TEST_ENV_NAME=openshift -export TEST_STORAGE_CLASS=gp2 -export TEST_BASE_DIR=/tmp -``` + ```console + export TEST_ENV_NAME=openshift + export TEST_STORAGE_CLASS=gp2 + export TEST_BASE_DIR=/tmp + ``` 5. Run the integration tests diff --git a/Documentation/Getting-Started/Prerequisites/authenticated-registry.md b/Documentation/Getting-Started/Prerequisites/authenticated-registry.md index e9f346dad..c382db2b3 100644 --- a/Documentation/Getting-Started/Prerequisites/authenticated-registry.md +++ b/Documentation/Getting-Started/Prerequisites/authenticated-registry.md @@ -38,7 +38,7 @@ Since it's the same procedure for all service accounts, here is just one example kubectl -n rook-ceph edit serviceaccount rook-ceph-default ``` -```yaml hl_lines="9-10" +```yaml apiVersion: v1 kind: ServiceAccount metadata: diff --git a/Documentation/Getting-Started/example-configurations.md b/Documentation/Getting-Started/example-configurations.md index b48d992be..3488ffab7 100644 --- a/Documentation/Getting-Started/example-configurations.md +++ b/Documentation/Getting-Started/example-configurations.md @@ -23,9 +23,9 @@ The examples all assume the operator and all Ceph daemons will be started in the After the common resources are created, the next step is to create the Operator deployment. Several spec file examples are provided in [this directory](https://github.com/rook/rook/blob/master/deploy/examples/): * [`operator.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/operator.yaml): The most common settings for production deployments - * `kubectl create -f operator.yaml` + * `kubectl create -f operator.yaml` * [`operator-openshift.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/operator-openshift.yaml): Includes all of the operator settings for running a basic Rook cluster in an OpenShift environment. You will also want to review the [OpenShift Prerequisites](../Getting-Started/ceph-openshift.md) to confirm the settings. - * `oc create -f operator-openshift.yaml` + * `oc create -f operator-openshift.yaml` Settings for the operator are configured through environment variables on the operator deployment. The individual settings are documented in [operator.yaml](https://github.com/rook/rook/blob/master/deploy/examples/operator.yaml). @@ -40,7 +40,9 @@ the cluster. These examples represent several different ways to configure the st * [`cluster-on-pvc.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/cluster-on-pvc.yaml): Common settings for backing the Ceph Mons and OSDs by PVs. Useful when running in cloud environments or where local PVs have been created for Ceph to consume. * [`cluster-external.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/external/cluster-external.yaml): Connect to an [external Ceph cluster](../CRDs/Cluster/ceph-cluster-crd.md#external-cluster) with minimal access to monitor the health of the cluster and connect to the storage. * [`cluster-external-management.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/cluster-external-management.yaml): Connect to an [external Ceph cluster](../CRDs/Cluster/ceph-cluster-crd.md#external-cluster) with the admin key of the external cluster to enable - remote creation of pools and configure services such as an [Object Store](../Storage-Configuration/Object-Storage-RGW/object-storage.md) or a [Shared Filesystem](../Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md). + remote creation of pools and configure services such as an + [Object Store](../Storage-Configuration/Object-Storage-RGW/object-storage.md) or a + [Shared Filesystem](../Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md). * [`cluster-stretched.yaml`](https://github.com/rook/rook/blob/master/deploy/examples/cluster-stretched.yaml): Create a cluster in "stretched" mode, with five mons stretched across three zones, and the OSDs across two zones. See the [Stretch documentation](../CRDs/Cluster/ceph-cluster-crd.md#stretch-cluster). See the [Cluster CRD](../CRDs/Cluster/ceph-cluster-crd.md) topic for more details and more examples for the settings. diff --git a/Documentation/Getting-Started/quickstart.md b/Documentation/Getting-Started/quickstart.md index 987108c9b..7c95603de 100644 --- a/Documentation/Getting-Started/quickstart.md +++ b/Documentation/Getting-Started/quickstart.md @@ -187,7 +187,7 @@ The Rook maintainers would like to receive telemetry reports for Rook clusters. The data is anonymous and does not include any identifying information. Enable the telemetry reporting feature with the following command in the toolbox: -``` +```console ceph telemetry on ``` diff --git a/Documentation/Storage-Configuration/Advanced/ceph-configuration.md b/Documentation/Storage-Configuration/Advanced/ceph-configuration.md index 86a0d4048..399b78a24 100644 --- a/Documentation/Storage-Configuration/Advanced/ceph-configuration.md +++ b/Documentation/Storage-Configuration/Advanced/ceph-configuration.md @@ -211,8 +211,7 @@ ceph osd pool set rbd pg_num 512 !!! warning The advised method for controlling Ceph configuration is to use the [`cephConfig:` structure](../../CRDs/Cluster/ceph-cluster-crd.md#ceph-config) in the `CephCluster` CRD. - - It is highly recommended that this only be used when absolutely necessary and that the `config` be +

It is highly recommended that this only be used when absolutely necessary and that the `config` be reset to an empty string if/when the configurations are no longer necessary. Configurations in the config file will make the Ceph cluster less configurable from the CLI and dashboard and may make future tuning or debugging difficult. @@ -235,7 +234,7 @@ has been initialized, each daemon will need to be restarted where you want the s * mons: ensure all three mons are online and healthy before restarting each mon pod, one at a time. * mgrs: the pods are stateless and can be restarted as needed, but note that this will disrupt the - Ceph dashboard during restart. + Ceph dashboard during restart. * OSDs: restart your the pods by deleting them, one at a time, and running `ceph -s` between each restart to ensure the cluster goes back to "active/clean" state. * RGW: the pods are stateless and can be restarted as needed. @@ -358,10 +357,10 @@ sizes vary. This should work for most use-cases, but the following situations could warrant weight changes: * Your cluster has some relatively slow OSDs or nodes. Lowering their weight can - reduce the impact of this bottleneck. + reduce the impact of this bottleneck. * You're using bluestore drives provisioned with Rook v0.3.1 or older. In this - case you may notice OSD weights did not get set relative to their storage - capacity. Changing the weight can fix this and maximize cluster capacity. + case you may notice OSD weights did not get set relative to their storage + capacity. Changing the weight can fix this and maximize cluster capacity. This example sets the weight of osd.0 which is 600GiB diff --git a/Documentation/Storage-Configuration/Advanced/ceph-mon-health.md b/Documentation/Storage-Configuration/Advanced/ceph-mon-health.md index ce80aa6fc..f39c931b3 100644 --- a/Documentation/Storage-Configuration/Advanced/ceph-mon-health.md +++ b/Documentation/Storage-Configuration/Advanced/ceph-mon-health.md @@ -118,7 +118,9 @@ $ ceph -s ## Automatic Monitor Failover -Rook will automatically fail over the mons when the following settings are updated in the CephCluster CR: +Rook will automatically fail over the mons when the following settings are updated in the +CephCluster CR: + - `spec.network.hostNetwork`: When enabled or disabled, Rook fails over all monitors, configuring them to enable or disable host networking. - `spec.network.Provider` : When updated from being empty to "host", Rook fails over all monitors, configuring them to enable or disable host networking. - `spec.network.multiClusterService`: When enabled or disabled, Rook fails over all monitors, configuring them to start (or stop) using service IPs compatible with the multi-cluster service. diff --git a/Documentation/Storage-Configuration/Advanced/configuration.md b/Documentation/Storage-Configuration/Advanced/configuration.md index b4be4ad26..0f17bbc7b 100644 --- a/Documentation/Storage-Configuration/Advanced/configuration.md +++ b/Documentation/Storage-Configuration/Advanced/configuration.md @@ -48,8 +48,7 @@ spec: pgp_num: '128' # this should at least match `pg_num` so that all PGs are used ``` - Ceph [OSD and Pool config -docs](https://docs.ceph.com/docs/master/rados/operations/placement-groups/#a-preselection-of-pg-num) +Ceph [OSD and Pool config docs](https://docs.ceph.com/docs/master/rados/operations/placement-groups/#a-preselection-of-pg-num) provide detailed information about how to tune these parameters. Nautilus [introduced the PG auto-scaler mgr module](https://ceph.com/rados/new-in-nautilus-pg-merging-and-autotuning/) diff --git a/Documentation/Storage-Configuration/Advanced/key-management-system.md b/Documentation/Storage-Configuration/Advanced/key-management-system.md index a56406946..841beb39b 100644 --- a/Documentation/Storage-Configuration/Advanced/key-management-system.md +++ b/Documentation/Storage-Configuration/Advanced/key-management-system.md @@ -2,7 +2,10 @@ title: Key Management System --- -Rook has the ability to encrypt OSDs of clusters running on PVC via the flag (`encrypted: true`) in your `storageClassDeviceSets` [template](#pvc-based-cluster). + + + +Rook has the ability to encrypt OSDs of clusters running on PVC via the flag (`encrypted: true`) in your `storageClassDeviceSets` [template](../../CRDs/Cluster/pvc-cluster.md). Rook also has the ability to rotate encryption keys of OSDs using a cron job per OSD. By default, the Key Encryption Keys (also known as Data Encryption Keys) are stored in a Kubernetes Secret. However, if a Key Management System exists Rook is capable of using it. @@ -10,30 +13,30 @@ However, if a Key Management System exists Rook is capable of using it. The `security` section contains settings related to encryption of the cluster. * `security`: - * `kms`: Key Management System settings - * `connectionDetails`: the list of parameters representing kms connection details - * `tokenSecretName`: the name of the Kubernetes Secret containing the kms authentication token - * `keyRotation`: Key Rotation settings - * `enabled`: whether key rotation is enabled or not, default is `false` - * `schedule`: the schedule, written in [cron format](https://en.wikipedia.org/wiki/Cron), with which key rotation [CronJob](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/) is created, default value is `"@weekly"`. + * `kms`: Key Management System settings + * `connectionDetails`: the list of parameters representing kms connection details + * `tokenSecretName`: the name of the Kubernetes Secret containing the kms authentication token + * `keyRotation`: Key Rotation settings + * `enabled`: whether key rotation is enabled or not, default is `false` + * `schedule`: the schedule, written in [cron format](https://en.wikipedia.org/wiki/Cron), with which key rotation [CronJob](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/) is created, default value is `"@weekly"`. !!! note Currently key rotation is only supported for the default type, where the Key Encryption Keys are stored in a Kubernetes Secret. Supported KMS providers: -* [Vault](#vault) - * [Authentication methods](#authentication-methods) - * [Token-based authentication](#token-based-authentication) - * [Kubernetes-based authentication](#kubernetes-based-authentication) - * [General Vault configuration](#general-vault-configuration) - * [TLS configuration](#tls-configuration) -* [IBM Key Protect](#ibm-key-protect) - * [Configuration](#configuration) -* [Key Management Interoperability Protocol](#key-management-interoperability-protocol) - * [Configuration](#configuration-1) -* [Azure Key Vault](#azure-key-vault) - * [Client Authentication](#client-authentication) +- [Vault](#vault) + - [Authentication methods](#authentication-methods) + - [Token-based authentication](#token-based-authentication) + - [Kubernetes-based authentication](#kubernetes-based-authentication) + - [General Vault configuration](#general-vault-configuration) + - [TLS configuration](#tls-configuration) +- [IBM Key Protect](#ibm-key-protect) + - [Configuration](#configuration) +- [Key Management Interoperability Protocol](#key-management-interoperability-protocol) + - [Configuration](#configuration-1) +- [Azure Key Vault](#azure-key-vault) + - [Client Authentication](#client-authentication) ## Vault @@ -44,12 +47,12 @@ Rook supports storing OSD encryption keys in [HashiCorp Vault KMS](https://www.v Rook support two authentication methods: * [token-based](#token-based-authentication): a token is provided by the user and is stored in a Kubernetes Secret. It's used to - authenticate the KMS by the Rook operator. This has several pitfalls such as: - * when the token expires it must be renewed, so the secret holding it must be updated - * no token automatic rotation + authenticate the KMS by the Rook operator. This has several pitfalls such as: + * when the token expires it must be renewed, so the secret holding it must be updated + * no token automatic rotation * [Kubernetes Service Account](#kubernetes-based-authentication) uses [Vault Kubernetes native - authentication](https://www.vaultproject.io/docs/auth/kubernetes) mechanism and alleviate some of the limitations from the token authentication such as token automatic renewal. This method is - generally recommended over the token-based authentication. + authentication](https://www.vaultproject.io/docs/auth/kubernetes) mechanism and alleviate some of the limitations from the token authentication such as token automatic renewal. This method is + generally recommended over the token-based authentication. #### Token-based authentication @@ -289,9 +292,9 @@ security: More options are supported such as: * `IBM_BASE_URL`: the base URL of the Key Protect instance, depending on your - [region](https://cloud.ibm.com/docs/key-protect?topic=key-protect-regions). Defaults to `https://us-south.kms.cloud.ibm.com`. + [region](https://cloud.ibm.com/docs/key-protect?topic=key-protect-regions). Defaults to `https://us-south.kms.cloud.ibm.com`. * `IBM_TOKEN_URL`: the URL of the Key Protect instance to retrieve the token. Defaults to - `https://iam.cloud.ibm.com/oidc/token`. Only needed for private instances. + `https://iam.cloud.ibm.com/oidc/token`. Only needed for private instances. ## Key Management Interoperability Protocol @@ -346,14 +349,14 @@ Rook supports storing OSD encryption keys in [Azure Key vault](https://learn.mic Different methods are available in Azure to authenticate a client. Rook supports Azure recommended method of authentication with Service Principal and a certificate. Refer the following Azure documentation to set up key vault and authenticate it via service principal and certtificate * [Create Azure Key Vault](https://learn.microsoft.com/en-us/azure/key-vault/general/quick-create-portal) - * `AZURE_VAULT_URL` can be retrieved at this step + * `AZURE_VAULT_URL` can be retrieved at this step * [Create Service Principal](https://learn.microsoft.com/en-us/entra/identity-platform/howto-create-service-principal) - * `AZURE_CLIENT_ID` and `AZURE_TENANT_ID` can be obtained after creating the service principal - * Ensure that the service principal is authenticated with a certificate and not with a client secret. + * `AZURE_CLIENT_ID` and `AZURE_TENANT_ID` can be obtained after creating the service principal + * Ensure that the service principal is authenticated with a certificate and not with a client secret. * [Set Azure Key Vault RBAC](https://learn.microsoft.com/en-us/azure/key-vault/general/rbac-guide?tabs=azure-cli#enable-azure-rbac-permissions-on-key-vault) - * Ensure that the role assigned to the key vault should be able to create, retrieve and delete secrets in the key vault. + * Ensure that the role assigned to the key vault should be able to create, retrieve and delete secrets in the key vault. Provide the following KMS connection details in order to connect with Azure Key Vault. diff --git a/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-async-disaster-recovery-failover-failback.md b/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-async-disaster-recovery-failover-failback.md index 36f8388dd..f18bd48ff 100644 --- a/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-async-disaster-recovery-failover-failback.md +++ b/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-async-disaster-recovery-failover-failback.md @@ -21,39 +21,37 @@ The following document will help to track the procedure for failover and failbac ### Relocation The Relocation operation is the process of switching production to a - backup facility(normally your recovery site) or vice versa. For relocation, - access to the image on the primary site should be stopped. +backup facility(normally your recovery site) or vice versa. For relocation, +access to the image on the primary site should be stopped. The image should now be made *primary* on the secondary cluster so that - the access can be resumed there. +the access can be resumed there. !!! note :memo: Periodic or one-time backup of the application should be available for restore on the secondary site (cluster-2). Follow the below steps for planned migration of workload from the primary - cluster to the secondary cluster: +cluster to the secondary cluster: * Scale down all the application pods which are using the - mirrored PVC on the Primary Cluster. + mirrored PVC on the Primary Cluster. * [Take a backup](rbd-mirroring.md#backup-&-restore) of PVC and PV object from the primary cluster. - This can be done using some backup tools like - [velero](https://velero.io/docs/main/). + This can be done using some backup tools like + [velero](https://velero.io/docs/main/). * [Update VolumeReplication CR](rbd-mirroring.md#create-a-volumereplication-cr) to set `replicationState` to `secondary` at the Primary Site. - When the operator sees this change, it will pass the information down to the - driver via GRPC request to mark the dataSource as `secondary`. + When the operator sees this change, it will pass the information down to the + driver via GRPC request to mark the dataSource as `secondary`. * If you are manually recreating the PVC and PV on the secondary cluster, - remove the `claimRef` section in the PV objects. (See [this](rbd-mirroring.md#restore-the-backup-on-cluster-2) for details) + remove the `claimRef` section in the PV objects. (See [this](rbd-mirroring.md#restore-the-backup-on-cluster-2) for details) * Recreate the storageclass, PVC, and PV objects on the secondary site. * As you are creating the static binding between PVC and PV, a new PV won’t - be created here, the PVC will get bind to the existing PV. + be created here, the PVC will get bind to the existing PV. * [Create the VolumeReplicationClass](rbd-mirroring.md#create-a-volume-replication-class-cr) on the secondary site. -* [Create VolumeReplications](rbd-mirroring.md#create-a-volumereplication-cr) for all the PVC’s for which mirroring - is enabled - * `replicationState` should be `primary` for all the PVC’s on - the secondary site. +* [Create VolumeReplications](rbd-mirroring.md#create-a-volumereplication-cr) for all the PVC’s for which mirroring is enabled + * `replicationState` should be `primary` for all the PVC’s on the secondary site. * [Check VolumeReplication CR status](rbd-mirroring.md#checking-replication-status) to verify if the image is marked `primary` on the secondary site. * Once the Image is marked as `primary`, the PVC is now ready - to be used. Now, we can scale up the applications to use the PVC. + to be used. Now, we can scale up the applications to use the PVC. !!! warning :memo: In Async Disaster recovery use case, we don't get @@ -72,35 +70,35 @@ Follow the below steps for planned migration of workload from the primary ### Failover (abrupt shutdown) In case of Disaster recovery, create VolumeReplication CR at the Secondary Site. - Since the connection to the Primary Site is lost, the operator automatically - sends a GRPC request down to the driver to forcefully mark the dataSource as `primary` - on the Secondary Site. +Since the connection to the Primary Site is lost, the operator automatically +sends a GRPC request down to the driver to forcefully mark the dataSource as `primary` +on the Secondary Site. * If you are manually creating the PVC and PV on the secondary cluster, remove - the claimRef section in the PV objects. (See [this](rbd-mirroring.md#restore-the-backup-on-cluster-2) for details) + the claimRef section in the PV objects. (See [this](rbd-mirroring.md#restore-the-backup-on-cluster-2) for details) * Create the storageclass, PVC, and PV objects on the secondary site. * As you are creating the static binding between PVC and PV, a new PV won’t be - created here, the PVC will get bind to the existing PV. + created here, the PVC will get bind to the existing PV. * [Create the VolumeReplicationClass](rbd-mirroring.md#create-a-volume-replication-class-cr) and [VolumeReplication CR](rbd-mirroring.md#create-a-volumereplication-cr) on the secondary site. * [Check VolumeReplication CR status](rbd-mirroring.md#checking-replication-status) to verify if the image is marked `primary` on the secondary site. * Once the Image is marked as `primary`, the PVC is now ready to be used. Now, - we can scale up the applications to use the PVC. + we can scale up the applications to use the PVC. ### Failback (post-disaster recovery) Once the failed cluster is recovered on the primary site and you want to failback - from secondary site, follow the below steps: +from secondary site, follow the below steps: * Scale down the running applications (if any) on the primary site. - Ensure that all persistent volumes in use by the workload are no - longer in use on the primary cluster. + Ensure that all persistent volumes in use by the workload are no + longer in use on the primary cluster. * [Update VolumeReplication CR](rbd-mirroring.md#create-a-volumereplication-cr) replicationState - from `primary` to `secondary` on the primary site. + from `primary` to `secondary` on the primary site. * Scale down the applications on the secondary site. * [Update VolumeReplication CR](rbd-mirroring.md#create-a-volumereplication-cr) replicationState state from `primary` to - `secondary` in secondary site. + `secondary` in secondary site. * On the primary site, [verify the VolumeReplication status](rbd-mirroring.md#checking-replication-status) is marked as - volume ready to use. + volume ready to use. * Once the volume is marked to ready to use, change the replicationState state - from `secondary` to `primary` in primary site. + from `secondary` to `primary` in primary site. * Scale up the applications again on the primary site. diff --git a/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-mirroring.md b/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-mirroring.md index 270edaf8b..2070bf71e 100644 --- a/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-mirroring.md +++ b/Documentation/Storage-Configuration/Block-Storage-RBD/rbd-mirroring.md @@ -12,16 +12,16 @@ One of the solutions, to achieve the same, is [RBD mirroring](https://docs.ceph. ## RBD Mirroring [RBD mirroring](https://docs.ceph.com/en/latest/rbd/rbd-mirroring/) - is an asynchronous replication of RBD images between multiple Ceph clusters. - This capability is available in two modes: +is an asynchronous replication of RBD images between multiple Ceph clusters. +This capability is available in two modes: * Journal-based: Every write to the RBD image is first recorded - to the associated journal before modifying the actual image. - The remote cluster will read from this associated journal and - replay the updates to its local image. + to the associated journal before modifying the actual image. + The remote cluster will read from this associated journal and + replay the updates to its local image. * Snapshot-based: This mode uses periodically scheduled or - manually created RBD image mirror-snapshots to replicate - crash-consistent RBD images between clusters. + manually created RBD image mirror-snapshots to replicate + crash-consistent RBD images between clusters. !!! note This document sheds light on rbd mirroring and how to set it up using rook. @@ -30,31 +30,30 @@ One of the solutions, to achieve the same, is [RBD mirroring](https://docs.ceph. ## Create RBD Pools In this section, we create specific RBD pools that are RBD mirroring - enabled for use with the DR use case. +enabled for use with the DR use case. -Execute the following steps on each peer cluster to create mirror - enabled pools: +Execute the following steps on each peer cluster to create mirror enabled pools: * Create a RBD pool that is enabled for mirroring by adding the section - `spec.mirroring` in the CephBlockPool CR: + `spec.mirroring` in the CephBlockPool CR: -```yaml -apiVersion: ceph.rook.io/v1 -kind: CephBlockPool -metadata: - name: mirrored-pool - namespace: rook-ceph -spec: - replicated: - size: 1 - mirroring: - enabled: true - mode: image -``` + ```yaml + apiVersion: ceph.rook.io/v1 + kind: CephBlockPool + metadata: + name: mirrored-pool + namespace: rook-ceph + spec: + replicated: + size: 1 + mirroring: + enabled: true + mode: image + ``` -```console -kubectl create -f pool-mirrored.yaml -``` + ```console + kubectl create -f pool-mirrored.yaml + ``` * Repeat the steps on the peer cluster. @@ -71,10 +70,9 @@ See the [CephBlockPool documentation](../../CRDs/Block-Storage/ceph-block-pool-c ## Bootstrap Peers In order for the rbd-mirror daemon to discover its peer cluster, the - peer must be registered and a user account must be created. +peer must be registered and a user account must be created. -The following steps enable bootstrapping peers to discover and - authenticate to each other: +The following steps enable bootstrapping peers to discover and authenticate to each other: * For Bootstrapping a peer cluster its bootstrap secret is required. To determine the name of the secret that contains the bootstrap secret execute the following command on the remote cluster (cluster-2) @@ -86,64 +84,63 @@ Here, `pool-peer-token-mirrored-pool` is the desired bootstrap secret name. * The secret pool-peer-token-mirrored-pool contains all the information related to the token and needs to be injected to the peer, to fetch the decoded secret: -```console -[cluster-2]$ kubectl get secret -n rook-ceph pool-peer-token-mirrored-pool -o jsonpath='{.data.token}'|base64 -d -eyJmc2lkIjoiNGQ1YmNiNDAtNDY3YS00OWVkLThjMGEtOWVhOGJkNDY2OTE3IiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFDZ3hmZGdxN013R0JBQWZzcUtCaGpZVjJUZDRxVzJYQm5kemc9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMzkuMzY6MzMwMCx2MToxOTIuMTY4LjM5LjM2OjY3ODldIn0= -``` + ```console + [cluster-2]$ kubectl get secret -n rook-ceph pool-peer-token-mirrored-pool -o jsonpath='{.data.token}'|base64 -d + eyJmc2lkIjoiNGQ1YmNiNDAtNDY3YS00OWVkLThjMGEtOWVhOGJkNDY2OTE3IiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFDZ3hmZGdxN013R0JBQWZzcUtCaGpZVjJUZDRxVzJYQm5kemc9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMzkuMzY6MzMwMCx2MToxOTIuMTY4LjM5LjM2OjY3ODldIn0= + ``` * With this Decoded value, create a secret on the primary site (cluster-1): -```console -[cluster-1]$ kubectl -n rook-ceph create secret generic rbd-primary-site-secret --from-literal=token=eyJmc2lkIjoiNGQ1YmNiNDAtNDY3YS00OWVkLThjMGEtOWVhOGJkNDY2OTE3IiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFDZ3hmZGdxN013R0JBQWZzcUtCaGpZVjJUZDRxVzJYQm5kemc9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMzkuMzY6MzMwMCx2MToxOTIuMTY4LjM5LjM2OjY3ODldIn0= --from-literal=pool=mirrored-pool -``` + ```console + [cluster-1]$ kubectl -n rook-ceph create secret generic rbd-primary-site-secret --from-literal=token=eyJmc2lkIjoiNGQ1YmNiNDAtNDY3YS00OWVkLThjMGEtOWVhOGJkNDY2OTE3IiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFDZ3hmZGdxN013R0JBQWZzcUtCaGpZVjJUZDRxVzJYQm5kemc9PSIsIm1vbl9ob3N0IjoiW3YyOjE5Mi4xNjguMzkuMzY6MzMwMCx2MToxOTIuMTY4LjM5LjM2OjY3ODldIn0= --from-literal=pool=mirrored-pool + ``` * This completes the bootstrap process for cluster-1 to be peered with cluster-2. * Repeat the process switching cluster-2 in place of cluster-1, to complete the bootstrap process across both peer clusters. For more details, refer to the official rbd mirror documentation on - [how to create a bootstrap peer](https://docs.ceph.com/en/latest/rbd/rbd-mirroring/#bootstrap-peers). +[how to create a bootstrap peer](https://docs.ceph.com/en/latest/rbd/rbd-mirroring/#bootstrap-peers). ## Configure the RBDMirror Daemon Replication is handled by the rbd-mirror daemon. The rbd-mirror daemon - is responsible for pulling image updates from the remote, peer cluster, - and applying them to image within the local cluster. +is responsible for pulling image updates from the remote, peer cluster, +and applying them to image within the local cluster. -Creation of the rbd-mirror daemon(s) is done through the custom resource - definitions (CRDs), as follows: +Creation of the rbd-mirror daemon(s) is done through the custom resource definitions (CRDs), as follows: * Create mirror.yaml, to deploy the rbd-mirror daemon -```yaml -apiVersion: ceph.rook.io/v1 -kind: CephRBDMirror -metadata: - name: my-rbd-mirror - namespace: rook-ceph -spec: - # the number of rbd-mirror daemons to deploy - count: 1 -``` + ```yaml + apiVersion: ceph.rook.io/v1 + kind: CephRBDMirror + metadata: + name: my-rbd-mirror + namespace: rook-ceph + spec: + # the number of rbd-mirror daemons to deploy + count: 1 + ``` * Create the RBD mirror daemon -```console -[cluster-1]$ kubectl create -f mirror.yaml -n rook-ceph -``` + ```console + [cluster-1]$ kubectl create -f mirror.yaml -n rook-ceph + ``` * Validate if `rbd-mirror` daemon pod is now up -```console -[cluster-1]$ kubectl get pods -n rook-ceph -rook-ceph-rbd-mirror-a-6985b47c8c-dpv4k 1/1 Running 0 10s -``` + ```console + [cluster-1]$ kubectl get pods -n rook-ceph + rook-ceph-rbd-mirror-a-6985b47c8c-dpv4k 1/1 Running 0 10s + ``` * Verify that daemon health is OK -```console -kubectl get cephblockpools.ceph.rook.io mirrored-pool -n rook-ceph -o jsonpath='{.status.mirroringStatus.summary}' -{"daemon_health":"OK","health":"OK","image_health":"OK","states":{"replaying":1}} -``` + ```console + kubectl get cephblockpools.ceph.rook.io mirrored-pool -n rook-ceph -o jsonpath='{.status.mirroringStatus.summary}' + {"daemon_health":"OK","health":"OK","image_health":"OK","states":{"replaying":1}} + ``` * Repeat the above steps on the peer cluster. @@ -172,41 +169,40 @@ kubectl create -f https://raw.githubusercontent.com/csi-addons/kubernetes-csi-ad ## Enable CSI Replication Sidecars To achieve RBD Mirroring, `csi-omap-generator` and `csi-addons` - containers need to be deployed in the RBD provisioner pods, which are not enabled by default. +containers need to be deployed in the RBD provisioner pods, which are not enabled by default. * **Omap Generator**: Omap generator is a sidecar container that when - deployed with the CSI provisioner pod, generates the internal CSI - omaps between the PV and the RBD image. This is required as static PVs are - transferred across peer clusters in the DR use case, and hence - is needed to preserve PVC to storage mappings. + deployed with the CSI provisioner pod, generates the internal CSI + omaps between the PV and the RBD image. This is required as static PVs are + transferred across peer clusters in the DR use case, and hence + is needed to preserve PVC to storage mappings. * **Volume Replication Operator**: Volume Replication Operator is a - kubernetes operator that provides common and reusable APIs for - storage disaster recovery. The volume replication operation is - supported by the [CSIAddons](https://github.com/csi-addons/kubernetes-csi-addons#readme) - It is based on [csi-addons/spec](https://github.com/csi-addons/spec) - specification and can be used by any storage provider. + kubernetes operator that provides common and reusable APIs for + storage disaster recovery. The volume replication operation is + supported by the [CSIAddons](https://github.com/csi-addons/kubernetes-csi-addons#readme) + It is based on [csi-addons/spec](https://github.com/csi-addons/spec) + specification and can be used by any storage provider. -Execute the following steps on each peer cluster to enable the - OMap generator and CSIADDONS sidecars: +Execute the following steps on each peer cluster to enable the OMap generator and CSIADDONS sidecars: -* Edit the `rook-ceph-operator-config` configmap and add the - following configurations +* Edit the `rook-ceph-operator-config` configmap and add the following configurations -```console -kubectl edit cm rook-ceph-operator-config -n rook-ceph -``` + ```console + kubectl edit cm rook-ceph-operator-config -n rook-ceph + ``` -Add the following properties if not present: + Add the following properties if not present: -```yaml -data: - CSI_ENABLE_OMAP_GENERATOR: "true" - CSI_ENABLE_CSIADDONS: "true" -``` + ```yaml + data: + CSI_ENABLE_OMAP_GENERATOR: "true" + CSI_ENABLE_CSIADDONS: "true" + ``` * After updating the configmap with those settings, two new sidecars - should now start automatically in the CSI provisioner pod. + should now start automatically in the CSI provisioner pod. + * Repeat the steps on the peer cluster. ## Volume Replication Custom Resources @@ -223,7 +219,7 @@ corresponding to the driver providing replication. ## Enable mirroring on a PVC Below guide assumes that we have a PVC (rbd-pvc) in BOUND state; created using - *StorageClass* with `Retain` reclaimPolicy. +**StorageClass** with `Retain` reclaimPolicy. ```console [cluster-1]$ kubectl get pvc @@ -248,7 +244,7 @@ In this case, we create a Volume Replication Class on cluster-1 ### Create a VolumeReplication CR * Once VolumeReplicationClass is created, create a Volume Replication for - the PVC which we intend to replicate to secondary cluster. + the PVC which we intend to replicate to secondary cluster. ```console [cluster-1]$ kubectl apply -f deploy/examples/volume-replication.yaml @@ -261,7 +257,7 @@ In this case, we create a Volume Replication Class on cluster-1 ### Checking Replication Status `replicationState` is the state of the volume being referenced. - Possible values are primary, secondary, and resync. +Possible values are primary, secondary, and resync. * `primary` denotes that the volume is primary. * `secondary` denotes that the volume is secondary. @@ -351,8 +347,8 @@ volumereplicationclass.replication.storage.openshift.io/rbd-volumereplicationcla ``` * If Persistent Volumes and Claims are created manually on the secondary cluster, - remove the `claimRef` on the backed up PV objects in yaml files; so that the - PV can get bound to the new claim on the secondary cluster. + remove the `claimRef` on the backed up PV objects in yaml files; so that the + PV can get bound to the new claim on the secondary cluster. ```yaml ... diff --git a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md index 11d041d26..87f802190 100644 --- a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md +++ b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md @@ -5,12 +5,12 @@ title: Ceph CSI Drivers There are three CSI drivers integrated with Rook that are used in different scenarios: * RBD: This block storage driver is optimized for RWO pod access where only one pod may access the - storage. [More information](../Block-Storage-RBD/block-storage.md). + storage. [More information](../Block-Storage-RBD/block-storage.md). * CephFS: This file storage driver allows for RWX with one or more pods accessing the same storage. - [More information](../Shared-Filesystem-CephFS/filesystem-storage.md). + [More information](../Shared-Filesystem-CephFS/filesystem-storage.md). * NFS (experimental): This file storage driver allows creating NFS exports that can be mounted on - pods, or directly via an NFS client from inside or outside the - Kubernetes cluster. [More information](../NFS/nfs-csi-driver.md) + pods, or directly via an NFS client from inside or outside the + Kubernetes cluster. [More information](../NFS/nfs-csi-driver.md) The Ceph Filesystem (CephFS) and RADOS Block Device (RBD) drivers are enabled automatically by the Rook operator. The NFS driver is disabled by default. All drivers will be started in the same @@ -182,29 +182,29 @@ which are not enabled by default. Execute the following to enable the CSI-Addons sidecars: * Update the `rook-ceph-operator-config` configmap and patch the - following configuration: + following configuration: -```console -kubectl patch cm rook-ceph-operator-config -nrook-ceph -p $'data:\n "CSI_ENABLE_CSIADDONS": "true"' -``` + ```console + kubectl patch cm rook-ceph-operator-config -nrook-ceph -p $'data:\n "CSI_ENABLE_CSIADDONS": "true"' + ``` * After enabling `CSI_ENABLE_CSIADDONS` in the configmap, a new sidecar container named `csi-addons` -will start automatically in the RBD CSI provisioner and nodeplugin pods. + will start automatically in the RBD CSI provisioner and nodeplugin pods. ### CSI-Addons Operations CSI-Addons supports the following operations: * Reclaim Space - * [Creating a ReclaimSpaceJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.8.0/docs/reclaimspace.md#reclaimspacejob) - * [Creating a ReclaimSpaceCronJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.8.0/docs/reclaimspace.md#reclaimspacecronjob) - * [Annotating PersistentVolumeClaims](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.8.0/docs/reclaimspace.md#annotating-perstentvolumeclaims) - * [Annotating Namespace](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.8.0/docs/reclaimspace.md#annotating-namespace) + * [Creating a ReclaimSpaceJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.8.0/docs/reclaimspace.md#reclaimspacejob) + * [Creating a ReclaimSpaceCronJob](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.8.0/docs/reclaimspace.md#reclaimspacecronjob) + * [Annotating PersistentVolumeClaims](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.8.0/docs/reclaimspace.md#annotating-perstentvolumeclaims) + * [Annotating Namespace](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.8.0/docs/reclaimspace.md#annotating-namespace) * Network Fencing - * [Creating a NetworkFence](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.8.0/docs/networkfence.md) + * [Creating a NetworkFence](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.8.0/docs/networkfence.md) * Volume Replication - * [Creating VolumeReplicationClass](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.8.0/docs/volumereplicationclass.md) - * [Creating VolumeReplication CR](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.8.0/docs/volumereplication.md) + * [Creating VolumeReplicationClass](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.8.0/docs/volumereplicationclass.md) + * [Creating VolumeReplication CR](https://github.com/csi-addons/kubernetes-csi-addons/blob/v0.8.0/docs/volumereplication.md) ## Enable RBD Encryption Support @@ -243,7 +243,7 @@ data: ``` * Update the `rook-ceph-operator-config` configmap and patch the - following configurations + following configurations ```console kubectl patch cm rook-ceph-operator-config -nrook-ceph -p $'data:\n "CSI_ENABLE_ENCRYPTION": "true"' diff --git a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot.md b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot.md index 900e36a61..f214eaaec 100644 --- a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot.md +++ b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-snapshot.md @@ -19,7 +19,7 @@ In short, as the documentation describes it: ## RBD Snapshots -### VolumeSnapshotClass +### RBD VolumeSnapshotClass In [VolumeSnapshotClass](https://github.com/rook/rook/tree/master/deploy/examples/csi/rbd/snapshotclass.yaml), the `csi.storage.k8s.io/snapshotter-secret-name` parameter should reference the @@ -62,7 +62,7 @@ rbd-pvc-snapshot true rbd-pvc 1Gi The snapshot will be ready to restore to a new PVC when the `READYTOUSE` field of the `volumesnapshot` is set to true. -### Restore the snapshot to a new PVC +### Restore the RBD snapshot to a new PVC In [pvc-restore](https://github.com/rook/rook/tree/master/deploy/examples/csi/rbd/pvc-restore.yaml), `dataSource` should be the name of the `VolumeSnapshot` previously @@ -95,7 +95,7 @@ kubectl delete -f deploy/examples/csi/rbd/snapshotclass.yaml ## CephFS Snapshots -### VolumeSnapshotClass +### CephFS VolumeSnapshotClass In [VolumeSnapshotClass](https://github.com/rook/rook/tree/master/deploy/examples/csi/cephfs/snapshotclass.yaml), the `csi.storage.k8s.io/snapshotter-secret-name` parameter should reference the @@ -138,7 +138,7 @@ cephfs-pvc-snapshot true cephfs-pvc 1Gi The snapshot will be ready to restore to a new PVC when `READYTOUSE` field of the `volumesnapshot` is set to true. -### Restore the snapshot to a new PVC +### Restore the CephFS snapshot to a new PVC In [pvc-restore](https://github.com/rook/rook/tree/master/deploy/examples/csi/cephfs/pvc-restore.yaml), diff --git a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-volume-clone.md b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-volume-clone.md index 23841070f..4434f2315 100644 --- a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-volume-clone.md +++ b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-volume-clone.md @@ -15,10 +15,7 @@ for more info. ## RBD Volume Cloning -### Volume Cloning - -In -[pvc-clone](https://github.com/rook/rook/tree/master/deploy/examples/csi/rbd/pvc-clone.yaml), +In [pvc-clone](https://github.com/rook/rook/tree/master/deploy/examples/csi/rbd/pvc-clone.yaml), `dataSource` should be the name of the `PVC` which is already created by RBD CSI driver. The `dataSource` kind should be the `PersistentVolumeClaim` and also storageclass should be same as the source `PVC`. @@ -35,11 +32,11 @@ kubectl create -f deploy/examples/csi/rbd/pvc-clone.yaml kubectl get pvc ``` ->``` ->NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE ->rbd-pvc Bound pvc-74734901-577a-11e9-b34f-525400581048 1Gi >RWO rook-ceph-block 34m ->rbd-pvc-clone Bound pvc-70473135-577f-11e9-b34f-525400581048 1Gi RWO rook-ceph-block 8s ->``` +```console +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +rbd-pvc Bound pvc-74734901-577a-11e9-b34f-525400581048 1Gi RWO rook-ceph-block 34m +rbd-pvc-clone Bound pvc-70473135-577f-11e9-b34f-525400581048 1Gi RWO rook-ceph-block 8s +``` ## RBD clone resource Cleanup @@ -53,13 +50,10 @@ kubectl delete -f deploy/examples/csi/rbd/pvc-clone.yaml ### Volume Clone Prerequisites - 1. Requires Kubernetes v1.16+ which supports volume clone. - 2. Ceph-csi diver v3.1.0+ which supports volume clone. +1. Requires Kubernetes v1.16+ which supports volume clone. +2. Ceph-csi diver v3.1.0+ which supports volume clone. -### Volume Cloning - -In -[pvc-clone](https://github.com/rook/rook/tree/master/deploy/examples/csi/cephfs/pvc-clone.yaml), +In [pvc-clone](https://github.com/rook/rook/tree/master/deploy/examples/csi/cephfs/pvc-clone.yaml), `dataSource` should be the name of the `PVC` which is already created by CephFS CSI driver. The `dataSource` kind should be the `PersistentVolumeClaim` and also storageclass should be same as the source `PVC`. @@ -76,11 +70,11 @@ kubectl create -f deploy/examples/csi/cephfs/pvc-clone.yaml kubectl get pvc ``` ->``` ->NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE ->cephfs-pvc Bound pvc-1ea51547-a88b-4ab0-8b4a-812caeaf025d 1Gi RWX rook-cephfs 39m ->cephfs-pvc-clone Bound pvc-b575bc35-d521-4c41-b4f9-1d733cd28fdf 1Gi RWX rook-cephfs 8s ->``` +```console +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +cephfs-pvc Bound pvc-1ea51547-a88b-4ab0-8b4a-812caeaf025d 1Gi RWX rook-cephfs 39m +cephfs-pvc-clone Bound pvc-b575bc35-d521-4c41-b4f9-1d733cd28fdf 1Gi RWX rook-cephfs 8s +``` ## CephFS clone resource Cleanup diff --git a/Documentation/Storage-Configuration/Monitoring/ceph-dashboard.md b/Documentation/Storage-Configuration/Monitoring/ceph-dashboard.md index 684a0bf00..cfd08757f 100755 --- a/Documentation/Storage-Configuration/Monitoring/ceph-dashboard.md +++ b/Documentation/Storage-Configuration/Monitoring/ceph-dashboard.md @@ -60,14 +60,14 @@ spec: ``` * `urlPrefix` If you are accessing the dashboard via a reverse proxy, you may - wish to serve it under a URL prefix. To get the dashboard to use hyperlinks - that include your prefix, you can set the `urlPrefix` setting. + wish to serve it under a URL prefix. To get the dashboard to use hyperlinks + that include your prefix, you can set the `urlPrefix` setting. * `port` The port that the dashboard is served on may be changed from the - default using the `port` setting. The corresponding K8s service exposing the - port will automatically be updated. + default using the `port` setting. The corresponding K8s service exposing the + port will automatically be updated. * `ssl` The dashboard may be served without SSL (useful for when you deploy the - dashboard behind a proxy already served using SSL) by setting the `ssl` option - to be false. + dashboard behind a proxy already served using SSL) by setting the `ssl` option + to be false. ## Visualization of 'Physical Disks' section in the dashboard diff --git a/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md index 61adf8a2f..badd8c52c 100644 --- a/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md +++ b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md @@ -23,6 +23,7 @@ A full explanation can be found in the [Prometheus operator repository on GitHub ```console kubectl create -f https://raw.githubusercontent.com/coreos/prometheus-operator/v0.71.1/bundle.yaml ``` + !!! note If the Prometheus Operator is already present in your cluster, the command provided above may fail. For a detailed explanation of the issue and a workaround, please refer to [this issue](https://github.com/rook/rook/issues/13459). @@ -68,7 +69,9 @@ kubectl -n rook-ceph get pod prometheus-rook-prometheus-0 ### Dashboard config -Configure the Prometheus endpoint so the dashboard can retrieve metrics from Prometheus with two settings: +Configure the Prometheus endpoint so the dashboard can retrieve metrics from Prometheus with two +settings: + - `prometheusEndpoint`: The url of the Prometheus instance - `prometheusEndpointSSLVerify`: Whether SSL should be verified if the Prometheus server is using https @@ -80,12 +83,12 @@ echo "http://$(kubectl -n rook-ceph -o jsonpath={.status.hostIP} get pod prometh Following is an example to configure the Prometheus endpoint in the CephCluster CR. - ```YAML - spec: - dashboard: - prometheusEndpoint: http://192.168.61.204:30900 - prometheusEndpointSSLVerify: true - ``` +```YAML +spec: + dashboard: + prometheusEndpoint: http://192.168.61.204:30900 + prometheusEndpointSSLVerify: true +``` !!! note It is not recommended to consume storage from the Ceph cluster for Prometheus. @@ -130,10 +133,10 @@ A guide to how you can write your own Prometheus consoles can be found on the of To enable the Ceph Prometheus alerts via the helm charts, set the following properties in values.yaml: * rook-ceph chart: - `monitoring.enabled: true` + - `monitoring.enabled: true` * rook-ceph-cluster chart: - `monitoring.enabled: true` - `monitoring.createPrometheusRules: true` + - `monitoring.enabled: true` + - `monitoring.createPrometheusRules: true` Alternatively, to enable the Ceph Prometheus alerts with example manifests follow these steps: diff --git a/Documentation/Storage-Configuration/NFS/nfs-csi-driver.md b/Documentation/Storage-Configuration/NFS/nfs-csi-driver.md index 22b2b21f9..6ba0fdd44 100644 --- a/Documentation/Storage-Configuration/NFS/nfs-csi-driver.md +++ b/Documentation/Storage-Configuration/NFS/nfs-csi-driver.md @@ -36,8 +36,7 @@ will expose the exports. RGWs cannot be used for the CSI driver. From the examples, `filesystem.yaml` creates a CephFilesystem called `myfs`, and `nfs.yaml` creates an NFS server called `my-nfs`. -You may need to enable or disable the Ceph orchestrator. Follow the same steps documented -[above](#enable-the-ceph-orchestrator-if-necessary) based on your Ceph version and desires. +You may need to enable or disable the Ceph orchestrator. You must also create a storage class. Ceph CSI is designed to support any arbitrary Ceph cluster, but we are focused here only on Ceph clusters deployed by Rook. Let's take a look at a portion of @@ -69,14 +68,14 @@ parameters: ``` 1. `provisioner`: **rook-ceph**.nfs.csi.ceph.com because **rook-ceph** is the namespace where the - CephCluster is installed + CephCluster is installed 2. `nfsCluster`: **my-nfs** because this is the name of the CephNFS 3. `server`: rook-ceph-nfs-**my-nfs**-a because Rook creates this Kubernetes Service for the CephNFS - named **my-nfs** + named **my-nfs** 4. `clusterID`: **rook-ceph** because this is the namespace where the CephCluster is installed 5. `fsName`: **myfs** because this is the name of the CephFilesystem used to back the NFS exports 6. `pool`: **myfs**-**replicated** because **myfs** is the name of the CephFilesystem defined in - `fsName` and because **replicated** is the name of a data pool defined in the CephFilesystem + `fsName` and because **replicated** is the name of a data pool defined in the CephFilesystem 7. `csi.storage.k8s.io/*`: note that these values are shared with the Ceph CSI CephFS provisioner ### Creating a PVC @@ -194,11 +193,11 @@ kubectl create -f deploy/examples/csi/nfs/pvc-clone.yaml kubectl get pvc ``` ->``` ->NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE ->nfs-pvc Bound pvc-1ea51547-a88b-4ab0-8b4a-812caeaf025d 1Gi RWX rook-nfs 39m ->nfs-pvc-clone Bound pvc-b575bc35-d521-4c41-b4f9-1d733cd28fdf 1Gi RWX rook-nfs 8s ->``` +```console +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +nfs-pvc Bound pvc-1ea51547-a88b-4ab0-8b4a-812caeaf025d 1Gi RWX rook-nfs 39m +nfs-pvc-clone Bound pvc-b575bc35-d521-4c41-b4f9-1d733cd28fdf 1Gi RWX rook-nfs 8s +``` ### Cleaning up clone resources diff --git a/Documentation/Storage-Configuration/NFS/nfs-security.md b/Documentation/Storage-Configuration/NFS/nfs-security.md index 0148bb8e8..4a3aa2bac 100644 --- a/Documentation/Storage-Configuration/NFS/nfs-security.md +++ b/Documentation/Storage-Configuration/NFS/nfs-security.md @@ -41,15 +41,17 @@ Methods of providing the configuration file are documented in the [NFS CRD security section](../../CRDs/ceph-nfs-crd.md#security). Recommendations: + - The SSSD sidecar only requires the namespace switch (a.k.a. "nsswitch" or "nss"). We recommend - enabling only the `nss` service to lower CPU usage. + enabling only the `nss` service to lower CPU usage. - NFS-Ganesha does not require user enumeration. We recommend leaving this option unset or setting - `enumerate = false` to speed up lookups and reduce RAM usage. + `enumerate = false` to speed up lookups and reduce RAM usage. - NFS exports created via documented methods do not require listing all members of groups. We - recommend setting `ignore_group_members = true` to speed up LDAP lookups. Only customized exports - that set `manage_gids` need to consider this option. + recommend setting `ignore_group_members = true` to speed up LDAP lookups. Only customized exports + that set `manage_gids` need to consider this option. A sample `sssd.conf` file is shown below. + ```ini [sssd] # Only the nss service is required for the SSSD sidecar. @@ -96,23 +98,26 @@ users are authenticated and not merely client machines. Kerberos authentication requires configuration files in order for the NFS-Ganesha server to authenticate to the Kerberos server (KDC). The requirements are two-parted: + 1. one or more kerberos configuration files that configures the connection to the Kerberos server. - This file follows the `krb5.conf` format documented in its - [man pages](https://linux.die.net/man/5/krb5.conf). + This file follows the `krb5.conf` format documented in its + [man pages](https://linux.die.net/man/5/krb5.conf). 2. a keytab file that provides credentials for the - [service principal](#nfs-service-principals) that NFS-Ganesha will use to authenticate with the - Kerberos server. + [service principal](#nfs-service-principals) that NFS-Ganesha will use to authenticate with the + Kerberos server. 3. a kerberos domain name which will be used to map kerberos credentials to uid/gid - [domain name](#kerberos-domain-name) that NFS-Ganesha will use to authenticate with the + [domain name](#kerberos-domain-name) that NFS-Ganesha will use to authenticate with the Methods of providing the configuration files are documented in the [NFS CRD security section](../../CRDs/ceph-nfs-crd.md#security). Recommendations: + - Rook configures Kerberos to log to stderr. We suggest removing logging sections from config files - to avoid consuming unnecessary disk space from logging to files. + to avoid consuming unnecessary disk space from logging to files. A sample Kerberos config file is shown below. + ```ini [libdefaults] default_realm = EXAMPLE.NET @@ -145,9 +150,10 @@ securely add files via annotations on the CephNFS spec (passed to the NFS server The Kerberos service principal used by Rook's CephNFS servers to authenticate with the Kerberos server is built up from 3 components: + 1. the configured from `spec.security.kerberos.principalName` that acts as the service name 2. the hostname of the server on which NFS-Ganesha is running which is in turn built up from the - namespace and name of the CephNFS resource, joined by a hyphen. e.g., `rooknamespace-nfsname` + namespace and name of the CephNFS resource, joined by a hyphen. e.g., `rooknamespace-nfsname` 3. the realm as configured by the kerberos config file(s) from `spec.security.kerberos.configFiles` The full service principal name is constructed as `/-@`. For diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md index 4e80e6eeb..c2be34c68 100644 --- a/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-claim.md @@ -35,15 +35,15 @@ spec: ``` 1. `name` of the `ObjectBucketClaim`. This name becomes the name of the Secret and ConfigMap. -1. `namespace`(optional) of the `ObjectBucketClaim`, which is also the namespace of the ConfigMap and Secret. -1. `bucketName` name of the `bucket`. +2. `namespace`(optional) of the `ObjectBucketClaim`, which is also the namespace of the ConfigMap and Secret. +3. `bucketName` name of the `bucket`. **Not** recommended for new buckets since names must be unique within an entire object store. -1. `generateBucketName` value becomes the prefix for a randomly generated name, if supplied then `bucketName` must be empty. +4. `generateBucketName` value becomes the prefix for a randomly generated name, if supplied then `bucketName` must be empty. If both `bucketName` and `generateBucketName` are supplied then `BucketName` has precedence and `GenerateBucketName` is ignored. If both `bucketName` and `generateBucketName` are blank or omitted then the storage class is expected to contain the name of an _existing_ bucket. It's an error if all three bucket related names are blank or omitted. -1. `storageClassName` which defines the StorageClass which contains the names of the bucket provisioner, the object-store and specifies the bucket retention policy. -1. `additionalConfig` is an optional list of key-value pairs used to define attributes specific to the bucket being provisioned by this OBC. This information is typically tuned to a particular bucket provisioner and may limit application portability. Options supported: +5. `storageClassName` which defines the StorageClass which contains the names of the bucket provisioner, the object-store and specifies the bucket retention policy. +6. `additionalConfig` is an optional list of key-value pairs used to define attributes specific to the bucket being provisioned by this OBC. This information is typically tuned to a particular bucket provisioner and may limit application portability. Options supported: * `maxObjects`: The maximum number of objects in the bucket * `maxSize`: The maximum size of the bucket, please note minimum recommended value is 4K. @@ -69,10 +69,10 @@ status: ``` 1. `namespace` where OBC got created. -1. `ObjectBucketName` generated OB name created using name space and OBC name. -1. the generated (in this case), unique `bucket name` for the new bucket. -1. name of the storage class from OBC got created. -1. phases of bucket creation: +2. `ObjectBucketName` generated OB name created using name space and OBC name. +3. the generated (in this case), unique `bucket name` for the new bucket. +4. name of the storage class from OBC got created. +5. phases of bucket creation: * _Pending_: the operator is processing the request. * _Bound_: the operator finished processing the request and linked the OBC and OB * _Released_: the OB has been deleted, leaving the OBC unclaimed but unavailable. @@ -98,8 +98,8 @@ spec: ``` 1. use `env:` if mapping of the defined key names to the env var names used by the app is needed. -1. makes available to the pod as env variables: `BUCKET_HOST`, `BUCKET_PORT`, `BUCKET_NAME` -1. makes available to the pod as env variables: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` +2. makes available to the pod as env variables: `BUCKET_HOST`, `BUCKET_PORT`, `BUCKET_NAME` +3. makes available to the pod as env variables: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` ### StorageClass @@ -119,11 +119,11 @@ reclaimPolicy: Delete [5] ``` 1. `label`(optional) here associates this `StorageClass` to a specific provisioner. -1. `provisioner` responsible for handling `OBCs` referencing this `StorageClass`. -1. **all** `parameter` required. -1. `bucketName` is required for access to existing buckets but is omitted when provisioning new buckets. -Unlike greenfield provisioning, the brownfield bucket name appears in the `StorageClass`, not the `OBC`. -1. rook-ceph provisioner decides how to treat the `reclaimPolicy` when an `OBC` is deleted for the bucket. See explanation as [specified in Kubernetes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#retain) +2. `provisioner` responsible for handling `OBCs` referencing this `StorageClass`. +3. **all** `parameter` required. +4. `bucketName` is required for access to existing buckets but is omitted when provisioning new buckets. + Unlike greenfield provisioning, the brownfield bucket name appears in the `StorageClass`, not the `OBC`. +5. rook-ceph provisioner decides how to treat the `reclaimPolicy` when an `OBC` is deleted for the bucket. See explanation as [specified in Kubernetes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#retain) * _Delete_ = physically delete the bucket. * _Retain_ = do not physically delete the bucket. diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-notifications.md b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-notifications.md index 9bc6bae4b..6c29b3bdf 100644 --- a/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-notifications.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/ceph-object-bucket-notifications.md @@ -68,38 +68,38 @@ spec: ``` 1. `name` of the `CephBucketTopic` - * In case of AMQP endpoint, the name is used for the AMQP topic (“routing key” for a topic exchange) - * In case of Kafka endpoint, the name is used as the Kafka topic -1. `namespace`(optional) of the `CephBucketTopic`. Should match the namespace of the CephBucketNotification associated with this CephBucketTopic, and the OBC with the label referencing the CephBucketNotification -1. `objectStoreName` is the name of the object store in which the topic should be created. This must be the same object store used for the buckets associated with the notifications referencing this topic. -1. `objectStoreNamespace` is the namespace of the object store in which the topic should be created -1. `opaqueData` (optional) is added to all notifications triggered by a notifications associated with the topic -1. `persistent` (optional) indicates whether notifications to this endpoint are persistent (=asynchronous) or sent synchronously (“false” by default) -1. `endpoint` to which to send the notifications to. Exactly one of the endpoints must be defined: `http`, `amqp`, `kafka` -1. `http` (optional) hold the spec for an HTTP endpoint. The format of the URI would be: `http[s]://[:][/]` - * port defaults to: 80/443 for HTTP/S accordingly -1. `disableVerifySSL` indicates whether the RGW is going to verify the SSL certificate of the HTTP server in case HTTPS is used ("false" by default) -1. `sendCloudEvents`: (optional) send the notifications with the [CloudEvents header](https://github.com/cloudevents/spec/blob/main/cloudevents/adapters/aws-s3.md). Supported for Ceph Quincy (v17) or newer ("false" by default) -1. `amqp` (optional) hold the spec for an AMQP endpoint. The format of the URI would be: `amqp[s]://[:@][:][/]` - * port defaults to: 5672/5671 for AMQP/S accordingly - * user/password defaults to: guest/guest - * user/password may only be provided if HTTPS is used with the RGW. If not, topic creation request will be rejected - * vhost defaults to: “/” -1. `disableVerifySSL` (optional) indicates whether the RGW is going to verify the SSL certificate of the AMQP server in case AMQPS is used ("false" by default) -1. `ackLevel` (optional) indicates what kind of ack the RGW is waiting for after sending the notifications: - * “none”: message is considered “delivered” if sent to broker - * “broker”: message is considered “delivered” if acked by broker (default) - * “routable”: message is considered “delivered” if broker can route to a consumer -1. `exchange` in the AMQP broker that would route the notifications. Different topics pointing to the same endpoint must use the same exchange -1. `kafka` (optional) hold the spec for a Kafka endpoint. The format of the URI would be: `kafka://[:@][:[:][/]` + + port defaults to: 80/443 for HTTP/S accordingly +9. `disableVerifySSL` indicates whether the RGW is going to verify the SSL certificate of the HTTP server in case HTTPS is used ("false" by default) +10. `sendCloudEvents`: (optional) send the notifications with the [CloudEvents header](https://github.com/cloudevents/spec/blob/main/cloudevents/adapters/aws-s3.md). Supported for Ceph Quincy (v17) or newer ("false" by default) +11. `amqp` (optional) hold the spec for an AMQP endpoint. The format of the URI would be: `amqp[s]://[:@][:][/]` + + port defaults to: 5672/5671 for AMQP/S accordingly + + user/password defaults to: guest/guest + + user/password may only be provided if HTTPS is used with the RGW. If not, topic creation request will be rejected + + vhost defaults to: “/” +12. `disableVerifySSL` (optional) indicates whether the RGW is going to verify the SSL certificate of the AMQP server in case AMQPS is used ("false" by default) +13. `ackLevel` (optional) indicates what kind of ack the RGW is waiting for after sending the notifications: + + “none”: message is considered “delivered” if sent to broker + + “broker”: message is considered “delivered” if acked by broker (default) + + “routable”: message is considered “delivered” if broker can route to a consumer +14. `exchange` in the AMQP broker that would route the notifications. Different topics pointing to the same endpoint must use the same exchange +15. `kafka` (optional) hold the spec for a Kafka endpoint. The format of the URI would be: `kafka://[:@][: radosgw-admin zonegroup rename --rgw-zonegroup=my-store --zonegroup-new-name= --rgw-realm= radosgw-admin zone rename --rgw-zone=my-store --zone-new-name= --rgw-zonegroup= --rgw-realm= radosgw-admin period update --commit ``` + !!! important Renaming in the toolbox must be performed **before** creating the multisite CRs diff --git a/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md b/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md index 06003244d..162aeb764 100644 --- a/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md +++ b/Documentation/Storage-Configuration/Object-Storage-RGW/cosi.md @@ -10,6 +10,7 @@ The Ceph COSI driver provisions buckets for object storage. This document instru ## Prerequisites COSI requires: + 1. A running Rook [object store](object-storage.md) 2. [COSI controller](https://github.com/kubernetes-sigs/container-object-storage-interface-controller#readme) diff --git a/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-mirroring.md b/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-mirroring.md index 214e20277..e6a50ae23 100644 --- a/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-mirroring.md +++ b/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-mirroring.md @@ -58,7 +58,7 @@ spec: ## Create the cephfs-mirror daemon -Launch the `rook-ceph-fs-mirror` pod on the source storage cluster, which deploys the `cephfs-mirror` daemon in the cluster: +Launch the `rook-ceph-fs-mirror` pod on the source storage cluster, which deploys the `cephfs-mirror` daemon in the cluster: ```console kubectl create -f deploy/examples/filesystem-mirror.yaml @@ -84,7 +84,7 @@ This secret can then be fetched like so: eyJmc2lkIjoiOTFlYWUwZGQtMDZiMS00ZDJjLTkxZjMtMTMxMWM5ZGYzODJiIiwiY2xpZW50X2lkIjoicmJkLW1pcnJvci1wZWVyIiwia2V5IjoiQVFEN1psOWZ3V1VGRHhBQWdmY0gyZi8xeUhYeGZDUTU5L1N0NEE9PSIsIm1vbl9ob3N0IjoiW3YyOjEwLjEwMS4xOC4yMjM6MzMwMCx2MToxMC4xMDEuMTguMjIzOjY3ODldIn0= ``` -### Import the token in the Destination cluster +### Import the token in the Destination cluster The decoded secret must be saved in a file before importing. @@ -99,7 +99,7 @@ Further refer to CephFS mirror documentation to [configure a directory for snaps ## Verify that the snapshots have synced -To check the `mirror daemon status`, please run the following command from the [toolbox](../../Troubleshooting/ceph-toolbox.md) pod. +To check the `mirror daemon status`, please run the following command from the [toolbox](../../Troubleshooting/ceph-toolbox.md) pod. For example : @@ -189,4 +189,4 @@ For getting `peer synchronization status`: "snaps_renamed": 0 } } -``` \ No newline at end of file +``` diff --git a/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md b/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md index a8e82bd37..ba6104671 100644 --- a/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md +++ b/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md @@ -228,8 +228,8 @@ You can do that using the following recipe. * In the `rook` namespace, create a copy of the secret `rook-csi-cephfs-node`, name it `rook-csi-cephfs-node-user` . * Edit your new secret, changing the name of the keys (keep the value as it is): - * `adminID` -> `userID` - * `adminKey` -> `userKey` + * `adminID` -> `userID` + * `adminKey` -> `userKey` * Create the PVC you want to share, for example: ```yaml @@ -295,12 +295,12 @@ spec: * On this PV, change the `persistentVolumeReclaimPolicy` parameter to `Retain` to avoid it from being deleted when you will delete PVCs. Don't forget to change it back to `Delete` when you want to remove the shared volume (see full procedure in the next section). * Copy the YAML content of the PV, and create a new static PV with the same information and some modifications. From the original YAML, you must: - * Modify the original name. To keep track, the best solution is to append to the original name the namespace name where you want your new PV. In this example `newnamespace`. - * Modify the volumeHandle. Again append the targeted namespace. - * Add the `staticVolume: "true"` entry to the volumeAttributes. - * Add the rootPath entry to the volumeAttributes, with the same content as `subvolumePath`. - * In the `nodeStageSecretRef` section, change the name to point to the secret you created earlier, `rook-csi-cephfs-node-user`. - * Remove the unnecessary information before applying the YAML (claimRef, managedFields,...): + * Modify the original name. To keep track, the best solution is to append to the original name the namespace name where you want your new PV. In this example `newnamespace`. + * Modify the volumeHandle. Again append the targeted namespace. + * Add the `staticVolume: "true"` entry to the volumeAttributes. + * Add the rootPath entry to the volumeAttributes, with the same content as `subvolumePath`. + * In the `nodeStageSecretRef` section, change the name to point to the secret you created earlier, `rook-csi-cephfs-node-user`. + * Remove the unnecessary information before applying the YAML (claimRef, managedFields,...): Your YAML should look like this: diff --git a/Documentation/Storage-Configuration/ceph-teardown.md b/Documentation/Storage-Configuration/ceph-teardown.md index 20ab5c0ca..b207efc47 100644 --- a/Documentation/Storage-Configuration/ceph-teardown.md +++ b/Documentation/Storage-Configuration/ceph-teardown.md @@ -5,7 +5,7 @@ title: Cleanup Rook provides the following clean up options: 1. [Uninstall: Clean up the entire cluster and delete all data](#cleaning-up-a-cluster) -1. [Force delete individual resources](#force-delete-resources) +2. [Force delete individual resources](#force-delete-resources) ## Cleaning up a Cluster @@ -44,31 +44,32 @@ kubectl delete storageclass csi-cephfs 1. To instruct Rook to wipe the host paths and volumes, edit the `CephCluster` and add the `cleanupPolicy`: -```console -kubectl -n rook-ceph patch cephcluster rook-ceph --type merge -p '{"spec":{"cleanupPolicy":{"confirmation":"yes-really-destroy-data"}}}' -``` + ```console + kubectl -n rook-ceph patch cephcluster rook-ceph --type merge -p '{"spec":{"cleanupPolicy":{"confirmation":"yes-really-destroy-data"}}}' + ``` -Once the cleanup policy is enabled, any new configuration changes in the CephCluster will be blocked. Nothing will happen until the deletion of the CR is requested, so this `cleanupPolicy` change can still be reverted if needed. + Once the cleanup policy is enabled, any new configuration changes in the CephCluster will be blocked. Nothing will happen until the deletion of the CR is requested, so this `cleanupPolicy` change can still be reverted if needed. -Checkout more details about the `cleanupPolicy` [here](../CRDs/Cluster/ceph-cluster-crd.md#cleanup-policy) + Checkout more details about the `cleanupPolicy` [here](../CRDs/Cluster/ceph-cluster-crd.md#cleanup-policy) 2. Delete the `CephCluster` CR. -```console -kubectl -n rook-ceph delete cephcluster rook-ceph -``` + ```console + kubectl -n rook-ceph delete cephcluster rook-ceph + ``` 3. Verify that the cluster CR has been deleted before continuing to the next step. -```console -kubectl -n rook-ceph get cephcluster -``` + ```console + kubectl -n rook-ceph get cephcluster + ``` -If the `cleanupPolicy` was applied, wait for the `rook-ceph-cleanup` jobs to be completed on all the nodes. -These jobs will perform the following operations: +4. If the `cleanupPolicy` was applied, wait for the `rook-ceph-cleanup` jobs to be completed on all the nodes. -* Delete the namespace directory under `dataDirHostPath`, for example `/var/lib/rook/rook-ceph`, on all the nodes -* Wipe the data on the drives on all the nodes where OSDs were running in this cluster + These jobs will perform the following operations: + + * Delete the namespace directory under `dataDirHostPath`, for example `/var/lib/rook/rook-ceph`, on all the nodes + * Wipe the data on the drives on all the nodes where OSDs were running in this cluster !!! note The cleanup jobs might not start if the resources created on top of Rook Cluster are not deleted completely. diff --git a/Documentation/Troubleshooting/ceph-common-issues.md b/Documentation/Troubleshooting/ceph-common-issues.md index 795708327..b7735e773 100644 --- a/Documentation/Troubleshooting/ceph-common-issues.md +++ b/Documentation/Troubleshooting/ceph-common-issues.md @@ -2,6 +2,73 @@ title: Ceph Common Issues --- + + + + +## Topics + +- [Troubleshooting Techniques](#troubleshooting-techniques) + - [Ceph Tools](#ceph-tools) + - [Tools in the Rook Toolbox](#tools-in-the-rook-toolbox) + - [Ceph Commands](#ceph-commands) +- [Cluster failing to service requests](#cluster-failing-to-service-requests) + - [Symptoms](#symptoms) + - [Investigation](#investigation) + - [Solution](#solution) +- [Monitors are the only pods running](#monitors-are-the-only-pods-running) + - [Symptoms](#symptoms-1) + - [Investigation](#investigation-1) + - [Operator fails to connect to the mon](#operator-fails-to-connect-to-the-mon) + - [Solution](#solution-1) + - [Failing mon pod](#failing-mon-pod) + - [Solution](#solution-2) +- [PVCs stay in pending state](#pvcs-stay-in-pending-state) + - [Symptoms](#symptoms-2) + - [Investigation](#investigation-2) + - [Confirm if there are OSDs](#confirm-if-there-are-osds) + - [OSD Prepare Logs](#osd-prepare-logs) + - [CSI Driver](#csi-driver) + - [Operator unresponsiveness](#operator-unresponsiveness) + - [Solution](#solution-3) +- [OSD pods are failing to start](#osd-pods-are-failing-to-start) + - [Symptoms](#symptoms-3) + - [Investigation](#investigation-3) + - [Solution](#solution-4) +- [OSD pods are not created on my devices](#osd-pods-are-not-created-on-my-devices) + - [Symptoms](#symptoms-4) + - [Investigation](#investigation-4) + - [Solution](#solution-5) +- [Node hangs after reboot](#node-hangs-after-reboot) + - [Symptoms](#symptoms-5) + - [Investigation](#investigation-5) + - [Solution](#solution-6) +- [Using multiple shared filesystem (CephFS) is attempted on a kernel version older than 4.7](#using-multiple-shared-filesystem-cephfs-is-attempted-on-a-kernel-version-older-than-47) + - [Symptoms](#symptoms-6) + - [Solution](#solution-7) +- [Set debug log level for all Ceph daemons](#set-debug-log-level-for-all-ceph-daemons) +- [Activate log to file for a particular Ceph daemon](#activate-log-to-file-for-a-particular-ceph-daemon) +- [A worker node using RBD devices hangs up](#a-worker-node-using-rbd-devices-hangs-up) + - [Symptoms](#symptoms-7) + - [Investigation](#investigation-6) + - [Solution](#solution-8) +- [Too few PGs per OSD warning is shown](#too-few-pgs-per-osd-warning-is-shown) + - [Symptoms](#symptoms-8) + - [Solution](#solution-9) +- [LVM metadata can be corrupted with OSD on LV-backed PVC](#lvm-metadata-can-be-corrupted-with-osd-on-lv-backed-pvc) + - [Symptoms](#symptoms-9) + - [Solution](#solution-10) +- [OSD prepare job fails due to low aio-max-nr setting](#osd-prepare-job-fails-due-to-low-aio-max-nr-setting) +- [Unexpected partitions created](#unexpected-partitions-created) + - [Symptoms](#symptoms-10) + - [Solution](#solution-11) + - [Recover from corruption (v1.6.0-v1.6.7)](#recover-from-corruption-v160-v167) +- [Operator environment variables are ignored](#operator-environment-variables-are-ignored) + - [Symptoms](#symptoms-11) + - [Investigation](#investigation-7) + - [Solution](#solution-12) + + Many of these problem cases are hard to summarize down to a short phrase that adequately describes the problem. Each problem will start with a bulleted list of symptoms. Keep in mind that all symptoms may not apply depending on the configuration of Rook. If the majority of the symptoms are seen there is a fair chance you are experiencing that problem. If after trying the suggestions found on this page and the problem is not resolved, the Rook team is very happy to help you troubleshoot the issues in their Slack channel. Once you have [registered for the Rook Slack](https://slack.rook.io), proceed to the `#ceph` channel to ask for assistance. @@ -13,7 +80,7 @@ See also the [CSI Troubleshooting Guide](../Troubleshooting/ceph-csi-common-issu There are two main categories of information you will need to investigate issues in the cluster: 1. Kubernetes status and logs documented [here](common-issues.md) -1. Ceph cluster status (see upcoming [Ceph tools](#ceph-tools) section) +2. Ceph cluster status (see upcoming [Ceph tools](#ceph-tools) section) ### Ceph Tools @@ -22,7 +89,7 @@ After you verify the basic health of the running pods, next you will want to run * Logs on a specific node to find why a PVC is failing to mount * See the [log collection topic](../Storage-Configuration/Advanced/ceph-configuration.md#log-collection) for a script that will help you gather the logs * Other artifacts: - * The monitors that are expected to be in quorum: `kubectl -n get configmap rook-ceph-mon-endpoints -o yaml | grep data` + * The monitors that are expected to be in quorum: `kubectl -n get configmap rook-ceph-mon-endpoints -o yaml | grep data` #### Tools in the Rook Toolbox @@ -110,13 +177,13 @@ There are several common causes for the mons failing to form quorum: * The operator pod does not have network connectivity to the mon pod(s). The network may be configured incorrectly. * One or more mon pods are in running state, but the operator log shows they are not able to form quorum * A mon is using configuration from a previous installation. See the [cleanup guide](../Storage-Configuration/ceph-teardown.md#delete-the-data-on-hosts) - for cleaning the previous cluster. + for cleaning the previous cluster. * A firewall may be blocking the ports required for the Ceph mons to form quorum. Ensure ports 6789 and 3300 are enabled. - See the [Ceph networking guide](https://docs.ceph.com/en/latest/rados/configuration/network-config-ref/) for more details. + See the [Ceph networking guide](https://docs.ceph.com/en/latest/rados/configuration/network-config-ref/) for more details. * There may be MTU mismatch between different networking components. Some networks may be more - susceptible to mismatch than others. If Kubernetes CNI or hosts enable jumbo frames (MTU 9000), - Ceph will use large packets to maximize network bandwidth. If other parts of the networking chain - don't support jumbo frames, this could result in lost or rejected packets unexpectedly. + susceptible to mismatch than others. If Kubernetes CNI or hosts enable jumbo frames (MTU 9000), + Ceph will use large packets to maximize network bandwidth. If other parts of the networking chain + don't support jumbo frames, this could result in lost or rejected packets unexpectedly. #### Operator fails to connect to the mon @@ -288,7 +355,7 @@ The common misconfigurations include: * If `useAllDevices: true`, Rook expects to find local devices attached to the nodes. If no devices are found, no OSDs will be created. * If `useAllDevices: false`, OSDs will only be created if `deviceFilter` is specified. * Only local devices attached to the nodes will be configurable by Rook. In other words, the devices must show up under `/dev`. - * The devices must not have any partitions or filesystems on them. Rook will only configure raw devices. Partitions are not yet supported. + * The devices must not have any partitions or filesystems on them. Rook will only configure raw devices. Partitions are not yet supported. ## OSD pods are failing to start @@ -655,16 +722,16 @@ As an example, you may have `/dev/sdb` with two unexpected partitions (`/dev/sdb as well as a second corrupted disk `/dev/sde` with one unexpected partition (`/dev/sde2`). 1. First, remove the OSDs associated with `/dev/sdb`, `/dev/sdb2`, and `/dev/sdb3`. There might be - only one, or up to 3 OSDs depending on how your system was affected. Again see the - [OSD management doc](../Storage-Configuration/Advanced/ceph-osd-mgmt.md#remove-an-osd). + only one, or up to 3 OSDs depending on how your system was affected. Again see the + [OSD management doc](../Storage-Configuration/Advanced/ceph-osd-mgmt.md#remove-an-osd). 2. Use `dd` to wipe the first sectors of the partitions followed by the disk itself. E.g., * `dd if=/dev/zero of=/dev/sdb2 bs=1M` * `dd if=/dev/zero of=/dev/sdb3 bs=1M` * `dd if=/dev/zero of=/dev/sdb bs=1M` 3. Then wipe clean `/dev/sdb` to prepare it for a new OSD. - See [the teardown document](../Storage-Configuration/ceph-teardown.md#zapping-devices) for details. + See [the teardown document](../Storage-Configuration/ceph-teardown.md#zapping-devices) for details. 4. After this, scale up the Rook operator to deploy a new OSD to `/dev/sdb`. This will allow Ceph to - use `/dev/sdb` for data recovery and replication while the next OSDs are removed. + use `/dev/sdb` for data recovery and replication while the next OSDs are removed. 5. Now Repeat steps 1-4 for `/dev/sde` and `/dev/sde2`, and continue for any other corrupted disks. If your Rook cluster does not have any critical data stored in it, it may be simpler to diff --git a/Documentation/Troubleshooting/ceph-csi-common-issues.md b/Documentation/Troubleshooting/ceph-csi-common-issues.md index fa8ce8af5..d02a8ab9a 100644 --- a/Documentation/Troubleshooting/ceph-csi-common-issues.md +++ b/Documentation/Troubleshooting/ceph-csi-common-issues.md @@ -255,8 +255,8 @@ When a user requests to create the application pod with PVC, there is a three-st ### csi-driver registration `csi-cephfsplugin-xxxx` or `csi-rbdplugin-xxxx` is a daemonset pod running on all the nodes - where your application gets scheduled. If the plugin pods are not running on the node where - your application is scheduled might cause the issue, make sure plugin pods are always running. +where your application gets scheduled. If the plugin pods are not running on the node where +your application is scheduled might cause the issue, make sure plugin pods are always running. Each plugin pod has two important containers: one is `driver-registrar` and `csi-rbdplugin` or `csi-cephfsplugin`. Sometimes there is also a `liveness-prometheus` container. diff --git a/Documentation/Troubleshooting/common-issues.md b/Documentation/Troubleshooting/common-issues.md index 22ef48ab3..7dc081481 100644 --- a/Documentation/Troubleshooting/common-issues.md +++ b/Documentation/Troubleshooting/common-issues.md @@ -18,19 +18,19 @@ Kubernetes status and logs are the main resources needed to investigate issues i Kubernetes status is the first line of investigating when something goes wrong with the cluster. Here are a few artifacts that are helpful to gather: * Rook pod status: - * `kubectl get pod -n -o wide` - * e.g., `kubectl get pod -n rook-ceph -o wide` + * `kubectl get pod -n -o wide` + * e.g., `kubectl get pod -n rook-ceph -o wide` * Logs for Rook pods - * Logs for the operator: `kubectl logs -n -l app=` - * e.g., `kubectl logs -n rook-ceph -l app=rook-ceph-operator` - * Logs for a specific pod: `kubectl logs -n `, or a pod using a label such as mon1: `kubectl logs -n -l ` - * e.g., `kubectl logs -n rook-ceph -l mon=a` - * Logs on a specific node to find why a PVC is failing to mount: - * Connect to the node, then get kubelet logs (if your distro is using systemd): `journalctl -u kubelet` - * Pods with multiple containers - * For all containers, in order: `kubectl -n logs --all-containers` - * For a single container: `kubectl -n logs -c ` - * Logs for pods which are no longer running: `kubectl -n logs --previous ` + * Logs for the operator: `kubectl logs -n -l app=` + * e.g., `kubectl logs -n rook-ceph -l app=rook-ceph-operator` + * Logs for a specific pod: `kubectl logs -n `, or a pod using a label such as mon1: `kubectl logs -n -l ` + * e.g., `kubectl logs -n rook-ceph -l mon=a` + * Logs on a specific node to find why a PVC is failing to mount: + * Connect to the node, then get kubelet logs (if your distro is using systemd): `journalctl -u kubelet` + * Pods with multiple containers + * For all containers, in order: `kubectl -n logs --all-containers` + * For a single container: `kubectl -n logs -c ` + * Logs for pods which are no longer running: `kubectl -n logs --previous ` Some pods have specialized init containers, so you may need to look at logs for different containers within the pod. diff --git a/Documentation/Troubleshooting/disaster-recovery.md b/Documentation/Troubleshooting/disaster-recovery.md index c512dd0a2..eefc08532 100644 --- a/Documentation/Troubleshooting/disaster-recovery.md +++ b/Documentation/Troubleshooting/disaster-recovery.md @@ -72,11 +72,11 @@ the CRs to their prior state without even necessarily suffering cluster downtime 2. **Verify that all critical resources are shown in the output.** The critical resources are these: - - Secrets: `rook-ceph-admin-keyring`, `rook-ceph-config`, `rook-ceph-mon`, `rook-ceph-mons-keyring` - - ConfigMap: `rook-ceph-mon-endpoints` - - Services: `rook-ceph-mon-*`, `rook-ceph-mgr-*` - - Deployments: `rook-ceph-mon-*`, `rook-ceph-osd-*`, `rook-ceph-mgr-*` - - PVCs (if applicable): `rook-ceph-mon-*` and the OSD PVCs (named `-*`, for example `set1-data-*`) + - Secrets: `rook-ceph-admin-keyring`, `rook-ceph-config`, `rook-ceph-mon`, `rook-ceph-mons-keyring` + - ConfigMap: `rook-ceph-mon-endpoints` + - Services: `rook-ceph-mon-*`, `rook-ceph-mgr-*` + - Deployments: `rook-ceph-mon-*`, `rook-ceph-osd-*`, `rook-ceph-mgr-*` + - PVCs (if applicable): `rook-ceph-mon-*` and the OSD PVCs (named `-*`, for example `set1-data-*`) 3. For each listed resource, remove the `ownerReferences` metadata field, in order to unlink it from the deleting `CephCluster` CR. @@ -125,9 +125,9 @@ the CRs to their prior state without even necessarily suffering cluster downtime 6. If there are other CRs in terminating state such as CephBlockPools, CephObjectStores, or CephFilesystems, follow the above steps as well for those CRs: - 1. Backup the CR - 2. Remove the finalizer and confirm the CR is deleted (the underlying Ceph resources will be preserved) - 3. Create the CR again + 1. Backup the CR + 2. Remove the finalizer and confirm the CR is deleted (the underlying Ceph resources will be preserved) + 3. Create the CR again 7. Scale up the operator @@ -172,26 +172,25 @@ Situations this section can help resolve: Assuming `dataHostPathData` is `/var/lib/rook`, and the `CephCluster` trying to adopt is named `rook-ceph`. 1. Make sure the old Kubernetes cluster is completely torn down and the new Kubernetes cluster is up and running without Rook Ceph. -1. Backup `/var/lib/rook` in all the Rook Ceph nodes to a different directory. Backups will be used later. -1. Pick a `/var/lib/rook/rook-ceph/rook-ceph.config` from any previous Rook Ceph node and save the old cluster `fsid` from its content. -1. Remove `/var/lib/rook` from all the Rook Ceph nodes. -1. Add identical `CephCluster` descriptor to the new Kubernetes cluster, especially identical `spec.storage.config` and `spec.storage.nodes`, except `mon.count`, which should be set to `1`. -1. Add identical `CephFilesystem` `CephBlockPool` `CephNFS` `CephObjectStore` descriptors (if any) to the new Kubernetes cluster. -1. Install Rook Ceph in the new Kubernetes cluster. -1. Watch the operator logs with `kubectl -n rook-ceph logs -f rook-ceph-operator-xxxxxxx`, and wait until the orchestration has settled. -1. **STATE**: Now the cluster will have `rook-ceph-mon-a`, `rook-ceph-mgr-a`, and all the auxiliary pods up and running, and zero (hopefully) `rook-ceph-osd-ID-xxxxxx` running. `ceph -s` output should report 1 mon, 1 mgr running, and all of the OSDs down, all PGs are in `unknown` state. Rook should not start any OSD daemon since all devices belongs to the old cluster (which have a different `fsid`). -1. Run `kubectl -n rook-ceph exec -it rook-ceph-mon-a-xxxxxxxx bash` to enter the `rook-ceph-mon-a` pod, +2. Backup `/var/lib/rook` in all the Rook Ceph nodes to a different directory. Backups will be used later. +3. Pick a `/var/lib/rook/rook-ceph/rook-ceph.config` from any previous Rook Ceph node and save the old cluster `fsid` from its content. +4. Remove `/var/lib/rook` from all the Rook Ceph nodes. +5. Add identical `CephCluster` descriptor to the new Kubernetes cluster, especially identical `spec.storage.config` and `spec.storage.nodes`, except `mon.count`, which should be set to `1`. +6. Add identical `CephFilesystem` `CephBlockPool` `CephNFS` `CephObjectStore` descriptors (if any) to the new Kubernetes cluster. +7. Install Rook Ceph in the new Kubernetes cluster. +8. Watch the operator logs with `kubectl -n rook-ceph logs -f rook-ceph-operator-xxxxxxx`, and wait until the orchestration has settled. +9. **STATE**: Now the cluster will have `rook-ceph-mon-a`, `rook-ceph-mgr-a`, and all the auxiliary pods up and running, and zero (hopefully) `rook-ceph-osd-ID-xxxxxx` running. `ceph -s` output should report 1 mon, 1 mgr running, and all of the OSDs down, all PGs are in `unknown` state. Rook should not start any OSD daemon since all devices belongs to the old cluster (which have a different `fsid`). +10. Run `kubectl -n rook-ceph exec -it rook-ceph-mon-a-xxxxxxxx bash` to enter the `rook-ceph-mon-a` pod, ```console mon-a# cat /etc/ceph/keyring-store/keyring # save this keyring content for later use mon-a# exit ``` -1. Stop the Rook operator by running `kubectl -n rook-ceph edit deploy/rook-ceph-operator` and set `replicas` to `0`. -1. Stop cluster daemons by running `kubectl -n rook-ceph delete deploy/X` where X is every deployment in namespace `rook-ceph`, except `rook-ceph-operator` and `rook-ceph-tools`. -1. Save the `rook-ceph-mon-a` address with `kubectl -n rook-ceph get cm/rook-ceph-mon-endpoints -o yaml` in the new Kubernetes cluster for later use. - -1. SSH to the host where `rook-ceph-mon-a` in the new Kubernetes cluster resides. +11. Stop the Rook operator by running `kubectl -n rook-ceph edit deploy/rook-ceph-operator` and set `replicas` to `0`. +12. Stop cluster daemons by running `kubectl -n rook-ceph delete deploy/X` where X is every deployment in namespace `rook-ceph`, except `rook-ceph-operator` and `rook-ceph-tools`. +13. Save the `rook-ceph-mon-a` address with `kubectl -n rook-ceph get cm/rook-ceph-mon-endpoints -o yaml` in the new Kubernetes cluster for later use. +14. SSH to the host where `rook-ceph-mon-a` in the new Kubernetes cluster resides. 1. Remove `/var/lib/rook/mon-a` 2. Pick a healthy `rook-ceph-mon-ID` directory (`/var/lib/rook/mon-ID`) in the previous backup, copy to `/var/lib/rook/mon-a`. `ID` is any healthy mon node ID of the old cluster. 3. Replace `/var/lib/rook/mon-a/keyring` with the saved keyring, preserving only the `[mon.]` section, remove `[client.admin]` section. @@ -213,13 +212,13 @@ Assuming `dataHostPathData` is `/var/lib/rook`, and the `CephCluster` trying to exit ``` -1. Tell Rook to run as old cluster by running `kubectl -n rook-ceph edit secret/rook-ceph-mon` and changing `fsid` to the original `fsid`. Note that the `fsid` is base64 encoded and must not contain a trailing carriage return. For example: +15. Tell Rook to run as old cluster by running `kubectl -n rook-ceph edit secret/rook-ceph-mon` and changing `fsid` to the original `fsid`. Note that the `fsid` is base64 encoded and must not contain a trailing carriage return. For example: ```console echo -n a811f99a-d865-46b7-8f2c-f94c064e4356 | base64 # Replace with the fsid from your old cluster. ``` -1. Disable authentication by running `kubectl -n rook-ceph edit cm/rook-config-override` and adding content below: +16. Disable authentication by running `kubectl -n rook-ceph edit cm/rook-config-override` and adding content below: ```yaml data: @@ -231,10 +230,10 @@ Assuming `dataHostPathData` is `/var/lib/rook`, and the `CephCluster` trying to auth supported = none ``` -1. Bring the Rook Ceph operator back online by running `kubectl -n rook-ceph edit deploy/rook-ceph-operator` and set `replicas` to `1`. -1. Watch the operator logs with `kubectl -n rook-ceph logs -f rook-ceph-operator-xxxxxxx`, and wait until the orchestration has settled. -1. **STATE**: Now the new cluster should be up and running with authentication disabled. `ceph -s` should report 1 mon & 1 mgr & all of the OSDs up and running, and all PGs in either `active` or `degraded` state. -1. Run `kubectl -n rook-ceph exec -it rook-ceph-tools-XXXXXXX bash` to enter tools pod: +17. Bring the Rook Ceph operator back online by running `kubectl -n rook-ceph edit deploy/rook-ceph-operator` and set `replicas` to `1`. +18. Watch the operator logs with `kubectl -n rook-ceph logs -f rook-ceph-operator-xxxxxxx`, and wait until the orchestration has settled. +19. **STATE**: Now the new cluster should be up and running with authentication disabled. `ceph -s` should report 1 mon & 1 mgr & all of the OSDs up and running, and all PGs in either `active` or `degraded` state. +20. Run `kubectl -n rook-ceph exec -it rook-ceph-tools-XXXXXXX bash` to enter tools pod: ```console vi key @@ -243,31 +242,31 @@ Assuming `dataHostPathData` is `/var/lib/rook`, and the `CephCluster` trying to rm key ``` -1. Re-enable authentication by running `kubectl -n rook-ceph edit cm/rook-config-override` and removing auth configuration added in previous steps. -1. Stop the Rook operator by running `kubectl -n rook-ceph edit deploy/rook-ceph-operator` and set `replicas` to `0`. -1. Shut down entire new cluster by running `kubectl -n rook-ceph delete deploy/X` where X is every deployment in namespace `rook-ceph`, except `rook-ceph-operator` and `rook-ceph-tools`, again. This time OSD daemons are present and should be removed too. -1. Bring the Rook Ceph operator back online by running `kubectl -n rook-ceph edit deploy/rook-ceph-operator` and set `replicas` to `1`. -1. Watch the operator logs with `kubectl -n rook-ceph logs -f rook-ceph-operator-xxxxxxx`, and wait until the orchestration has settled. -1. **STATE**: Now the new cluster should be up and running with authentication enabled. `ceph -s` output should not change much comparing to previous steps. +21. Re-enable authentication by running `kubectl -n rook-ceph edit cm/rook-config-override` and removing auth configuration added in previous steps. +22. Stop the Rook operator by running `kubectl -n rook-ceph edit deploy/rook-ceph-operator` and set `replicas` to `0`. +23. Shut down entire new cluster by running `kubectl -n rook-ceph delete deploy/X` where X is every deployment in namespace `rook-ceph`, except `rook-ceph-operator` and `rook-ceph-tools`, again. This time OSD daemons are present and should be removed too. +24. Bring the Rook Ceph operator back online by running `kubectl -n rook-ceph edit deploy/rook-ceph-operator` and set `replicas` to `1`. +25. Watch the operator logs with `kubectl -n rook-ceph logs -f rook-ceph-operator-xxxxxxx`, and wait until the orchestration has settled. +26. **STATE**: Now the new cluster should be up and running with authentication enabled. `ceph -s` output should not change much comparing to previous steps. ## Backing up and restoring a cluster based on PVCs into a new Kubernetes cluster It is possible to migrate/restore an rook/ceph cluster from an existing Kubernetes cluster to a new one without resorting to SSH access or ceph tooling. This allows doing the migration using standard kubernetes resources only. This guide assumes the following: 1. You have a CephCluster that uses PVCs to persist mon and osd data. Let's call it the "old cluster" -1. You can restore the PVCs as-is in the new cluster. Usually this is done by taking regular snapshots of the PVC volumes and using a tool that can re-create PVCs from these snapshots in the underlying cloud provider. [Velero](https://github.com/vmware-tanzu/velero) is one such tool. -1. You have regular backups of the secrets and configmaps in the rook-ceph namespace. Velero provides this functionality too. +2. You can restore the PVCs as-is in the new cluster. Usually this is done by taking regular snapshots of the PVC volumes and using a tool that can re-create PVCs from these snapshots in the underlying cloud provider. [Velero](https://github.com/vmware-tanzu/velero) is one such tool. +3. You have regular backups of the secrets and configmaps in the rook-ceph namespace. Velero provides this functionality too. Do the following in the new cluster: 1. Stop the rook operator by scaling the deployment `rook-ceph-operator` down to zero: `kubectl -n rook-ceph scale deployment rook-ceph-operator --replicas 0` and deleting the other deployments. An example command to do this is `k -n rook-ceph delete deployment -l operator!=rook` -1. Restore the rook PVCs to the new cluster. -1. Copy the keyring and fsid secrets from the old cluster: `rook-ceph-mgr-a-keyring`, `rook-ceph-mon`, `rook-ceph-mons-keyring`, `rook-ceph-osd-0-keyring`, ... -1. Delete mon services and copy them from the old cluster: `rook-ceph-mon-a`, `rook-ceph-mon-b`, ... Note that simply re-applying won't work because the goal here is to restore the `clusterIP` in each service and this field is immutable in `Service` resources. -1. Copy the endpoints configmap from the old cluster: `rook-ceph-mon-endpoints` -1. Scale the rook operator up again : `kubectl -n rook-ceph scale deployment rook-ceph-operator --replicas 1` -1. Wait until the reconciliation is over. +2. Restore the rook PVCs to the new cluster. +3. Copy the keyring and fsid secrets from the old cluster: `rook-ceph-mgr-a-keyring`, `rook-ceph-mon`, `rook-ceph-mons-keyring`, `rook-ceph-osd-0-keyring`, ... +4. Delete mon services and copy them from the old cluster: `rook-ceph-mon-a`, `rook-ceph-mon-b`, ... Note that simply re-applying won't work because the goal here is to restore the `clusterIP` in each service and this field is immutable in `Service` resources. +5. Copy the endpoints configmap from the old cluster: `rook-ceph-mon-endpoints` +6. Scale the rook operator up again : `kubectl -n rook-ceph scale deployment rook-ceph-operator --replicas 1` +7. Wait until the reconciliation is over. ## Restoring the Rook cluster after the Rook namespace is deleted diff --git a/Documentation/Troubleshooting/kubectl-plugin.md b/Documentation/Troubleshooting/kubectl-plugin.md index 0cb210fad..5ba866858 100644 --- a/Documentation/Troubleshooting/kubectl-plugin.md +++ b/Documentation/Troubleshooting/kubectl-plugin.md @@ -18,41 +18,41 @@ See the [kubectl-rook-ceph documentation](https://github.com/rook/kubectl-rook-c - Install [krew](https://krew.sigs.k8s.io/docs/user-guide/setup/install/) - Install Rook plugin - ```console + ```console kubectl krew install rook-ceph - ``` + ``` ## Ceph Commands - Run any `ceph` command with `kubectl rook-ceph ceph `. For example, get the Ceph status: - ```console + ```console kubectl rook-ceph ceph status - ``` + ``` - Output: + Output: - ```console - cluster: - id: a1ac6554-4cc8-4c3b-a8a3-f17f5ec6f529 - health: HEALTH_OK + ```console + cluster: + id: a1ac6554-4cc8-4c3b-a8a3-f17f5ec6f529 + health: HEALTH_OK - services: - mon: 3 daemons, quorum a,b,c (age 11m) - mgr: a(active, since 10m) - mds: 1/1 daemons up, 1 hot standby - osd: 3 osds: 3 up (since 10m), 3 in (since 8d) + services: + mon: 3 daemons, quorum a,b,c (age 11m) + mgr: a(active, since 10m) + mds: 1/1 daemons up, 1 hot standby + osd: 3 osds: 3 up (since 10m), 3 in (since 8d) - data: - volumes: 1/1 healthy - pools: 6 pools, 137 pgs - objects: 34 objects, 4.1 KiB - usage: 58 MiB used, 59 GiB / 59 GiB avail - pgs: 137 active+clean + data: + volumes: 1/1 healthy + pools: 6 pools, 137 pgs + objects: 34 objects, 4.1 KiB + usage: 58 MiB used, 59 GiB / 59 GiB avail + pgs: 137 active+clean - io: - client: 1.2 KiB/s rd, 2 op/s rd, 0 op/s wr - ``` + io: + client: 1.2 KiB/s rd, 2 op/s rd, 0 op/s wr + ``` Reference: [Ceph Status](https://github.com/rook/kubectl-rook-ceph/blob/master/README.md#run-a-ceph-command) @@ -62,14 +62,14 @@ Debug mode can be useful when a MON or OSD needs advanced maintenance operations - Start the debug pod for mon b - ```console + ```console kubectl rook-ceph debug start rook-ceph-mon-b - ``` + ``` - Stop the debug pod for mon b - ```console + ```console kubectl rook-ceph debug stop rook-ceph-mon-b - ``` + ``` Reference: [Debug Mode](https://github.com/rook/kubectl-rook-ceph/blob/master/README.md#debug-mode) diff --git a/Documentation/Troubleshooting/openshift-common-issues.md b/Documentation/Troubleshooting/openshift-common-issues.md index cdb88b632..5b82a3d50 100644 --- a/Documentation/Troubleshooting/openshift-common-issues.md +++ b/Documentation/Troubleshooting/openshift-common-issues.md @@ -8,14 +8,14 @@ OpenShift Console uses OpenShift Prometheus for monitoring and populating data i 1. Change the monitoring namespace to `openshift-monitoring` - Change the namespace of the RoleBinding `rook-ceph-metrics` from `rook-ceph` to `openshift-monitoring` for the `prometheus-k8s` ServiceAccount in [rbac.yaml](https://github.com/rook/rook/blob/master/deploy/examples/monitoring/rbac.yaml#L70). + Change the namespace of the RoleBinding `rook-ceph-metrics` from `rook-ceph` to `openshift-monitoring` for the `prometheus-k8s` ServiceAccount in [rbac.yaml](https://github.com/rook/rook/blob/master/deploy/examples/monitoring/rbac.yaml#L70). -```yaml -subjects: -- kind: ServiceAccount - name: prometheus-k8s - namespace: openshift-monitoring -``` + ```yaml + subjects: + - kind: ServiceAccount + name: prometheus-k8s + namespace: openshift-monitoring + ``` 2. Enable Ceph Cluster monitoring diff --git a/Documentation/Troubleshooting/performance-profiling.md b/Documentation/Troubleshooting/performance-profiling.md index db3603af3..67b3e7124 100644 --- a/Documentation/Troubleshooting/performance-profiling.md +++ b/Documentation/Troubleshooting/performance-profiling.md @@ -28,6 +28,7 @@ To collect this information, please follow these steps: ``` * Install `gdb` , `perf` and `git` inside the pod. For example: + ```console # dnf install gdb git perf -y ``` diff --git a/Documentation/Upgrade/ceph-upgrade.md b/Documentation/Upgrade/ceph-upgrade.md index fc8cdb860..64abb8050 100644 --- a/Documentation/Upgrade/ceph-upgrade.md +++ b/Documentation/Upgrade/ceph-upgrade.md @@ -18,7 +18,7 @@ until all the daemons have been updated. ## Considerations * **WARNING**: Upgrading a Rook cluster is not without risk. There may be unexpected issues or - obstacles that damage the integrity and health of the storage cluster, including data loss. + obstacles that damage the integrity and health of the storage cluster, including data loss. * The Rook cluster's storage may be unavailable for short periods during the upgrade process. * Read this document in full before undertaking a Rook cluster upgrade. @@ -51,10 +51,10 @@ Official Ceph container images can be found on [Quay](https://quay.io/repository These images are tagged in a few ways: * The most explicit form of tags are full-ceph-version-and-build tags (e.g., `v18.2.2-20240311`). - These tags are recommended for production clusters, as there is no possibility for the cluster to - be heterogeneous with respect to the version of Ceph running in containers. + These tags are recommended for production clusters, as there is no possibility for the cluster to + be heterogeneous with respect to the version of Ceph running in containers. * Ceph major version tags (e.g., `v18`) are useful for development and test clusters so that the - latest version of Ceph is always available. + latest version of Ceph is always available. **Ceph containers other than the official images from the registry above will not be supported.** diff --git a/Documentation/Upgrade/health-verification.md b/Documentation/Upgrade/health-verification.md index bf36587e6..2a759ca6d 100644 --- a/Documentation/Upgrade/health-verification.md +++ b/Documentation/Upgrade/health-verification.md @@ -35,7 +35,8 @@ kubectl -n $ROOK_CLUSTER_NAMESPACE exec -it $TOOLS_POD -- ceph status ``` The output should look similar to the following: -``` + +```console cluster: id: a3f4d647-9538-4aff-9fd1-b845873c3fe9 health: HEALTH_OK @@ -61,7 +62,7 @@ The output should look similar to the following: In the output above, note the following indications that the cluster is in a healthy state: * Cluster health: The overall cluster status is `HEALTH_OK` and there are no warning or error status - messages displayed. + messages displayed. * Monitors (mon): All of the monitors are included in the `quorum` list. * Manager (mgr): The Ceph manager is in the `active` state. * OSDs (osd): All OSDs are `up` and `in`. diff --git a/Documentation/Upgrade/rook-upgrade.md b/Documentation/Upgrade/rook-upgrade.md index 57dc8c487..c91880c1b 100644 --- a/Documentation/Upgrade/rook-upgrade.md +++ b/Documentation/Upgrade/rook-upgrade.md @@ -51,18 +51,18 @@ those releases. ## Breaking changes in v1.14 * The minimum supported version of Kubernetes is v1.25. - Upgrade to Kubernetes v1.25 or higher before upgrading Rook. + Upgrade to Kubernetes v1.25 or higher before upgrading Rook. * The Rook operator config `CSI_ENABLE_READ_AFFINITY` was removed. v1.13 clusters that have modified - this value to be `"true"` must set the option as desired in each CephCluster as documented - [here](https://rook.github.io/docs/rook/v1.14/CRDs/Cluster/ceph-cluster-crd/#csi-driver-options) - before upgrading to v1.14. + this value to be `"true"` must set the option as desired in each CephCluster as documented + [here](https://rook.github.io/docs/rook/v1.14/CRDs/Cluster/ceph-cluster-crd/#csi-driver-options) + before upgrading to v1.14. * Rook is beginning the process of deprecating CSI network "holder" pods. - If there are pods named `csi-*plugin-holder-*` in the Rook operator namespace, see the - [detailed documentation](../CRDs/Cluster/network-providers.md#holder-pod-deprecation) - to disable them. This is optional for v1.14, but will be required in a future release. + If there are pods named `csi-*plugin-holder-*` in the Rook operator namespace, see the + [detailed documentation](../CRDs/Cluster/network-providers.md#holder-pod-deprecation) + to disable them. This is optional for v1.14, but will be required in a future release. * In the operator helm chart, the images for the CSI driver are now specified with separate - `repository` and `tag` values. If the CSI images have been customized, convert them from the - `image` value to the separated `repository` and `tag` values. + `repository` and `tag` values. If the CSI images have been customized, convert them from the + `image` value to the separated `repository` and `tag` values. ## Considerations @@ -70,9 +70,9 @@ those releases. With this upgrade guide, there are a few notes to consider: * **WARNING**: Upgrading a Rook cluster is not without risk. There may be unexpected issues or - obstacles that damage the integrity and health the storage cluster, including data loss. + obstacles that damage the integrity and health the storage cluster, including data loss. * The Rook cluster's storage may be unavailable for short periods during the upgrade process for - both Rook operator updates and for Ceph version updates. + both Rook operator updates and for Ceph version updates. * Read this document in full before undertaking a Rook cluster upgrade. ## Patch Release Upgrades @@ -103,7 +103,7 @@ manifests before any update. The common resources and CRDs might not be updated release, but Kubernetes will only apply updates to the ones that changed. Also update optional resources like Prometheus monitoring noted more fully in the -[upgrade section below](#updates-for-optional-resources). +[upgrade section below](#prometheus-updates). ## Helm @@ -115,7 +115,7 @@ The upgrade steps in this guide will clarify what Helm handles automatically. config `csi.disableHolderPods: false` in the values.yaml before upgrading to v1.14. The `rook-ceph` helm chart upgrade performs the Rook upgrade. -The `rook-ceph-cluster` helm chart upgrade performs a [Ceph upgrade](#ceph-version-upgrades) if the Ceph image is updated. +The `rook-ceph-cluster` helm chart upgrade performs a [Ceph upgrade](./ceph-upgrade.md) if the Ceph image is updated. The `rook-ceph` chart should be upgraded before `rook-ceph-cluster`, so the latest operator has the opportunity to update custom resources as necessary. @@ -127,8 +127,8 @@ custom resources as necessary. In order to successfully upgrade a Rook cluster, the following prerequisites must be met: * The cluster should be in a healthy state with full functionality. Review the - [health verification guide](health-verification.md) in order to verify a CephCluster is in a good - starting state. + [health verification guide](health-verification.md) in order to verify a CephCluster is in a good + starting state. * All pods consuming Rook storage should be created, running, and in a steady state. ## Rook Operator Upgrade diff --git a/README.md b/README.md index 4ff629f1f..5fa7ae403 100644 --- a/README.md +++ b/README.md @@ -47,10 +47,10 @@ For further details, please see the complete [security release process](SECURITY Please use the following to reach members of the community: -- Slack: Join our [slack channel](https://slack.rook.io) -- GitHub: Start a [discussion](https://github.com/rook/rook/discussions) or open an [issue](https://github.com/rook/rook/issues) -- Twitter: [@rook_io](https://twitter.com/rook_io) -- Security topics: [cncf-rook-security@lists.cncf.io](#reporting-security-vulnerabilities) +- Slack: Join our [slack channel](https://slack.rook.io) +- GitHub: Start a [discussion](https://github.com/rook/rook/discussions) or open an [issue](https://github.com/rook/rook/issues) +- Twitter: [@rook_io](https://twitter.com/rook_io) +- Security topics: [cncf-rook-security@lists.cncf.io](#reporting-security-vulnerabilities) ## Community Meeting @@ -61,9 +61,9 @@ Any changes to the meeting schedule will be added to the [agenda doc](https://do Anyone who wants to discuss the direction of the project, design and implementation reviews, or general questions with the broader community is welcome and encouraged to join. -- Meeting link: -- [Current agenda and past meeting notes](https://docs.google.com/document/d/1exd8_IG6DkdvyA0eiTtL2z5K2Ra-y68VByUUgwP7I9A/edit?usp=sharing) -- [Past meeting recordings](https://www.youtube.com/playlist?list=PLP0uDo-ZFnQP6NAgJWAtR9jaRcgqyQKVy) +- Meeting link: +- [Current agenda and past meeting notes](https://docs.google.com/document/d/1exd8_IG6DkdvyA0eiTtL2z5K2Ra-y68VByUUgwP7I9A/edit?usp=sharing) +- [Past meeting recordings](https://www.youtube.com/playlist?list=PLP0uDo-ZFnQP6NAgJWAtR9jaRcgqyQKVy) ## Official Releases diff --git a/tests/scripts/markdownlint-admonitions.js b/tests/scripts/markdownlint-admonitions.js new file mode 100644 index 000000000..177e29a6c --- /dev/null +++ b/tests/scripts/markdownlint-admonitions.js @@ -0,0 +1,52 @@ +const admonition_identifier = "!!!" +const body_tab_space_indent = 4 + +module.exports = { + "names": [ "mkdocs-admonitions" ], + "description": "Enforce mkdocs admonitions are formatted properly", + "tags": [ "test" ], + "parser": "none", + "function": function rule(params, onError) { + const { lines } = params; + const countTabSpaces = (line) => { + start_spaces = /^\s*/; + return line.match(start_spaces)[0].length + }; + lines.forEach((line, i) => { + const thisLineNumber = i+1 // "lineNumber" field is ones-based + const bodyLineIndex = i+1 // body line should start after header + const bodyLineNumber = bodyLineIndex + 1 + + const isAdmonitionHeader = ( line.trimLeft().startsWith(admonition_identifier) ) + if ( !isAdmonitionHeader ) return; + + const admonition_start = line.indexOf(admonition_identifier) + const expected_body_tab_spaces = ''.padStart(admonition_start+body_tab_space_indent, ' ') + if ( lines[i+1].trim().length == 0 ) { + // body should immediately follow header + onError({ + "lineNumber": thisLineNumber + 1, + "detail": "found blank line after admonition header -- body text should immediately follow header", + "context": line, + "fixInfo": { + "lineNumber": thisLineNumber + 1, + "deleteCount": -1, // delete the line + }, + }); + } else if ( lines[bodyLineIndex] != expected_body_tab_spaces + lines[bodyLineIndex].trimStart() ) { + // body should be indented exactly 4 spaces from start of header + got_tab_spaces = countTabSpaces(lines[bodyLineIndex]) + onError({ + "lineNumber": bodyLineNumber, + "detail": "admonition/callout body is not indented properly", + "context": lines[bodyLineIndex], + "fixInfo": { + "lineNumber": bodyLineNumber, + "deleteCount": got_tab_spaces, + "insertText": expected_body_tab_spaces, // insert at beginning of lineNumber + }, + }); + } + }); + } +}; diff --git a/tests/scripts/markdownlint-tab-spacing.js b/tests/scripts/markdownlint-tab-spacing.js new file mode 100644 index 000000000..6c8c56ba5 --- /dev/null +++ b/tests/scripts/markdownlint-tab-spacing.js @@ -0,0 +1,77 @@ +// enforce that lines must be indented to a strict number of tab-spaces +// this lint rule is largely non-contextual and requires nearly all lines to adhere to the rule +// this is even true for ordered/unordered lists in which mkdocs **usually** allows spacing to match +// that of the parent line. but this rule is quick and dirty, and it does not harm to enforce +// indenting to 4-space boundaries to be entirely certain that mkdocs will render as intended. + +const expected_tab_spaces = 4 + +// it's hard to know exactly what spacing the user intended, but we can make an educated guess +// considering 2 things: +// 1. most code editors are likely to indent to 2 spaces by default. therefore, suggest +// that anything indented 2 spaces or more should be indented all the way +// 2. some users put a single space in front of bullets/numbers when starting a list. +// therefore, suggest that anything indented only a single space should be non-indented +const suggested_spacing_indent_cutoff = 2 + +module.exports = { + "names": [ "strict-tab-spacing" ], + "description": "Enforce strict tab spacing", + "tags": [ "test" ], + "parser": "none", + "function": function rule(params, onError) { + const { lines } = params; + const countTabSpaces = (line) => { + start_spaces = /^\s*/; + return line.match(start_spaces)[0].length + }; + const generateTabSpaces = (count) => { + return ''.padStart(count, ' ') + } + const isCodeBlockEnd = (line) => { + trimmed = line.trim() + return ( trimmed === "```" ) + } + const isCodeBlockStart = (line) => { + if ( isCodeBlockEnd(line) ) return false; + trimmed = line.trim() + return ( trimmed.startsWith("```") ) + } + let code_block_depth = 0 + lines.forEach((line, i) => { + // don't strictly enforce tab spacing inside code blocks + if ( isCodeBlockStart(line) ) { + code_block_depth++ + // code block start lines must be indented to a tab-space boundary + } else if ( isCodeBlockEnd(line) ) { + code_block_depth-- + // code block end lines must also be indented to a tab-space boundary + } else if ( code_block_depth > 0 ) { + // if inside of a code block, but not a start/end line, don't warn about spacing + return + } + + const thisLineNumber = i+1 // "lineNumber" field is ones-based + got_tab_spaces = countTabSpaces(line) + const floor = got_tab_spaces / expected_tab_spaces + const remainder = got_tab_spaces % expected_tab_spaces + if ( remainder != 0 ) { + // begin with the assumption that the line should be non-indented + suggested_spacing = generateTabSpaces(floor) + if ( remainder >= suggested_spacing_indent_cutoff ) { + suggested_spacing = generateTabSpaces(floor + expected_tab_spaces) + } + onError({ + "lineNumber": thisLineNumber, + "detail": "lines must be indented to a strict boundary of " + expected_tab_spaces + " spaces", + "context": line, + "fixInfo": { + "lineNumber": thisLineNumber, + "deleteCount": got_tab_spaces, + "insertText": suggested_spacing, // insert at beginning of lineNumber + }, + }); + } + }); + } + };