From a950feb01ea4bf429c138d5f2ad0ed35c58e9353 Mon Sep 17 00:00:00 2001 From: Blaine Gardner Date: Thu, 7 Apr 2022 12:47:33 -0600 Subject: [PATCH] docs: update NFS docs including latest CSI work Update the NFS docs with the latest available info for Ceph Quincy including the CSI provisioner driver. Signed-off-by: Blaine Gardner --- Documentation/ceph-csi-drivers.md | 21 +- Documentation/ceph-nfs-crd.md | 487 ++++++++++--------------- Documentation/ceph-upgrade.md | 9 +- deploy/examples/nfs-load-balancer.yaml | 19 + deploy/examples/nfs-test.yaml | 11 +- deploy/examples/nfs.yaml | 77 ++-- 6 files changed, 272 insertions(+), 352 deletions(-) create mode 100644 deploy/examples/nfs-load-balancer.yaml diff --git a/Documentation/ceph-csi-drivers.md b/Documentation/ceph-csi-drivers.md index 91b2a6f2a..bb5f5be85 100644 --- a/Documentation/ceph-csi-drivers.md +++ b/Documentation/ceph-csi-drivers.md @@ -6,18 +6,19 @@ indent: true {% include_relative branch.liquid %} # Ceph CSI Drivers -There are two CSI drivers integrated with Rook that will enable different scenarios: +There are three CSI drivers integrated with Rook that will enable different scenarios: -* RBD: This driver is optimized for RWO pod access where only one pod may access the storage -* CephFS: This driver allows for RWX with one or more pods accessing the same storage +* RBD: This block storage driver is optimized for RWO pod access where only one pod may access the + storage. [More information](ceph-block.md). +* CephFS: This file storage driver allows for RWX with one or more pods accessing the same storage. + [More information](ceph-filesystem.md). +* NFS (experimental): This file storage driver allows creating NFS exports that can be mounted to + pods, or the exports can be mounted directly via an NFS client from inside or outside the + Kubernetes cluster. [More information](ceph-nfs-crd.md#ceph-csi-nfs-provisioner-and-nfs-csi-driver). -The drivers are enabled automatically with the Rook operator. They will be started -in the same namespace as the operator when the first CephCluster CR is created. - -For documentation on consuming the storage: - -* RBD: See the [Block Storage](ceph-block.md) topic -* CephFS: See the [Shared Filesystem](ceph-filesystem.md) topic +The Ceph Filesysetem (CephFS) and RADOS Block Device (RBD) drivers are enabled automatically with +the Rook operator. The NFS driver is disabled by default. All drivers will be started in the same +namespace as the operator when the first CephCluster CR is created. ## Supported Versions The supported Ceph CSI version is 3.3.0 or greater with Rook. Refer to ceph csi [releases](https://github.com/ceph/ceph-csi/releases) diff --git a/Documentation/ceph-nfs-crd.md b/Documentation/ceph-nfs-crd.md index 8f7a9e6dc..578159dd2 100644 --- a/Documentation/ceph-nfs-crd.md +++ b/Documentation/ceph-nfs-crd.md @@ -3,6 +3,7 @@ title: NFS CRD weight: 3100 indent: true --- +{% include_relative branch.liquid %} # Ceph NFS Server CRD @@ -13,13 +14,12 @@ resource definition. This will spin up a cluster of via shared RADOS objects. The servers will be configured for NFSv4.1+ access only, as serving earlier protocols can inhibit responsiveness after a server restart. -> **WARNING**: We do not recommend using NFS in Ceph v16.2.0 through v16.2.6. If you are using Ceph -> v15, we encourage you to upgrade directly to Ceph Pacific v16.2.7. -> [Upgrade steps are outlined below.](#upgrading-from-ceph-v15-to-v16) +> **WARNING**: Due to a number of Ceph issues and changes, Rook officially only supports Ceph +> v16.2.7 or higher for CephNFS. If you are using an earlier version, upgrade your Ceph version +> following the advice given in Rook's +> [v1.8 NFS docs](https://rook.github.io/docs/rook/v1.8/ceph-nfs-crd.html). ## Samples -The following sample assumes Ceph v16 and will create a two-node active-active cluster of NFS -Ganesha gateways. ```yaml apiVersion: ceph.rook.io/v1 @@ -28,99 +28,88 @@ metadata: name: my-nfs namespace: rook-ceph spec: - # For Ceph v15, the rados block is required. It is ignored for Ceph v16. - rados: - # RADOS pool where NFS configs are stored. - # In this example the data pool for the "myfs" filesystem is used. - # If using the object store example, the data pool would be "my-store.rgw.buckets.data". - # Note that this has nothing to do with where exported file systems or object stores live. - pool: myfs-data0 - # RADOS namespace where NFS client recovery data is stored in the pool. - namespace: nfs-ns - # Settings for the NFS server server: - # the number of active NFS servers - active: 2 - # A key/value list of annotations + active: 1 + + placement: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: role + operator: In + values: + - nfs-node + topologySpreadConstraints: + tolerations: + - key: nfs-node + operator: Exists + podAffinity: + podAntiAffinity: + annotations: - # key: value - # where to run the NFS server - placement: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: role - # operator: In - # values: - # - mds-node - # tolerations: - # - key: mds-node - # operator: Exists - # podAffinity: - # podAntiAffinity: - # topologySpreadConstraints: + my-annotation: something + + labels: + my-label: something - # The requests and limits set here allow the ganesha pod(s) to use half of one CPU core and 1 gigabyte of memory resources: - # limits: - # cpu: "500m" - # memory: "1024Mi" - # requests: - # cpu: "500m" - # memory: "1024Mi" - # the priority class to set to influence the scheduler's pod preemption + limits: + cpu: "500m" + memory: "1024Mi" + requests: + cpu: "500m" + memory: "1024Mi" + priorityClassName: + + logLevel: NIV_INFO ``` ## NFS Settings -### RADOS Settings -NFS configuration is stored in a Ceph pool so that it is highly available and protected. How that is -configured changes depending on the Ceph version. Configuring the pool is done via the `rados` config. +### Server +The `server` spec sets configuration for Rook-created NFS-Ganesha servers. -> **WARNING**: Do not use [erasure coded (EC) pools](ceph-pool-crd.md#erasure-coded) for NFS. -> NFS-Ganesha uses OMAP which is not supported by Ceph's erasure coding. - -#### For Ceph v16 or newer -* `poolConfig`: (optional) The pool settings to use for the RADOS pool. - It matches the [CephBlockPool](ceph-block.md) specification. - The settings will be applied to a pool named `.nfs` on Ceph v16.2.7 or newer. - -#### For Ceph v15 -* `pool`: (mandatory) The Ceph pool where NFS configuration is stored. -* `namespace`: (mandatory) The namespace in the `pool` where configuration objects will be stored. - -Rook ignores both `pool` and `namespace` ([see above](#for-ceph-v15)) settings when running Ceph v16 -or newer. +* `active`: The number of active NFS servers. Rook supports creating more than one active NFS + server, but cannot guarantee high availability. For values greater than 1, see the + [known issue](#serveractive-count-greater-than-1) below. +* `placement`: Kubernetes placement restrictions to apply to NFS server Pod(s). This is similar to + placement defined for daemons configured by the + [CephCluster CRD](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/cluster.yaml). +* `annotations`: Kubernetes annotations to apply to NFS server Pod(s) +* `labels`: Kubernetes labels to apply to NFS server Pod(s) +* `resources`: Kubernetes resource requests and limits to set on NFS server Pod(s) +* `priorityClassName`: Set priority class name for the NFS server Pod(s) +* `logLevel`: The log level that NFS-Ganesha servers should output.
+ Default value: NIV_INFO
+ Supported values: NIV_NULL | NIV_FATAL | NIV_MAJ | NIV_CRIT | NIV_WARN | NIV_EVENT | NIV_INFO | NIV_DEBUG | NIV_MID_DEBUG | NIV_FULL_DEBUG | NB_LOG_LEVEL ## Creating Exports When a CephNFS is first created, all NFS daemons within the CephNFS cluster will share a configuration with no exports defined. -### For Ceph v16 or newer -For Ceph v16.2.0 through v16.2.6, exports cannot be managed through the Ceph dashboard, and -newly-created Ceph command line tools are lacking. We highly recommend using Ceph v16.2.7 or higher -with NFS, which fixes bugs and streamlines export management, allowing exports to be created via the -Ceph Dashboard and the Ceph CLI. With v16.2.7 or higher, the Ceph dashboard and Ceph CLI will be -able to manage the same NFS exports interchangeably as desired. - -#### Using the Ceph Dashboard +### Using the Ceph Dashboard Exports can be created via the [Ceph dashboard](https://docs.ceph.com/en/latest/mgr/dashboard/#nfs-ganesha-management) for Ceph v16 as well. To enable and use the Ceph dashboard in Rook, see [here](ceph-dashboard.md). -#### Using the Ceph CLI +### Using the Ceph CLI The Ceph CLI can be used from the Rook toolbox pod to create and manage NFS exports. To do so, first -ensure the necessary Ceph mgr modules are enabled and that the Ceph orchestrator backend is set to -Rook. -```console -ceph mgr module enable rook -ceph mgr module enable nfs -ceph orch set backend rook -``` +ensure the necessary Ceph mgr modules are enabled, if necessary, and that the Ceph orchestrator +backend is set to Rook. + +> #### **Enable the Ceph orchestrator if necessary** +> * Required for Ceph v16.2.7 and below +> * Optional for Ceph v16.2.8 and above +> * Must be disabled for Ceph v17.2.0 due to a [Ceph regression](#ceph-v1720) +> ```console +> ceph mgr module enable rook +> ceph mgr module enable nfs +> ceph orch set backend rook +> ``` [Ceph's NFS CLI](https://docs.ceph.com/en/latest/mgr/nfs/#export-management) can create NFS exports that are backed by [CephFS](https://docs.ceph.com/en/latest/cephfs/nfs/) (a CephFilesystem) or @@ -178,7 +167,7 @@ If you are done managing NFS exports and don't need the Ceph orchestrator module anything else, it may be preferable to disable the Rook and NFS mgr modules to free up a small amount of RAM in the Ceph mgr Pod. ```console -ceph mgr module disable nfs +ceph orch set backend "" ceph mgr module disable rook ``` @@ -194,217 +183,11 @@ exports at once to a mount location. mount -t nfs4 -o proto=tcp :/ ``` -### For Ceph v15 -Exports can be created via the -[Ceph dashboard](https://docs.ceph.com/en/octopus/mgr/dashboard/#dashboard-nfs-ganesha-management) -for Ceph v15. To enable and use the Ceph dashboard in Rook, see [here](ceph-dashboard.md). -Enable the creation of NFS exports in the dashboard for a given cephfs or object gateway pool by -running the following command in the toolbox container: -* [For a single CephNFS cluster](https://docs.ceph.com/en/octopus/mgr/dashboard/#configuring-nfs-ganesha-in-the-dashboard) - ```console - ceph dashboard set-ganesha-clusters-rados-pool-namespace [/] - ``` -* [For multiple CephNFS clusters](https://docs.ceph.com/en/octopus/mgr/dashboard/#support-for-multiple-nfs-ganesha-clusters) - ```console - ceph dashboard set-ganesha-clusters-rados-pool-namespace :[/](,:[/])* - ``` - For each of the multiple entries above, `cephnfs-name` is the name given to CephNFS resource by the - manifest's `metadata.name`: `my-nfs` for the example earlier in this document. `pool` and - `namespace` are the same configured via the CephNFS spec's `rados` block. - -You should now be able to create exports from the -[Ceph dashboard](https://docs.ceph.com/en/octopus/mgr/dashboard/#ceph-dashboard). - -**You may need to enable exports created by the dashboard before they will work!** - -Creating exports via the dashboard does not necessarily enable them. Newer versions of Ceph v15 -enable the exports automatically, but not all. To ensure the exports are created automatically, use -Ceph v15.2.15 or higher. Otherwise, you must take the manual steps below to ensure the exports are -enabled. - -To enable exports, we are going to modify the Ceph RADOS object (stored in Ceph) that defines the -configuration shared by all NFS daemons. - -Please note that `` and `` will continue to refer to the configured `rados` spec's -`pool` and `namespace` for a particular CephNFS cluster. - -List the shared configuration objects in a Ceph pool with this command from the Ceph toolbox. -```console -rados --pool --namespace ls -``` - -The output may look something like below after you have created two exports. Here we have used the -`my-nfs` example CephNFS. -``` -conf-nfs.my-nfs -export-1 -export-2 -grace -rec-0000000000000002:my-nfs.a -``` - -The configuration of NFS daemons, and enabling exports, is controlled by the `conf-nfs.my-nfs` -object in this example. The object name follows the `conf-nfs.` pattern. - -Get the contents of the config file, which may be empty as in this example. -```console -rados --pool --namespace get conf-nfs.my-nfs my-nfs.conf -cat my-nfs.conf -``` - -Modify the `my-nfs.conf` file above to add URLs for enabling exports. -``` -%url "rados:////export-1" -%url "rados:////export-2" -``` - -Then write the modified file to the RADOS config object. -```console -rados --pool --namespace put conf-nfs.my-nfs my-nfs.conf -``` - -Verify the changes are saved by getting the config again, just as before. -```console -rados --pool --namespace get conf-nfs.my-nfs my-nfs.conf -cat my-nfs.conf -``` -``` -%url "rados:////export-1" -%url "rados:////export-2" -``` - - -## Upgrading from Ceph v15 to v16 -We do not recommend using NFS in Ceph v16.2.0 through v16.2.6 due to bugs in Ceph's NFS -implementation. If you are using Ceph v15, we encourage you to upgrade directly to Ceph v16.2.7. - -### Prep -To upgrade, first follow the [usual Ceph upgrade steps](ceph-upgrade.md#ceph-version-upgrades). When -the upgrade completes, this will result in NFS exports that no longer work. The dashboard's NFS -management will also be broken. We must now migrate the NFS exports to Ceph's new management method. - -We will do all work from the toolbox pod. Exec into an interactive session there. - -First, unset the previous dashboard configuration with the below command. -```sh -ceph dashboard set-ganesha-clusters-rados-pool-namespace "" -``` - -Also ensure the necessary Ceph mgr modules are enabled and that the Ceph orchestrator backend is set -to Rook. -```console -ceph mgr module enable rook -ceph mgr module enable nfs -ceph orch set backend rook -``` - -### Step 1 -Pick a CephNFS to work with and make a note of the `spec.rados.pool` and `spec.rados.namespace`. If -the `pool` is not set, it is `.nfs`. We will refer to these as pool/`` and -namespace/`` for the remainder of the steps. Also note the name of the CephNFS resource, -which will be referred to as CephNFS name or ``. - -### Step 2 -List the exports defined in the pool. -```sh -rados --pool --namespace ls -``` - -This may look something like below. -``` -grace -rec-0000000000000002:my-nfs.a -export-1 -export-2 -conf-nfs.my-nfs -``` - -### Step 3 -For each export above, save the export to an `.conf` file. -``` -EXPORT="export-1" # "export-2", "export-3", etc. -rados --pool --namespace get "$EXPORT" "/tmp/$EXPORT.conf" -``` - -The file should contain content similar to what is shown here. -``` -$ cat /tmp/export-1.conf -EXPORT { - export_id = 1; - path = "/"; - pseudo = "/test"; - access_type = "RW"; - squash = "no_root_squash"; - protocols = 4; - transports = "TCP"; - FSAL { - name = "CEPH"; - user_id = "admin"; - filesystem = "myfs"; - secret_access_key = "AQAyr69hwddJERAAE9WdFCmY10fqehzK3kabFw=="; - } - -} -``` - -### Step 4 -We will now import each export into Ceph's new format. Perform this step for each export you wish to -migrate. - -First remove the `FSAL` configuration block's `user_id` and `secret_access_key` configuration items. -It is sufficient to delete the lines in the `/tmp/.conf` file using `vi` or some other -editor. The file should look similar to below when the edit is finished. -``` -$ cat /tmp/.conf -EXPORT { - export_id = 1; - path = "/"; - pseudo = "/test"; - access_type = "RW"; - squash = "no_root_squash"; - protocols = 4; - transports = "TCP"; - FSAL { - name = "CEPH"; - filesystem = "myfs"; - } - -} -``` - -Now that the old user and access key are removed, import the export. There should be no errors, but -if there are, follow the error message instructions to proceed. -```sh -ceph nfs export apply -i /tmp/.conf -``` - -### Step 5 -Once all exports have been migrated for the current CephNFS, it is good to verify the exports. Use -`ceph nfs export ls ` to list all exports (identified by the pseudo path), and use -`ceph nfs export info ` to inspect the configuration. An export -configuration may look something like below. The [v16 CLI section above](#using-the-ceph-cli) shows -this in more detail. - -### Step 6 -Repeat these [steps](#step-1) for each other CephNFS. - -Clean up all `.conf` files before moving onto subsequent CephNFSes to avoid confusion. -``` -rm -f /tmp/export-*.conf -``` - -### Wrap-up -Once you are finished migrating all CephNFSes, the migration is complete. If you wish to use the -Ceph dashboard to manage exports, you should now be able to find them all listed there. - -If you are done managing NFS exports via the CLI and don't need the Ceph orchestrator module enabled -for anything else, it may be preferable to disable the Rook and NFS mgr modules to free up a small -amount of RAM in the Ceph mgr Pod. -```console -ceph mgr module disable nfs -ceph mgr module disable rook -``` +## Exposing the NFS server outside of the Kubernetes cluster +Use a LoadBalancer Service to expose an NFS server (and its exports) outside of the Kubernetes +cluster. We provide an example here: +[`deploy/examples/nfs-load-balancer.yaml`](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples). ## Scaling the active server count @@ -417,18 +200,52 @@ started. Scaling down the cluster requires that clients be migrated from servers eliminated to others. That process is currently a manual one and should be performed before reducing the size of the cluster. +> **WARNING:** see the [known issue](#serveractive-count-greater-than-1) below about setting this +> value greater than one. + + +## Known issues +### server.active count greater than 1 +* Active-active scale out does not work well with the NFS protocol. If one NFS server in a cluster + is offline, other servers may block client requests until the offline server returns, which may + not always happen due to the Kubernetes scheduler. + * Workaround: It is safest to run only a single NFS server, but we do not limit this if it + benefits your use case. + +### Ceph v17.2.0 +* Ceph NFS management with the Rook mgr module enabled has a breaking regression with the Ceph + Quincy v17.2.0 release. + * Workaround: Leave Ceph's Rook orchestrator mgr module disabled. If you have enabled it, you must + disable it using the snippet below from the toolbox. + ```console + ceph orch set backend "" + ceph mgr module disable rook + ``` + ## Advanced configuration -All CephNFS daemons are configured using shared configuration objects stored in Ceph. In general, -users should only need to modify the configuration object. Exports can be created via the simpler -Ceph-provided means documented above. +All CephNFS daemons are configured using shared RADOS objects stored in a Ceph pool named `.nfs`. +Users can modify the configuration object for each CephNFS cluster if they wish to customize the +configuration. -For configuration and advanced usage, the format for these objects is documented in the -[NFS Ganesha](https://github.com/nfs-ganesha/nfs-ganesha/wiki) project. +### Changing configuration of the .nfs pool +By default, Rook creates the `.nfs` pool with Ceph's default configuration. If you wish to change +the configuration of this pool (for example to change its failure domain or replication factor), you +can create a CephBlockPool with the `spec.name` field set to `.nfs`. This pool **must** be +replicated and **cannot** be erasure coded. +[`deploy/examples/nfs.yaml`](https://github.com/rook/rook/blob/{{ branchName }}/deploy/examples/nfs.yaml) +contains a sample for reference. + +### Adding custom NFS-Ganesha config file changes +The NFS-Ganesha config file format for these objects is documented in the +[NFS-Ganesha project](https://github.com/nfs-ganesha/nfs-ganesha/wiki). Use Ceph's `rados` tool from the toolbox to interact with the configuration object. The below -command will get you started by dumping the contents of the config object to stdout. The output may -look something like the example shown. +command will get you started by dumping the contents of the config object to stdout. The output will +look something like the example shown if you have already created two exports as documented above. +It is best not to modify any of the export objects created by Ceph so as not to cause errors with +Ceph's export management. + ```console rados --pool --namespace get conf-nfs. - ``` @@ -440,5 +257,83 @@ rados --pool --namespace get conf-nfs. - `rados ls` and `rados put` are other commands you will want to work with the other shared configuration objects. -Of note, it is possible to pre-populate the NFS configuration and export objects prior to starting -NFS servers. +Of note, it is possible to pre-populate the NFS configuration and export objects prior to creating +CephNFS server clusters. + + +## Ceph CSI NFS provisioner and NFS CSI driver +> **EXPERIMENTAL**: this feature is experimental, and we do not guarantee it is bug-free, nor will +> we support upgrades to future versions + +In version 1.9.1, Rook is able to deploy the experimental NFS Ceph CSI driver. This requires Ceph +CSI version 3.6.0 or above. We recommend Ceph v16.2.7 or above. + +For this section, we will refer to Rook's deployment examples in the +[deploy/examples](https://github.com/rook/rook/tree/{{ branchName }}/deploy/examples) directory. + +The Ceph CSI NFS provisioner and driver require additional RBAC to operate. Apply the +`deploy/examples/csi/nfs/rbac.yaml` manifest to deploy the additional resources. + +Rook will only deploy the Ceph CSI NFS provisioner and driver components when the +`ROOK_CSI_ENABLE_NFS` config is set to `"true"` in the `rook-ceph-operator-config` configmap. Change +the value in your manifest, or patch the resource as below. +```sh +kubectl --namespace rook-ceph patch configmap rook-ceph-operator-config --type merge --patch '{"data":{"ROOK_CSI_ENABLE_NFS": "true"}}' +``` + +> **NOTE:** The rook-ceph operator Helm chart will deploy the required RBAC and enable the driver +> components if `csi.nfs.enabled` is set to `true`. + +In order to create NFS exports via the CSI driver, you must first create a CephFilesystem to serve +as the underlying storage for the exports, and you must create a CephNFS to run an NFS server that +will expose the exports. + +From the examples, `filesystem.yaml` creates a CephFilesystem called `myfs`, and `nfs.yaml` creates +an NFS server called `my-nfs`. + +You may need to enable or disable the Ceph orchestrator. Follow the same steps documented +[above](#enable-the-ceph-orchestrator-if-necessary) based on your Ceph version and desires. + +You must also create a storage class. Ceph CSI is designed to support any arbitrary Ceph cluster, +but we are focused here only on Ceph clusters deployed by Rook. Let's take a look at a portion of +the example storage class found at `deploy/examples/csi/nfs/storageclass.yaml` and break down how +the values are determined. + +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: rook-nfs +provisioner: rook-ceph.nfs.csi.ceph.com # [1] +parameters: + nfsCluster: my-nfs # [2] + server: rook-ceph-nfs-my-nfs-a # [3] + clusterID: rook-ceph # [4] + fsName: myfs # [5] + pool: myfs-replicated # [6] + + # [7] (entire csi.storage.k8s.io/* section immediately below) + csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph + csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph + +# ... some fields omitted ... +``` + +1. `provisioner`: **rook-ceph**.nfs.csi.ceph.com because **rook-ceph** is the namespace where the + CephCluster is installed +2. `nfsCluster`: **my-nfs** because this is the name of the CephNFS +3. `server`: rook-ceph-nfs-**my-nfs**-a because Rook creates this Kubernetes Service for the CephNFS + named **my-nfs** +4. `clusterID`: **rook-ceph** because this is the namespace where the CephCluster is installed +5. `fsName`: **myfs** because this is the name of the CephFilesystem used to back the NFS exports +6. `pool`: **myfs**-**replicated** because **myfs** is the name of the CephFilesystem defined in + `fsName` and because **replicated** is the name of a data pool defined in the CephFilesystem +7. `csi.storage.k8s.io/*`: note that these values are shared with the Ceph CSI CephFS provisioner + +See `deploy/examples/csi/nfs/pvc.yaml` for an example of how to create a PVC that will create an NFS +export. See `deploy/examples/csi/nfs/pod.yaml` for an example of how this can be connected to an +application pod. Both are quite standard. diff --git a/Documentation/ceph-upgrade.md b/Documentation/ceph-upgrade.md index 27ede2d49..e37a0d1d2 100644 --- a/Documentation/ceph-upgrade.md +++ b/Documentation/ceph-upgrade.md @@ -57,6 +57,11 @@ those releases. * Rook no longer deploys Prometheus rules from the operator. If you have been relying on Rook to deploy prometheus rules in the past, please follow the upgrade instructions [below](#prometheus). +* Due to a number of Ceph issues and changes, Rook officially only supports Ceph + v16.2.7 or higher for CephNFS. If you are using an earlier version, upgrade your Ceph version + following the advice given in Rook's + [v1.8 NFS docs](https://rook.github.io/docs/rook/v1.8/ceph-nfs-crd.html). + ## Considerations With this upgrade guide, there are a few notes to consider: @@ -438,8 +443,8 @@ will need two extra steps after the Ceph upgrade is complete. Once upgrade is co 2. Delete the old CephBlockPool that represents the `device_health_metrics` pool. ### **Important consideration for CephNFS users** -Users of CephNFS need to take additional steps to upgrade Ceph versions. Please see the -[NFS documentation](ceph-nfs-crd.md#upgrading-from-ceph-v15-to-v16) for full details. +Ceph Quincy v17.2.0 has a potentially breaking regression with CephNFS. See the NFS documentation's +[known issue](ceph-nfs-crd.md#ceph-v1720) for more detail. ### **Ceph images** diff --git a/deploy/examples/nfs-load-balancer.yaml b/deploy/examples/nfs-load-balancer.yaml new file mode 100644 index 000000000..4b7f82afe --- /dev/null +++ b/deploy/examples/nfs-load-balancer.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: rook-ceph-nfs-my-nfs-load-balancer + namespace: rook-ceph +spec: + ports: + - name: nfs + port: 2049 + type: LoadBalancer + externalTrafficPolicy: Local + selector: + app: rook-ceph-nfs + + # Use the name of the CephNFS here + ceph_nfs: my-nfs + + # It is safest to send clients to a single NFS server instance. Instance "a" always exists. + instance: a diff --git a/deploy/examples/nfs-test.yaml b/deploy/examples/nfs-test.yaml index 87b2433ee..15caf742b 100644 --- a/deploy/examples/nfs-test.yaml +++ b/deploy/examples/nfs-test.yaml @@ -1,19 +1,16 @@ +# This example is for Ceph v16 and above only. If you are using Ceph v15, see Rook v1.8 examples +# from the 'release-1.8' branch + apiVersion: ceph.rook.io/v1 kind: CephNFS metadata: name: my-nfs namespace: rook-ceph # namespace:cluster spec: - # this setting is ignored for Ceph v16+ - rados: - pool: myfs-replicated - namespace: nfs-ns # Settings for the NFS server server: - # the number of active NFS servers active: 1 - # The logging levels: NIV_NULL | NIV_FATAL | NIV_MAJ | NIV_CRIT | NIV_WARN | NIV_EVENT | NIV_INFO | NIV_DEBUG | NIV_MID_DEBUG |NIV_FULL_DEBUG |NB_LOG_LEVEL - logLevel: NIV_INFO + logLevel: NIV_DEBUG --- apiVersion: ceph.rook.io/v1 kind: CephBlockPool diff --git a/deploy/examples/nfs.yaml b/deploy/examples/nfs.yaml index bad0e3297..f7afd0990 100644 --- a/deploy/examples/nfs.yaml +++ b/deploy/examples/nfs.yaml @@ -2,6 +2,9 @@ # Create a Ceph pool with settings for replication in production environments. A minimum of 3 OSDs on # different hosts are required in this example. # kubectl create -f nfs.yaml +# +# This example is for Ceph v16 and above only. If you are using Ceph v15, see Rook v1.8 examples +# from the 'release-1.8' branch ################################################################################################################# apiVersion: ceph.rook.io/v1 @@ -10,49 +13,49 @@ metadata: name: my-nfs namespace: rook-ceph # namespace:cluster spec: - # For Ceph v15, the rados block is required. It is ignored for Ceph v16. - rados: - # RADOS pool where NFS configs are stored. - # In this example the data pool for the "myfs" filesystem is used. - # If using the object store example, the data pool would be "my-store.rgw.buckets.data". - # Note that this has nothing to do with where exported file systems or object stores live. - pool: myfs-replicated - # RADOS namespace where NFS client recovery data is stored in the pool. - namespace: nfs-ns - # Settings for the NFS server server: - # the number of active NFS servers - active: 3 + # The number of active NFS servers + # Rook supports creating more than one active NFS server, but cannot guarantee high availability + active: 1 + # where to run the NFS server placement: - # nodeAffinity: - # requiredDuringSchedulingIgnoredDuringExecution: - # nodeSelectorTerms: - # - matchExpressions: - # - key: role - # operator: In - # values: - # - mds-node - # topologySpreadConstraints: - # tolerations: - # - key: mds-node - # operator: Exists - # podAffinity: - # podAntiAffinity: - # A key/value list of annotations + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: role + # operator: In + # values: + # - nfs-node + # topologySpreadConstraints: + # tolerations: + # - key: nfs-node + # operator: Exists + # podAffinity: + # podAntiAffinity: + + # A key/value list of annotations to apply to NFS server pods annotations: - # key: value - # The requests and limits set here allow the ganesha pod(s) to use half of one CPU core and 1 gigabyte of memory + # key: value + + # A key/value list of labels to apply to NFS server pods + labels: + # key: value + + # Resource equests and limits to apply to NFS server pods resources: - # limits: - # cpu: "500m" - # memory: "1024Mi" - # requests: - # cpu: "500m" - # memory: "1024Mi" - # the priority class to set to influence the scheduler's pod preemption - #priorityClassName: + # limits: + # cpu: "500m" + # memory: "1024Mi" + # requests: + # cpu: "500m" + # memory: "1024Mi" + + # Set priority class to set to influence the scheduler's pod preemption + # priorityClassName: + # The logging levels: NIV_NULL | NIV_FATAL | NIV_MAJ | NIV_CRIT | NIV_WARN | NIV_EVENT | NIV_INFO | NIV_DEBUG | NIV_MID_DEBUG |NIV_FULL_DEBUG |NB_LOG_LEVEL logLevel: NIV_INFO # ---